def get_versioninfo_files(srcdir):
'''
Read the header of all python scripts and returns the highest number of any file as results.
The highest number is determined by LastChangedRevision.
Version number must be in the first 100 lines.
'''
versionnumber = 0
filename_versionnumber = None
# Make a list of all files.
filelist = os.listdir(srcdir)
for filename in filelist:
# Check only the python files.
if (os.path.splitext(filename)[1].upper() == ".PY"):
lines = my_sys.my_readfile(os.path.join(srcdir,filename))
for item in range(len(lines)):
if (item > 100):
break
if ("$LastChangedRevision:" in lines[item]):
# Get version number. Information is given between the $ signs.
ind_end = lines[item].rfind("$")
ind_start = lines[item].find("$")
version = int(lines[item][ind_start+1:ind_end].split(":")[1])
if (version > versionnumber):
versionnumber = version
filename_versionnumber = filename
break
return versionnumber, filename_versionnumber
def get_versioninfo(obj,outputdir):
'''
Get version (svn) information of directories and return a list of strings.
'''
try:
my_sys.my_system(os.path.join("../../tools/sliksvn/bin","svn") + " info " + str(obj) + " 1> " +\
os.path.join(outputdir,"tempuit1.txt") + " 2> " +\
os.path.join(outputdir,"tempuit.err"),fatal=1)
my_sys.my_system(os.path.join("../../tools","grep") + " Rev " +\
os.path.join(outputdir,"tempuit1.txt") + " 1> " +\
os.path.join(outputdir,"tempuit.txt") + " 2> " +\
os.path.join(outputdir,"tempuit.err"),fatal=1)
list = my_sys.my_readfile(os.path.join(outputdir,"tempuit.txt"))
my_sys.my_removefile(os.path.join(outputdir,"tempuit.txt"))
my_sys.my_removefile(os.path.join(outputdir,"tempuit1.txt"))
my_sys.my_removefile(os.path.join(outputdir,"tempuit.err"))
list.insert(0,"Directory " +str(obj))
except (MyError,OSError):
try:
#Try another time but with the assumption that the system has svn installed.
my_sys.my_system("svn info " + str(obj) + " 1> " +\
os.path.join(outputdir,"tempuit1.txt") + " 2> " +\
os.path.join(outputdir,"tempuit.err"),fatal=1)
my_sys.my_system("grep Rev " +\
os.path.join(outputdir,"tempuit1.txt") + " 1> " +\
os.path.join(outputdir,"tempuit.txt") + " 2> " +\
os.path.join(outputdir,"tempuit.err"),fatal=1)
list = my_sys.my_readfile(os.path.join(outputdir,"tempuit.txt"))
my_sys.my_removefile(os.path.join(outputdir,"tempuit.txt"))
my_sys.my_removefile(os.path.join(outputdir,"tempuit1.txt"))
my_sys.my_removefile(os.path.join(outputdir,"tempuit.err"))
list.insert(0,"Directory " +str(obj))
except (MyError,OSError):
list = []
list.append("Directory "+str(obj) + " has no version info.")
# Remove temporary files which are created here
if (os.path.isfile(os.path.join(outputdir,"tempuit.txt"))):
my_sys.my_removefile(os.path.join(outputdir,"tempuit.txt"))
if (os.path.isfile(os.path.join(outputdir,"tempuit1.txt"))):
my_sys.my_removefile(os.path.join(outputdir,"tempuit1.txt"))
if (os.path.isfile(os.path.join(outputdir,"tempuit.err"))):
my_sys.my_removefile(os.path.join(outputdir,"tempuit.err"))
return list
def get_versioninfo(obj,outputdir):
'''
Get version (svn) information of directories and return a list of strings.
'''
try:
my_sys.my_system(os.path.join("../../tools/sliksvn/bin","svn") + " info " + str(obj) + " 1> " +\
os.path.join(outputdir,"tempuit1.txt") + " 2> " +\
os.path.join(outputdir,"tempuit.err"),fatal=1)
my_sys.my_system(os.path.join("../../tools","grep") + " Rev " +\
os.path.join(outputdir,"tempuit1.txt") + " 1> " +\
os.path.join(outputdir,"tempuit.txt") + " 2> " +\
os.path.join(outputdir,"tempuit.err"),fatal=1)
list = my_sys.my_readfile(os.path.join(outputdir,"tempuit.txt"))
my_sys.my_removefile(os.path.join(outputdir,"tempuit.txt"))
my_sys.my_removefile(os.path.join(outputdir,"tempuit1.txt"))
my_sys.my_removefile(os.path.join(outputdir,"tempuit.err"))
list.insert(0,"Directory " +str(obj))
except (MyError,OSError):
try:
#Try another time but with the assumption that the system has svn installed.
my_sys.my_system("svn info " + str(obj) + " 1> " +\
os.path.join(outputdir,"tempuit1.txt") + " 2> " +\
os.path.join(outputdir,"tempuit.err"),fatal=1)
my_sys.my_system("grep Rev " +\
os.path.join(outputdir,"tempuit1.txt") + " 1> " +\
os.path.join(outputdir,"tempuit.txt") + " 2> " +\
os.path.join(outputdir,"tempuit.err"),fatal=1)
list = my_sys.my_readfile(os.path.join(outputdir,"tempuit.txt"))
my_sys.my_removefile(os.path.join(outputdir,"tempuit.txt"))
my_sys.my_removefile(os.path.join(outputdir,"tempuit1.txt"))
my_sys.my_removefile(os.path.join(outputdir,"tempuit.err"))
list.insert(0,"Directory " +str(obj))
except (MyError,OSError):
list = []
list.append("Directory "+str(obj) + " has no version info.")
# Remove temporary files which are created here
if (os.path.isfile(os.path.join(outputdir,"tempuit.txt"))):
my_sys.my_removefile(os.path.join(outputdir,"tempuit.txt"))
if (os.path.isfile(os.path.join(outputdir,"tempuit1.txt"))):
my_sys.my_removefile(os.path.join(outputdir,"tempuit1.txt"))
if (os.path.isfile(os.path.join(outputdir,"tempuit.err"))):
my_sys.my_removefile(os.path.join(outputdir,"tempuit.err"))
return list
def read_mandist_ini(inputdir):
'''
Read the ini file from the mandist directory and stores it in a dictionary.
'''
# Read manure.ini file
lines = my_sys.my_readfile(os.path.join(inputdir,'manure.ini'))
dict_out = {}
for line in lines:
fields = line.split(":")
if (len(fields) != 2):
# Skip this line
continue
else:
dict_out[fields[0].strip()] = fields[1].strip()
return dict_out
def read_file(filename, sep, my_object, keyitem="basinid"):
'''
Read the file and put everything in a dictionary with
int(basinid) as key and a instance of a class (PMouth or NMouth) as value
'''
dict = {}
# Open file and read all the lines
lines = my_sys.my_readfile(filename, my_sys_ldebug=0, fatal=1)
# The header gives the name of the column, which is the same as the item of the class.
header = lines[0].split(sep)
# Find key:
ikey = -1
for item in range(len(header)):
if (header[item] == keyitem):
ikey = item
print(ikey, header[ikey])
for line in range(1, len(lines)):
fields = lines[line].split(sep)
try:
key = int(fields[ikey])
except ValueError:
key = fields[ikey]
dict[key] = my_object.copy()
# Put key element in the class
setattr(dict[key], keyitem, key)
for item in range(min(len(fields), len(header))):
if (item != ikey):
# Put all the values in the Mouth object
try:
dum = int(fields[item])
setattr(dict[key], header[item], int(fields[item]))
except ValueError:
try:
dum = float(fields[item])
setattr(dict[key], header[item], float(fields[item]))
except ValueError:
setattr(dict[key], header[item], str(fields[item]))
return dict
def read_file(filename,sep,my_object,keyitem="basinid"):
'''
Read the file and put everything in a dictionary with int(basinid) as key and a instance of a class (PMouth or NMouth) as value
'''
dict = {}
# Open file and read all the lines
lines = my_sys.my_readfile(filename,my_sys_ldebug=0,fatal=1)
# The header gives the name of the column, which is the same as the item of the class.
header = lines[0].split(sep)
# Find key:
ikey = -1
for item in range(len(header)):
if (header[item] == keyitem):
ikey = item
print ikey, header[ikey]
for line in range(1,len(lines)):
fields = lines[line].split(sep)
try:
key = int(fields[ikey])
except ValueError:
key = fields[ikey]
dict[key] = my_object.copy()
# Put key element in the class
setattr(dict[key],keyitem,key)
for item in range(min(len(fields),len(header))):
if (item != ikey):
# Put all the values in the Mouth object
try:
dum = int(fields[item])
setattr(dict[key],header[item],int(fields[item]))
except ValueError:
try:
dum = float(fields[item])
setattr(dict[key],header[item],float(fields[item]))
except ValueError:
setattr(dict[key],header[item],str(fields[item]))
return dict
def print_versioninfo(filename, fp=None):
'''
if pointer to file (argument fp) is equal to None then info is written to screen.
Else it is written to the file pointer.
Version info is fetched from filename.
'''
labels = [
"$LastChangedDate:", "$LastChangedRevision:", "$LastChangedBy:",
"$HeadURL:"
]
lfound = len(labels) * [False]
# Read filename
lines = my_sys.my_readfile(filename)
# Open version information
if (fp == None):
print("******************************************************")
print(("## Version information from file: " + filename))
else:
fp.write("******************************************************")
fp.write("## Version information from file: " + filename)
# Get version information from file
for line in lines:
for item in range(len(labels)):
if (not lfound[item]):
if (labels[item] in line):
if (fp == None):
print(line)
else:
fp.write(line)
lfound[item] = True
break
# Close version information
if (fp == None):
print("******************************************************")
else:
fp.write("******************************************************")
def read_general_file(filename,sep=";",key=None,out_type="dict"):
'''
Reads a table file.
First line is the header which are used to name the attributes.
Returns a dictionary, with the key found in the first column when key=None. Otherwise the given key will be used.
Each dictionary contains a common class, with attributes given in the header and the values given in the table.
'''
# Read input file.
print "Reading input file: ",filename
dat = my_sys.my_readfile(filename)
# Find header
headline = -1
for linenum in xrange(len(dat)):
# Look whether the first column contains a ! or a #. When it does, then it is comment. Otherwise it is the header.
fields = dat[linenum].split(sep)
if (len(fields) == 0):
continue
if not ("!" in fields[0].lstrip() or "#" in fields[0].lstrip()):
# Header found.
headline = linenum
number_of_fields = len(fields)
# Put header in a list
header = []
for item in range(len(fields)):
header.append(fields[item])
break
if (headline < 0):
raise MyError("Input file " + filename + " is not correct.",\
"Header is not found.")
# Store information from file in the General class
if (out_type == 'dict'):
out = {}
elif(out_type == 'list'):
out = []
else:
raise MyError("Reading input file " + filename + " goes wrong.",\
"Wrong value for argument out_type.")
# Make a standard empty General class for this file.
G = General()
# Make a list of all attributes
key_item = 1000000
for item in range(0,len(header)):
if (len(str(header[item]))!=0):
if (type(out) == dict):
if (key == None and item == 0):
# This is the key of the dictionary
key_item = item
continue
else:
if (str(header[item]) == key):
key_item = item
continue
G.add_item(header[item],None)
if (type(out) == dict and key_item == 1000000):
raise MyError("Error in reading file: " + filename,\
" Given key for dictionay is not found. ",\
" Available keys are: ",\
" ".join(G.names))
for linenum in xrange(headline+1,len(dat)):
if (dat[linenum].startswith("!") or dat[linenum].startswith("#")):
# Comment line, so skip
continue
lineinfo = G.copy()
# Split line into fields
fields = dat[linenum].split(sep)
if (len(fields) == 0):
# Empty line
continue
# Check whether we have the same amount of data fields as header fields.
if (len(fields) != number_of_fields):
raise MyError("Input file " + filename + " is not correct.",\
"Number of headers fields found: " + str(number_of_fields),\
"Number of data fields found: " + str(len(fields)) + " on linenumber: " +str(linenum+1))
for item in range(0,number_of_fields):
try:
if (len(str(header[item]))==0):
# Header is empty, so no data. Skip this column.
continue
elif (type(out) == dict and item == key_item):
dictkey = fields[item]
else:
lineinfo.set_val(header[item],fields[item])
except AttributeError:
raise MyError("Input file " + filename + " is not correct.",\
"Error on linenumber: " + str(linenum+1) + " and column number: " + str(item+1))
if (type(out) == dict):
# Add line to the output dictionary
try:
qq = out[dictkey]
# Key does exist, so information is overwritten.
raise MyError("READ_GENERAL_FILE: Double entry found in file: " + filename,\
" Key which is double: " + str(dictkey))
except KeyError:
# Key does not exist.
if (len(dictkey) > 0):
out[dictkey] = lineinfo
elif (type(out) == list):
# Add line to the output list
out.append(lineinfo)
# Return the list with General objects.
return out
def read_general_file(filename,
sep=";",
key=None,
out_type="dict",
ldebug=False):
'''
Reads a table file.
First line is the header which are used to name the attributes.
Returns a dictionary, with the key found in the first column when key=None. Otherwise the given key will be used.
Each dictionary contains a common class, with attributes given in the header and the values given in the table.
'''
# Read input file.
if (ldebug):
print("Reading input file: ", filename)
dat = my_sys.my_readfile(filename)
# Find header
headline = -1
for linenum in range(len(dat)):
# Look whether the first column contains a ! or a #. When it does, then it is comment. Otherwise it is the header.
fields = dat[linenum].split(sep)
if (len(fields) == 0):
continue
if ((len(fields) == 1) and (len(fields[0].lstrip()) == 0)):
continue
if not ("!" in fields[0].lstrip() or "#" in fields[0].lstrip()):
# Header found.
headline = linenum
number_of_fields = len(fields)
# Put header in a list
header = []
for item in range(len(fields)):
header.append(fields[item])
break
if (headline < 0):
raise MyError("Input file " + filename + " is not correct.",\
"Header is not found.")
# Store information from file in the General class
if (out_type == 'dict'):
out = {}
elif (out_type == 'list'):
out = []
else:
raise MyError("Reading input file " + filename + " goes wrong.",\
"Wrong value for argument out_type.")
# Make a standard empty General class for this file.
G = General()
# Make a list of all attributes
key_item = 1000000
for item in range(0, len(header)):
if (len(str(header[item])) != 0):
if (type(out) == dict):
if (key == None and item == 0):
# This is the key of the dictionary
key_item = item
continue
else:
if (str(header[item]) == key):
key_item = item
continue
G.add_item(header[item], None)
if (type(out) == dict and key_item == 1000000):
raise MyError("Error in reading file: " + filename,\
" Given key for dictionay is not found. ",\
" Available keys are: ",\
" ".join(G.names))
for linenum in range(headline + 1, len(dat)):
lineinfo = G.copy()
# Split line into fields
fields = dat[linenum].split(sep)
if (len(fields) == 0):
# Empty line
continue
# Check whether we have the same amount of data fields as header fields.
if (len(fields) != number_of_fields):
# Try to fix the problem, that field separator is used when it i splaced between quotes.
fields = merge_fields(fields)
if (len(fields) != number_of_fields):
print(fields)
raise MyError("Input file " + filename + " is not correct. ",\
"Number of headers fields found: " + str(number_of_fields),\
". Number of data fields found: " + str(len(fields)) + " on linenumber: " +str(linenum+1)+".")
for item in range(0, number_of_fields):
try:
if (len(str(header[item])) == 0):
# Header is empty, so no data. Skip this column.
continue
elif (type(out) == dict and item == key_item):
dictkey = fields[item]
else:
lineinfo.set_val(header[item], fields[item])
except AttributeError:
raise MyError("Input file " + filename + " is not correct.",\
" Error on linenumber: " + str(linenum+1) + " and column number: " + str(item+1)+".")
if (type(out) == dict):
# Add line to the output dictionary
try:
qq = out[dictkey]
# Key does exist, so information is overwritten.
raise MyError("READ_GENERAL_FILE: Double entry found in file: " + filename + ".",\
" Key which is double: " + str(dictkey))
except KeyError:
# Key does not exist.
if (len(dictkey) > 0):
out[dictkey] = lineinfo
elif (type(out) == list):
# Add line to the output list
out.append(lineinfo)
# Return the list with General objects.
return out
def read_datafile(filename,sep=";",return_value_None=0):
'''
Read datafile and returns a list of entries with Param objects and time dependent TimeParam objects.
@filename: name of file with data
'''
# Read input file.
dat = my_sys.my_readfile(filename)
# Find header on the population sheet
headline = -1
for linenum in xrange(len(dat)):
# Look whether the first column contains a ! or a #. When it does, then it is comment. Otherwise it is the header.
fields = dat[linenum].split(sep)
if (len(fields) == 0):
continue
if not ("!" in fields[0].lstrip() or "#" in fields[0].lstrip()):
# Header found.
headline = linenum
number_of_fields = len(fields)
# Put header in a list
header = []
for item in range(len(fields)):
header.append(fields[item])
break
if (headline < 0):
raise MyError("Input file " + filename + " is not correct.",\
"Header is not found.")
# Store information from file in the Param class
out = []
for linenum in xrange(headline+1,len(dat)):
lineinfo = Param()
# Make param list empty.
lineinfo.param = []
# Split line into fields
fields = dat[linenum].split(sep)
if (len(fields) == 0):
# Empty line
continue
# Check whether we have the same amount of data fields as header fields.
if (len(fields) != number_of_fields):
raise MyError("Input file " + filename + " is not correct.",\
"Number of headers fields found: " + str(number_of_fields),\
"Number of data fields found: " + str(len(fields)) + " on linenumber: " +str(linenum+1))
for item in range(number_of_fields):
try:
lwarning = (linenum == headline+1)
if (len(str(header[item]))==0):
# Header is empty, so no data. Skip this column.
continue
lfound = lineinfo.fill(header[item],fields[item],lwarning=False)
if (not lfound):
# Must be a year or a non existent header
try:
year = float(header[item])
lineinfo.param.append(TimeParam(year=header[item],val=fields[item]))
except (ValueError,TypeError):
if (lwarning):
print "Kolom " + header[item] + " is not used of the input file: " + filename
except AttributeError:
raise MyError("Input file " + filename + " is not correct.",\
"Error on linenumber: " + str(linenum+1) + " and column number: " + str(item+1))
# Add production line to the production database when it has a
# valid isocode
try:
qq = int(float(lineinfo.isocode))
out.append(lineinfo)
except:
pass
# Return the list with Param objects.
return out
