def get_versioninfo_files(srcdir): ''' Read the header of all python scripts and returns the highest number of any file as results. The highest number is determined by LastChangedRevision. Version number must be in the first 100 lines. ''' versionnumber = 0 filename_versionnumber = None # Make a list of all files. filelist = os.listdir(srcdir) for filename in filelist: # Check only the python files. if (os.path.splitext(filename)[1].upper() == ".PY"): lines = my_sys.my_readfile(os.path.join(srcdir,filename)) for item in range(len(lines)): if (item > 100): break if ("$LastChangedRevision:" in lines[item]): # Get version number. Information is given between the $ signs. ind_end = lines[item].rfind("$") ind_start = lines[item].find("$") version = int(lines[item][ind_start+1:ind_end].split(":")[1]) if (version > versionnumber): versionnumber = version filename_versionnumber = filename break return versionnumber, filename_versionnumber
def get_versioninfo(obj,outputdir): ''' Get version (svn) information of directories and return a list of strings. ''' try: my_sys.my_system(os.path.join("../../tools/sliksvn/bin","svn") + " info " + str(obj) + " 1> " +\ os.path.join(outputdir,"tempuit1.txt") + " 2> " +\ os.path.join(outputdir,"tempuit.err"),fatal=1) my_sys.my_system(os.path.join("../../tools","grep") + " Rev " +\ os.path.join(outputdir,"tempuit1.txt") + " 1> " +\ os.path.join(outputdir,"tempuit.txt") + " 2> " +\ os.path.join(outputdir,"tempuit.err"),fatal=1) list = my_sys.my_readfile(os.path.join(outputdir,"tempuit.txt")) my_sys.my_removefile(os.path.join(outputdir,"tempuit.txt")) my_sys.my_removefile(os.path.join(outputdir,"tempuit1.txt")) my_sys.my_removefile(os.path.join(outputdir,"tempuit.err")) list.insert(0,"Directory " +str(obj)) except (MyError,OSError): try: #Try another time but with the assumption that the system has svn installed. my_sys.my_system("svn info " + str(obj) + " 1> " +\ os.path.join(outputdir,"tempuit1.txt") + " 2> " +\ os.path.join(outputdir,"tempuit.err"),fatal=1) my_sys.my_system("grep Rev " +\ os.path.join(outputdir,"tempuit1.txt") + " 1> " +\ os.path.join(outputdir,"tempuit.txt") + " 2> " +\ os.path.join(outputdir,"tempuit.err"),fatal=1) list = my_sys.my_readfile(os.path.join(outputdir,"tempuit.txt")) my_sys.my_removefile(os.path.join(outputdir,"tempuit.txt")) my_sys.my_removefile(os.path.join(outputdir,"tempuit1.txt")) my_sys.my_removefile(os.path.join(outputdir,"tempuit.err")) list.insert(0,"Directory " +str(obj)) except (MyError,OSError): list = [] list.append("Directory "+str(obj) + " has no version info.") # Remove temporary files which are created here if (os.path.isfile(os.path.join(outputdir,"tempuit.txt"))): my_sys.my_removefile(os.path.join(outputdir,"tempuit.txt")) if (os.path.isfile(os.path.join(outputdir,"tempuit1.txt"))): my_sys.my_removefile(os.path.join(outputdir,"tempuit1.txt")) if (os.path.isfile(os.path.join(outputdir,"tempuit.err"))): my_sys.my_removefile(os.path.join(outputdir,"tempuit.err")) return list
def read_mandist_ini(inputdir): ''' Read the ini file from the mandist directory and stores it in a dictionary. ''' # Read manure.ini file lines = my_sys.my_readfile(os.path.join(inputdir,'manure.ini')) dict_out = {} for line in lines: fields = line.split(":") if (len(fields) != 2): # Skip this line continue else: dict_out[fields[0].strip()] = fields[1].strip() return dict_out
def read_file(filename, sep, my_object, keyitem="basinid"): ''' Read the file and put everything in a dictionary with int(basinid) as key and a instance of a class (PMouth or NMouth) as value ''' dict = {} # Open file and read all the lines lines = my_sys.my_readfile(filename, my_sys_ldebug=0, fatal=1) # The header gives the name of the column, which is the same as the item of the class. header = lines[0].split(sep) # Find key: ikey = -1 for item in range(len(header)): if (header[item] == keyitem): ikey = item print(ikey, header[ikey]) for line in range(1, len(lines)): fields = lines[line].split(sep) try: key = int(fields[ikey]) except ValueError: key = fields[ikey] dict[key] = my_object.copy() # Put key element in the class setattr(dict[key], keyitem, key) for item in range(min(len(fields), len(header))): if (item != ikey): # Put all the values in the Mouth object try: dum = int(fields[item]) setattr(dict[key], header[item], int(fields[item])) except ValueError: try: dum = float(fields[item]) setattr(dict[key], header[item], float(fields[item])) except ValueError: setattr(dict[key], header[item], str(fields[item])) return dict
def read_file(filename,sep,my_object,keyitem="basinid"): ''' Read the file and put everything in a dictionary with int(basinid) as key and a instance of a class (PMouth or NMouth) as value ''' dict = {} # Open file and read all the lines lines = my_sys.my_readfile(filename,my_sys_ldebug=0,fatal=1) # The header gives the name of the column, which is the same as the item of the class. header = lines[0].split(sep) # Find key: ikey = -1 for item in range(len(header)): if (header[item] == keyitem): ikey = item print ikey, header[ikey] for line in range(1,len(lines)): fields = lines[line].split(sep) try: key = int(fields[ikey]) except ValueError: key = fields[ikey] dict[key] = my_object.copy() # Put key element in the class setattr(dict[key],keyitem,key) for item in range(min(len(fields),len(header))): if (item != ikey): # Put all the values in the Mouth object try: dum = int(fields[item]) setattr(dict[key],header[item],int(fields[item])) except ValueError: try: dum = float(fields[item]) setattr(dict[key],header[item],float(fields[item])) except ValueError: setattr(dict[key],header[item],str(fields[item])) return dict
def print_versioninfo(filename, fp=None): ''' if pointer to file (argument fp) is equal to None then info is written to screen. Else it is written to the file pointer. Version info is fetched from filename. ''' labels = [ "$LastChangedDate:", "$LastChangedRevision:", "$LastChangedBy:", "$HeadURL:" ] lfound = len(labels) * [False] # Read filename lines = my_sys.my_readfile(filename) # Open version information if (fp == None): print("******************************************************") print(("## Version information from file: " + filename)) else: fp.write("******************************************************") fp.write("## Version information from file: " + filename) # Get version information from file for line in lines: for item in range(len(labels)): if (not lfound[item]): if (labels[item] in line): if (fp == None): print(line) else: fp.write(line) lfound[item] = True break # Close version information if (fp == None): print("******************************************************") else: fp.write("******************************************************")
def read_general_file(filename,sep=";",key=None,out_type="dict"): ''' Reads a table file. First line is the header which are used to name the attributes. Returns a dictionary, with the key found in the first column when key=None. Otherwise the given key will be used. Each dictionary contains a common class, with attributes given in the header and the values given in the table. ''' # Read input file. print "Reading input file: ",filename dat = my_sys.my_readfile(filename) # Find header headline = -1 for linenum in xrange(len(dat)): # Look whether the first column contains a ! or a #. When it does, then it is comment. Otherwise it is the header. fields = dat[linenum].split(sep) if (len(fields) == 0): continue if not ("!" in fields[0].lstrip() or "#" in fields[0].lstrip()): # Header found. headline = linenum number_of_fields = len(fields) # Put header in a list header = [] for item in range(len(fields)): header.append(fields[item]) break if (headline < 0): raise MyError("Input file " + filename + " is not correct.",\ "Header is not found.") # Store information from file in the General class if (out_type == 'dict'): out = {} elif(out_type == 'list'): out = [] else: raise MyError("Reading input file " + filename + " goes wrong.",\ "Wrong value for argument out_type.") # Make a standard empty General class for this file. G = General() # Make a list of all attributes key_item = 1000000 for item in range(0,len(header)): if (len(str(header[item]))!=0): if (type(out) == dict): if (key == None and item == 0): # This is the key of the dictionary key_item = item continue else: if (str(header[item]) == key): key_item = item continue G.add_item(header[item],None) if (type(out) == dict and key_item == 1000000): raise MyError("Error in reading file: " + filename,\ " Given key for dictionay is not found. ",\ " Available keys are: ",\ " ".join(G.names)) for linenum in xrange(headline+1,len(dat)): if (dat[linenum].startswith("!") or dat[linenum].startswith("#")): # Comment line, so skip continue lineinfo = G.copy() # Split line into fields fields = dat[linenum].split(sep) if (len(fields) == 0): # Empty line continue # Check whether we have the same amount of data fields as header fields. if (len(fields) != number_of_fields): raise MyError("Input file " + filename + " is not correct.",\ "Number of headers fields found: " + str(number_of_fields),\ "Number of data fields found: " + str(len(fields)) + " on linenumber: " +str(linenum+1)) for item in range(0,number_of_fields): try: if (len(str(header[item]))==0): # Header is empty, so no data. Skip this column. continue elif (type(out) == dict and item == key_item): dictkey = fields[item] else: lineinfo.set_val(header[item],fields[item]) except AttributeError: raise MyError("Input file " + filename + " is not correct.",\ "Error on linenumber: " + str(linenum+1) + " and column number: " + str(item+1)) if (type(out) == dict): # Add line to the output dictionary try: qq = out[dictkey] # Key does exist, so information is overwritten. raise MyError("READ_GENERAL_FILE: Double entry found in file: " + filename,\ " Key which is double: " + str(dictkey)) except KeyError: # Key does not exist. if (len(dictkey) > 0): out[dictkey] = lineinfo elif (type(out) == list): # Add line to the output list out.append(lineinfo) # Return the list with General objects. return out
def read_general_file(filename, sep=";", key=None, out_type="dict", ldebug=False): ''' Reads a table file. First line is the header which are used to name the attributes. Returns a dictionary, with the key found in the first column when key=None. Otherwise the given key will be used. Each dictionary contains a common class, with attributes given in the header and the values given in the table. ''' # Read input file. if (ldebug): print("Reading input file: ", filename) dat = my_sys.my_readfile(filename) # Find header headline = -1 for linenum in range(len(dat)): # Look whether the first column contains a ! or a #. When it does, then it is comment. Otherwise it is the header. fields = dat[linenum].split(sep) if (len(fields) == 0): continue if ((len(fields) == 1) and (len(fields[0].lstrip()) == 0)): continue if not ("!" in fields[0].lstrip() or "#" in fields[0].lstrip()): # Header found. headline = linenum number_of_fields = len(fields) # Put header in a list header = [] for item in range(len(fields)): header.append(fields[item]) break if (headline < 0): raise MyError("Input file " + filename + " is not correct.",\ "Header is not found.") # Store information from file in the General class if (out_type == 'dict'): out = {} elif (out_type == 'list'): out = [] else: raise MyError("Reading input file " + filename + " goes wrong.",\ "Wrong value for argument out_type.") # Make a standard empty General class for this file. G = General() # Make a list of all attributes key_item = 1000000 for item in range(0, len(header)): if (len(str(header[item])) != 0): if (type(out) == dict): if (key == None and item == 0): # This is the key of the dictionary key_item = item continue else: if (str(header[item]) == key): key_item = item continue G.add_item(header[item], None) if (type(out) == dict and key_item == 1000000): raise MyError("Error in reading file: " + filename,\ " Given key for dictionay is not found. ",\ " Available keys are: ",\ " ".join(G.names)) for linenum in range(headline + 1, len(dat)): lineinfo = G.copy() # Split line into fields fields = dat[linenum].split(sep) if (len(fields) == 0): # Empty line continue # Check whether we have the same amount of data fields as header fields. if (len(fields) != number_of_fields): # Try to fix the problem, that field separator is used when it i splaced between quotes. fields = merge_fields(fields) if (len(fields) != number_of_fields): print(fields) raise MyError("Input file " + filename + " is not correct. ",\ "Number of headers fields found: " + str(number_of_fields),\ ". Number of data fields found: " + str(len(fields)) + " on linenumber: " +str(linenum+1)+".") for item in range(0, number_of_fields): try: if (len(str(header[item])) == 0): # Header is empty, so no data. Skip this column. continue elif (type(out) == dict and item == key_item): dictkey = fields[item] else: lineinfo.set_val(header[item], fields[item]) except AttributeError: raise MyError("Input file " + filename + " is not correct.",\ " Error on linenumber: " + str(linenum+1) + " and column number: " + str(item+1)+".") if (type(out) == dict): # Add line to the output dictionary try: qq = out[dictkey] # Key does exist, so information is overwritten. raise MyError("READ_GENERAL_FILE: Double entry found in file: " + filename + ".",\ " Key which is double: " + str(dictkey)) except KeyError: # Key does not exist. if (len(dictkey) > 0): out[dictkey] = lineinfo elif (type(out) == list): # Add line to the output list out.append(lineinfo) # Return the list with General objects. return out
def read_datafile(filename,sep=";",return_value_None=0): ''' Read datafile and returns a list of entries with Param objects and time dependent TimeParam objects. @filename: name of file with data ''' # Read input file. dat = my_sys.my_readfile(filename) # Find header on the population sheet headline = -1 for linenum in xrange(len(dat)): # Look whether the first column contains a ! or a #. When it does, then it is comment. Otherwise it is the header. fields = dat[linenum].split(sep) if (len(fields) == 0): continue if not ("!" in fields[0].lstrip() or "#" in fields[0].lstrip()): # Header found. headline = linenum number_of_fields = len(fields) # Put header in a list header = [] for item in range(len(fields)): header.append(fields[item]) break if (headline < 0): raise MyError("Input file " + filename + " is not correct.",\ "Header is not found.") # Store information from file in the Param class out = [] for linenum in xrange(headline+1,len(dat)): lineinfo = Param() # Make param list empty. lineinfo.param = [] # Split line into fields fields = dat[linenum].split(sep) if (len(fields) == 0): # Empty line continue # Check whether we have the same amount of data fields as header fields. if (len(fields) != number_of_fields): raise MyError("Input file " + filename + " is not correct.",\ "Number of headers fields found: " + str(number_of_fields),\ "Number of data fields found: " + str(len(fields)) + " on linenumber: " +str(linenum+1)) for item in range(number_of_fields): try: lwarning = (linenum == headline+1) if (len(str(header[item]))==0): # Header is empty, so no data. Skip this column. continue lfound = lineinfo.fill(header[item],fields[item],lwarning=False) if (not lfound): # Must be a year or a non existent header try: year = float(header[item]) lineinfo.param.append(TimeParam(year=header[item],val=fields[item])) except (ValueError,TypeError): if (lwarning): print "Kolom " + header[item] + " is not used of the input file: " + filename except AttributeError: raise MyError("Input file " + filename + " is not correct.",\ "Error on linenumber: " + str(linenum+1) + " and column number: " + str(item+1)) # Add production line to the production database when it has a # valid isocode try: qq = int(float(lineinfo.isocode)) out.append(lineinfo) except: pass # Return the list with Param objects. return out