def readFiles (path, pNum): #Set path based on particle number pPath = path + "/particles/particleNum" + str(pNum) + ".mel" rManPath = path + "/rmanCurves/rManCurve" # Open file for reading based on path, read content filePart = open(pPath, 'r') fileLength = len(filePart.readlines()) # Initialize counter for getLine command line = 0 # Initialize array for storing point positions readPart = [] # Cycle through individual particle files # Read each line to get position information for n in range(fileLength): line = n + 1 readPart.append(linecache.getline(pPath, line)) # Remove end of line character from entry if readPart[n][-1] == '\n': readPart[n] = readPart[n][:-1] # Clear cache for future values linecache.clearcache() # Call on drawRmanCurves to generate curves, pass array drawRmanCurves(readPart, pNum, rManPath)
def get_nth_line(f,n): # with open(f) as ff: line = linecache.getline(f,n) [key, gt, conf] = line.split("\t") if n % 10 == 0 and n > 10: linecache.clearcache() return key,gt,conf.strip()
def Merger(dir,oldname,newname): linecache.clearcache() # num_lines = file_len(dir+oldname) # print num_lines lineNo = 1 text=[] while(lineNo < 265000): # print lineNo line1 = linecache.getline(dir+oldname, lineNo) line2 = linecache.getline(dir+oldname, lineNo+1) # if len(line1)<2 and len(line2)<2 and len(line3)<2 and len(line4)<2: # break # print line2 if len(line1.split(','))>3: if len(line2.split(','))<3: line1=line1.strip()+line2 text.append(line1) lineNo=lineNo+2 else: text.append(line1) # text.append(line2) lineNo=lineNo+1 else: # print "1"+text[-1] text[-1]=(text[-1].strip()) # print "2"+text[-1] text[-1]=text[-1]+line1 # print "3"+text[-1] lineNo=lineNo+1 for item in text: new_file = open(newname,'a+') new_file.write(item)
def generate_async(self, **kwargs): namespace = self._get_namespace(**kwargs) exec_in(self.compiled, namespace) execute = gen.coroutine(namespace["_tt_execute"]) linecache.clearcache() result = yield execute() return result
def split_csv_texts(input_fpath, output1_fpath, output2_fpath, split_percent): # Check the parameters if (not os.path.exists(input_fpath)) or (split_percent < 0) or (split_percent > 1): print "Error: wrong input arguments." return # Open the files input_file = open(input_fpath, "r") output1_file = open(output1_fpath,"w") output2_file = open(output2_fpath,"w") # Get number of lines input_number = len(input_file.readlines()) output1_number = int(input_number * split_percent) print input_fpath, ":", input_number, "texts" print output1_fpath, ":", output1_number, "texts" print output2_fpath, ":", input_number - output1_number, "texts" # Get a random sample of line numbers input_lines = range(1, input_number + 1) output1_lines = random.sample(input_lines, output1_number) # Save the lines in two separate files for line in input_lines: if(line in output1_lines): output1_file.write(linecache.getline(input_fpath, line)) else: output2_file.write(linecache.getline(input_fpath, line)) linecache.clearcache() input_file.close() output1_file.close() output2_file.close()
def ReadTailLines(sFileName, iNum=5): # 读取文件尾部几行 import linecache linecache.checkcache(sFileName) lsText = linecache.getlines(sFileName)[-iNum:] linecache.clearcache() return lsText
def Get_nRxn(nRxn_file,Z,N,nFission_filename): # this goes through each neutron reaction file, ID's reaction types, and daughter products. nRxnTypes, fission products and yields returned as dictionaries and nRxns not tracked are returned as a list. nRxnType = {} FissP_ID = {} FissP_Yield = {} Rxns_not_Tracked = [] with open(nRxn_file,'r') as file1: tmp = linecache.getline(nRxn_file,5,None) #pulls line 2 from current_file into cache a = tmp.split() skip = a[4] # grabs the number of records of descriptive text (NWD in ENDF7.1 manual) for i in range(int(skip)+5): #skip over the descriptive records... file1.__next__() for line in file1: # ...and go straight to the (MF, MT, NC, MOD) descriptive lines a = line.split() if (a[0]=='3'): # ID's which neutron reactions are tabulated. if (a[1]=='18'): #or (a[1]=='19') or (a[1]=='20') or (a[1]=='21') or (a[1]=='38') fissType, FissP_ID, FissP_Yield = trls.MT_fission(int(a[1]),nFission_filename) else: Ztmp,Ntmp,RxnType,noTrack = trls.MT(int(a[1]),Z,N) # using base Z & N values of isotope, get new Z & N for isotope post neutron reaction if noTrack == ' ': pass else: Rxns_not_Tracked.append(noTrack) # append which reactions are tabulated but not tracked if RxnType == ' ': pass else: nRxnType[RxnType] = trls.NuclideIdentifier(Ztmp,Ntmp) elif ('1 099999' in line): # if you hit the end of the (MF, MT, NC, MOD) records, quit out of function and move onto next nuclide file linecache.clearcache() return(nRxnType,FissP_ID, FissP_Yield,Rxns_not_Tracked)
def __getitem__(self, key): if key in self.deprecation_messages: import warnings import linecache # DeprecationWarnings are ignored by default. Clear the filter so # they are not: previous_warning_filters = warnings.filters[:] try: warnings.resetwarnings() # Hacky stuff to get it to work from within execfile() with # correct line data: linecache.clearcache() caller = sys._getframe(1) globals = caller.f_globals lineno = caller.f_lineno module = globals['__name__'] filename = globals.get('__file__') fnl = filename.lower() if fnl.endswith((".pyc", ".pyo")): filename = filename[:-1] message = self.deprecation_messages[key] warnings.warn_explicit(message, DeprecationWarning, filename, lineno, module) finally: # Restore the warnings filter: warnings.filters[:] = previous_warning_filters return dict.__getitem__(self, key)
def RunNoahModel(self,strtxtForcePathIn): #1 run model fileset = formic.FileSet(include="**/*.txt", directory=strtxtForcePathIn) nFile = 0 # print process file ID os.chdir('F:\\worktemp\\Permafrost(Change)\\Run(S)\\') outputPath = None for file_name in fileset: nFile+=1 print "Current file is : " + file_name + "; It is the " + str(nFile) print "################################################################" # get array of lines if outputPath is None: with open(file_name, 'r') as txtFile: updateLine = 5 # from 1 ~ not 0 lineTxt = linecache.getline(file_name, updateLine) lineTxtS = lineTxt.split("=") outputPath = lineTxtS[1].strip()[1:-1] linecache.clearcache() #very important InputFileDir,InputFile = os.path.split(file_name) filename, file_extension = os.path.splitext(InputFile) items = [outputPath,filename,".nc"] outputFileName = ''.join(items) if os.path.isfile(outputFileName) and os.access(outputFileName, os.R_OK): continue #... updateOutputDirLine command = 'simple_driver.exe '+ file_name subprocess.call(command, shell=True) #call 7z command
def check_words(files,text,counter,result_list): #three different scopes, #1.pick a tv show and a season and search through them for file in files: fp = open (file, 'r') for i, line in enumerate(fp): if re.search(r'\b%s\b'%(text),line.lower()): #'\bfoo\b' matches 'foo', 'foo.', '(foo)', 'bar foo baz' but not 'foobar' or 'foo3'. counter = counter + 1 filename = os.path.basename(fp.name) lines_presented = -5 result_list.append( "<Number %d match>\n<In line %d in the file>\n<In file '%s'>" % (counter,i,filename), ) while lines_presented <= 5: if re.search(r'^\w',linecache.getline(file, i+lines_presented)): if re.search(r'^\D',linecache.getline(file, i+lines_presented)): result_list.append(linecache.getline(file, i+lines_presented)) if re.search(r'-->',linecache.getline(file, i+lines_presented)): result_list.append(linecache.getline(file, i+lines_presented)) lines_presented = lines_presented + 1 result_list.append( "------------------------------------------------------\n" ) linecache.clearcache() fp.close() for item in result_list: print item print "Total %d matches for \"%s\"" % (counter,text) print_or_not(counter,text,result_list)
def load_module(self, fullname): if self.mod is None: mod = self.mod = imp.new_module(self.name) else: mod = self.mod log_buffer = [] mod.logging = mod.logger = logging.Logger(self.name) mod.logger.addHandler(SaveLogHandler(log_buffer)) mod.log_buffer = log_buffer mod.__file__ = '<%s>' % self.name mod.__loader__ = self mod.__project__ = self.project mod.__package__ = '' code = self.get_code(fullname) exec code in mod.__dict__ linecache.clearcache() if '__handler_cls__' not in mod.__dict__: for each in mod.__dict__.values(): if inspect.isclass(each) and each is not base_handler.BaseHandler \ and issubclass(each, base_handler.BaseHandler): mod.__dict__['__handler_cls__'] = each return mod
def setUp(self): linecache.clearcache() zipimport._zip_directory_cache.clear() self.path = sys.path[:] self.meta_path = sys.meta_path[:] self.path_hooks = sys.path_hooks[:] sys.path_importer_cache.clear()
def update(self): '''Write the in-memory tnodes to file.''' with open(self.tdict_path, 'r+') as tdict_file: tdata = tdict_file.readlines() line_num = len(tdata) + 1 # Get updated / new terms terms = {} for term in self: if (term in self) and (self[term]['tnode'] is not None): terms[term] = self[term] # Modify existing term data in tdata for term in terms.keys(): if self[term]['loc'] is not None: tdata[self[term]['loc']-1] = ( '%s\n' % self[term]['tnode'].serialize() ) del terms[term] # Write changes to file with open(self.tdict_path, 'w+') as tdict_file: # Write modified tdict_file.writelines(tdata) # Add new for term in terms: tdict_file.write('%s\n' % self[term]['tnode'].serialize()) self[term]['loc'] = line_num line_num += 1 linecache.clearcache()
def get_crds_from_gms(logfile): unit = 'angs' #Coordinates will use angs unit in default ln = 1 fp = open(logfile, 'r') for line in fp: if ' ATOM ATOMIC COORDINATES (BOHR)' in line: bln = ln + 2 unit = 'bohr' #Means using Bohr unit elif ' ATOM ATOMIC COORDINATES (ANGS' in line: bln = ln + 2 unit = 'angs' elif ' INTERNUCLEAR DISTANCES' in line: eln = ln - 2 ln = ln + 1 fp.close() crdl = [] for i in range(bln, eln+1): line = linecache.getline(logfile, i) line = line.strip('\n') line = line.split() if unit == 'bohr': crdl.append(float(line[2])) crdl.append(float(line[3])) crdl.append(float(line[4])) elif unit == 'angs': crdl.append(float(line[2])/B_TO_A) crdl.append(float(line[3])/B_TO_A) crdl.append(float(line[4])/B_TO_A) linecache.clearcache() return crdl
def dash_R_cleanup(fs, ps, pic): import gc, copy_reg import _strptime, linecache, dircache import urlparse, urllib, urllib2, mimetypes, doctest import struct, filecmp from distutils.dir_util import _path_created # Restore some original values. warnings.filters[:] = fs copy_reg.dispatch_table.clear() copy_reg.dispatch_table.update(ps) sys.path_importer_cache.clear() sys.path_importer_cache.update(pic) # Clear assorted module caches. _path_created.clear() re.purge() _strptime._regex_cache.clear() urlparse.clear_cache() urllib.urlcleanup() urllib2.install_opener(None) dircache.reset() linecache.clearcache() mimetypes._default_mime_types() struct._cache.clear() filecmp._cache.clear() doctest.master = None # Collect cyclic trash. gc.collect()
def generate(self, **kwargs): """Generate this template with the given arguments.""" namespace = { "escape": escape.xhtml_escape, "xhtml_escape": escape.xhtml_escape, "url_escape": escape.url_escape, "json_encode": escape.json_encode, "squeeze": escape.squeeze, "linkify": escape.linkify, "datetime": datetime, "_utf8": escape.utf8, # for internal use "_string_types": (unicode, bytes_type), # __name__ and __loader__ allow the traceback mechanism to find # the generated source code. "__name__": self.name.replace('.', '_'), "__loader__": ObjectDict(get_source=lambda name: self.code), } namespace.update(self.namespace) namespace.update(kwargs) exec self.compiled in namespace execute = namespace["_execute"] # Clear the traceback module's cache of source data now that # we've generated a new template (mainly for this module's # unittests, where different tests reuse the same name). linecache.clearcache() try: return execute() except Exception: formatted_code = _format_code(self.code).rstrip() logging.error("%s code:\n%s", self.name, formatted_code) raise
def _load_plugins(self, plugins, first_run=False): # A dictionary of loaded plugins loaded_plugins = {} # A list of plugins that failed to load failed_plugins = [] # Turn this into a list if it isn't one if isinstance(plugins, basestring): plugins = [plugins] linecache.clearcache() for plugin in plugins: loaded_plugins[plugin] = {} # Import each plugin with a custom _import_module function. try: module = self._import_module(self.loaded_plugins[plugin]['module'] if plugin in self.loaded_plugins else plugin) except Exception, e: print >> sys.stderr, "ERROR: Could not load plugin module: %s (%s)" % (plugin, e) failed_plugins.append(plugin) continue # Import each config with the same _import_module function. try: self.config[plugin] = self._import_module(self.config[plugin] if plugin in self.config else plugin, config=True) except ImportError: self.config[plugin] = None except Exception, e: self.config[plugin] = None print >> sys.stderr, "WARNING: Could not load plugin config: %s (%s)" % (plugin, e)
def test_lazycache_provide_after_failed_lookup(self): linecache.clearcache() lines = linecache.getlines(NONEXISTENT_FILENAME, globals()) linecache.clearcache() linecache.getlines(NONEXISTENT_FILENAME) linecache.lazycache(NONEXISTENT_FILENAME, globals()) self.assertEqual(lines, linecache.updatecache(NONEXISTENT_FILENAME))
def litterRowsTail(): # 文件行数 fileRowNums = util.countFileRows(filepath) # 开始行数 start = max(fileRowNums - 10, 0) + 1 while True: try: import linecache, time line_str = linecache.getline(filepath, start) if line_str: print line_str, start = start + 1 time.sleep(0.01) else: linecache.clearcache() except ImportError: print '' print 'error!' sys.exit(1) except KeyboardInterrupt: print '' print 'quit!' sys.exit(1)
def test_lazycache_already_cached(self): linecache.clearcache() lines = linecache.getlines(NONEXISTENT_FILENAME, globals()) self.assertEqual( False, linecache.lazycache(NONEXISTENT_FILENAME, globals())) self.assertEqual(4, len(linecache.cache[NONEXISTENT_FILENAME]))
def GetActiveFileName(bAutoSave = 1): """Gets the file name for the active frame, saving it if necessary. Returns None if it cant be found, or raises KeyboardInterrupt. """ pathName = None active = GetActiveView() if active is None: return None try: doc = active.GetDocument() pathName = doc.GetPathName() if bAutoSave and \ (len(pathName)>0 or \ doc.GetTitle()[:8]=="Untitled" or \ doc.GetTitle()[:6]=="Script"): # if not a special purpose window if doc.IsModified(): try: doc.OnSaveDocument(pathName) pathName = doc.GetPathName() # clear the linecache buffer linecache.clearcache() except win32ui.error: raise KeyboardInterrupt except (win32ui.error, AttributeError): pass if not pathName: return None return pathName
def get_file_metadata(fname): import linecache metadata = {} grid_metadata = linecache.getline(fname, 1).split() n_grids = int(grid_metadata[0]) # Trajectory metadata present after grid metadata trajectory_metadata = linecache.getline(fname, n_grids+2).split() metadata['n_trajectories'] = int(trajectory_metadata[0]) # Get starting lat/lon/alt of each trajectory metadata['trajectories'] = {} for t in range(metadata['n_trajectories']): tstart = linecache.getline(fname, n_grids+3+t).split() # Save trajectories according to numbering in file metadata['trajectories'][t+1] = (tstart[-3], tstart[-2], tstart[-1]) metadata['data_start'] = n_grids + metadata['n_trajectories'] + 3 # Get custom variable names variable_names = linecache.getline(fname, metadata['data_start']).split()[2:] metadata['labels'] = hysplit_default_var + variable_names metadata['custom_labels'] = variable_names linecache.clearcache() return metadata
def check_line_exist(self, line_no): if linecache.getline(self.filename, line_no) == "": linecache.clearcache() return False else: linecache.clearcache() return True
def load_module(self, fullname): if self.mod is None: mod = self.mod = imp.new_module(self.name) else: mod = self.mod log_buffer = [] mod.logging = mod.logger = logging.Logger(self.name) handler = SaveLogHandler(log_buffer) handler.setFormatter(LogFormatter(color=False)) mod.logger.addHandler(handler) mod.log_buffer = log_buffer mod.__file__ = '<%s>' % self.name mod.__loader__ = self mod.__project__ = self.project mod.__package__ = '' code = self.get_code(fullname) six.exec_(code, mod.__dict__) linecache.clearcache() if '__handler_cls__' not in mod.__dict__: BaseHandler = mod.__dict__.get('BaseHandler', base_handler.BaseHandler) for each in list(six.itervalues(mod.__dict__)): if inspect.isclass(each) and each is not BaseHandler \ and issubclass(each, BaseHandler): mod.__dict__['__handler_cls__'] = each return mod
def forwardon(self): import linecache import Global print "actionforward" if Global.NAV_P == Global.NAV_NO: print "no method forward!" else: linecache.clearcache() Global.NAV_P += 1 i = 2*Global.NAV_P classname = linecache.getline('1.txt',i) pathindex = linecache.getline('1.txt',i-1) pathindex = pathindex.strip() classname = linecache.getline('1.txt',i) classname = classname[:-1] print "get from 1.txt" print pathindex print classname method = self.path2method[pathindex] print "the type of method is %s, the type of classname is %s" %(type(method),type(classname)) self.displayMethod(method,classname) Global.currentclass = classname QMessageBox.information(self ,'Current Class', classname) Global.currentmethod = method QMessageBox.information(self ,'Current method', method) print method print classname
def __dreload(mdl): """Recursively reload modules.""" nonlocal _s nonlocal _base_path nonlocal _reloaded_files for name in dir(mdl): mm = getattr(mdl, name) if type(mm) is not ModuleType: if (hasattr(mm, '__module__') and mm.__module__ is not None): mm = sys.modules[mm.__module__] if (not hasattr(mm, '__file__') or not os.path.realpath(mm.__file__).startswith(_base_path) or mm.__name__[0] == '_' or '._' in mm.__name__ or mm.__name__ in _s or any(mm.__name__.startswith(bln) for bln in dreload_blacklist_starting_with)): continue _s.add(mm.__name__) __dreload(mm) _reloaded_files.append(os.path.realpath(mdl.__file__)) if not just_visit: log.info('reloading: ' + str(mdl.__name__)) linecache.clearcache() imp.reload(mdl) else: log.info('visiting: ' + str(mdl.__name__))
def _pl_eox(): try: import linecache linecache.clearcache() except (ImportError, AttributeError): # ignore if linecache doesn't exist pass
def generate(self, **kwargs): """Generate this template with the given arguments.""" namespace = { "escape": escape.xhtml_escape, "xhtml_escape": escape.xhtml_escape, "url_escape": escape.url_escape, "json_encode": escape.json_encode, "squeeze": escape.squeeze, "linkify": escape.linkify, "datetime": datetime, "_tt_utf8": escape.utf8, # for internal use "_tt_string_types": (unicode_type, bytes_type), # __name__ and __loader__ allow the traceback mechanism to find # the generated source code. "__name__": self.name.replace(".", "_"), "__loader__": ObjectDict(get_source=lambda name: self.code), } namespace.update(self.namespace) namespace.update(kwargs) exec_in(self.compiled, namespace) execute = namespace["_tt_execute"] # Clear the traceback module's cache of source data now that # we've generated a new template (mainly for this module's # unittests, where different tests reuse the same name). linecache.clearcache() return execute()
def readWellData(self, dataType): #print("readWellRates") start = self.readDataFor("WELLS") end = self.readDataFor("ENDWELLS") #print ("start %s end %s" % (start, end)) wellData = numpy.zeros(self.particles) for lineCounter in range(start+2, end+1): #print("here") line = linecache.getline(self.myDataFile, lineCounter) #print("line %s start %s end %s lineCounter %s" % (line, start, end, lineCounter)) propertyList = line.split() if dataType=="FLOWRATE": if propertyList[1] == "FLOWRATE": wellData[int(propertyList[0])] = propertyList[2] if dataType=="SKIN": if propertyList[1] == "SKIN": wellData[int(propertyList[0])] = propertyList[2] if dataType=="RW": if propertyList[1] == "RW": wellData[int(propertyList[0])] = propertyList[2] if dataType=="PERFTHICK": if propertyList[1] == "PERFTHICK": wellData[int(propertyList[0])] = propertyList[2] linecache.clearcache() return wellData
def backon(self): import linecache import Global print "actionbackactionback" if Global.NAV_P == 0 or Global.NAV_P == 1: print "no history!" QMessageBox.warning(self ,'warning', 'no history!') else: linecache.clearcache() Global.NAV_P -= 1 i = 2*Global.NAV_P print "NAV_P=" print Global.NAV_P print "NAV_NO=" print Global.NAV_NO # method = self.path2method[1] pathindex = linecache.getline('1.txt',i-1) pathindex = pathindex.strip() classname = linecache.getline('1.txt',i) classname = classname[:-1] Global.current print "get from 1.txt" print pathindex print classname method = self.path2method[pathindex] Global.currentmethod = method QMessageBox.information(self ,'Current Method', method) print "the type of method is %s, the type of classname is %s" %(type(method),type(classname)) self.displayMethod(method,classname) Global.currentclass = classname QMessageBox.information(self ,'Current Class', classname) Global.currentmethod = method QMessageBox.information(self ,'Current method', method) print method print classname
def test_lazy_lines(self): linecache.clearcache() f = traceback.FrameSummary("f", 1, "dummy", lookup_line=False) self.assertEqual(None, f._line) linecache.lazycache("f", globals()) self.assertEqual('"""Test cases for traceback module"""', f.line)
def __del__(self): self.epicsfile.close() linecache.clearcache()
def show_func(filename, start_lineno, func_name, timings, unit, output_unit=None, stream=None, stripzeros=False): """ Show results for a single function. """ if stream is None: stream = sys.stdout template = '%6s %9s %12s %8s %8s %-s' d = {} total_time = 0.0 linenos = [] for lineno, nhits, time in timings: total_time += time linenos.append(lineno) if stripzeros and total_time == 0: return if output_unit is None: output_unit = unit scalar = unit / output_unit stream.write("Total time: %g s\n" % (total_time * unit)) if os.path.exists(filename) or filename.startswith("<ipython-input-"): stream.write("File: %s\n" % filename) stream.write("Function: %s at line %s\n" % (func_name, start_lineno)) if os.path.exists(filename): # Clear the cache to ensure that we get up-to-date results. linecache.clearcache() all_lines = linecache.getlines(filename) sublines = inspect.getblock(all_lines[start_lineno - 1:]) else: stream.write("\n") stream.write("Could not find file %s\n" % filename) stream.write( "Are you sure you are running this program from the same directory\n" ) stream.write("that you ran the profiler from?\n") stream.write("Continuing without the function's contents.\n") # Fake empty lines so we can see the timings, if not the code. nlines = max(linenos) - min(min(linenos), start_lineno) + 1 sublines = [''] * nlines for lineno, nhits, time in timings: d[lineno] = (nhits, '%5.1f' % (time * scalar), '%5.1f' % (float(time) * scalar / nhits), '%5.1f' % (100 * time / total_time)) linenos = range(start_lineno, start_lineno + len(sublines)) empty = ('', '', '', '') header = template % ('Line #', 'Hits', 'Time', 'Per Hit', '% Time', 'Line Contents') stream.write("\n") stream.write(header) stream.write("\n") stream.write('=' * len(header)) stream.write("\n") for lineno, line in zip(linenos, sublines): nhits, time, per_hit, percent = d.get(lineno, empty) txt = template % (lineno, nhits, time, per_hit, percent, line.rstrip('\n').rstrip('\r')) stream.write(txt) stream.write("\n") stream.write("\n")
def setUp(self): # We're reusing the zip archive path, so we must clear the # cached directory info and linecache linecache.clearcache() zipimport._zip_directory_cache.clear() ImportHooksBaseTestCase.setUp(self)
def show_func(self, filename, start_lineno, func_name, timings, stream=None, stripzeros=False): """ Show results for a single function. """ if stream is None: stream = sys.stdout template = '%6s %8s %8s %-s' d = {} total_hits = 0.0 linenos = [] for lineno, nhits in six.iteritems(timings): total_hits += nhits linenos.append(lineno) if stripzeros and total_hits == 0: return stream.write("Total hits: %g s\n" % total_hits) if os.path.exists(filename) or filename.startswith("<ipython-input-"): stream.write("File: %s\n" % filename) stream.write("Function: %s at line %s\n" % (func_name, start_lineno)) if os.path.exists(filename): # Clear the cache to ensure that we get up-to-date results. linecache.clearcache() all_lines = linecache.getlines(filename) try: sublines = inspect.getblock(all_lines[start_lineno - 1:]) except tokenize.TokenError: # inspect.getblock fails on multi line dictionary comprehensions sublines = all_lines[start_lineno - 1:max(linenos)] else: stream.write("\n") stream.write("Could not find file %s\n" % filename) stream.write( "Are you sure you are running this program from the same directory\n" ) stream.write("that you ran the profiler from?\n") stream.write("Continuing without the function's contents.\n") # Fake empty lines so we can see the timings, if not the code. nlines = max(linenos) - min(min(linenos), start_lineno) + 1 sublines = [''] * nlines for lineno, nhits in six.iteritems(timings): d[lineno] = (nhits, '%5.1f' % (100 * nhits / total_hits)) linenos = range(start_lineno, start_lineno + len(sublines)) empty = ('', '') header = template % ('Line #', 'Hits', '% Hits', 'Line Contents') stream.write("\n") stream.write(header) stream.write("\n") stream.write('=' * len(header)) stream.write("\n") for lineno, line in zip(linenos, sublines): nhits, percent = d.get(lineno, empty) txt = template % (lineno, nhits, percent, line.rstrip('\n').rstrip('\r')) stream.write(txt) stream.write("\n") stream.write("\n")
def readCSV(File, Line): lines = linecache.getline(File, Line) linecache.clearcache() data = lines.strip().split(";") return data
import sys import os import linecache file_relative_dir=sys.argv[1] linenumber=int(sys.argv[2]) while linenumber: text=linecache.getline(file_relative_dir,linenumber) if 'class' in text: escape_index=text.find(' ') brackets_index=text.find('(') class_name=text[escape_index:brackets_index] linecache.clearcache() break else: linenumber-=1 commands=[ '/Library/Frameworks/Python.framework/Versions/3.8/bin/python3', # my anaconda dir -> to replace it with yours 'manim.py', file_relative_dir, class_name ] os.system(' '.join(commands))
def test_lazycache_no_globals(self): lines = linecache.getlines(FILENAME) linecache.clearcache() self.assertEqual(False, linecache.lazycache(FILENAME, None)) self.assertEqual(lines, linecache.getlines(FILENAME))
def setUp(self): linecache.clearcache() zipimport._zip_directory_cache.clear() ImportHooksBaseTestCase.setUp(self)
def dash_R_cleanup(fs, ps, pic, zdc, abcs): import gc, copyreg import _strptime, linecache import urllib.parse, urllib.request, mimetypes, doctest import struct, filecmp, collections.abc from distutils.dir_util import _path_created from weakref import WeakSet # Clear the warnings registry, so they can be displayed again for mod in sys.modules.values(): if hasattr(mod, '__warningregistry__'): del mod.__warningregistry__ # Restore some original values. warnings.filters[:] = fs copyreg.dispatch_table.clear() copyreg.dispatch_table.update(ps) sys.path_importer_cache.clear() sys.path_importer_cache.update(pic) try: import zipimport except ImportError: pass # Run unmodified on platforms without zipimport support else: zipimport._zip_directory_cache.clear() zipimport._zip_directory_cache.update(zdc) # clear type cache sys._clear_type_cache() # Clear ABC registries, restoring previously saved ABC registries. for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: if not isabstract(abc): continue for obj in abc.__subclasses__() + [abc]: obj._abc_registry = abcs.get(obj, WeakSet()).copy() obj._abc_cache.clear() obj._abc_negative_cache.clear() # Flush standard output, so that buffered data is sent to the OS and # associated Python objects are reclaimed. for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): if stream is not None: stream.flush() # Clear assorted module caches. _path_created.clear() re.purge() _strptime._regex_cache.clear() urllib.parse.clear_cache() urllib.request.urlcleanup() linecache.clearcache() mimetypes._default_mime_types() filecmp._cache.clear() struct._clearcache() doctest.master = None try: import ctypes except ImportError: # Don't worry about resetting the cache if ctypes is not supported pass else: ctypes._reset_cache() # Collect cyclic trash and read memory statistics immediately after. func1 = sys.getallocatedblocks func2 = sys.gettotalrefcount gc.collect() return func1(), func2(), fd_count()
async def processPacket(self, packetData): print('Processing packet') # Packet data comes in as hex, need to convet to binary to parse binaryDataLength = len(packetData) * 4 print('bin data len' + str(binaryDataLength)) binaryData = format(int(packetData,16), 'b').zfill(binaryDataLength) secretKey = b'SECRETKEY' if binaryData[0:8] == '00000000': # This is a TX Schedule packet. print("TX Schedule Packet") # Get window start delta T windowStartBinary = binaryData[8:40] windowStartDecimal = int(windowStartBinary,2) print("Window start in seconds: ", windowStartDecimal) # Get window duration windowDurationBinary = binaryData[40:56] windowDurationDecimal = int(windowDurationBinary,2) print("Window duration in seconds: ", windowDurationDecimal) # Get data type dataTypeBinary = binaryData[56:64] dataTypeDecimal = int(dataTypeBinary,2) print("Data type: ", dataTypeDecimal) # Get picture number pictureNumberBinary = binaryData[64:80] pictureNumberDecimal = int(pictureNumberBinary,2) print("Picture number: ", pictureNumberDecimal) #Get index print("index will be:", int(binaryData[80:112], 2)) if int(binaryData[80:112], 2) == 0: index = -1 else: index = int(binaryData[80:112], 2) print("Indexing to:", index) # Get the appended hash - it is a 16 byte (128 bit) value receivedHash = binaryData[112:] print("Received Hash: ", receivedHash) # Generated hash from received data generatedHash = hmac.new(secretKey, bytes(binaryData[0:112], 'utf-8'), digestmod=hashlib.md5) generatedHashHex = generatedHash.hexdigest() generatedHashLength = len(generatedHashHex) * 4 generatedHashBinary = format(int(generatedHashHex,16), 'b').zfill(generatedHashLength) print("Generated hash: ", generatedHashBinary) if receivedHash == generatedHashBinary: print("Hashes match! Writing window") self.writeTXWindow(windowStartDecimal, windowDurationDecimal, dataTypeDecimal, pictureNumberDecimal, index) else: print("Hashes do not match, will not save window!") else: # This is a command packet print("Command packet") # Validate HMAC Hash # Note, hash is 16 bytes (128 bits). Command packet is 1 byte (8 bits) receivedHash = binaryData[-128:] print("Received Hash: ", receivedHash) # Generated hash from received data generatedHash = hmac.new(secretKey, bytes(binaryData[0:-128], 'utf-8'), digestmod=hashlib.md5) generatedHashHex = generatedHash.hexdigest() generatedHashLength = len(generatedHashHex) * 4 generatedHashBinary = format(int(generatedHashHex,16), 'b').zfill(generatedHashLength) print("Generated hash: ", generatedHashBinary) if receivedHash == generatedHashBinary: print("Hashes match! Executing commands") if binaryData[8:16] == '00000000': # Turn off Transmitter print("Turn off Transmissions") self.disableTransmissions() else: #Turn on Transmitter print("Turn on Transmitter") self.enableTransmissions() if binaryData[16:24] == '00000000': # DO NOT Clear TX Schedule and Progress print("Do NOT Clear TX Schedule and Progress") else: # Clear TX Schedule & Progress print("Clear TX Schedule and Progress") self.clearTXFile() self.clearTXProgress() if binaryData[24:32] == '00000000': # Do not take picture print("Do not take picture") else: # Take picture print("Take picture") self.__cam.takePicture() if binaryData[32:40] == '00000000': # Do not deploy boom print("Do not deploy boom") else: # Deploy boom print("Deploy boom") deployer = boomDeployer.BoomDeployer() await deployer.deploy() if binaryData[40:48] == '00000000': # Do not reboot print("Do not reboot") else: #Send reboot command to Beetle print("Reboot") os.system("sudo reboot") fileChecker.checkFile(self.__bootRecordsPath) reboots = int(linecache.getline(self.__bootRecordsPath, 1)) skip = int(linecache.getline(self.__bootRecordsPath, 3)) linecache.clearcache() if skip != 4: if binaryData[56:64] == '00000000': #Chose whether or not to skip to post boom deploy print("Running flight logic normally.") else: print("Skipping to post boom deploy.") bootRecords = open(self.__bootRecordsPath, 'w+') bootRecords.write(str(reboots) + "\n1\n4\n") bootRecords.close() os.system("sudo reboot") if binaryData[64:72] == '00000000': # Keep Pictures print("Keeping Pictures") else: #Delete Pictures print("Deleting Pictures") self.deletePictures() if binaryData[72:80] == '00000000': # Keep Data print("Keeping Data") else: # Delete Data print("Deleting Data") self.deleteData() #Beacon commands have been removed and now send directly to the transceiver else: print("Hashes do not match, will not execute commands!")
def calculateFitness(self, population): rraCMCTaskSetFilePath = 'H:\\Northwestern-RIC\\Modeling\\OpenSim\\GenericFiles\\gait2392_RRA_CMCTaskSet.xml' dom = parse(rraCMCTaskSetFilePath) cmcJointElements = dom.getElementsByTagName('CMC_Joint') fitnesses = [] logReport = [] for individual in population: logReportLine = [ str(self.currentGen), str(population.index(individual) + 1) ] chromosomes = individual.split('-') # Write weights to XML for i in range(len(chromosomes)): index = self.decode(chromosomes[i]) value = self.variableValues[index] name = self.variableNames[i] logReportLine.append(value) for elem in cmcJointElements: if elem.getAttribute('name') == name: elem.getElementsByTagName( 'weight')[0].firstChild.nodeValue = ' ' + value break xmlString = dom.toxml('UTF-8') xmlFile = open(rraCMCTaskSetFilePath, 'w') xmlFile.write(xmlString) xmlFile.close() # Specify trial subDir = self.subDir trialName = self.trialName # Run simulation subprocess.Popen((subDir + 'Run.bat'), shell=True, cwd=subDir) startTime = time.time() while True: # Check for simulation result file if os.access(subDir + trialName + '_RRA_controls.xml', os.F_OK): break # Timeout after 2 minutes if file doesn't exist yet (simulation probably failed) elif (time.time() - startTime) > 120: break # Wait else: time.sleep(5) # Process simulation output try: weightSum = 0 # Residuals txtline = linecache.getline( subDir + trialName + '_RRA_Actuation_force.sto', 23) headerList = txtline.rstrip().split('\t') residualNames = headerList[1:7] linecache.clearcache() residuals = numpy.loadtxt(subDir + trialName + '_RRA_Actuation_force.sto', skiprows=23, usecols=(1, 2, 3, 4, 5, 6)) maxResiduals = residuals.__abs__().max(0) rmsResiduals = numpy.sqrt( numpy.sum(numpy.square(residuals), 0) / numpy.size(residuals, 0)) ###residuals = numpy.genfromtxt(subDir+trialName+'_RRA_Actuation_force.sto',dtype=float,skip_header=22,usecols=(1,2,3,4,5,6),names=True) ###residualNames = residuals.dtype.names ###residualsArray = residuals.view((float, len(residuals.dtype.names))) for k in range(len(residualNames)): if residualNames[k] == 'FX' or residualNames[ k] == 'FY' or residualNames[k] == 'FZ': # Max if maxResiduals[k] <= 10: weightSum += 0.02 elif maxResiduals[k] <= 25: weightSum += 0.25 else: weightSum += 10 # RMS if rmsResiduals[k] <= 5: weightSum += 0.02 elif rmsResiduals[k] <= 10: weightSum += 0.25 else: weightSum += 10 elif residualNames[k] == 'MX' or residualNames[ k] == 'MY' or residualNames[k] == 'MZ': # Max if maxResiduals[k] <= 50: weightSum += 0.02 elif maxResiduals[k] <= 75: weightSum += 0.25 else: weightSum += 10 # RMS if rmsResiduals[k] <= 30: weightSum += 0.02 elif rmsResiduals[k] <= 50: weightSum += 0.25 else: weightSum += 10 # Position Errors txtline = linecache.getline( subDir + trialName + '_RRA_pErr.sto', 7) headerList = txtline.rstrip().split('\t') posErrNames = headerList[1:] linecache.clearcache() # (Column indices to remove -- after removing first column) removeIndices = [ posErrNames.index('subtalar_angle_r'), posErrNames.index('mtp_angle_r'), posErrNames.index('subtalar_angle_l'), posErrNames.index('mtp_angle_l') ] removeIndices.reverse() for k in removeIndices: del posErrNames[k] posErrors = numpy.loadtxt(subDir + trialName + '_RRA_pErr.sto', skiprows=7) posErrors = numpy.delete(posErrors, 0, 1) for k in removeIndices: posErrors = numpy.delete(posErrors, k, 1) maxPosErr = posErrors.__abs__().max(0) rmsPosErr = numpy.sqrt( numpy.sum(numpy.square(posErrors), 0) / numpy.size(posErrors, 0)) for k in range(len(posErrNames)): if posErrNames[k] == 'pelivs_tx' or posErrNames[ k] == 'pelvis_ty' or posErrNames[k] == 'pelvis_tz': # Convert from m to cm maxPosErr[k] *= 100 rmsPosErr[k] *= 100 # Max if maxPosErr[k] <= 2: weightSum += 0.02 elif maxPosErr[k] <= 5: weightSum += 0.25 elif maxPosErr[k] <= 15: weightSum += 10 else: weightSum += 25 # RMS if rmsPosErr[k] <= 2: weightSum += 0.02 elif rmsPosErr[k] <= 4: weightSum += 0.25 else: weightSum += 10 else: # Convert from rad to deg maxPosErr[k] *= 180 / numpy.pi rmsPosErr[k] *= 180 / numpy.pi # Max if maxPosErr[k] <= 2: weightSum += 0.02 elif maxPosErr[k] <= 5: weightSum += 0.25 else: weightSum += 10 # RMS if rmsPosErr[k] <= 2: weightSum += 0.02 elif rmsPosErr[k] <= 5: weightSum += 0.25 else: weightSum += 10 # Update log maxResiduals = maxResiduals.tolist() rmsResiduals = rmsResiduals.tolist() maxPosErr = maxPosErr.tolist() rmsPosErr = rmsPosErr.tolist() # FX, FY, FZ for k in range(3): logReportLine.append(str(maxResiduals[k])) for k in range(3): logReportLine.append(str(rmsResiduals[k])) # MX, MY, MZ for k in range(3, 6): logReportLine.append(str(maxResiduals[k])) for k in range(3, 6): logReportLine.append(str(rmsResiduals[k])) # Translations for k in range(3): logReportLine.append(str(maxPosErr[k])) for k in range(3): logReportLine.append(str(rmsPosErr[k])) # Angles for k in range(3, len(maxPosErr)): logReportLine.append(str(maxPosErr[k])) for k in range(3, len(rmsPosErr)): logReportLine.append(str(rmsPosErr[k])) # Individual fitness fitnesses.append(1 / weightSum) logReportLine.append(str(1 / weightSum)) # Remove simulation output files try: os.remove(subDir + trialName + '_RRA.log') os.remove(subDir + trialName + '_AdjustedCOM.osim') os.remove(subDir + 'err.log') os.remove(subDir + 'out.log') except: pass rraSpecifiers = ('Actuation_force.sto', 'Actuation_power.sto', 'Actuation_speed.sto', 'BodyKinematics_acc_global.sto', 'BodyKinematics_pos_global.sto', 'BodyKinematics_vel_global.sto', 'Kinematics_dudt.sto', 'Kinematics_q.sto', 'Kinematics_u.sto', 'avgResiduals.txt', 'controls.sto', 'controls.xml', 'pErr.sto', 'states.sto') for fspec in rraSpecifiers: try: os.remove(subDir + trialName + '_RRA_' + fspec) except: break # If simulation failed except: fitnesses.append(0.0000000001) # Append to log logReport.append('\t'.join(logReportLine) + '\n') # Write to output file logFile = open(self.log, 'a') logFile.writelines(logReport) logFile.close() # Summary summaryFile = open(self.summary, 'a') summaryFile.write('\t'.join([ str(self.currentGen), str(numpy.mean(fitnesses)), str(numpy.max(fitnesses)), time.strftime('%H:%M:%S', time.localtime()) ]) + '\n') summaryFile.close() # Print to screen print('Generation ' + str(self.currentGen) + ' is complete. Max fitness is ' + str(numpy.max(fitnesses)) + '.') # Return return fitnesses
def fuse_duplicate_clients_to_file(file_paths): ###EXPERIMENT#####-----> ip_alla = list() ip_list = list() for clients_file in file_paths: ip_list.append(set()) file = open(clients_file, 'r') for f in file: f_json = json.loads(f) ip_list[-1].add(f_json['ipaddress']) for ip in ip_list[-1]: ip_alla.append(ip) ip_alla.sort() ip_dublicates = set() last_ip = '' for ip in ip_alla: if ip in last_ip: ip_dublicates.add(ip) last_ip = ip for ip in ip_dublicates: print(ip) ####<-------EXPERIMENT for clients_file in file_paths: dub_list = list() # get all ip addresses of client_objects numbers_lines = 0 file = open(clients_file, 'r') ip_list = list() for f in file: f_json = json.loads(f) ip_list.append(f_json['ipaddress']) numbers_lines += 1 file.close() file = None added_list = list() for number in range(1, numbers_lines): if number not in added_list: added_list.append(number) line = linecache.getline(clients_file, number) dub_list.append(list()) dub_list[-1].append(number) line_json = json.loads(line) ip_main = line_json['ipaddress'] count = 0 for ip in ip_list: count += 1 #If matches add that number to the dublist. if (ip_main in ip) and (count not in added_list): dub_list[-1].append(count) added_list.append(count) #fuse all cow_client dublicates and send to file file = open(clients_file + '.clog', 'w') file.write('') file.close() for dub in dub_list: if len(dub) != 0: count = 0 while count < len(dub): if count == 0: line = linecache.getline(clients_file, dub[count]) client_new = make_client_from_text(line) elif count > 0: line = linecache.getline(clients_file, dub[count]) client_tmp = make_client_from_text(line) sessions = client_tmp.get_sessions() for ses in sessions: client_new.add_new_session(ses) count += 1 with open(clients_file + '.clog', 'a') as file: jdump = json.dumps(client_new, default=lambda x: x.__dict__) file.write(jdump + '\n') file.close() file = None linecache.clearcache() #delete tmp file os.remove(clients_file)
def read_frcmod_file(frcmodf): #Get range of each parameter part in the frcmodf rfrcmodf = open(frcmodf, 'r') cardlist = ['MASS', 'BOND', 'ANGL', 'DIHE', 'IMPR', 'NONB'] lnlist1 = [] lnlist2 = [] ln = 1 for line in rfrcmodf: for card in cardlist: if line[0:len(card)] == card: lnlist1.append(card) lnlist2.append(ln) ln = ln + 1 tln = ln - 1 rfrcmodf.close() lndict = {} for i in range(0, len(lnlist1)-1): lndict[lnlist1[i]] = (lnlist2[i]+1, lnlist2[i+1]) lndict[lnlist1[-1]] = (lnlist2[-1]+1, tln) #Read the parameter into dicts massparms = {} bondparms = {} angparms = {} dihparms = {} impparms = {} nbparms = {} for i in list(lndict.keys()): if i == "MASS": for j in range(lndict[i][0],lndict[i][1]): line = linecache.getline(frcmodf, j) massparms = readmass(massparms, line) elif i == "BOND": for j in range(lndict[i][0], lndict[i][1]): line = linecache.getline(frcmodf, j) bondparms = readbond(bondparms, line) elif i == "ANGL": for j in range(lndict[i][0], lndict[i][1]): line = linecache.getline(frcmodf, j) angparms = readang(angparms, line) elif i == "DIHE": for j in range(lndict[i][0], lndict[i][1]): line = linecache.getline(frcmodf, j) dihparms = readdih(dihparms, line) elif i == "IMPR": for j in range(lndict[i][0], lndict[i][1]): line = linecache.getline(frcmodf, j) impparms = readimp(impparms, line) elif i == "NONB": for j in range(lndict[i][0], lndict[i][1]): line = linecache.getline(frcmodf, j) nbparms = readnb(nbparms, line) linecache.clearcache() parmdict = Parms(massparms, bondparms, angparms, dihparms, impparms, nbparms) return parmdict
''' #---enter # of nodes in mesh nnodes = 165290 #----nodes of interest node_list = [60643, 63290] #---number of recordings tsteps = 289 #directories to loop through if desired (one only should work fine) dirs = ['P:\\02\\LakeOntario\\Storm\\20080203','P:\\02\\LakeOntario\\Storm\\19701120', 'P:\\02\\LakeOntario\\Storm\\19710210','P:\\02\\LakeOntario\\Storm\\19731103', 'P:\\02\\LakeOntario\\Storm\\19710124'] clearcache() a = datetime.now() for i, d in enumerate(dirs): print d os.chdir(d) #----Read fort.63 clearcache() for cur_node in node_list: name = str(cur_node) with open('fort'+str(i) +'_'+ name +'.txt', 'w') as f: for j in range(cur_node,(tsteps-1)*nnodes,nnodes+1): if j == cur_node: f.write(getline('fort.63',j+3).rstrip() + '\n') else: #print i, getline('fort.63',i+3).rstrip()
def get_parm_dict(ffchoice, gaff, frcmodfs): #------------------------------------------------------------------------- #1. Read the parm*.dat file #------------------------------------------------------------------------- global add if ffchoice == 'ff94': #parm94 parmf = add + 'parm94.dat' lndict = {'MASS': (2, 57), 'BOND': (59, 142), 'ANGL': (143, 334), 'DIHE': (335, 416), 'IMPR': (417, 448), 'EQUA': (451, 453), 'NONB': (455, 489)} elif ffchoice in ['ff99', 'ff99SB', 'ff03', 'ff03.r1']: parmf = add + 'parm99.dat' lndict = {'MASS': (2, 66), 'BOND': (68, 184), 'ANGL': (185, 466), 'DIHE': (467, 631), 'IMPR': (632, 670), 'EQUA': (673, 675), 'NONB': (677, 719)} elif ffchoice in ['ff10', 'ff12SB', 'ff14SB']: parmf = add + 'parm10.dat' lndict = {'MASS': (2, 65), 'BOND': (67, 218), 'ANGL': (219, 619), 'DIHE': (620, 895), 'IMPR': (896, 955), 'EQUA': (958, 960), 'NONB': (962, 1001)} #define the parameter dicts massparms = {} bondparms = {} angparms = {} dihparms = {} impparms = {} eqdict = {} nbparms = {} for i in list(lndict.keys()): if i == "MASS": for j in range(lndict[i][0],lndict[i][1]): line = linecache.getline(parmf, j) massparms = readmass(massparms, line) elif i == "BOND": for j in range(lndict[i][0], lndict[i][1]): line = linecache.getline(parmf, j) bondparms = readbond(bondparms, line) elif i == "ANGL": for j in range(lndict[i][0], lndict[i][1]): line = linecache.getline(parmf, j) angparms = readang(angparms, line) elif i == "DIHE": for j in range(lndict[i][0], lndict[i][1]): line = linecache.getline(parmf, j) dihparms = readdih(dihparms, line) elif i == "IMPR": for j in range(lndict[i][0], lndict[i][1]): line = linecache.getline(parmf, j) impparms = readimp(impparms, line) elif i == "EQUA": for j in range(lndict[i][0], lndict[i][1]): line = linecache.getline(parmf, j) eqdict = readeqnb(eqdict, line) elif i == "NONB": for j in range(lndict[i][0], lndict[i][1]): line = linecache.getline(parmf, j) nbparms = readnb(nbparms, line) linecache.clearcache() #Deal with the equil atoms for i in list(eqdict.keys()): for j in eqdict[i]: if len(i) == 1: nbparms[j] = nbparms[i + ' '] else: nbparms[j] = nbparms[i] parmdict = Parms(massparms, bondparms, angparms, dihparms, impparms, nbparms) #------------------------------------------------------------------------- #2. Read the frcmod file for each force field #------------------------------------------------------------------------- if ffchoice in ['ff03', 'ff03.r1', 'ff99SB', 'ff12SB', 'ff14SB']: if ffchoice in ['ff03', 'ff03.r1']: #Year: 2003 parmf1 = add + 'frcmod.ff03' elif ffchoice == 'ff99SB': #Year: 2006 parmf1 = add + 'frcmod.ff99SB' elif ffchoice == 'ff12SB': #Year: 2012 parmf1 = add + 'frcmod.ff12SB' elif ffchoice == 'ff14SB': #Year: 2014 parmf1 = add + 'frcmod.ff14SB' parmdict1 = read_frcmod_file(parmf1) parmdict.combine(parmdict1) #------------------------------------------------------------------------- #3. GAFF #------------------------------------------------------------------------- if gaff == 1: parmf2 = add + 'gaff.dat' gaff_vsinfo = linecache.getline(parmf2, 1) if 'Version 1.7, Nov 2013' in gaff_vsinfo: lndict2 = {'MASS': (2, 73), 'BOND': (75, 882), 'ANGL': (883, 5131), 'DIHE': (5132, 5848), 'IMPR': (5849, 5887), 'NONB': (5892, 5963)} elif 'Version 1.8, Mar 2015' in gaff_vsinfo: lndict2 = {'MASS': (2, 73), 'BOND': (75, 907), 'ANGL': (908, 5526), 'DIHE': (5527, 6267), 'IMPR': (6268, 6306), 'NONB': (6311, 6382)} massparms2 = {} bondparms2 = {} angparms2 = {} dihparms2 = {} impparms2 = {} nbparms2 = {} for i in list(lndict2.keys()): if i == "MASS": for j in range(lndict2[i][0], lndict2[i][1]): line = linecache.getline(parmf2, j) massparms2 = readmass(massparms2, line) elif i == "BOND": for j in range(lndict2[i][0], lndict2[i][1]): line = linecache.getline(parmf2, j) bondparms2 = readbond(bondparms2, line) elif i == "ANGL": for j in range(lndict2[i][0], lndict2[i][1]): line = linecache.getline(parmf2, j) angparms2 = readang(angparms2, line) elif i == "DIHE": for j in range(lndict2[i][0], lndict2[i][1]): line = linecache.getline(parmf2, j) dihparms2 = readgaffdih(dihparms2, line) elif i == "IMPR": for j in range(lndict2[i][0], lndict2[i][1]): line = linecache.getline(parmf2, j) impparms2 = readimp(impparms2, line) elif i == "NONB": for j in range(lndict2[i][0], lndict2[i][1]): line = linecache.getline(parmf2, j) nbparms2 = readnb(nbparms2, line) linecache.clearcache() parmdict2 = Parms(massparms2, bondparms2, angparms2, dihparms2, impparms2, nbparms2) parmdict.combine(parmdict2) #------------------------------------------------------------------------- #4. Additional frcmod file #------------------------------------------------------------------------- for i in frcmodfs: parmdict3 = read_frcmod_file(i) parmdict.combine(parmdict3) return parmdict
def data_processor(case): # First, we call the structure of the model: table1 = xlrd.open_workbook( "0_Model Structure/" + str(structural_list[case])) # works for all scenarios print("$$$$$$") print(str(structural_list[case])) print("$$$$$$") sheet_sets_structure = table1.sheet_by_index(0) sheet_params_structure = table1.sheet_by_index(1) sheet_vars_structure = table1.sheet_by_index(2) # S_DICT_sets_structure = { 'set': [], 'initial': [], 'number_of_elements': [], 'elements_list': [] } for col in range(1, 11 + 1): S_DICT_sets_structure['set'].append( sheet_sets_structure.cell_value(rowx=0, colx=col)) S_DICT_sets_structure['initial'].append( sheet_sets_structure.cell_value(rowx=1, colx=col)) S_DICT_sets_structure['number_of_elements'].append( int(sheet_sets_structure.cell_value(rowx=2, colx=col))) # element_number = int(sheet_sets_structure.cell_value(rowx=2, colx=col)) this_elements_list = [] if element_number > 0: for n in range(1, element_number + 1): this_elements_list.append( sheet_sets_structure.cell_value(rowx=2 + n, colx=col)) S_DICT_sets_structure['elements_list'].append(this_elements_list) # S_DICT_params_structure = { 'category': [], 'parameter': [], 'number_of_elements': [], 'index_list': [] } param_category_list = [] for col in range(1, 30 + 1): if str(sheet_params_structure.cell_value(rowx=0, colx=col)) != '': param_category_list.append( sheet_params_structure.cell_value(rowx=0, colx=col)) S_DICT_params_structure['category'].append(param_category_list[-1]) S_DICT_params_structure['parameter'].append( sheet_params_structure.cell_value(rowx=1, colx=col)) S_DICT_params_structure['number_of_elements'].append( int(sheet_params_structure.cell_value(rowx=2, colx=col))) # index_number = int(sheet_params_structure.cell_value(rowx=2, colx=col)) this_index_list = [] for n in range(1, index_number + 1): this_index_list.append( sheet_params_structure.cell_value(rowx=2 + n, colx=col)) S_DICT_params_structure['index_list'].append(this_index_list) # S_DICT_vars_structure = { 'category': [], 'variable': [], 'number_of_elements': [], 'index_list': [] } var_category_list = [] for col in range(1, 43 + 1): if str(sheet_vars_structure.cell_value(rowx=0, colx=col)) != '': var_category_list.append( sheet_vars_structure.cell_value(rowx=0, colx=col)) S_DICT_vars_structure['category'].append(var_category_list[-1]) S_DICT_vars_structure['variable'].append( sheet_vars_structure.cell_value(rowx=1, colx=col)) S_DICT_vars_structure['number_of_elements'].append( int(sheet_vars_structure.cell_value(rowx=2, colx=col))) # index_number = int(sheet_vars_structure.cell_value(rowx=2, colx=col)) this_index_list = [] for n in range(1, index_number + 1): this_index_list.append( sheet_vars_structure.cell_value(rowx=2 + n, colx=col)) S_DICT_vars_structure['index_list'].append(this_index_list) #-------------------------------------------# # all_vars = S_DICT_vars_structure['variable'] # all_vars_output_dict = [{} for e in range(len(txt_list))] # output_header = [ 'Run.ID', 'Fuel', 'Fuel.DESCRIPTION', 'Technology', 'Technology.DESCRIPTION', 'Emission', 'Emission.DESCRIPTION', 'Year' ] #-------------------------------------------------------# for v in range(len(all_vars)): output_header.append(all_vars[v]) #-------------------------------------------------------# vars_as_appear = [] case_name = txt_list[case].replace('.txt', '') # data_name = './2_Scenarios_Outputs/' + str(case_name) + '_output.txt' print(data_name) # ini_line = 5741012 + 2 end_line = 11438802 # for n in range(ini_line, end_line, 2): structure_line_raw = linecache.getline(data_name, n) structure_list_raw = structure_line_raw.split(' ') # print( structure_line_raw, data_name, n, ini_line, end_line ) # time.sleep(20) structure_list_raw_2 = [ s_line for s_line in structure_list_raw if s_line != '' ] structure_line = structure_list_raw_2[1] structure_list = structure_line.split('[') the_variable = structure_list[0] # if the_variable in all_vars: set_list = structure_list[1].replace(']', '').replace('\n', '').split(',') #--% index = S_DICT_vars_structure['variable'].index(the_variable) this_variable_indices = S_DICT_vars_structure['index_list'][index] # if 'y' in this_variable_indices: data_line = linecache.getline(data_name, n + 1) data_line_list_raw = data_line.split(' ') data_line_list = [ data_cell for data_cell in data_line_list_raw if data_cell != '' ] useful_data_cell = data_line_list[1] #--% if useful_data_cell != '0': # if the_variable not in vars_as_appear: vars_as_appear.append(the_variable) all_vars_output_dict[case].update({the_variable: {}}) all_vars_output_dict[case][the_variable].update( {the_variable: []}) # for n in range(len(this_variable_indices)): all_vars_output_dict[case][the_variable].update( {this_variable_indices[n]: []}) #--% this_variable = vars_as_appear[-1] all_vars_output_dict[case][this_variable][ this_variable].append(useful_data_cell) for n in range(len(this_variable_indices)): all_vars_output_dict[case][the_variable][ this_variable_indices[n]].append(set_list[n]) # # elif 'y' not in this_variable_indices: data_line = linecache.getline(data_name, n + 1) data_line_list_raw = data_line.split(' ') data_line_list = [ data_cell for data_cell in data_line_list_raw if data_cell != '' ] useful_data_cell = data_line_list[1] #--% if useful_data_cell != '0': # if the_variable not in vars_as_appear: vars_as_appear.append(the_variable) all_vars_output_dict[case].update({the_variable: {}}) all_vars_output_dict[case][the_variable].update( {the_variable: []}) # for n in range(len(this_variable_indices)): all_vars_output_dict[case][the_variable].update( {this_variable_indices[n]: []}) #--% this_variable = vars_as_appear[-1] all_vars_output_dict[case][this_variable][ this_variable].append(useful_data_cell) for n in range(len(this_variable_indices)): all_vars_output_dict[case][the_variable][ this_variable_indices[n]].append(set_list[n]) #--% else: pass # linecache.clearcache() #%% output_adress = './2_Scenarios_Outputs' combination_list = [] # [fuel, technology, emission, year] data_row_list = [] for var in range(len(vars_as_appear)): this_variable = vars_as_appear[var] this_var_dict = all_vars_output_dict[case][this_variable] #--% index = S_DICT_vars_structure['variable'].index(this_variable) this_variable_indices = S_DICT_vars_structure['index_list'][index] #--------------------------------------# for k in range(len(this_var_dict[this_variable])): this_combination = [] # if 'f' in this_variable_indices: this_combination.append(this_var_dict['f'][k]) else: this_combination.append('') # if 't' in this_variable_indices: this_combination.append(this_var_dict['t'][k]) else: this_combination.append('') # if 'e' in this_variable_indices: this_combination.append(this_var_dict['e'][k]) else: this_combination.append('') # if 'l' in this_variable_indices: this_combination.append('') else: this_combination.append('') # if 'y' in this_variable_indices: this_combination.append(this_var_dict['y'][k]) else: this_combination.append('') # if this_combination not in combination_list: combination_list.append(this_combination) data_row = ['' for n in range(len(output_header))] # print('check', len(data_row), len(run_id) ) data_row[0] = 0 data_row[1] = this_combination[0] data_row[3] = this_combination[1] data_row[5] = this_combination[2] # data_row[7] = this_combination[3] data_row[7] = this_combination[4] # var_position_index = output_header.index(this_variable) data_row[var_position_index] = this_var_dict[this_variable][k] data_row_list.append(data_row) else: ref_index = combination_list.index(this_combination) this_data_row = deepcopy(data_row_list[ref_index]) # var_position_index = output_header.index(this_variable) # if 'l' in this_variable_indices: # if str(this_data_row[var_position_index]) != '' and str( this_var_dict[this_variable][k]) != '' and ( 'Rate' not in this_variable): this_data_row[var_position_index] = str( float(this_data_row[var_position_index]) + float(this_var_dict[this_variable][k])) elif str(this_data_row[var_position_index]) == '' and str( this_var_dict[this_variable][k]) != '': this_data_row[var_position_index] = str( float(this_var_dict[this_variable][k])) elif str(this_data_row[var_position_index]) != '' and str( this_var_dict[this_variable][k]) == '': pass else: this_data_row[var_position_index] = this_var_dict[ this_variable][k] # data_row_list[ref_index] = deepcopy(this_data_row) # non_year_combination_list = [] non_year_combination_list_years = [] for n in range(len(combination_list)): this_combination = combination_list[n] this_non_year_combination = [ this_combination[0], this_combination[1], this_combination[2] ] if this_combination[ 4] != '' and this_non_year_combination not in non_year_combination_list: non_year_combination_list.append(this_non_year_combination) non_year_combination_list_years.append([this_combination[4]]) elif this_combination[ 4] != '' and this_non_year_combination in non_year_combination_list: non_year_combination_list_years[non_year_combination_list.index( this_non_year_combination)].append(this_combination[4]) # # complete_years = [ '2015', '2019', '2025', '2030', '2035', '2040', '2045', '2050' ] complete_years = [str(a_year) for a_year in range(2015, 2050 + 1, 1)] # for n in range(len(non_year_combination_list)): if len(non_year_combination_list_years[n]) != len(complete_years): # this_existing_combination = non_year_combination_list[n] # print('flag 1', this_existing_combination ) this_existing_combination.append('') # print('flag 2', this_existing_combination ) this_existing_combination.append( non_year_combination_list_years[n][0]) # print('flag 3', this_existing_combination ) ref_index = combination_list.index(this_existing_combination) this_existing_data_row = deepcopy(data_row_list[ref_index]) # for n2 in range(len(complete_years)): # if complete_years[n2] not in non_year_combination_list_years[ n]: # data_row = ['' for n in range(len(output_header))] data_row[0] = 0 data_row[1] = non_year_combination_list[n][0] data_row[3] = non_year_combination_list[n][1] data_row[5] = non_year_combination_list[n][2] data_row[7] = complete_years[n2] # for n3 in range(len(vars_as_appear)): this_variable = vars_as_appear[n3] this_var_dict = all_vars_output_dict[case][ this_variable] index = S_DICT_vars_structure['variable'].index( this_variable) this_variable_indices = S_DICT_vars_structure[ 'index_list'][index] # var_position_index = output_header.index(this_variable) # print_true = False # if ( 'f' in this_variable_indices and str(non_year_combination_list[n][0]) != '' ): # or ( 'f' not in this_variable_indices and str(non_year_combination_list[n][0]) == '' ): print_true = True else: pass # if ( 't' in this_variable_indices and str(non_year_combination_list[n][1]) != '' ): # or ( 't' not in this_variable_indices and str(non_year_combination_list[n][1]) == '' ): print_true = True else: pass # if ( 'e' in this_variable_indices and str(non_year_combination_list[n][2]) != '' ): # or ( 'e' not in this_variable_indices and str(non_year_combination_list[n][2]) == '' ): print_true = True else: pass # if 'y' in this_variable_indices and ( str(this_existing_data_row[var_position_index]) != '') and print_true == True: data_row[var_position_index] = '0' # else: pass # data_row_list.append(data_row) with open(output_adress + '/' + case_name + '_Output' + '.csv', 'w', newline='') as csvfile: csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL) csvwriter.writerow(output_header) for n in range(len(data_row_list)): csvwriter.writerow(data_row_list[n]) gc.collect(generation=2) time.sleep(0.05)
def readLine(path, line=0): re_Line = linecache.getline(path, line) linecache.clearcache() return re_Line
def facesConstructor(facesFile,boundaryFile,ownerFile,neighbourFile,points): # construct an array of object faces faces = [] # read how many faces n = int(linecache.getline(facesFile,19)) print 'there is ' + str(n) + ' faces' for i in range(0,n): line = linecache.getline(facesFile,21+i) newFace = face(i) # construct new face object. for j in range(0,4): pointNumber = int(line[2:-2].split()[j]) #print pointNumber newPoint = points[pointNumber] #print newPoint.x,newPoint.y,newPoint.z,newFace.faceID newFace.addPoint(newPoint) faces.append(newFace) linecache.clearcache() # read owner file for i in range(0,n): line = linecache.getline(ownerFile,22+i) cellID = int(line) faces[i].ownerCell(cellID) # read neighbour file n = int(linecache.getline(neighbourFile,20)) print 'there is ' + str(n) + ' faces in neighbour file' for i in range(0,n): line = linecache.getline(neighbourFile,22+i) cellID = int(line) faces[i].neighbourCell(cellID) # now, add boundary information into face in faces. n = int(linecache.getline(boundaryFile,18)) print 'there is ' + str(n) + ' boundaries' # patches contains all patches as a patch object patches = [] lineIndicator = 20 # for i in range(0,n): newPatch = patch(i) line = linecache.getline(boundaryFile,lineIndicator) # adding patch name newPatch.patchName = line.strip() lineIndicator = lineIndicator +2 line = linecache.getline(boundaryFile,lineIndicator) # adding patch type newPatch.addPatchType(line[:-2].split()[1]) lineIndicator = lineIndicator +1 line = linecache.getline(boundaryFile,lineIndicator) newPatch.nFaces = int(line[:-2].split()[1]) lineIndicator = lineIndicator +1 line = linecache.getline(boundaryFile,lineIndicator) newPatch.startFace = int(line[:-2].split()[1]) if(newPatch.ifCyclic): lineIndicator = lineIndicator +2 line = linecache.getline(boundaryFile,lineIndicator) # adding neighbour patch name newPatch.neighbourPatchName = line[:-2].split()[1] print 'neighbourPatchName is ',newPatch.neighbourPatchName lineIndicator = lineIndicator +2 patches.append(newPatch) # now, all patches has been added to paches. # find cyclic neighbour patch. for i in range(0,n): for j in range(0,n): if (patches[i].neighbourPatchName == patches[j].patchName): patches[i].neighbourPatch = patches[j] for i in range(0,n): nFaces = patches[i].nFaces startFace = patches[i].startFace for j in range(startFace,startFace+nFaces): faces[j].ifBoundary = True faces[j].addPatchType(patches[i].patchType) # forward patchType from patch to face if(faces[j].ifCyclic): # if this face is cyclic, find its cyclic face ID faces[j].pFaceID = (j - startFace)+ patches[i].neighbourPatch.startFace linecache.clearcache() return faces
def __del__(self): linecache.clearcache()
def AllinOneExit1(): print("程序5秒后自动退出") linecache.clearcache() time.sleep(5) os._exit(0)
def __init__(self, intervals_file, fasta_file, dnase_file, cell_line=None, mappability_file=None, GENCODE_dir=None, use_linecache=True): # intervals if use_linecache: linecache.clearcache() BT = BedToolLinecache else: BT = BedTool self.bt = BT(intervals_file) # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later # DNase self.dnase_file = dnase_file self.dnase_extractor = None # mappability if mappability_file is None: # download the mappability file if not existing mappability_file = os.path.join( this_dir, "../../template/dataloader_files", "wgEncodeDukeMapabilityUniqueness35bp.bigWig") if not os.path.exists(mappability_file): print("Downloading the mappability file") urlretrieve( "http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", mappability_file) print("Download complete") self.mappability_file = mappability_file self.mappability_extractor = None # Gencode features if GENCODE_dir is None: gp = os.path.join(this_dir, "dataloader_files/gencode_features/") else: gp = GENCODE_dir self.gencode_beds = [ ("cpg", BedTool(gp + '/cpgisland.bed.gz')), ("cds", BedTool(gp + '/wgEncodeGencodeBasicV19.cds.merged.bed.gz')), ("intron", BedTool(gp + '/wgEncodeGencodeBasicV19.intron.merged.bed.gz')), ("promoter", BedTool(gp + '/wgEncodeGencodeBasicV19.promoter.merged.bed.gz')), ("utr5", BedTool(gp + '/wgEncodeGencodeBasicV19.utr5.merged.bed.gz')), ("utr3", BedTool(gp + '/wgEncodeGencodeBasicV19.utr3.merged.bed.gz')), ] # Overlap beds - could be done incrementally print("Overlapping all the bed-files") # The BT() and .fn are there in order to leverage BedToolLinecache self.overlap_beds = [(b, BT(self.bt.intersect(v, wa=True, c=True).fn)) for b, v in self.gencode_beds] print("Assesing the file") assert len(self.overlap_beds[1][1]) == len(self.bt)
def vmd_network_creator(filename, cube_files=None, render=False, iso=(-0.01, 0.01), abspath=False, **kwargs): '''Creates a VMD script file from a list of cube files provided. **Parameters:** filename : str Contains the base name of the output file. cube_files : None or list of str Specifies the cube files which serve as input for the VMD script. If None, searches the directory for '.cb' and '.cube' files. render : bool If True, the VMD script will automatically create '.tga' files for each cube file. iso : tuple Specifies the isovalue for the blue and the red isosurface, respectively. abspath : bool If True, the paths of the cube files will be expanded to absolute file paths. ''' from os import path, listdir import linecache from orbkit import vmd_network_draft if cube_files is None: display( 'No list of cube (.cb or .cube) filenames provided. Checking the directory' + ' of the outputfile...') cube_files = [] for fid in listdir(path.dirname(filename)): if fid.endswith('.cb') or fid.endswith('.cube'): cube_files.append(fid) if cube_files == []: raise IOError('Could not find valid cube files in %s' % path.dirname(filename)) elif isinstance(cube_files, str): cube_files = [cube_files] elif not isinstance(cube_files, list): raise IOError('`cube_files` has to be a list of strings.') title = [] mo = '' for i, f in enumerate(cube_files): title = linecache.getline(f, 2) if title.split() == []: title = path.splitext(path.basename(f))[0] else: title = title.replace('\n', '').replace(' ', '') linecache.clearcache() pid = path.abspath(f) if abspath else path.relpath( f, path.dirname(filename)) mo += vmd_network_draft.mo_string % { 'c': i, 'n1': pid, 'n2': title, 'isored': iso[0], 'isoblue': iso[1], 'render': '' if render else '#' } f = open('%(f)s.vmd' % {'f': filename}, 'w') f.write(vmd_network_draft.vmd_string % {'mo': mo}) f.close()
def rebuild(module, doLog=1): """ Reload a module and do as much as possible to replace its references. """ global lastRebuild lastRebuild = time.time() if hasattr(module, 'ALLOW_TWISTED_REBUILD'): # Is this module allowed to be rebuilt? if not module.ALLOW_TWISTED_REBUILD: raise RuntimeError("I am not allowed to be rebuilt.") if doLog: log.msg('Rebuilding {}...'.format(str(module.__name__))) # Safely handle adapter re-registration from twisted.python import components components.ALLOW_DUPLICATES = True d = module.__dict__ _modDictIDMap[id(d)] = module newclasses = {} classes = {} functions = {} values = {} if doLog: log.msg(' (scanning {}): '.format(str(module.__name__))) for k, v in d.items(): if _isClassType(type(v)): # ClassType exists on Python 2.x and earlier. # Failure condition -- instances of classes with buggy # __hash__/__cmp__ methods referenced at the module level... if v.__module__ == module.__name__: classes[v] = 1 if doLog: log.logfile.write("c") log.logfile.flush() elif type(v) == types.FunctionType: if v.__globals__ is module.__dict__: functions[v] = 1 if doLog: log.logfile.write("f") log.logfile.flush() elif isinstance(v, type): if v.__module__ == module.__name__: newclasses[v] = 1 if doLog: log.logfile.write("o") log.logfile.flush() values.update(classes) values.update(functions) fromOldModule = values.__contains__ newclasses = newclasses.keys() classes = classes.keys() functions = functions.keys() if doLog: log.msg('') log.msg(' (reload {})'.format(str(module.__name__))) # Boom. reload(module) # Make sure that my traceback printing will at least be recent... linecache.clearcache() if doLog: log.msg(' (cleaning {}): '.format(str(module.__name__))) for clazz in classes: if getattr(module, clazz.__name__) is clazz: log.msg("WARNING: class {} not replaced by reload!".format( reflect.qual(clazz))) else: if doLog: log.logfile.write("x") log.logfile.flush() clazz.__bases__ = () clazz.__dict__.clear() clazz.__getattr__ = __getattr__ clazz.__module__ = module.__name__ if newclasses: import gc for nclass in newclasses: ga = getattr(module, nclass.__name__) if ga is nclass: log.msg("WARNING: new-class {} not replaced by reload!".format( reflect.qual(nclass))) else: for r in gc.get_referrers(nclass): if getattr(r, '__class__', None) is nclass: r.__class__ = ga if doLog: log.msg('') log.msg(' (fixing {}): '.format(str(module.__name__))) modcount = 0 for mk, mod in sys.modules.items(): modcount = modcount + 1 if mod == module or mod is None: continue if not hasattr(mod, '__file__'): # It's a builtin module; nothing to replace here. continue if hasattr(mod, '__bundle__'): # PyObjC has a few buggy objects which segfault if you hash() them. # It doesn't make sense to try rebuilding extension modules like # this anyway, so don't try. continue changed = 0 for k, v in mod.__dict__.items(): try: hash(v) except Exception: continue if fromOldModule(v): if _isClassType(type(v)): if doLog: log.logfile.write("c") log.logfile.flush() nv = latestClass(v) else: if doLog: log.logfile.write("f") log.logfile.flush() nv = latestFunction(v) changed = 1 setattr(mod, k, nv) else: # Replace bases of non-module classes just to be sure. if _isClassType(type(v)): for base in v.__bases__: if fromOldModule(base): latestClass(v) if doLog and not changed and ((modcount % 10) == 0): log.logfile.write(".") log.logfile.flush() components.ALLOW_DUPLICATES = False if doLog: log.msg('') log.msg(' Rebuilt {}.'.format(str(module.__name__))) return module
def test_lazycache_check(self): linecache.clearcache() linecache.lazycache(NONEXISTENT_FILENAME, globals()) linecache.checkcache()
def __init__(self, intervals_file, fasta_file, dnase_file, cell_line=None, RNAseq_PC_file=None, mappability_file=None, use_linecache=True): # intervals if use_linecache: linecache.clearcache() BT = BedToolLinecache else: BT = BedTool self.bt = BT(intervals_file) # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later # DNase self.dnase_file = dnase_file self.dnase_extractor = None # mappability if mappability_file is None: # download the mappability file if not existing common_dl_dir = os.path.join( this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile( url= "http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", md5="1d15ddafe2c8df51cf08495db96679e7") mappability_file = os.path.join( common_dl_dir, "wgEncodeDukeMapabilityUniqueness35bp.bigWig") if not os.path.exists(mappability_file) or not rf.validate( mappability_file): # download the path rf.get_file(mappability_file) self.mappability_file = mappability_file self.mappability_extractor = None # Get the metadata features if cell_line is None: if RNAseq_PC_file is None: raise ValueError( "RNAseq_PC_file has to be specified when cell_line=None") assert os.path.exists(RNAseq_PC_file) else: # Using the pre-defined cell-line output_dir = os.path.join( this_dir, "../../template/downloaded/dataloader_files/RNAseq_features/") makedir_exist_ok(output_dir) RNAseq_PC_file = os.path.join(output_dir, cell_line, "meta.txt") url_template = ( 'https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/' 'FactorNet/dataloader_files/RNAseq_features/{}/meta.txt') # rf = RemoteFile(url=url_template.format(cell_line)) if not os.path.exists( RNAseq_PC_file): # or not rf.validate(mappability_file): # download the path download_url(url_template.format(cell_line), os.path.join(output_dir, cell_line), "meta.txt") # rf.get_file(RNAseq_PC_file) self.meta_feat = pd.read_csv(RNAseq_PC_file, sep="\t", header=None)[0].values
def test_lazycache_bad_filename(self): linecache.clearcache() self.assertEqual(False, linecache.lazycache('', globals())) self.assertEqual(False, linecache.lazycache('<foo>', globals()))
def test_lazycache_already_cached(self): linecache.clearcache() lines = linecache.getlines(NONEXISTENT_FILENAME, globals()) self.assertEqual(False, linecache.lazycache(NONEXISTENT_FILENAME, globals())) self.assertEqual(4, len(linecache.cache[NONEXISTENT_FILENAME]))
def __init__(self, aPath): ## ''' Purging routines ''' def prune(): print "prune iteration" self.model_numbers = self.original_history[:, self.history_labels. index("model_number")] i = len(self.model_numbers) - 1 while (self.model_numbers[i] > self.model_numbers[i - 1] and i > 0): i = i - 1 # print i # Reference line numbers as log output print "Model#", self.model_numbers[ i], "is greater than or equal to model#", self.model_numbers[i - 1] if i == 0: return 0 Drop2 = i - 1 # print "Drop2 " + str(Drop2) FindDrop1 = self.model_numbers[i] i = i - 1 while self.model_numbers[i] != FindDrop1: i = i - 1 Drop1 = i - 1 # print "Drop1 " + str(Drop1) print "Model Cleaning: Dropping model number #" + str(Drop2) piece1 = self.original_history[0:Drop1] piece2 = self.original_history[Drop2 + 1:-1] self.original_history = np.concatenate([piece1, piece2]) return 1 ## ''' END Purging routines '''' def get_columns_names(filename): return linecache.getline(filename, 6).split() def get_GlobalInfo_names(filename): return linecache.getline(filename, 2).split() def get_GlobalInfo_values(filename): return linecache.getline(filename, 3).split() """Collects the output from a MESA run""" print aPath ## Checks if load_all_profiles is true or false in default_settings.py, then handles situation appropriately self.num_profiles = len( np.loadtxt(aPath + os.path.sep + 'profiles.index', skiprows=1)) if ds.load_all_profiles: print "Loading all profiles(" + str( self.num_profiles ) + ") (load_all_profiles=False in default_settings.py.)" else: if ds.num_profiles_to_load <= self.num_profiles: self.num_profiles = ds.num_profiles_to_load print "Loading " + str( self.num_profiles ) + " profiles (load_all_profiles=True in default_settings.py.)" else: print "num_profiles_to_load greater than actual number of profiles, loading all profiles(" + str( self.num_profiles) + ")." # profile=np.loadtxt(aPath+os.path.sep+'profiles.index',skiprows=1) # if os.path.isfile(FM.path_folder + os.path.sep + 'star.log'): if os.path.isfile(aPath + os.path.sep + 'star.log'): self.history_path = aPath + os.path.sep + 'star.log' profileversion = 'log' # elif os.path.isfile(FM.path_folder + os.path.sep + 'history.data'): elif os.path.isfile(aPath + os.path.sep + 'history.data'): self.history_path = aPath + os.path.sep + 'history.data' profileversion = 'profile' else: print 'Non-standard MESA data names. We will eventually support this, so please let us know there\'s a need' self.profile_labels = get_columns_names(aPath + os.path.sep + profileversion + '1.data') self.profile_GlobalInfo_names = get_GlobalInfo_names(aPath + os.path.sep + profileversion + '1.data') self.history_size = len(np.loadtxt(self.history_path, skiprows=6)) self.history_labels = get_columns_names(self.history_path) self.history_GlobalInfo_names = get_GlobalInfo_names(self.history_path) self.star_age_index = self.profile_GlobalInfo_names.index('star_age') ##History_star_data = np.loadtxt(history_path,skiprows=6, usecols=(history_labels.index('star_age'),)).tolist() History_star_data = np.loadtxt( self.history_path, skiprows=6, usecols=(self.history_labels.index('star_age'), )) self.original_history = np.loadtxt( self.history_path, skiprows=6) #this is the history file before purging original_history_length = len(self.original_history) self.model_numbers = self.original_history[:, self.history_labels. index("model_number")] # print type(self.model_numbers) #Builds an array called profile with #the hystory file as its zero entry and profile_n as its nth entry. while prune(): pass final_history_length = len(self.original_history) N_deleted_lines = original_history_length - final_history_length self.profile = [self.original_history] self.profile_GlobalInfo_values = [ get_GlobalInfo_values(self.history_path) ] self.profileAge = [0] self.listAges = [[0, 0]] self.profile_information = [""] ## profile_age_in_History is a list of the approximated values of the profile ages as they appear in the history file ## the nth position in the list corresponds to the the age (as it appear in the history file) of the nth profile ## profile_age_index_in_History is a list of positions in the history file of the profile ages. ##the nth element of the list corresponds to the position in history where the age of the profile n appears. self.profile_age_in_History = [0] self.profile_age_index_in_History = [0] profile_size = [self.history_size] print 'Imported history file' ##Loads the content of the profiles for i in range(1, self.num_profiles + 1): newPath = aPath + os.path.sep + profileversion + str(i) + '.data' self.profile.append(np.loadtxt(newPath, skiprows=6)) profile_size.append(len(np.loadtxt(newPath, skiprows=6))) self.profile_GlobalInfo_values.append( get_GlobalInfo_values(newPath)) self.profileAge.append( self.profile_GlobalInfo_values[i][self.star_age_index]) self.listAges.append([ float(self.profile_GlobalInfo_values[i][self.star_age_index]), i ]) self.profile_age_in_History.append( self.find_nearest(History_star_data, float(self.profileAge[i]))) self.profile_age_index_in_History.append( History_star_data.tolist().index( self.profile_age_in_History[i])) self.new_profile_information = "" for j in range(len(self.profile_GlobalInfo_names)): self.new_profile_information = self.new_profile_information + str( self.profile_GlobalInfo_names[j]) + ": " + str( self.profile_GlobalInfo_values[i][j]) + "\n" self.profile_information.append(self.new_profile_information) print 'Imported profile #' + str(i) + ' / ' + str( self.num_profiles) ##the function sorted_profile_number takes n and returns the number of the nth profie in order of age. ##Example, sorted_profile_number(1) returns the profile number of the youngest profile. self.max_profile_size = max(profile_size[1:]) ## This is the column of the history file corresponding to the star age ##History_star_data = profile[0][:,history_labels.index('star_age')].tolist() ##History_star_data = np.loadtxt(history_path,skiprows=6, usecols=(history_labels.index('star_age'),)).tolist() # print type(History_star_data) ##print History_star_data.tolist() ##History_star_data = np.loadtxt(history_path,skiprows=6, usecols=(1,)) ##History_star_data.where(array==item) ##Clean the cash linecache.clearcache() # print 'Downloaded '+ str(self.num_profiles)+ ' profiles + 1 hystory file' print 'Original history file has ' + str( self.history_size) + ' lines (excluding the header)' print 'deleted ' + str( N_deleted_lines) + ' lines from the original history file' ## print "HERE" ## print "now "+str(findAnomly(self.profile[0][:,self.history_labels.index("model_number")])) ## print type(self.profile) ##print history_labels.index('star_age') ##print type(History_star_data) ##print History_star_data print self.profile_age_in_History[4] print self.profile_age_index_in_History[4] ##print type(profileAge[3]) self.star_information = "" for i in range(len(self.history_GlobalInfo_names)): self.star_information = self.star_information + str( self.history_GlobalInfo_names[i]) + ": " + str( self.profile_GlobalInfo_values[0][i]) + "\n" print self.profile[0][0:len(self.profile[0]) - 2, self.history_labels.index("model_number")]
def parse_variable(self): lineNumber = 0 start_flag = True end_flag = False with open(self.var_path, 'r') as f: lines = f.readlines() for line in lines: # print(line) if line.startswith('#') and len(line)>1 and start_flag: # print('start line:', lineNumber, line) lineNumber += 1 start_flag = False end_flag = True self.sign_numb_list.append(lineNumber) continue elif line.startswith('#') and end_flag: lineNumber += 1 if len(line) == 2 or len(line) == 1: start_flag = True end_flag = False self.sign_numb_list.append(lineNumber) # print('end line:', lineNumber, line) else: raise Exception('Please make sure the component name, DLC, variables defined in #...#!\n' 'Error occurs in line %s' %lineNumber) else: lineNumber += 1 # print(start_flag, end_flag) print('line number:', self.sign_numb_list) if len(self.sign_numb_list)%2 == 0: for j in range(len(self.sign_numb_list)): # 清除缓存 linecache.clearcache() if j%2 == 0: start = self.sign_numb_list[j] end = self.sign_numb_list[j+1] file_name = linecache.getline(self.var_path, start).strip()[1:] comp_name = file_name.split(',')[0] dlc_list = [lc.strip() for lc in linecache.getline(self.var_path, start+1).strip().split(',')] # print(dlc_list) self.component_list.append(comp_name) self.component_dlcs[comp_name] = dlc_list self.dlc_list += dlc_list self.component_type[comp_name] = [_.strip() for _ in file_name.split(',')[1:]] # hub, gearbox and so on if 'section' not in comp_name.lower(): # read variable var_lines = linecache.getlines(self.var_path)[start+1:end-1] vars_list = [] unit_list = [] var_no_unit = [] # tower_list = [] # tower_sec = [] for var in var_lines: if "!" not in var: temp = var.strip().split(',') if len(temp)>=2: vars_list.append(temp[0].strip()) unit_list.append(temp[1].strip()) # if 'tower' in temp[0].lower: # tower_list.append(temp[0]) # tower_sec.append(temp[2]) else: var_no_unit.append(var.strip()) print(var.strip()) vars_list.insert(0, 'Time from start of output') unit_list.insert(0, 's') self.component_vars[comp_name] = vars_list self.component_unit[comp_name] = unit_list # self.component_secs[comp_name] = tower_sec if var_no_unit: self.comp_varwounit[comp_name] = var_no_unit raise Exception('Please define unit for the variables:\n%s' % (','.join([var for k, v in self.comp_varwounit.items() for var in v]))) # blade and tower else: # from third line to the last three line var_lines = linecache.getlines(self.var_path)[start+1:end-2] vars_list = [] unit_list = [] # the last two line sec_line = linecache.getlines(self.var_path)[end-2:end-1][0] self.section_comp.append(comp_name) # tower-offshore and onshore if 'tower' in comp_name.lower(): # section or mbr sec_mbr = sec_line.strip().split(',')[0].strip() # tower type: bottom to top or top to bottom, t2b/b2t tr_type = sec_line.strip().split(',')[1].strip() # section list or mbr list in %25 file sec_list = sec_line.strip().split(',')[2:] if sec_list[0] != 'a': sec_list = [sec.strip() for sec in sec_list] # print(sec_list) self.component_secs[comp_name] = [sec_mbr, tr_type, sec_list] # blade else: sec_list = sec_line.strip().split(',')[1:] # for blade, all or [1,2,...] if sec_list[0] != 'a': self.component_secs[comp_name] = [sec.strip() for sec in sec_list] else: self.component_secs[comp_name] = sec_list var_no_unit = [] for var in var_lines: if "!" not in var: temp = var.strip().split(',') if len(temp)>=2: vars_list.append(temp[0].strip()) unit_list.append(temp[1].strip()) # print(temp[0].strip(), temp[1].strip()) else: var_no_unit.append(var.strip()) vars_list.insert(0, 'Time from start of output') unit_list.insert(0, 's') self.component_vars[comp_name] = vars_list self.component_unit[comp_name] = unit_list if var_no_unit: self.comp_varwounit[comp_name] = var_no_unit raise Exception('Please define unit for the variables:\n%s' %(','.join([var for k,v in self.comp_varwounit.items() for var in v]))) else: raise Exception('Please make sure the variable defined between #!') self.dlc_list = list(set(self.dlc_list)) self.dlc_list.sort()