def readFile(self, **kwargs): asciiLines = None try: with mcopen(self.filename, 'r') as fd: asciiLines = fd.readlines() except UnicodeDecodeError: with mcopen(self.filename, 'r', encoding = 'latin1') as fd: asciiLines = fd.readlines() self.parseLines(asciiLines, **kwargs)
def hashFile(filename, hasher, blocksize=65536): os.path.abspath(filename) with mcopen(filename, 'rb') as fd: buf = fd.read(blocksize) while len(buf) > 0: hasher.update(buf) buf = fd.read(blocksize) return hasher.hexdigest(), os.path.basename(filename)
def _writeContribs(self, mcResult): # Writes the contribution parameters to a pickled file. # Can be used to continue or reanalyse a previously fitted file fn = self._outFn.filenameVerbose("contributions", "Model contribution parameters", extension='.pickle') with mcopen(fn, 'wb') as fh: pickle.dump(mcResult['contribs'], fh)
def updateFile(module, newversion): """Updates the version string within a given module. Replaces the version string in the source code textually. Assumes there is an AppVersion constructor call in the module. """ fn = os.path.abspath(inspect.getfile(module)) if not os.path.exists(fn): return if os.path.splitext(fn)[-1].endswith("pyc"): fn = fn[:-1] # read current file content content = None with mcopen(fn) as fd: content = fd.read() # get version number, parse and update it match = re.search(r'(versionNumber\s+=\s+(\"[^\"]*\"))', content) oldversion = match.group(2) # replace version string and write back newversion = '"{0}"'.format(newversion) content = content.replace(oldversion, newversion) with mcopen(fn, 'wb') as fd: fd.write(content)
def loadParams(self, fname = None): """ writes the default definitions and bounds for the configuration parameters to self.parameters Can also be used to update existing parameters from supplied filename """ with mcopen(fname, 'r') as jfile: logging.info('loading parameters from file: {}'.format(fname)) parDict = json.load(jfile) if self.parameters is None: # create if it does not exist yet self.parameters = lambda: None # now we cast this information into the Parameter class: for kw in list(parDict.keys()): subDict = parDict[kw] name = kw value = subDict.pop("value", None) default = subDict.pop("default", None) if value is None and default is not None: value = default # determine parameter class: cls = subDict.pop("cls", None) if cls == "int": subDict.update(cls = ParameterNumerical) elif cls == "float": subDict.update(cls = ParameterFloat) elif cls == "bool": subDict.update(cls = ParameterBoolean) elif cls == "str": subDict.update(cls = ParameterString) else: logging.warning('parameter type {} for parameter {} not ' ' understood from {}'.format(cls, kw, fname)) if kw in self.parameterNames: #value exists, should be updated with supplied kwargs self.set(kw,**subDict) logging.info('successfully updated parameter: {}'.format(kw)) else: #logging.info('ingesting parameter {} with value {}'.format(name, value)) temp = Parameter(name, value, **subDict) setattr(self.parameters,kw,temp) self.parameterNames.append(kw) logging.info('successfully ingested parameter: {}'.format(kw))
def archive(self, targetPath): targetPath = os.path.abspath(targetPath) if not os.path.isdir(targetPath): return None fnPackage = targetPath + "." + self._ext fnLog = os.path.abspath(self.getLogFilename()) with mcopen(fnLog, 'w') as fd: retcode = subprocess.call([ self._path, "a", "-t" + self._type, "-mx=9", os.path.relpath(fnPackage), os.path.relpath(targetPath) ], stdout=fd, stderr=fd) if not os.path.exists(fnPackage): fnPackage = None return fnPackage
def archive(self, targetPath): """Expects an absolute target directory path""" targetPath = os.path.abspath(targetPath) if not os.path.isdir(targetPath): return None fnPackage = targetPath + ".zip" fnLog = os.path.abspath(self.getLogFilename()) with mcopen(fnLog, 'w') as fd: retcode = subprocess.call([ self._path, "-r9", os.path.relpath(fnPackage), os.path.relpath(targetPath) ], stdout=fd, stderr=fd) if not os.path.exists(fnPackage): fnPackage = None return fnPackage
def writeConfig(self, fname): """ writes the configuration to a settings file. Required input parameter is the filename to write to. """ #create a dictionary containing one (sub-)dictionary per parameter parDict = dict() for kw in self.parameterNames: subDict = dict() par = self.getPar(kw) for aName in par.attributeNames(): subDict[aName] = par.get(aName) parDict[kw] = subDict #write dictionary to file with mcopen(fname, 'wb') as jfile: json.dump(parDict, jfile, cls=ExtendedEncoder, indent = 4, sort_keys=True)
def _writeSettings(self, mcargs, dataset): if self.model is None: return [] fn = self._outFn.filenameVerbose("settings", "algorithm settings", extension='.cfg') config = configparser.RawConfigParser() sectionName = "I/O Settings" config.add_section(sectionName) # the absolute filename with extension, see SASData.load() config.set(sectionName, 'fileName', dataset.filename) # the filename with timestamp of results config.set(sectionName, 'outputBaseName', self._outFn.basename) sectionName = "MCSAS Settings" config.add_section(sectionName) for key, value in mcargs.items(): config.set(sectionName, key, value) for p in self.algo.params(): config.set(sectionName, p.name(), p.value()) config.set(sectionName, "model", self.model.name()) # We don't have to do anything with these yet, but storing them for now: if isinstance(dataset, SASData): # useful with SAS data only config.set(sectionName, "X0 limits", str(dataset.x0.limit)) sectionName = "Model Settings" config.add_section(sectionName) for p in self.modelParams(): if isActiveFitParam(p): config.set(sectionName, p.name() + "_min", p.min()) config.set(sectionName, p.name() + "_max", p.max()) else: config.set(sectionName, p.name(), p.value()) with mcopen(fn, 'w') as configfile: config.write(configfile)
if not len(files): files = glob.glob("build/*.app") if len(files): PACKAGEFN = files[0] # rename package possibly fn, ext = os.path.splitext(PACKAGEFN) if os.path.exists(PACKAGEFN) and fn != TARGETDIR: try: newfn = "".join((TARGETDIR, ext)) os.rename(PACKAGEFN, newfn) PACKAGEFN = newfn except OSError as e: logging.exception("rename '{src}' -> '{dst}':".format( src=PACKAGEFN, dst=newfn)) pass logging.info("Created package '{0}'.".format(PACKAGEFN)) if PACKAGEFN is not None and os.path.isfile(PACKAGEFN): hashValue = hashFile(PACKAGEFN, hashlib.sha256()) # write the checksum to file with mcopen(os.path.abspath(PACKAGEFN + ".sha256"), 'w') as fd: fd.write(" *".join(hashValue)) # always restore initially modified version if version.number() != OLDVERSIONNUM: version.updateFile(gui.version, OLDVERSIONNUM) logging.info("done.") # vim: set ts=4 sw=4 sts=4 tw=0:
def pickleStore(filename, somedata): """Writes python object to a file.""" fh = mcopen(filename, 'w') pickle.dump(somedata, fh) fh.close() return
def pickleLoad(filename): """Loads data from a pickle file""" fh = mcopen(filename) output = pickle.load(fh) fh.close() return output
def _write(filename, mode, asciiData): with mcopen(filename, mode) as fd: fd.write(asciiData)