def check(self, filename): """ Check the settings currently set against default.cfg """ default = Parser() default.readfp(open(os.path.join(self.config_location, "default.cfg"))) local = Parser() local.readfp(open(filename)) local_ok = True diff = set(local.sections()) - set(default.sections()) for section in diff: logging.warning("Section {} does not exist in {}".format( section, "default.cfg")) local_ok = False for section in local.sections(): if not default.has_section(section): continue diff = set(local.options(section)) - set(default.options(section)) for option in diff: logging.warning( "Option {} in section {} does not exist in {}".format( option, section, "default.cfg")) local_ok = False if local_ok: logging.info("{} is OK".format(filename)) else: logging.warning("{} contains errors.".format(filename)) return local_ok
def populate_random(random_file, random_templates=None, saml_info=None): """Populate random.ini Create missing random values according to the template Do not change existing values""" from base64 import b64encode from os import urandom from assembl.auth.make_saml import (make_saml_key, make_saml_cert, cleanup_x509_text) base = Parser() assert random_templates, "Please give one or more templates" for template in random_templates: assert exists(template), "Cannot find template " + template base.read(template) existing = Parser() if exists(random_file): existing.read(random_file) combine_ini(base, existing) saml_keys = {} changed = False for section in base.sections(): for key, value in base.items(section): keyu = key.upper() # too much knowdledge, but hard to avoid if "SAML" in keyu and keyu.endswith("_PRIVATE_KEY"): prefix = keyu[:-12] if value == "{saml_key}": saml_key_text, saml_key = make_saml_key() saml_key_text = cleanup_x509_text(saml_key_text) base.set(section, key, saml_key_text) saml_keys[prefix] = saml_key changed = True else: saml_keys[prefix] = value elif value.startswith('{random') and value.endswith("}"): size = int(value[7:-1]) assert 0 < size < 100 value = b64encode(urandom(size)) base.set(section, key, value) changed = True # Do certs in second pass, to be sure keys are set for section in base.sections(): for key, value in base.items(section): keyu = key.upper() if ("SAML" in keyu and keyu.endswith("_PUBLIC_CERT") and value == '{saml_cert}'): assert saml_info prefix = keyu[:-12] # If key is not there, it IS a mismatch and and error. saml_key = saml_keys[prefix] saml_cert_text, _ = make_saml_cert(saml_key, **saml_info) saml_cert_text = cleanup_x509_text(saml_cert_text) base.set(section, key, saml_cert_text) changed = True if changed: with open(random_file, 'w') as f: base.write(f) return base
def save(self, filename): """ Save the changed settings to local.cfg """ current = CascadingConfigParser(self.config_files) # Build a list of changed values to_save = [] for section in self.sections(): #logging.debug(section) for option in self.options(section): if self.get(section, option) != current.get(section, option): old = current.get(section, option) val = self.get(section, option) to_save.append((section, option, val, old)) # Update local config with changed values local = Parser() # Start each file with revision identification local.add_section("Configuration") local.set("Configuration", "version", "1") local.readfp(open(filename, "r")) for opt in to_save: (section, option, value, old) = opt if not local.has_section(section): local.add_section(section) local.set(section, option, value) logging.info("Update setting: {} from {} to {} ".format( option, old, value)) # Save changed values to file local.write(open(filename, "w+"))
def train(): opt = Parser().parse() print("Loading dataset") dataset = Dataset(opt, data_mode=DATA_TRAIN) print("Generating model") model = standard_cnn(opt) print("Preparing for training...") #(images, labels) = dataset.get_train_data() #Not efficent, loads all training data into memory model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"]) print("Starting training") tensorboard_callback = tf.keras.callbacks.TensorBoard( log_dir=opt.tensorboard_logdir, histogram_freq=0, write_graph=True, write_images=True, update_freq="batch") save_callback = CustomSaveCallback(opt.model_dir + "cnn") model.fit_generator(generator=dataset, steps_per_epoch=(len(dataset.training_files) // opt.batch_size), epochs=opt.num_epochs, verbose=1, callbacks=[tensorboard_callback, save_callback]) print(model.summary()) print("Model trained and saved to " + opt.model_dir)
def rc_to_ini(rc_info, default_section=SECTION): """Convert a .rc file to a ConfigParser (.ini-like object) Items are assumed to be in app:assembl section, unless prefixed by "{section}__" . Keys prefixed with an underscore are not passed on. Keys prefixed with a star are put in the global (DEFAULT) section. Value of '__delete_key__' is eliminated if existing. """ p = Parser() for key, val in rc_info.iteritems(): if not key or key.startswith('_'): continue if key[0] == '*': # Global keys section = "DEFAULT" key = key[1:] elif '__' in key: section, key = key.split('__', 1) else: section = default_section ensureSection(p, section) if val == '__delete_key__': # Allow to remove a variable from rc # so we can fall back to underlying ini p.remove_option(section, key) else: p.set(section, key, val) return p
def config_from_filepath(path): config = Parser() config.readfp(open(path)) config.get('credentials', 'user') config.get('credentials', 'password') config.get('site', 'remote') return config
def loadMaterialList(cfgFile): """Load a list of materials from a file The file uses the config file format. See ConfigParser module. """ p = Parser() p.read(cfgFile) # # Each section defines a material # names = p.sections() matList = [Material(n, p) for n in names] # Sort the list matList = sorted(matList, key=lambda x: x.name) return matList
def compose(rc_filename, random_file=None): """Compose local.ini from the given .rc file""" rc_info = combine_rc(rc_filename) ini_sequence = rc_info.get('ini_files', None) assert ini_sequence, "Define ini_files" ini_sequence = ini_sequence.split() base = Parser() random_file = random_file or rc_info.get('random_file', RANDOM_FILE) for overlay in ini_sequence: if overlay == 'RC_DATA': overlay = rc_to_ini(rc_info) elif overlay.startswith('RANDOM'): templates = overlay.split(':')[1:] overlay = populate_random(random_file, templates, extract_saml_info(rc_info)) combine_ini(base, overlay) return base
def get_default_settings(self): fs = [] for config_file in self.config_files: if os.path.isfile(config_file): c_file = os.path.basename(config_file) cp = Parser() cp.readfp(open(config_file)) fs.append((c_file, cp)) lines = [] for section in self.sections(): for option in self.options(section): for (name, cp) in fs: if cp.has_option(section, option): line = [name, section, option, cp.get(section, option)] lines.append(line) return lines
def __init__(self, cfile=None): service.MultiService.__init__(self) parser = Parser(self.defaults) read = parser.read(map(expanduser, self.paths)) if cfile: parser.readfp(open(cfile, 'r')) if not cfile and len(read) == 0: # no user configuration given so load some defaults parser.add_section('recceiver') parser.set('recceiver', 'procs', 'show') elif not parser.has_section('recceiver'): parser.add_section('recceiver') pnames = parser.get('recceiver', 'procs').split(',') plugs = {} for plug in plugin.getPlugins(interfaces.IProcessorFactory): _log.debug('Available plugin: %s', plug.name) plugs[plug.name] = plug self.procs = [] for P in pnames: P = P.strip() plugname, _, instname = P.partition(':') if not instname: instname = plugname plug = plugs[plugname] if not parser.has_section(instname): parser.add_section(instname) inst = plug.build(instname, ConfigAdapter(parser, instname)) self.procs.append(inst) self.addService(inst) self._C = parser
def diff_ini(first, second, diff=None, existing_only=False): """Diff ini files Generate a parser with any value in the second that is different in the first. Returns a ConfigParser. Takes interpolation into account. Does not include values that disappeared.""" from ConfigParser import _Chainmap first = asParser(first) second = asParser(second) # TODO: Look at first both in raw and formatted versions diff = diff or Parser() interpolating = SafeConfigParser() for section in second.sections(): if section != 'DEFAULT' and not first.has_section(section): if not existing_only: diff.add_section(section) for option in second.options(section): value = second.get(section, option) diff.set(section, option, value) else: vars = _Chainmap(second._sections[section], first._sections[section], second._defaults, first._defaults) for option, value2 in second.items(section): if not first.has_option(section, option): if not existing_only: ensureSection(diff, section) diff.set(section, option, value2) continue value1 = first.get(section, option) if value1 != value2 and '%(' in value1: # try to interpolate, and see if it would amount to the same. try: value1 = interpolating._interpolate( section, option, value1, vars) except Exception as e: pass if value1 != value2: ensureSection(diff, section) diff.set(section, option, value2) return diff
def main(source_ini, update_ini): parser = Parser() # By default, the parser is case insensitve and rewrites config # keys to lowercase. reddit is case sensitive, however # See: http://docs.python.org/library/configparser.html#ConfigParser.RawConfigParser.optionxform parser.optionxform = str # parser.read() will "fail" silently if the file is # not found; use open() and parser.readfp() to fail # on missing (or unreadable, etc.) file parser.readfp(open(source_ini)) with open(update_ini) as f: updates = f.read() try: # Existing *.update files don't include section # headers; inject a [DEFAULT] header if the parsing # fails parser.readfp(StringIO(updates)) except MissingSectionHeaderError: updates = "[DEFAULT]\n" + updates parser.readfp(StringIO(updates)) print HEADER parser.write(sys.stdout)
def compose(rc_filename, random_file=None): """Compose local.ini from the given .rc file""" rc_info = combine_rc(rc_filename) rc_info['*code_root'] = code_root(rc_info) # Special case: uwsgi does not do internal computations. rc_info['uwsgi__virtualenv'] = venv_path(rc_info) ini_sequence = rc_info.get('ini_files', None) assert ini_sequence, "Define ini_files" ini_sequence = ini_sequence.split() base = Parser() random_file = random_file or rc_info.get('random_file', RANDOM_FILE) for overlay in ini_sequence: if overlay == 'RC_DATA': overlay = rc_to_ini(rc_info) elif overlay.startswith('RANDOM'): templates = overlay.split(':')[1:] overlay = populate_random(random_file, templates, extract_saml_info(rc_info)) else: overlay = find_ini_file(overlay, dirname(rc_filename)) assert overlay, "Cannot find " + overlay combine_ini(base, overlay) return base
log.readout('Creating summary pdf') summary_main(parser_dict, workflow, target, qc_dir, genome_bam_dir, paired_end, fastq_dir, R1_suffix, R2_suffix, samtools, log) if clean_intermediates: from summary import cleanup_intermediate_folders log.readout('Removing intermediate files and folders') cleanup_intermediate_folders(output_dir, parser_dict, target) log.readout('Completed summary of %s', target) log.removeFilter(summary) return args = parseArgs() pipeline_config = args['pipeline_config'] parser = Parser() if int(sys.version[0]) == 3: try: parser.read_file( open(pipeline_config)) #reading in the config file except IOError as e: print('Missing config file!') sys.exit() try: parser.read_file(open(parser.get('i/o', 'static_config'))) except configparser.NoOptionError: pass elif int(sys.version[0]) == 2: try: parser.readfp(open(pipeline_config)) except IOError as e:
import os from model import load_from_path from Dataset import Dataset, DATA_TEST from ConfigParser import Parser def predict(model, image_data): return model.predict(image_data) if __name__ == "__main__": opt = Parser().parse() print("Loading model") model = load_from_path(opt.model_path) dataset = Dataset(opt, data_mode=DATA_TEST) totalPredictions = 0 correctPredictions = 0 for sample in dataset: prediction = round(predict(model, sample["data"])[0][0]) == 1 print ("Predicted file {0}: {1} Label: {2}".format(sample["file"], prediction, sample["label"])) totalPredictions += 1 if prediction == sample["label"]: correctPredictions += 1 print ("Predicted {0} files, correct predictions={1}, accuracy={2}".format(totalPredictions,correctPredictions, correctPredictions / totalPredictions))
ap = ArgumentParser( description= """Combine a source ini file with an overlay ini file into a target ini file. You cannot edit in-place (use same source and destination). Source and overlay cannot both be stdin. Overlay values with a {randomN} format (N an integer), will be replaced with a random string of length 4*ceil(N/3).""") ap.add_argument('source', type=FileType('r'), help='The source file (use - for stdin)') ap.add_argument('overlay', type=FileType('r'), help='The overlay file (use - for stdin)') ap.add_argument('result', type=FileType('w'), help='The resulting file (use - for stdout)') args = ap.parse_args() source = Parser() source.readfp(args.source) overlay = Parser() overlay.readfp(args.overlay) for section in overlay.sections(): if section != 'DEFAULT' and not source.has_section(section): source.add_section(section) for option in overlay.options(section): value = overlay.get(section, option) # Special case: {random66} => random string of 66*(4/3) characters if value.startswith('{random') and value.endswith("}"): value = b64encode(urandom(int(value[7:-1]))) source.set(section, option, value) source.write(args.result)
def loadProject(self): """Load any project data to initialize (in development) This should implemented and documented somewhere else, but for now, let's start it here. FILE FORMAT name: experiment.cfg sections: [load] material-list = <filename> reader-list = <filename> [options] # options for this experiment """ cfgFile = 'experiment.cfg' exp = wx.GetApp().ws # if os.path.exists(cfgFile): p = Parser() p.read(cfgFile) # # Each section defines a material # sec = 'load' if p.has_section(sec): opt = 'material-list' try: fname = p.get(sec, opt) exp.loadMaterialList(fname) print 'loaded materials list from "%s"\n' % fname except: wx.MessageBox('failed to autoload materials list') pass opt = 'reader-list' try: fname = p.get(sec, opt) exp.loadReaderList(fname) print 'loaded readers list from "%s"\n' % fname except: wx.MessageBox('failed to autoload readers list') pass opt = 'detector' try: fname = p.get(sec, opt) exp.loadDetector(fname) print 'loaded detector from "%s"\n' % fname except: wx.MessageBox('failed to autoload detector list') pass pass sec = 'options' if p.has_section(sec): opt = 'start-tab' try: val = p.getint(sec, opt) self.nBook.SetSelection(val) print 'starting on tab number %d\n' % val except: pass pass return
def __init(self): self.file = self.opencfg(UIConfig.cfg) self.cfg = Parser() self.cfg.readfp(self.file) self.file.close()