def check(self, filename):
""" Check the settings currently set against default.cfg """
default = Parser()
default.readfp(open(os.path.join(self.config_location, "default.cfg")))
local = Parser()
local.readfp(open(filename))
local_ok = True
diff = set(local.sections()) - set(default.sections())
for section in diff:
logging.warning("Section {} does not exist in {}".format(
section, "default.cfg"))
local_ok = False
for section in local.sections():
if not default.has_section(section):
continue
diff = set(local.options(section)) - set(default.options(section))
for option in diff:
logging.warning(
"Option {} in section {} does not exist in {}".format(
option, section, "default.cfg"))
local_ok = False
if local_ok:
logging.info("{} is OK".format(filename))
else:
logging.warning("{} contains errors.".format(filename))
return local_ok
def populate_random(random_file, random_templates=None, saml_info=None):
"""Populate random.ini
Create missing random values according to the template
Do not change existing values"""
from base64 import b64encode
from os import urandom
from assembl.auth.make_saml import (make_saml_key, make_saml_cert,
cleanup_x509_text)
base = Parser()
assert random_templates, "Please give one or more templates"
for template in random_templates:
assert exists(template), "Cannot find template " + template
base.read(template)
existing = Parser()
if exists(random_file):
existing.read(random_file)
combine_ini(base, existing)
saml_keys = {}
changed = False
for section in base.sections():
for key, value in base.items(section):
keyu = key.upper()
# too much knowdledge, but hard to avoid
if "SAML" in keyu and keyu.endswith("_PRIVATE_KEY"):
prefix = keyu[:-12]
if value == "{saml_key}":
saml_key_text, saml_key = make_saml_key()
saml_key_text = cleanup_x509_text(saml_key_text)
base.set(section, key, saml_key_text)
saml_keys[prefix] = saml_key
changed = True
else:
saml_keys[prefix] = value
elif value.startswith('{random') and value.endswith("}"):
size = int(value[7:-1])
assert 0 < size < 100
value = b64encode(urandom(size))
base.set(section, key, value)
changed = True
# Do certs in second pass, to be sure keys are set
for section in base.sections():
for key, value in base.items(section):
keyu = key.upper()
if ("SAML" in keyu and keyu.endswith("_PUBLIC_CERT")
and value == '{saml_cert}'):
assert saml_info
prefix = keyu[:-12]
# If key is not there, it IS a mismatch and and error.
saml_key = saml_keys[prefix]
saml_cert_text, _ = make_saml_cert(saml_key, **saml_info)
saml_cert_text = cleanup_x509_text(saml_cert_text)
base.set(section, key, saml_cert_text)
changed = True
if changed:
with open(random_file, 'w') as f:
base.write(f)
return base
def save(self, filename):
""" Save the changed settings to local.cfg """
current = CascadingConfigParser(self.config_files)
# Build a list of changed values
to_save = []
for section in self.sections():
#logging.debug(section)
for option in self.options(section):
if self.get(section, option) != current.get(section, option):
old = current.get(section, option)
val = self.get(section, option)
to_save.append((section, option, val, old))
# Update local config with changed values
local = Parser()
# Start each file with revision identification
local.add_section("Configuration")
local.set("Configuration", "version", "1")
local.readfp(open(filename, "r"))
for opt in to_save:
(section, option, value, old) = opt
if not local.has_section(section):
local.add_section(section)
local.set(section, option, value)
logging.info("Update setting: {} from {} to {} ".format(
option, old, value))
# Save changed values to file
local.write(open(filename, "w+"))
def train():
opt = Parser().parse()
print("Loading dataset")
dataset = Dataset(opt, data_mode=DATA_TRAIN)
print("Generating model")
model = standard_cnn(opt)
print("Preparing for training...")
#(images, labels) = dataset.get_train_data() #Not efficent, loads all training data into memory
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
print("Starting training")
tensorboard_callback = tf.keras.callbacks.TensorBoard(
log_dir=opt.tensorboard_logdir,
histogram_freq=0,
write_graph=True,
write_images=True,
update_freq="batch")
save_callback = CustomSaveCallback(opt.model_dir + "cnn")
model.fit_generator(generator=dataset,
steps_per_epoch=(len(dataset.training_files) //
opt.batch_size),
epochs=opt.num_epochs,
verbose=1,
callbacks=[tensorboard_callback, save_callback])
print(model.summary())
print("Model trained and saved to " + opt.model_dir)
def rc_to_ini(rc_info, default_section=SECTION):
"""Convert a .rc file to a ConfigParser (.ini-like object)
Items are assumed to be in app:assembl section,
unless prefixed by "{section}__" .
Keys prefixed with an underscore are not passed on.
Keys prefixed with a star are put in the global (DEFAULT) section.
Value of '__delete_key__' is eliminated if existing.
"""
p = Parser()
for key, val in rc_info.iteritems():
if not key or key.startswith('_'):
continue
if key[0] == '*':
# Global keys
section = "DEFAULT"
key = key[1:]
elif '__' in key:
section, key = key.split('__', 1)
else:
section = default_section
ensureSection(p, section)
if val == '__delete_key__':
# Allow to remove a variable from rc
# so we can fall back to underlying ini
p.remove_option(section, key)
else:
p.set(section, key, val)
return p
def config_from_filepath(path):
config = Parser()
config.readfp(open(path))
config.get('credentials', 'user')
config.get('credentials', 'password')
config.get('site', 'remote')
return config
def loadMaterialList(cfgFile):
"""Load a list of materials from a file
The file uses the config file format. See ConfigParser module.
"""
p = Parser()
p.read(cfgFile)
#
# Each section defines a material
#
names = p.sections()
matList = [Material(n, p) for n in names]
# Sort the list
matList = sorted(matList, key=lambda x: x.name)
return matList
def compose(rc_filename, random_file=None):
"""Compose local.ini from the given .rc file"""
rc_info = combine_rc(rc_filename)
ini_sequence = rc_info.get('ini_files', None)
assert ini_sequence, "Define ini_files"
ini_sequence = ini_sequence.split()
base = Parser()
random_file = random_file or rc_info.get('random_file', RANDOM_FILE)
for overlay in ini_sequence:
if overlay == 'RC_DATA':
overlay = rc_to_ini(rc_info)
elif overlay.startswith('RANDOM'):
templates = overlay.split(':')[1:]
overlay = populate_random(random_file, templates,
extract_saml_info(rc_info))
combine_ini(base, overlay)
return base
def get_default_settings(self):
fs = []
for config_file in self.config_files:
if os.path.isfile(config_file):
c_file = os.path.basename(config_file)
cp = Parser()
cp.readfp(open(config_file))
fs.append((c_file, cp))
lines = []
for section in self.sections():
for option in self.options(section):
for (name, cp) in fs:
if cp.has_option(section, option):
line = [name, section, option, cp.get(section, option)]
lines.append(line)
return lines
def __init__(self, cfile=None):
service.MultiService.__init__(self)
parser = Parser(self.defaults)
read = parser.read(map(expanduser, self.paths))
if cfile:
parser.readfp(open(cfile, 'r'))
if not cfile and len(read) == 0:
# no user configuration given so load some defaults
parser.add_section('recceiver')
parser.set('recceiver', 'procs', 'show')
elif not parser.has_section('recceiver'):
parser.add_section('recceiver')
pnames = parser.get('recceiver', 'procs').split(',')
plugs = {}
for plug in plugin.getPlugins(interfaces.IProcessorFactory):
_log.debug('Available plugin: %s', plug.name)
plugs[plug.name] = plug
self.procs = []
for P in pnames:
P = P.strip()
plugname, _, instname = P.partition(':')
if not instname:
instname = plugname
plug = plugs[plugname]
if not parser.has_section(instname):
parser.add_section(instname)
inst = plug.build(instname, ConfigAdapter(parser, instname))
self.procs.append(inst)
self.addService(inst)
self._C = parser
def diff_ini(first, second, diff=None, existing_only=False):
"""Diff ini files
Generate a parser with any value in the second that is different in the first.
Returns a ConfigParser.
Takes interpolation into account. Does not include values that disappeared."""
from ConfigParser import _Chainmap
first = asParser(first)
second = asParser(second)
# TODO: Look at first both in raw and formatted versions
diff = diff or Parser()
interpolating = SafeConfigParser()
for section in second.sections():
if section != 'DEFAULT' and not first.has_section(section):
if not existing_only:
diff.add_section(section)
for option in second.options(section):
value = second.get(section, option)
diff.set(section, option, value)
else:
vars = _Chainmap(second._sections[section],
first._sections[section], second._defaults,
first._defaults)
for option, value2 in second.items(section):
if not first.has_option(section, option):
if not existing_only:
ensureSection(diff, section)
diff.set(section, option, value2)
continue
value1 = first.get(section, option)
if value1 != value2 and '%(' in value1:
# try to interpolate, and see if it would amount to the same.
try:
value1 = interpolating._interpolate(
section, option, value1, vars)
except Exception as e:
pass
if value1 != value2:
ensureSection(diff, section)
diff.set(section, option, value2)
return diff
def main(source_ini, update_ini):
parser = Parser()
# By default, the parser is case insensitve and rewrites config
# keys to lowercase. reddit is case sensitive, however
# See: http://docs.python.org/library/configparser.html#ConfigParser.RawConfigParser.optionxform
parser.optionxform = str
# parser.read() will "fail" silently if the file is
# not found; use open() and parser.readfp() to fail
# on missing (or unreadable, etc.) file
parser.readfp(open(source_ini))
with open(update_ini) as f:
updates = f.read()
try:
# Existing *.update files don't include section
# headers; inject a [DEFAULT] header if the parsing
# fails
parser.readfp(StringIO(updates))
except MissingSectionHeaderError:
updates = "[DEFAULT]\n" + updates
parser.readfp(StringIO(updates))
print HEADER
parser.write(sys.stdout)
def compose(rc_filename, random_file=None):
"""Compose local.ini from the given .rc file"""
rc_info = combine_rc(rc_filename)
rc_info['*code_root'] = code_root(rc_info)
# Special case: uwsgi does not do internal computations.
rc_info['uwsgi__virtualenv'] = venv_path(rc_info)
ini_sequence = rc_info.get('ini_files', None)
assert ini_sequence, "Define ini_files"
ini_sequence = ini_sequence.split()
base = Parser()
random_file = random_file or rc_info.get('random_file', RANDOM_FILE)
for overlay in ini_sequence:
if overlay == 'RC_DATA':
overlay = rc_to_ini(rc_info)
elif overlay.startswith('RANDOM'):
templates = overlay.split(':')[1:]
overlay = populate_random(random_file, templates,
extract_saml_info(rc_info))
else:
overlay = find_ini_file(overlay, dirname(rc_filename))
assert overlay, "Cannot find " + overlay
combine_ini(base, overlay)
return base
log.readout('Creating summary pdf')
summary_main(parser_dict, workflow, target, qc_dir, genome_bam_dir,
paired_end, fastq_dir, R1_suffix, R2_suffix, samtools,
log)
if clean_intermediates:
from summary import cleanup_intermediate_folders
log.readout('Removing intermediate files and folders')
cleanup_intermediate_folders(output_dir, parser_dict, target)
log.readout('Completed summary of %s', target)
log.removeFilter(summary)
return
args = parseArgs()
pipeline_config = args['pipeline_config']
parser = Parser()
if int(sys.version[0]) == 3:
try:
parser.read_file(
open(pipeline_config)) #reading in the config file
except IOError as e:
print('Missing config file!')
sys.exit()
try:
parser.read_file(open(parser.get('i/o', 'static_config')))
except configparser.NoOptionError:
pass
elif int(sys.version[0]) == 2:
try:
parser.readfp(open(pipeline_config))
except IOError as e:
import os
from model import load_from_path
from Dataset import Dataset, DATA_TEST
from ConfigParser import Parser
def predict(model, image_data):
return model.predict(image_data)
if __name__ == "__main__":
opt = Parser().parse()
print("Loading model")
model = load_from_path(opt.model_path)
dataset = Dataset(opt, data_mode=DATA_TEST)
totalPredictions = 0
correctPredictions = 0
for sample in dataset:
prediction = round(predict(model, sample["data"])[0][0]) == 1
print ("Predicted file {0}: {1} Label: {2}".format(sample["file"], prediction, sample["label"]))
totalPredictions += 1
if prediction == sample["label"]:
correctPredictions += 1
print ("Predicted {0} files, correct predictions={1}, accuracy={2}".format(totalPredictions,correctPredictions, correctPredictions / totalPredictions))
ap = ArgumentParser(
description=
"""Combine a source ini file with an overlay ini file into a target ini file.
You cannot edit in-place (use same source and destination). Source and overlay cannot both be stdin.
Overlay values with a {randomN} format (N an integer), will be replaced with
a random string of length 4*ceil(N/3).""")
ap.add_argument('source',
type=FileType('r'),
help='The source file (use - for stdin)')
ap.add_argument('overlay',
type=FileType('r'),
help='The overlay file (use - for stdin)')
ap.add_argument('result',
type=FileType('w'),
help='The resulting file (use - for stdout)')
args = ap.parse_args()
source = Parser()
source.readfp(args.source)
overlay = Parser()
overlay.readfp(args.overlay)
for section in overlay.sections():
if section != 'DEFAULT' and not source.has_section(section):
source.add_section(section)
for option in overlay.options(section):
value = overlay.get(section, option)
# Special case: {random66} => random string of 66*(4/3) characters
if value.startswith('{random') and value.endswith("}"):
value = b64encode(urandom(int(value[7:-1])))
source.set(section, option, value)
source.write(args.result)
def loadProject(self):
"""Load any project data to initialize (in development)
This should implemented and documented somewhere else,
but for now, let's start it here.
FILE FORMAT
name: experiment.cfg
sections:
[load]
material-list = <filename>
reader-list = <filename>
[options] # options for this experiment
"""
cfgFile = 'experiment.cfg'
exp = wx.GetApp().ws
#
if os.path.exists(cfgFile):
p = Parser()
p.read(cfgFile)
#
# Each section defines a material
#
sec = 'load'
if p.has_section(sec):
opt = 'material-list'
try:
fname = p.get(sec, opt)
exp.loadMaterialList(fname)
print 'loaded materials list from "%s"\n' % fname
except:
wx.MessageBox('failed to autoload materials list')
pass
opt = 'reader-list'
try:
fname = p.get(sec, opt)
exp.loadReaderList(fname)
print 'loaded readers list from "%s"\n' % fname
except:
wx.MessageBox('failed to autoload readers list')
pass
opt = 'detector'
try:
fname = p.get(sec, opt)
exp.loadDetector(fname)
print 'loaded detector from "%s"\n' % fname
except:
wx.MessageBox('failed to autoload detector list')
pass
pass
sec = 'options'
if p.has_section(sec):
opt = 'start-tab'
try:
val = p.getint(sec, opt)
self.nBook.SetSelection(val)
print 'starting on tab number %d\n' % val
except:
pass
pass
return
def __init(self):
self.file = self.opencfg(UIConfig.cfg)
self.cfg = Parser()
self.cfg.readfp(self.file)
self.file.close()
