def get_mad_dummy(app, data):
"""
instantiate a dummy - only used to save.
"""
global STORES
if STORES is None:
initialize_stores(app)
if 'madname' in data:
del data['madname']
if 'fullmadpath' in data:
del data['fullmadpath']
data_all = fantail.Fantail(data)
data_core = fantail.Fantail()
for kw in list(data_all.keys()):
tra = app.conf['keywords'][kw].get('transient', False)
if not tra:
data_core[kw] = data_all[kw]
lg.debug("instantiating dummy madfile")
return MadDummy(data_all=data_all,
data_core=data_core,
stores=STORES,
hook_method=app.run_hook)
def setUp(self):
self.a = fantail.Fantail()
self.b = fantail.Fantail()
self.s = fantail.Fanstack([self.a,self.b])
self.a['a.b.c'] = 1
self.a['a.b.d'] = 2
self.b['a.b.e'] = 3
def test_yaml_file_save(self):
y = fantail.Fantail()
y['a']['b']['c'] = 1
y['d.e'] = '2'
tf = tempfile.NamedTemporaryFile(delete=False)
tf.close()
fantail.yaml_file_save(y, tf.name)
tf.close()
z = fantail.yaml_file_loader(tf.name)
self.assertEqual(y, z)
def __init__(self, *args, **kwargs):
if len(args) == 0:
name = os.path.split(sys.argv[0])[1]
if name[:3] == 'kea_':
name = name[3:]
else:
name = args[0]
# Call Leip - we do not need the Leip argparser:
super(Kea, self).__init__('kea', disable_commands=True)
# replace the config by a stack so we can backfill
self.conf = fantail.Fanstack([self.conf, fantail.Fantail()])
# hack - if kea verbose is set - do that early:
verbose_flag = self.conf['arg_prefix'] + '-v'
if verbose_flag in sys.argv:
lg.setLevel(logging.DEBUG)
#default executors
self.executors = executors
lg.debug("start kea initialization")
# different hooks!
self.hook_order = [
'pre_argparse',
'argparse',
'post_argparse',
'prepare',
'pre_run',
'run',
'finish']
# for the kea argparse (starting with app.conf.arg_prefix)
# default prefix = '---'
# need to define how many arguments are taken from the command
# line for each flag - the rest is handled by argparse
self.kea_arg_harvest_extra = {}
self.kea_argparse = argparse.ArgumentParser(
prog='(kea){}'.format(name),
description='Kea wrapper for: {}'.format(name),
epilog='NOTE: Prefix all Kea arguments with: "' +
self.conf['arg_prefix'] + '"')
self._madapp = None # hold the Leip Mad Application
self.conf['appname'] = name
self.conf['kea_executable'] = sys.argv[0]
self.discover(globals())
def __init__(self,
inputfile,
stores=None,
sha1sum=None,
base=fantail.Fantail(),
hook_method=dummy_hook_method):
self.stores = stores
self.hook_method = hook_method
lg.debug('madfile start %s', inputfile)
super(MadFile, self).__init__(stack=[fantail.Fantail(), base.copy()])
self.dirmode = False
if os.path.isdir(inputfile):
self.dirmode = True
dirname = inputfile
filename = ''
else:
dirname = os.path.dirname(inputfile)
filename = os.path.basename(inputfile)
if os.path.islink(inputfile):
self.all['orphan'] = not os.path.exists(inputfile)
lg.debug("Instantiating a madfile for '{}' / '{}'".format(
dirname, filename))
self.all['inputfile'] = inputfile
self.all['dirname'] = os.path.abspath(dirname)
self.all['filename'] = filename
self.all['fullpath'] = os.path.abspath(os.path.realpath(inputfile))
self.hook_method('madfile_init', self)
for s in self.stores:
store = self.stores[s]
store.prepare(self)
self.load()
def __init__(self, app, args, template='.', argv=[], transient=False):
lg.debug("Instantiate job with template: %s", template)
self.app = app
self.transient = transient
self.runargs = args
self.skipped = False
self.argv = argv
# template name this object got called with
self.template = template
self._template_file = None
# the context will be used for parameter expaions
self.ctx = fantail.Fantail()
def _get_recursive_dir_data(pth):
global RECURSE_CACHE
lg.debug("start recursive data load for {}".format(pth))
pth = os.path.abspath(pth)
if os.path.exists(pth) and not os.path.isdir(pth):
here = os.path.dirname(pth)
else:
here = pth
here = here.rstrip('/')
conf = []
# find existing directory configuration from the perspective
# of the madfile
last = here
while True:
if os.path.isdir(here):
here_c = os.path.join(here, 'mad.config')
if os.path.exists(here_c):
conf.append(here_c)
parent = os.path.dirname(here)
if parent == '/': # no config in the root - that would be evil!
break
last = here
here = parent
rv = fantail.Fantail()
for c in conf[::-1]:
lg.debug("read .config file: %s", c)
fullname = os.path.expanduser(os.path.abspath(c))
if fullname in RECURSE_CACHE:
y = RECURSE_CACHE[fullname]
else:
#print('start load dir', fullname)
y = fantail.yaml_file_loader(fullname)
RECURSE_CACHE[fullname] = y
rv.update(y)
lg.debug(rv.pretty())
return rv
def test_setlevelgetitem(self):
y = fantail.Fantail()
y['a']['b'] = 1
self.assertEqual(y['a.b'], 1)
y['c.d'] = 1
self.assertEqual(y['c']['d'], 1)
def test_setgetitem(self):
y = fantail.Fantail()
y['a'] = 1
self.assertEqual(y['a'], 1)
def test_load(self):
y = fantail.Fantail()
def test_store_dict(self):
f = fantail.Fantail()
f['a'] = 1
self.assertEqual(f['a'], 1)
def d():
return fantail.Fantail(test_set_1)
def basic_command_line_generator(app):
"""
Command line generator that expands globs & ranges
"""
info = fantail.Fantail()
stdin_file = app.kea_args.stdin
stdout_file = app.kea_args.stdout
stderr_file = app.kea_args.stderr
#cl = [app.conf['executable']] + sys.argv[1:]
cl = sys.argv[1:]
max_no_jobs = app.kea_args.maxjobs
#check if there are map arguments in here
mapins = []
mapcount = 0
replacements = 0
## find all map definitions
for i, arg in enumerate(cl):
fipa = RE_FIND_MAPINPUT.search(arg)
if not fipa:
continue
if fipa.groups()[2] is None:
replacements += 1
continue
#print fipa.groups()
mapins.append(i)
# no map definitions found - then simply return the cl & execute
if len(mapins) == 0:
newcl = map(_unescape, cl)
info['cl'] = newcl
info['iteration'] = 0
info['stdin_file'] = stdin_file
info['stdout_file'] = stdout_file
info['stderr_file'] = stderr_file
yield info
return
#define iterators for each of the definitions
mapiters = []
for arg_pos in mapins:
map_info = {}
map_info['pos'] = arg_pos
map_info['arg'] = cl[arg_pos]
map_info['re_search'] = RE_FIND_MAPINPUT.search(cl[arg_pos])
name, operator, pattern = map_info['re_search'].groups()
if operator is None:
if ':' in pattern:
operator = '='
else:
operator = '~' #default to file type glob
if name is None:
name = ""
lg.debug("cl expand 1")
lg.debug(" - name : %s", name)
lg.debug(" - operator : %s", operator)
lg.debug(" - pattern : %s", pattern)
map_info['name'] = name
map_info['operator'] = operator
map_info['pattern'] = pattern
re_from_re = r'({' + map_info['name'] + r'[\~\=]?[^}]*})'
map_info['re_from'] = \
re.compile(re_from_re)
map_info['re_replace'] = \
re.compile(r'({' + map_info['name'] + r'})')
map_info['start'] = map_info['re_search'].start()
map_info['tail'] = len(cl[arg_pos]) - map_info['re_search'].end()
if map_info['operator'] == '~':
map_info['items'] = map_glob_expand(map_info, cl)
elif map_info['operator'] == '=':
map_info['items'] = map_range_expand(map_info, cl)
mapiters.append(map_iter(map_info))
for i, map_info_set in enumerate(itertools.product(*mapiters)):
if i >= max_no_jobs:
break
newcl = copy.copy(cl)
newinfo = copy.copy(info)
newstdin = stdin_file
newstdout = stdout_file
newstderr = stderr_file
for map_info in map_info_set:
newcl = apply_map_info_to_cl(newcl, map_info)
if not newstdin is None:
newstdin = apply_map_info_to_cl([newstdin], map_info)[0]
if not newstdout is None:
newstdout = apply_map_info_to_cl([newstdout], map_info)[0]
if not newstderr is None:
newstderr = apply_map_info_to_cl([newstderr], map_info)[0]
newcl = map(_unescape, cl)
newinfo['cl'] = newcl
newinfo['iteration'] = i
newinfo['stdin_file'] = newstdin
newinfo['stdout_file'] = newstdout
newinfo['stderr_file'] = newstderr
yield newinfo
def madset(app, args):
"""
Set a key/value for one or more files.
Use this command to set a key value pair for one or more files.
This command can take the following forms::
mad set project test genome.fasta
ls *.fasta | mad set project test
find . -size +10k | mad set project test
"""
key = args.key
val = args.value
if args.prompt or args.editor:
if not args.value is None:
# when asking for a prompt - the next item on sys.argv
# is assumed to be a file, and needs to be pushed
# into args.file
args.file = [args.value] + args.file
madfiles = []
# gather all madfiles for later parsing
use_stdin = not (args.prompt or args.editor)
if args.dir:
for m in get_filenames(args, use_stdin, allow_dirs=True):
if not os.path.isdir(m):
continue
fn = os.path.join(m, 'mad.config')
mf = fantail.Fantail()
if os.path.exists(fn):
mf = fantail.yaml_file_loader(fn)
mf._mad_dir_name = m
mf._mad_file_name = fn
madfiles.append(mf)
else:
for m in get_all_mad_files(app, args, use_stdin):
madfiles.append(m)
# check if mad needs to show a prompt or editor
if val is None and not (args.prompt or args.editor):
args.prompt = True
# show prompt or editor
if args.prompt or args.editor:
# get a value from the user
default = ''
# Show a prompt asking for a value
data = madfiles[0]
default = madfiles[0].get(key, "")
if args.prompt:
sys.stdin = open('/dev/tty')
val = mad2.ui.askUser(key, default, data)
sys.stdin = sys.__stdin__
elif args.editor:
editor = os.environ.get('EDITOR', 'vim')
tmp_file = tempfile.NamedTemporaryFile('wb', delete=False)
# write default value to the tmp file
if default:
tmp_file.write(default + "\n")
else:
tmp_file.write("\n")
tmp_file.close()
tty = open('/dev/tty')
subprocess.call('{} {}'.format(editor, tmp_file.name),
stdin=tty,
shell=True)
sys.stdin = sys.__stdin__
# read value back in
with open(tmp_file.name, 'r') as F:
# removing trailing space
val = F.read().rstrip()
# remove tmp file
os.unlink(tmp_file.name)
# process key & val
key, val, list_mode = _getkeyval(app, key, val, args.force)
lg.info('set %s to %s', key, val)
if list_mode:
lg.info("List modus")
# Now process madfiles
lg.debug("processing %d files" % len(madfiles))
for madfile in madfiles:
if list_mode:
if not key in madfile:
oldval = []
else:
oldval = madfile.get(key)
if not isinstance(oldval, list):
oldval = [oldval]
if args.dir:
madfile[key] = oldval + [val]
else:
madfile.mad[key] = oldval + [val]
else:
#not listmode
if args.dir:
madfile[key] = val
else:
madfile.mad[key] = val
if args.echo:
if args.dir:
print((madfile._mad_dir_name))
else:
print((madfile['filename']))
if args.dir:
fantail.yaml_file_save(madfile, madfile._mad_file_name)
else:
madfile.save()
def expand(self):
self.ctx['epilog'] = []
self.ctx['prolog'] = []
lg.debug("start expansion")
if self.data.get('mode') in ['start', 'reduce']:
lg.warning('%s mode - generate one job', self.data['mode'])
self.ctx['i'] = 0
for io in self.data['io']:
if 'expanded' in io:
self.ctx[io['name']] = io['expanded']
else:
self.ctx[io['name']] = io['pattern']
for par in self.data['parameters']:
if 'expanded' in par:
self.ctx[par['name']] = par['expanded']
else:
self.ctx[par['name']] = par['pattern']
self.app.run_hook('expanded', self)
yield self
return
# assume map mode
nojobs = set()
expfields = []
for io in self.data['io']:
if 'expanded' in io:
expfields.append(io['name'])
nojobs.add(len(io['expanded']))
else:
expfields.append(None)
nojobs.add(1)
for par in self.data['parameters']:
if 'expanded' in par:
expfields.append(par['name'])
nojobs.add(len(par['expanded']))
if len(nojobs) == 0:
nojobs = 0
elif len(nojobs) == 1:
nojobs = list(nojobs)[0]
else:
raise Exception("Invalid job")
lg.info("generating %d jobs", nojobs)
for i in range(nojobs):
# copying shallowly...
newjob = copy.copy(self)
# ...but ensure it has its own context
newjob.ctx = fantail.Fantail()
newjob.ctx.update(self.ctx)
newjob.ctx['i'] = i
# fill the io data into the ctx
for io in self.data['io']:
if 'expanded' in io:
newjob.ctx[io['name']] = io['expanded'][i]
else:
newjob.ctx[io['name']] = io['pattern']
for par in self.data['parameters']:
if 'expanded' in par:
newjob.ctx[par['name']] = par['expanded'][i]
else:
newjob.ctx[par['name']] = par['pattern']
self.app.run_hook('expanded', newjob)
yield newjob
def test_nonstringkeys(self):
y = fantail.Fantail()
y[1] = 12
self.assertEqual(y[1], 12)
def test_get(self):
y = fantail.Fantail()
y['a'] = 1
self.assertEqual(y.get('a'), 1)
self.assertEqual(y.get('a', 2), 1)
self.assertEqual(y.get('b', 3), 3)
def get_empty_yaco(self):
return fantail.Fantail()