def get_cfg(dep_json):
def load_cfg(cfg_path):
return demjson.decode_file(cfg_path)
return (F(get_config_path) >> F(load_cfg))(
dep_json["project_name"],
dep_json["config_name"] if "config_name" in dep_json else None)
def main():
(F(get_data)>> \
F(write_data, \
get_db_client(get_connection_info()), \
get_db_name(get_data_file())\
)
)(get_data_file())
def run(argv):
opts = util.parse_docopt(__doc__, argv, False)
biobox = opts['<biobox_type>']
image = opts['<image>']
task = opts['--task']
verbose = opts['--verbose']
log = opts['--log']
if not behave.features_available(biobox):
error.err_exit("unknown_command", {
"command_type": "biobox type",
"command": biobox
})
ctn.exit_if_no_image_available(image)
if verbose:
results = behave.run(biobox, image, task, False)
else:
results = behave.run(biobox, image, task)
if verbose:
if log:
sys.stdout = open(log, "w+")
statuses = fn.thread([
behave.get_scenarios_and_statuses(results),
F(map, lambda x: name_and_status(*x)),
F(list)
])
longest_name = fn.thread(
[statuses, F(map, fn.first),
F(map, len), max])
def justify(x, y):
return x.ljust(longest_name), y
output = fn.thread([
statuses,
F(map, lambda x: justify(*x)),
F(map, F(" ".join)), fn.unique,
F("\n".join)
])
print(output)
if behave.is_failed(results):
exit(1)
elif behave.is_failed(results):
if log:
sys.stderr = open(log, "w+")
msg = fn.thread([
behave.get_failing_scenarios(results),
F(map, behave.scenario_name),
F("\n".join)
])
error.err_exit('failed_verification', {
'image': image,
'error': msg,
'biobox': biobox
})
def iter_edges(paths, normalize=None, prefilter=None, postfilter=None):
parse = F(imap, iter_parse) >> chain.from_iterable >> F(ifilter, prefilter)
edges = parse(paths)
counter = _count(edges)
extract = F(imap, normalize) >> F(ifilter, postfilter)
return extract(counter.itervalues())
def input_args():
"""
Get command line args excluding those consisting of only whitespace
"""
return fn.thread([
sys.argv[1:],
F(map, string.strip),
F(filter, fn.is_not_empty)])
def test_composition(self):
def f(x): return x * 2
def g(x): return x + 10
self.assertEqual(30, (F(f) << g)(5))
def z(x): return x * 20
self.assertEqual(220, (F(f) << F(g) << F(z))(5))
def processing():
while 1:
res = (F() >>
F(lambda img: cv2.cvtColor(img, code=cv2.COLOR_BGR2GRAY)) >>
F(lambda img: cv2.flip(img, flipCode=1)) >> do_auto_canny >>
show_and_pipe)(get_frame())
if res is not None:
yield res
else:
break
def get_defaults_by_name(rolename):
def get_defaults(path):
return yaml.load(open(path, 'r')) if os.path.exists(path) else {}
return (F(lambda title, content: simple(content, title), "get_defaults_by_name_path") >> \
F(get_defaults) >> \
F(lambda title, content: simple(content, title), "get_defaults_by_name") >> \
F(init_root, rolename) >> \
F(lambda title, content: simple(content, title), "init_root") \
)(os.path.join(parent_folder, "roles", rolename, "defaults", "main.yml")) \
if len(role_names) == 0 or rolename in role_names else {}
def build_deploy_script_internal(project_name, config_name, only_structure=False, remote_dict=None):
deploy_path = lambda:get_deploy_path(project_name, config_name)
deployInfo = DeployInfo(deploy_path())
def yml_file_folder():
return put_folder(os.path.abspath(deploy_path()))
def get_roles_data():
return roles_load(logger.title("config_path").debug(get_config_path(project_name, config_name)))
def write_playbook(content):
open(put_file(deployInfo.playbook_path()), 'w').write(content)
# write data to file
write_playbook( \
roles_build( \
get_roles_data() \
, remote_host=None if remote_dict is None else 'remote' \
, remote_name=remote_dict["remote_user"] if remote_dict else None \
) \
)
# build link for role folders
roles_link(get_roles_data(), \
logger.title("link_root").debug(yml_file_folder()) \
)
# link src folder to deploy/roles/main/files/src for deployment
link_src_to_deploy(get_roles_data())
def default_vals():
return all_defaults(yml_file_folder(), [role["name"] for role in get_roles_data()])
# build inventory files on roles
write_defaults(deployInfo.host_file_path(), default_vals(), remote_addr=None if remote_dict is None else remote_dict["remote_addr"])
# write the defaults to cfg file
(F(lambda config_path: demjson.decode_file(config_path)) >> \
F(lambda new_vals, old_json: update_dict(old_json, new_vals), {} if only_structure else {"predefined_variables": default_vals()}) >> \
F(lambda json_data: json.dumps(json_data, indent=4, ensure_ascii=False)) >> \
F(lambda content: open(get_config_path(project_name, config_name), 'w').write(content)))(get_config_path(project_name, config_name))
def build_ansible_cfg():
if remote_dict is None:
return
open(get_deploy_ansible_cfg_path(project_name, config_name), 'w').write('''[defaults]
remote_user={remote_user}
private_key_file={key_file}
host_key_checking=False
'''.format(remote_user=remote_dict["remote_user"], key_file=remote_dict["key"]))
build_ansible_cfg()
def load_all_config(config_path):
'''
config_path: the path of the banyan configuration file
'''
def get_cfg(dep_json):
def load_cfg(cfg_path):
return demjson.decode_file(cfg_path)
return (F(get_config_path) >> F(load_cfg))(
dep_json["project_name"],
dep_json["config_name"] if "config_name" in dep_json else None)
def get_roles(cfg_json, project_path):
return [{"project_name": os.path.split(project_path)[1], \
"project_path": project_path,
"role_name": role_name} for role_name in cfg_json["roles_seq"]] if "roles_seq" in __logger__.title("get_roles cf_json").debug(cfg_json) else []
def handle(cfg_json, depended_roles, project_path):
try:
return [y for x in [handle(get_cfg(dep), depended_roles, get_project_path(dep["project_name"])) for dep in cfg_json["dependencies"]] for y in x] \
+ get_roles(cfg_json, __logger__.title("get_roles_path_recursive").debug(project_path)) \
if "dependencies" in cfg_json \
else depended_roles + get_roles( \
__logger__.title("no dep json").debug(cfg_json) \
, __logger__.title("no dep path").debug(project_path) \
)
except Exception:
__logger__.error(
"error happen in handle with project_path:%s\ncfg_json:%s" %
(project_path, cfg_json))
raise
def put_all_db_first(roles):
def get_db_roles():
return list(filter(lambda n: is_db_role(n), roles))
def get_non_db_roles():
return list(filter(lambda n: not is_db_role(n), roles))
return get_db_roles() + get_non_db_roles()
return __logger__.title('load_all_config.result') \
.debug( \
(F(handle) \
>> F(put_all_db_first) \
)( \
demjson.decode_file(config_path) \
, [] \
, extract_project_path(config_path) \
) \
)
def test_pushing_simplad(self):
add_maybe = SM.push_simplad(simplad=MS())
add_log = SM.push_simplad(simplad=LogSimplad())
sm1 = (F() >> add_maybe >> add_maybe >> add_log)(SM.make())
add_simplads = SM.add_simplads({
'1': MS(),
'2': MS(),
'3': LogSimplad(),
})
order = SM.set_simplad_order(['1','2','3'])
sm2 = (F() >> add_simplads >> order)(SM.make())
expect(SM.unit(sm1)(4)).to_equal(SM.unit(sm2)(4))
def scenario_status(scenario):
"""
Returns the status of the last step in a scenario
"""
status = fn.thread([
scenario['steps'],
F(map, fn.get("result")),
F(filter, fn.is_not_none),
F(map, fn.get("status")),
])
if fn.is_empty(status):
return "not run"
else:
return status[-1]
def unbin(binned: Iterable[Iterable[T]], bins: Iterable[Iterable[int]]) \
-> List[T]:
"""
Revert binning: transform a nested Iterable of objects (i.e. objects packed
into bins) into a list of objects ordered the same way as the original
Sequence
:param binned: a nested Iterable of binned objects
:param bins: a nested Iterable of bins: Iterables of indices referencing
objects in the original Sequence
:return:
"""
return (F(map, chain.from_iterable) >>
(lambda x: zip(*x)) >> F(sorted, key=op.itemgetter(0)) >>
(map, op.itemgetter(1)) >> list)([bins, binned])
def run(deployInfo):
def build_base_command():
return "sudo ansible-playbook ./{yml_file} -i ./{host_file}".format(yml_file=deployInfo.playbook_name(), host_file=deployInfo.host_file_name())
def build_tag(base_command):
return base_command if not tags else \
'{base_command} --tags "{tags}"'.format(base_command=base_command, tags=",".join(tags))
p = subprocess.Popen(logger.title('cli').debug((F(build_base_command) >> F(build_tag))()), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=put_folder(deployInfo.deploy_folder_path()))
while p.poll() is None:
line = p.stdout.readline()
if line:
print(line.strip())
def __get__(self, instance, owner):
"""
For compatibility with methods (beside functions),
it works like below:
```python
#----- define
class MyClazz(object):
@Repeat # ..... (1)
def my_method(self, a, b, prefix='>>>'):
for i in range(a, b):
yield '{}{}'.format(prefix, i)
obj = MyClazz()
my_gen = obj.my_method(0, 3, prefix=':') # ..... (2)
#----- is equivalent to
r = Repeat # the `Repeat` part at (1)
MyClazz.my_method = r(MyClazz.my_method) # the `@` part at (1)
got_fn = Repeat.__get__(r, obj, MyClazz) # the `obj.my_method` part at (2)
my_gen = got_fn(0, 3, prefix=':') # the `(0, 3, prefix=':')` part at (2)
```
:param instance: the calling object
:param owner: not used
:return:
"""
self.bound_fn = F(self.raw_fn, instance)
return self
def test_bind_maybe(self):
add_simplads = SM.add_simplads({
'1': MS(),
'2': MS(),
'3': MS(),
'4': MS(),
'5': MS()
})
order = SM.set_simplad_order(['1','2','3','4','5'])
sm = (F() >> add_simplads >> order)(SM.make())
val = 4
bound = SM.unit(sm)(val)
def double_simplad_result(i):
return SimpladResult(val=i*2, delta_map={})
bound = SM.bind(double_simplad_result)(
(sm, BindArgs(bound=bound, deltas=[])))
bound = SM.bind(double_simplad_result)(bound)
bound = SM.bind(double_simplad_result)(bound)
bound = SM.bind(double_simplad_result)(bound)
bound = SM.bind(double_simplad_result)(bound)
bound = SM.bind(double_simplad_result)(bound)
bound = SM.bind(double_simplad_result)(bound)
expect(bound[1].bound).to_equal(
Bound(unbound=Bound(unbound=Bound(unbound=Bound(unbound=Bound(
unbound=512,
annotation=MaybeType.has_value),
annotation=MaybeType.has_value),
annotation=MaybeType.has_value),
annotation=MaybeType.has_value),
annotation=MaybeType.has_value))
def test_bind_with_fail(self):
add_simplads = SM.add_simplads({
'1': MS(),
'2': MS(),
'3': ListSimplad(),
'4': ListSimplad(),
'5': MS()
})
order = SM.set_simplad_order(['1','2','3','4','5'])
sm = (F() >> add_simplads >> order)(SM.make())
val = [[4]]
bound = SM.unit(sm)(val)
def double_simplad_result(i):
return SimpladResult(val=i*2, delta_map={})
def double_simplad_result_fail_2(i):
return SimpladResult(val=i*2, delta_map={
'5': MaybeType.no_value
})
bound = SM.bind(double_simplad_result)(
(sm, BindArgs(bound=bound, deltas=[])))
bound = SM.bind(double_simplad_result)(bound)
bound = SM.bind(double_simplad_result)(bound)
bound = SM.bind(double_simplad_result_fail_2)(bound)
bound = SM.bind(double_simplad_result)(bound)
bound = SM.bind(double_simplad_result)(bound)
bound = SM.bind(double_simplad_result)(bound)
expect(bound[1].bound).to_equal(Bound(unbound=None,
annotation=MaybeType.no_value))
class MRows(DataFrameMonoid):
_rbind = (F(filter, lambda x: x is not None) >> list >>
(lambda x: pd.concat(x, axis=0, sort=True) if x else None))
def binder(self, a: pd.DataFrame, b: pd.DataFrame) -> pd.DataFrame:
return self._rbind([a, b])
def format_scenario_name(name):
return fn.thread([
string.replace(name, "Should ", ""),
lambda x: x[0].upper() + x[1:],
F(flip(string.split), '--'),
fn.first,
string.strip])
def eval_variables(analyses, node):
"""
Replace all variables in a node s-expression with their referenced literal
value.
Args:
analysis (dict): A dictionary corresponding to the values referenced in the
given s-expression.
node (list): An s-expression list in the form of [operator, arg1, arg2, ...].
Yields:
A node expression with referenced values replaced with their literal values.
Examples:
>>> eval_variables({a: 1}, [>, :a, 2])
[>, 1, 2]
"""
def _eval(n):
if var.is_variable(n):
return var.get_variable_value(analyses, n)
else:
return n
return map(fn.recursive_apply(F(eval_variables, analyses), _eval), node)
def parse_mutagenesis(lines: str):
if not lines.strip().startswith('>'):
raise ValueError('The input does not start with a subrecord entry')
def parse_subrecord(recs: List[List], line: str):
if line.startswith('>'):
return recs.append([line.lstrip('> ')]) or recs
cls, *spec = map(str.strip, line.split('|'))
if cls.upper().strip('[]?') not in MUTCLASSES:
raise ValueError(f'Unsupported effect class: {cls}')
if cls.upper() == 'UNK' and spec:
raise ValueError(f"UNK effects can't have specs: {line}")
if len(spec) > 3:
raise ValueError(f'Incorrect annotation format: {line}')
lvl, target, assoc = (
[val if val.lstrip('!') != '?' else None
for val in spec] + [None] * (3 - len(spec)))
if lvl and lvl.lstrip('!') not in MUTLEVELS:
raise ValueError(f'Unsupported effect level: {lvl}')
return recs[-1].append([cls.upper(), lvl, target, assoc]) or recs
subrecords = OrderedDict(
(entry, anno)
for entry, *anno in reduce(parse_subrecord, (F(str.splitlines) >> (
map, str.strip) >> (filter, bool) >> list)(lines), []))
if not all(subrecords.values()):
raise ValueError('Unannotated and empty subrecords are not allowed')
return subrecords
def check_mut(mut: parser.Mutation) -> parser.Mutation:
if not (mut.subrecs and all(mut.subrecs)):
raise ValueError(
f'malformed subrecord(s) in {record.protein}:{mut.id}')
return parser.Mutation(mut.id, mut.start, mut.stop, mut.ref, mut.alt,
mut.description,
list(map(F(check_subrec, mut.id), mut.subrecs)))
def stitchpoints(intervals: Sequence[Interval], targets: Sequence[Interval]):
"""
Find breakpoints that have to be stitched in order to recover target
intervals from finer subintervals. For a set of intervals [iv_1, ..., iv_n]
that must be stitched to obtain a target t1, the function returns
[(iv_1).stop-1, ..., (iv_n-1).stop-1]. The function groups all intervals
intersecting a target together and merges them. Take note, that an ideal
reconstruction might not be achievable. In this case it is only guaranteed
that all merged intervals will contain the entire span of the corresponding
target, but not the other way around.
:param intervals:
:param targets:
:return:
"""
intervals_ = sorted(intervals, key=lambda iv: iv.start)
stitched_ = sorted(targets, key=lambda iv: iv.start)
inbreaks = F(droplast, 1) >> breakpoints
def grouper(acc: Tuple[List[int], Iterable[Interval]], iv: Interval):
# find breakpoints to stitch
breaks, ivs = acc
grouped, remainder = splitby(
iv.intersects, dropwhile(lambda x: x.stop <= iv.start, ivs))
return breaks.extend(inbreaks(grouped)) or breaks, list(remainder)
return reduce(grouper, stitched_, ([], intervals_))[0]
def count_multiple_samples(
kmer_size: int,
input_file: str,
output_dir: str,
num_jobs: int = DEFAULT_NUM_JOBS,
delimiter: str = DEFAULT_DELIMITER,
num_bins: int = DEFAULT_NUM_BINS) -> Dict[str, SparseArray]:
"""
Count k-mers of several samples located in the same input_file.
:param input_file: Input file name
:param output_dir: Output directory
:param kmer_size: Size of k-mer
:param num_jobs: Number of jobs to run in parallel
:param delimiter: Pattern for splitting a sequence record description to parse a sample name
:param num_bins: Optional number of bins to split k-mers between while counting
:return: List of sparse arrays containing counted k-mers encoded as uint64_t numbers
"""
sample_files = _split_sequences(input_file,
output_dir,
delimiter=delimiter)
counter = F(count_sample, kmer_size, num_bins=num_bins)
if num_jobs == 1:
return dict(zip(sample_files, map(counter, sample_files)))
else:
pool = mp.Pool(num_jobs)
return dict(zip(sample_files, pool.map(counter, sample_files)))
def rename_associations(mapping: Mapping[str, str], record: parser.Record) \
-> parser.Record:
"""
Rename associations in all subrecords.
:param mapping:
:param record:
:return:
:raises ValueError: if there are missing/malformed entries at any level,
if the mapping is malformed or non-exhaustive.
"""
is_association = F(fnor, ASSOC_PAT.match, PHANTOM_ASSOC_PAT.match)
if not (F(map, is_association) >> all)(mapping.values()):
raise ValueError(f'malformed association mapping for {record.protein}')
def check_mut(mut: parser.Mutation) -> parser.Mutation:
if not (mut.subrecs and all(mut.subrecs)):
raise ValueError(
f'malformed subrecord(s) in {record.protein}:{mut.id}')
return parser.Mutation(mut.id, mut.start, mut.stop, mut.ref, mut.alt,
mut.description,
list(map(F(check_subrec, mut.id), mut.subrecs)))
def check_subrec(mutid: int, subrec: parser.SubRecord) -> parser.SubRecord:
if not (subrec.effects and all(subrec.effects)):
raise ValueError(
f'malformed effect(s) in {record.protein}:{mutid}:{subrec.id}')
effects = list(map(rename_assoc, subrec.effects))
return parser.SubRecord(subrec.id, subrec.description, effects)
def rename_assoc(effect: parser.Effect) \
-> parser.Effect:
try:
associations = [mapping[assoc] for assoc in effect.associations]
return parser.Effect(effect.cls, effect.level, effect.target,
associations)
except KeyError as err:
raise ValueError(
f'non-exhaustive association mapping in {record.protein}; {err}'
)
if not (record.mutations and all(record.mutations)):
raise ValueError(f'malformed mutations in {record.protein}')
return parser.Record(record.protein, list(map(check_mut,
record.mutations)))
def test_pipe_composition(self):
def f(x):
return x * 2
def g(x):
return x + 10
self.assertEqual(20, (F() >> f >> g)(5))
def mismatched_join(input_files, left_on, right_on, how):
data = map(F(load_datasets, delimiter), input_files[0:2])
transformed = p.merge(data[0],
data[1],
left_on=left_on,
right_on=right_on,
how=how)
return transformed
def readlines(path: str,
tokeniser: Callable[[str], List[intervals.Interval[str]]],
maxsteps: int) \
-> List[List[intervals.Interval[str]]]:
with open(path) as lines:
cleanlines = (F(map, str.strip) >> (filter, bool))(lines)
tokenised = map(tokeniser, cleanlines)
return [tokens[:maxsteps] for tokens in tokenised]
def apply_function(function, group):
"""@todo: Docstring for apply_functionn.
:group: @todo
:function: @todo
:returns: @todo
"""
fmap = {
'agg': F(group.agg, [np.sum, np.mean, np.std]),
'aggregate': group.aggregate,
'boxplot': group.boxplot,
'corr': group.corr,
'corrwith': group.corrwith,
'cov': group.cov,
'cumcount': group.cumcount,
'cummax': group.cummax,
'cummin': group.cummin,
'cumprod': group.cumprod,
'cumsum': group.cumsum,
'describe': group.describe,
'diff': group.diff,
'ffill': group.ffill,
'fillna': group.fillna,
'filter': group.filter,
'first': group.first,
'get_group': group.get_group,
'groups': group.groups,
'head': group.head,
'hist': group.hist,
'idxmax': group.idxmax,
'idxmin': group.idxmin,
'irow': group.irow,
'last': group.last,
'mad': group.mad,
'max': group.max,
'mean': group.mean,
'median': group.median,
'min': group.min,
'name': group.name,
'ngroups': group.ngroups,
'nth': group.nth,
'ohlc': group.ohlc,
'pct_change': group.pct_change,
'plot': group.plot,
'prod': group.prod,
'quantile': group.quantile,
'rank': group.rank,
'resample': group.resample,
'shift': group.shift,
'size': group.size,
'skew': group.skew,
'std': group.std,
'sum': group.sum,
'tail': group.tail,
'var': group.var
}
return fmap[function]().reset_index()
def main():
key = F(str.lower) << itemgetter(0)
for arg in argv[1:]:
if isdir(arg):
many = sorted(listdir(arg), key=key)
groups = groupby(many, key=key)
dest = join(dirname(arg), 'one.html')
with open(dest, 'w') as one:
one.write(make_index(make_all(groups)))
