def test_prepinput_exc(job0, tmpdir):
infile1 = tmpdir / 'test_prepinput_not_exists.txt'
job0.proc.input = Diot(
infile=('file', [[]]), # no a strin gor path or input [infile:file]
)
with pytest.raises(JobInputParseError):
job0._prepInput()
job0.proc.input = Diot(
nefile=('file', [infile1]), # not exists
)
with pytest.raises(JobInputParseError):
job0._prepInput()
job0.proc.input = Diot(
nlfiles=('files', [1]), # not a list
)
with pytest.raises(JobInputParseError):
job0._prepInput()
job0.proc.input = Diot(
npfiles=('files', [[None]]), # not a path
)
with pytest.raises(JobInputParseError):
job0._prepInput()
job0.proc.input = Diot(nefiles=('files', [[infile1]]))
with pytest.raises(JobInputParseError):
job0._prepInput()
def fixt_fileflush(request, fd_fileflush):
fd_read, fd_append = fd_fileflush
if request.param == 0:
return Diot(filed=fd_read, residue='', expt_lines=[], expt_residue='')
if request.param == 1:
fd_append.write('abcde')
fd_append.flush()
return Diot(filed=fd_read,
residue='',
expt_lines=[],
expt_residue='abcde')
if request.param == 2:
fd_append.write('ccc\ne1')
fd_append.flush()
return Diot(filed=fd_read,
residue='abcde',
expt_lines=['abcdeccc\n'],
expt_residue='e1')
if request.param == 3:
fd_append.write('ccc')
fd_append.flush()
return Diot(filed=fd_read,
residue='',
end=True,
expt_lines=['ccc\n'],
expt_residue='')
if request.param == 4:
return Diot(filed=fd_read,
residue='end',
end=True,
expt_lines=['end\n'],
expt_residue='')
def test_get():
bx = Diot(diot_nest=[dict, list])
bx["c"] = {}
assert isinstance(bx.get("c"), Diot)
assert isinstance(bx.get("b", {}), Diot)
assert "a" in bx.get("a", Diot(a=1))
assert isinstance(bx.get("a", [{"a": 1}])[0], Diot)
def test_trydeepcopy():
def tryDeepCopy(obj, _recurvise=True):
"""
Try do deepcopy an object. If fails, just do a shallow copy.
@params:
obj (any): The object
_recurvise (bool): A flag to avoid deep recursion
@returns:
The copied object
"""
if _recurvise and isinstance(obj, dict):
# do a shallow copy first
# we don't start with an empty dictionary, because obj may be
# an object from a class extended from dict
ret = obj.copy()
for key, value in obj.items():
ret[key] = tryDeepCopy(value, False)
return ret
if _recurvise and isinstance(obj, list):
ret = obj[:]
for i, value in enumerate(obj):
ret[i] = tryDeepCopy(value, False)
return ret
try:
return deepcopy(obj)
except TypeError:
return obj
dt = Diot(a=Diot(b=Diot(c=1)))
dt3 = tryDeepCopy(dt)
assert dt3 == dt
def test_update():
a = Diot(**test_dict)
a.grand = 1000
a.update({'key1': {'new': 5}, 'Key 2': {"add_key": 6}, 'lister': ['a']})
a.update([('asdf', 'fdsa')])
a.update(testkey=66)
a.update({'items': {'test': 'pme'}})
a.update({'key1': {'gg': 4}})
b = Diot(diot_nest=[list, dict])
b.update(item=1)
assert a.grand == 1000
assert a['grand'] == 1000
assert isinstance(a['items'], Diot)
assert a['items'].test == 'pme'
assert a['Key 2'].add_key == 6
assert isinstance(a.key1, Diot)
assert isinstance(a.lister, list)
assert a.asdf == 'fdsa'
assert a.testkey == 66
assert a.key1.new == 5 # On regular dict update this shouldn't happen
assert a.key1.gg == 4
c = Diot(diot_nest=[dict])
c.a = [1, 2]
c.update({'b': [3, 4]})
assert c.a == [1, 2]
assert isinstance(c.b, list)
def fixt_chmodx(request, tmp_path):
if request.param == 'not_a_file':
xfile = tmp_path / 'chmodxtest_not_a_file'
xfile.mkdir()
return Diot(file=xfile, expt=OSError)
elif request.param == 'success':
xfile = tmp_path / 'chmodxtest_success'
xfile.write_text('')
return Diot(file=xfile, expt=[str(xfile)])
elif getuid() == 0:
pytest.skip('I am root, I cannot fail chmod and read from shebang')
elif request.param == 'failed_to_chmod':
xfile = '/etc/passwd'
return Diot(file=xfile, expt=OSError)
elif request.param == 'from_shebang':
if not path.isfile('/bin/zcat'):
pytest.skip('/bin/zcat not exists.')
else:
return Diot(file='/bin/zcat', expt=['/bin/sh', '/bin/zcat'])
elif request.param == 'unierr_shebang':
xfile = '/bin/bash'
if not path.isfile('/bin/bash'):
pytest.skip('/bin/bash not exists.')
else:
return Diot(file=xfile, expt=OSError) # UnicodeDecodeError
def on_filetree_req(self, data):
"""Request the content of a folder or a file item"""
# data: proc, job, type, path, eleid, rootid
# resp: proc, job, type, eleid, path, rootid, content
data = Diot(data)
resp = data.copy()
workdir = pipeline_data.procs[data['proc']].props.workdir
jobdir = Path(workdir) / str(data.job)
path = jobdir.joinpath(data.path)
resp.name = path.name
if data.type == 'folder':
resp.content = []
for item in path.iterdir():
resp.content.append(
Diot(name=item.name,
path=(f"{data.path}/{item.name}"
if data.path else item.name),
type='folder' if item.is_dir() else _filetype(item)))
elif data.type == 'image':
with open(path, 'rb') as fimg:
resp.content = fimg.read()
elif path.stat().st_size > 1024 * 1024:
resp.type = False
else:
with open(path, 'rt') as fpath:
resp.content = fpath.read()
emit('filetree_resp', resp)
def test_basic_box():
a = Diot(one=1, two=2, three=3)
b = Diot({'one': 1, 'two': 2, 'three': 3})
c = Diot((zip(['one', 'two', 'three'], [1, 2, 3])))
d = Diot(([('two', 2), ('one', 1), ('three', 3)]))
e = Diot(({'three': 3, 'one': 1, 'two': 2}))
assert a == b == c == d == e
def on_job_script_save_req(self, data):
"""Save job.script"""
# proc, job, script
data = Diot(data)
workdir = pipeline_data.procs[data['proc']].props.workdir
jobscript = Path(workdir) / str(int(data.job)) / 'job.script'
resp = Diot(proc=data.proc,
job=data.job,
ok=True,
msg='',
run=data.run,
eleid=data.eleid)
try:
cmdy.cp(jobscript, str(jobscript) + '.pyppl_web.bak')
jobscript.write_text(data.script)
if data.run:
self.on_run_request(
Diot(eleid=data.eleid,
proc=data.proc,
job=data.job,
target='job.script',
cmd=''))
except BaseException as ex:
resp.ok = False
resp.msg = str(ex)
emit('job_script_save_resp', resp)
def on_run_request(self, data):
"""Running a command/script"""
logger.debug('Got request run_request from client '
f'{request.remote_addr}')
# eleid: logger.id,
# proc: that.proc,
# job: that.job,
# target: type,
# cmd: $(this).val()});
data = Diot(data)
running = pipeline_data.setdefault('running', Diot())
if data.eleid in running:
# is already running
return
cmd = data.cmd
if 'target' in data:
workdir = pipeline_data.procs[data['proc']].props.workdir
target = Path(workdir) / str(int(data.job)) / str(data.target)
target = repr(str(target))
cmd = cmd + ' ' + target if cmd else target
running[data.eleid] = {
'cmdy': cmdy.bash(c=cmd, _raise=False).iter,
'buffer': ''
}
def proc_init(proc):
"""Add config"""
proc.add_config('report_template',
default='',
converter=report_template_converter)
proc.add_config('report_envs',
default=Diot(),
converter=lambda envs: envs or Diot())
def test_box_inits():
a = Diot({'data': 2, 'count': 5})
b = Diot(data=2, count=5)
c = Diot({'data': 2, 'count': 1}, count=5)
d = Diot([('data', 2), ('count', 5)])
e = Diot({'a': [{'item': 3}, {'item': []}]}, diot_nest=[dict, list])
assert e.a[1].item == []
assert a == b == c == d
def test_pop_items():
bx = Diot(a=4)
assert bx.popitem() == ('a', 4)
with pytest.raises(KeyError):
assert bx.popitem()
bx = Diot({'a_@_b': 1})
assert bx.popitem() == ('a_@_b', 1)
def test_submit(job0, caplog):
job0.isRunningImpl = lambda: True
assert job0.submit()
assert 'is already running at' in caplog.text
job0.isRunningImpl = lambda: False
job0.submitImpl = lambda: Diot(rc=0)
assert job0.submit()
job0.submitImpl = lambda: Diot(rc=1, cmd='', stderr='')
caplog.clear()
assert not job0.submit()
assert 'Submission failed' in caplog.text
def test_is_in():
bx = Diot()
dbx = Diot()
assert "a" not in bx
assert "a" not in dbx
bx["b"] = 1
dbx["b"] = {}
assert "b" in bx
assert "b" in dbx
bx["a_@_b"] = 1
assert "a__b" in bx
assert "a_@_b" in bx
delattr(bx, "a_@_b")
def __init__(self, *procs, **kwargs):
"""@API
Constructor
@params:
*procs (Proc) : the set of processes
**kwargs: Other arguments to instantiate a `ProcSet`
depends (bool): Whether auto deduce depends. Default: `True`
id (str): The id of the procset. Default: `None` (the variable name)
tag (str): The tag of the processes. Default: `None`
copy (bool): Whether copy the processes or just use them. Default: `True`
"""
self.__dict__['id'] = kwargs.get('id') or varname(context = 101)
self.__dict__['tag'] = kwargs.get('tag')
self.__dict__['starts'] = Proxy()
self.__dict__['ends'] = Proxy()
self.__dict__['delegates'] = OrderedDiot()
self.__dict__['procs'] = OrderedDiot()
self.__dict__['modules'] = Diot(diot_nest = False)
# save initial states before a module is called
# states will be resumed before each module is called
self.__dict__['initials'] = Diot(diot_nest = False)
ifcopy = kwargs.get('copy', True)
depends = kwargs.get('depends', True)
prevproc = None
for proc in procs:
assert hasattr(proc, 'id') and hasattr(proc, 'tag'), \
'Argument has to be a Proc object: %r.' % proc
if ifcopy:
self.procs[proc.id] = proc.copy(proc.id,
tag = (self.tag or proc.tag.split('@', 1)[0]) + '@' + self.id)
else:
self.procs[proc.id] = proc
proc.config.tag = (self.tag or proc.tag.split('@', 1)[0]) + '@' + self.id
if depends and prevproc is None:
self.starts.add(self[proc.id])
if depends and prevproc:
self.procs[proc.id].depends = prevproc
prevproc = self.procs[proc.id]
if depends and prevproc:
self.ends.add(prevproc)
self.delegate('input', 'starts')
self.delegate('depends', 'starts')
self.delegate('ex*', 'ends')
def fixt_funcsig(request):
if request.param == 0:
def func1():
pass
return Diot(func=func1, expt="def func1(): pass")
if request.param == 1:
func2 = lambda: True
return Diot(func=func2, expt='func2 = lambda: True')
if request.param == 2:
return Diot(func="", expt="None")
if request.param == 3:
return Diot(func="A", expt="None")
def on_logger_request(self, data):
"""When requesting the output of a running command"""
logger.debug('Got request logger_request from client '
f'{request.remote_addr}')
# proc: that.proc,
# job: that.job_index(),
# reqlog: reqlog,
# eleid: this.id
# resp
# // data.log: the message got to show
# // data.isrunning: whether the process is still running
data = Diot(data)
running = pipeline_data.setdefault('running', Diot())
resp = data
resp.isrunning = False
resp.log = ''
if data.eleid in running:
rdata = running[data.eleid]
resp.isrunning = True
resp.pid = rdata['cmdy'].pid
buffer, done = read_cmdout(rdata['cmdy'])
rdata['buffer'] = rdata.get('buffer', '') + buffer
if data.reqlog == 'all':
resp.log = rdata['buffer']
elif data.reqlog == 'more':
resp.log = buffer
elif data.reqlog == 'kill':
cmdy.kill({'9': resp.pid}, _raise=False)
# killing succeeded?
resp.isrunning = 'killed'
resp.log = ''
done = False
try: # this could be deleted by previous request
del running[data.eleid]
except KeyError:
pass
if done:
resp.isrunning = 'done'
try:
del running[data.eleid]
except KeyError:
pass
logger.debug('Sending response logger_response to client '
f'{request.remote_addr}')
emit('logger_response', resp)
def test_fix_popen_config():
config = Diot({'envs': {'x': 1}, 'env': {'x': 2}})
with pytest.warns(UserWarning):
_cmdy_fix_popen_config(config)
assert 'envs' not in config
assert config['env']['x'] == '2'
config = Diot({'envs': {'x': True}})
_cmdy_fix_popen_config(config)
assert 'envs' not in config
assert config['env']['x'] == '1'
# other envs loaded
assert len(config['env']) > 1
def test_report(job0, caplog):
job0.proc._log.shorten = 10
job0.input = Diot(
a=('var', 'abcdefghijkmnopq'),
bc=('files', ['/long/path/to/file1']),
de=('file', '/long/path/to/file2'),
)
job0.output = Diot(outfile=('file', '/path/to/output/file1'),
outfiles=('files', ['/path/to/output/file2']))
job0.report()
assert 'pProc: [1/1] a => ab ... pq' in caplog.text
assert 'pProc: [1/1] bc => [ /l/p/t/file1 ]' in caplog.text
assert 'pProc: [1/1] de => /l/p/t/file2' in caplog.text
assert 'pProc: [1/1] outfile => /p/t/o/file1' in caplog.text
assert 'pProc: [1/1] outfiles => [ /p/t/o/file2 ]' in caplog.text
def __init__(
self, # pylint: disable=too-many-arguments
filename,
prev,
config,
stream=None,
shared_code=None,
code=None):
"""@API
Constructor for LiquidParser
@params:
filename (str): The filename of the template
prev (LiquidParse): The previous parser
config (LiquidConfig): The configuration
stream (stream): The stream to parse instead of the file of filename
shared_code (LiquidCode): The shared code
code (LiquidCode): The code object
"""
self.stream = stream or LiquidStream.from_file(filename)
self.shared_code = shared_code
self.code = code or LiquidCode()
# previous lineno and parser, to get call stacks
self.prev = prev
nstack = prev[1].context.nstack + 1 if prev else 1
LOGGER.debug("[PARSER%2s] Initialized from %s", nstack, filename)
# deferred nodes
self.deferred = []
self.config = config
self.context = Diot(
filename=filename,
lineno=1,
history=[],
# the data passed on during parsing
data=Diot(),
stacks=[],
# which parser stack I am at
nstack=nstack,
# attach myself to the context
parser=self)
# previous closing tag
# we need it to do compact for next literal
self.endtag = None
if nstack >= LIQUID_MAX_STACKS:
raise LiquidSyntaxError(f'Max stacks ({nstack}) reached',
self.context)
def fixt_filesig(request, tmp_path):
if not request.param:
return Diot(file='', expt=['', 0])
if request.param == 'a_file':
afile = tmp_path / 'filesig_afile'
afile.write_text('')
return Diot(file=afile, expt=[str(afile), int(path.getmtime(afile))])
if request.param == 'nonexists':
return Diot(file='/path/to/__non_exists__', expt=False)
if request.param == 'a_link':
alink = tmp_path / 'filesig_alink'
alink_orig = tmp_path / 'filesig_alink_orig'
alink_orig.write_text('')
alink.symlink_to(alink_orig)
return Diot(file=alink,
expt=[str(alink),
int(path.getmtime(alink_orig))])
if request.param == 'a_link_to_dir':
alink = tmp_path / 'filesig_alink_to_dir'
adir = tmp_path / 'filesig_adir'
adir.mkdir()
alink.symlink_to(adir)
return Diot(file=alink, expt=[str(alink), int(path.getmtime(adir))])
if request.param == 'a_dir_with_subdir':
adir = tmp_path / 'filesig_another_dir'
adir.mkdir()
utime(adir, (path.getmtime(adir) + 100, ) * 2)
asubdir = adir / 'filesig_another_subdir'
asubdir.mkdir()
return Diot(file=adir, expt=[str(adir), int(path.getmtime(adir))])
if request.param == 'a_dir_with_file':
adir = tmp_path / 'filesig_another_dir4'
adir.mkdir()
utime(adir, (path.getmtime(adir) - 100, ) * 2)
afile = adir / 'filesig_another_file4'
afile.write_text('')
return Diot(file=adir, expt=[str(adir), int(path.getmtime(afile))])
if request.param == 'a_dir_subdir_newer':
adir = tmp_path / 'filesig_another_dir2'
adir.mkdir()
utime(adir, (path.getmtime(adir) - 100, ) * 2)
asubdir = adir / 'filesig_another_subdir2'
asubdir.mkdir()
return Diot(file=adir, expt=[str(adir), int(path.getmtime(asubdir))])
if request.param == 'a_dir_subdir_newer_dirsig_false':
adir = tmp_path / 'filesig_another_dir3'
adir.mkdir()
utime(adir, (path.getmtime(adir) - 100, ) * 2)
asubdir = adir / 'filesig_another_subdir3'
asubdir.mkdir()
return Diot(file=adir,
dirsig=False,
expt=[str(adir), int(path.getmtime(adir))])
def test_missing_handler():
d = Diot(a=1, b=2, diot_missing=None)
assert d.c is None
d = Diot(a=1, b=2, diot_missing=RuntimeError)
with pytest.raises(RuntimeError):
d["c"]
d = Diot(a=1, b=2, diot_missing=RuntimeError("abc"))
with pytest.raises(RuntimeError, match="abc"):
d["c"]
d = Diot(a=1, b=2, diot_missing=lambda key, diot: diot.a + diot.b)
assert d.c == 3
assert "c" not in d
def test_module():
config = Diot(batch_size=10, data_tvt=1, max_epochs=2, num_classes=2)
trainer = Trainer.from_config(config)
model = Model(config)
data = DataModuleTest4(config=config)
trainer.fit(model, data)
trainer.test()
def test_module_regression():
config = Diot(batch_size=3, data_tvt=1, max_epochs=2, num_classes=1)
trainer = Trainer.from_config(config)
model = ModelRegr(config)
data = DataModuleTest4(config=config)
trainer.fit(model, data)
trainer.test()
def test_to_toml():
import rtoml
a = Diot(**test_dict)
assert rtoml.loads(a.to_toml()) == {
key: val
for key, val in test_dict.items() if key != 'diot_nest'
}
def test_to_yaml():
import yaml
a = Diot(**test_dict)
assert yaml.load(a.to_yaml(), Loader=yaml.SafeLoader) == {
key: val
for key, val in test_dict.items() if key != 'diot_nest'
}
def parse(self, force=False):
# type: (bool) -> None
"""Parse the include template"""
if not super().parse(force):
return
path = self.parsed[0] # pylint: disable=access-member-before-definition
path = str(path)
try:
include_template = find_template(path, self.context.path,
self.parser.config.include_dir)
if not include_template or not include_template.is_file():
raise OSError
except OSError:
raise LiquidSyntaxError(
f'Cannot find template: {path!r} ({self!r})', self.context,
self.parser) from None
meta = template_meta(include_template)
inc_parser = self.parser.__class__(
meta, self.parser.config,
Diot(name=meta.name,
stream=meta.stream,
path=meta.path,
colno=0,
lineno=0,
level=self.context.level + 1))
inc_parser.parse()
inc_parser.parent = self.parser
inc_parser.config.update_logger()
# pylint: disable=attribute-defined-outside-init
self.parsed = inc_parser, self.parsed[1]
def do_case(name, case):
"""Do scatter with one case"""
case = Diot(
convert_args=convert_args,
threshold=threshold,
min_probes=min_probes,
male_reference=male_reference,
sample_sex=sample_sex or False,
no_shift_xy=no_shift_xy,
title=title,
) | case
conv_args = case.pop("convert_args")
pdffile = Path(outdir).joinpath(f"{name}.heatmap.pdf")
pngfile = Path(outdir).joinpath(f"{name}.heatmap.png")
cmdy.cnvkit.diagram(
**case,
s=cnsfile,
o=pdffile,
_=cnrfile,
_exe=cnvkit,
).fg()
cmdy.convert(
**conv_args,
_=[pdffile, pngfile],
_prefix="-",
_exe=convert,
).fg()
def do_case(name, case):
"""Do heatmap with one case"""
case = Diot(
by_bin=by_bin,
chromosome=chromosome,
desaturate=desaturate,
male_reference=male_reference,
sample_sex=sample_sex or False,
no_shift_xy=no_shift_xy,
convert_args=convert_args,
) | case
conv_args = case.pop("convert_args")
pdffile = Path(outdir).joinpath(f"{name}.heatmap.pdf")
pngfile = Path(outdir).joinpath(f"{name}.heatmap.png")
cmdy.cnvkit.heatmap(
**case,
o=pdffile,
_=segfiles,
_exe=cnvkit,
).fg()
cmdy.convert(
**conv_args,
_=[pdffile, pngfile],
_prefix="-",
_exe=convert,
).fg()