def new_state(self, name, dependencies, readonly=False):
"""
creates a new context from an existing context.
if subdir is True it declares a new read-write nested under the old
context's read-write and adds all read-write and read-only locations
of the old context as read-only. This is recommended as it makes rolling
back changes to the global state made in this context easy.
else the same readwrite/readonly configuration as the parent context is used
"""
readwrites = []
if not readonly:
readwrites = (
["{}/{}".format(self.base.readwrite[0], name)]
if self.nest
else self.base.readwrite
)
log.debug("new state is: rw: %s", readwrites)
state_identifier = name.replace("/", "_") # replace in case name is nested path
newstate = LocalFSState(
readwrite=readwrites, identifier=state_identifier, dependencies=dependencies
)
if not readonly and self.ensure:
newstate.ensure()
return newstate
def test_setup_filebacked(tmpdir):
from yadage.state_providers.localposix import LocalFSProvider
from packtivity.statecontexts.posixfs_context import LocalFSState
import os
localprovider = LocalFSProvider(
LocalFSState(readwrite=[
os.path.join(str(tmpdir), 'rw_one'),
os.path.join(str(tmpdir), 'rw_two')
],
readonly=[
os.path.join(str(tmpdir), 'ro_one'),
os.path.join(str(tmpdir), 'ro_two')
]),
init_states=[
LocalFSState(readwrite=[
os.path.join(str(tmpdir), 'init_rw_one'),
os.path.join(str(tmpdir), 'init_rw_two')
],
readonly=[
os.path.join(str(tmpdir), 'init_ro_one'),
os.path.join(str(tmpdir), 'init_ro_two')
])
])
assert LocalFSProvider.fromJSON(localprovider.json(),
{}).json() == localprovider.json()
def handle_init_spec(pathbase,spec):
subinits = {}
for k,v in spec.items():
if isinstance(v,string_types):
subinits[k] = {'_data': LocalFSState(readonly = [os.path.join(pathbase,v)])}
if isinstance(v,dict):
subinits.setdefault(k,{})['_data'] = LocalFSState(readonly = [os.path.join(pathbase,v.pop('_data'))]) if '_data' in v else None
for kk,vv in handle_init_spec(pathbase,v).items():
subinits.setdefault(k,{})[kk] = vv
return subinits
def setup_provider(dataarg, dataopts):
workdir = dataarg.split(":", 2)[1]
read = dataopts.get("read", None)
nest = dataopts.get("nest", True)
ensure = dataopts.get("ensure", True)
overwrite = dataopts.get("overwrite", False)
subinits = dataopts.get("subinits", {})
pathbase = dataopts.get("pathbase")
if overwrite and os.path.exists(workdir):
log.info("work directory %s will be purged and overwritten", workdir)
shutil.rmtree(workdir)
init_states = []
initdir = os.path.join(pathbase or workdir, dataopts.get("initdir", "init"))
inputarchive = dataopts.get("inputarchive", None)
if initdir:
if inputarchive:
prepare_workdir_from_archive(
initdir, inputarchive, match=dataopts.get("archivematch")
)
init_state = LocalFSState(readonly=[initdir])
init_states.append(init_state)
def handle_init_spec(pathbase, spec):
subinits = {}
for k, v in spec.items():
if isinstance(v, string_types):
subinits[k] = {
"_data": LocalFSState(readonly=[os.path.join(pathbase, v)])
}
if isinstance(v, dict):
subinits.setdefault(k, {})["_data"] = (
LocalFSState(readonly=[os.path.join(pathbase, v.pop("_data"))])
if "_data" in v
else None
)
for kk, vv in handle_init_spec(pathbase, v).items():
subinits.setdefault(k, {})[kk] = vv
return subinits
if subinits:
subinits = handle_init_spec(pathbase or workdir, subinits)
writable_state = LocalFSState([workdir])
return LocalFSProvider(
read,
writable_state,
ensure=ensure,
nest=nest,
init_states=init_states,
sub_init_states=subinits,
)
def test_applicable():
data = yadage.workflow_loader.workflow('workflow.yml',
'tests/testspecs/local-helloworld')
wflow = YadageWorkflow.createFromJSON(
data, LocalFSProvider(LocalFSState(['/workdir']), ensure=False))
wflow.view().init({'par': 'value'})
assert wflow.rules[-1].applicable(wflow) == True
def test_getRule():
data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/nestedmapreduce')
wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False))
wflow.view().init({'input':[1,2,3]})
assert wflow.view().getRule(identifier = wflow.rules[0].identifier) == wflow.rules[0]
assert wflow.view().getRule(name = wflow.rules[0].rule.name, offset = '') == wflow.rules[0]
assert wflow.view().getRule(name = 'nonexistent') == None
def test_serialize_offsetstage():
data = yadage.workflow_loader.workflow("workflow.yml",
"tests/testspecs/nestedmapreduce")
wflow = YadageWorkflow.createFromJSON(
data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False))
wflow.view().init({"input": [1, 2, 3]})
for x in wflow.rules:
assert OffsetStage.fromJSON(x.json()).json() == x.json()
def test_applicable():
data = yadage.workflow_loader.workflow(
"workflow.yml", "tests/testspecs/local-helloworld"
)
wflow = YadageWorkflow.createFromJSON(
data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False)
)
wflow.view().init({"par": "value"})
assert wflow.rules[-1].applicable(wflow) == True
def setup_provider(dataarg, dataopts):
workdir = dataarg.split(':',2)[1]
read = dataopts.get('read',None)
nest = dataopts.get('nest',True)
ensure = dataopts.get('ensure',True)
overwrite = dataopts.get('overwrite',False)
subinits = dataopts.get('subinits',{})
pathbase = dataopts.get('pathbase')
if overwrite and os.path.exists(workdir):
shutil.rmtree(workdir)
init_states = []
initdir = os.path.join(pathbase or workdir,dataopts.get('initdir','init'))
inputarchive = dataopts.get('inputarchive',None)
if initdir:
if inputarchive:
prepare_workdir_from_archive(initdir, inputarchive)
init_state = LocalFSState(readonly = [initdir])
init_states.append(init_state)
def handle_init_spec(pathbase,spec):
subinits = {}
for k,v in spec.items():
if isinstance(v,string_types):
subinits[k] = {'_data': LocalFSState(readonly = [os.path.join(pathbase,v)])}
if isinstance(v,dict):
subinits.setdefault(k,{})['_data'] = LocalFSState(readonly = [os.path.join(pathbase,v.pop('_data'))]) if '_data' in v else None
for kk,vv in handle_init_spec(pathbase,v).items():
subinits.setdefault(k,{})[kk] = vv
return subinits
if subinits:
subinits = handle_init_spec(pathbase or workdir,subinits)
writable_state = LocalFSState([workdir])
return LocalFSProvider(read,writable_state,
ensure = ensure, nest = nest, init_states = init_states,
sub_init_states = subinits
)
def test_serialize_deserialize():
data = yadage.workflow_loader.workflow('workflow.yml',
'tests/testspecs/local-helloworld')
wflow = YadageWorkflow.createFromJSON(
data, LocalFSProvider(LocalFSState(['/workdir']), ensure=False))
wflow.view().init({'hello': 'world'})
assert wflow.rules[0].rule.json()
assert wflow.rules[1].rule.json()
assert JsonStage.fromJSON(
wflow.rules[0].rule.json()).json() == wflow.rules[0].rule.json()
def test_getRule():
data = yadage.workflow_loader.workflow("workflow.yml",
"tests/testspecs/nestedmapreduce")
wflow = YadageWorkflow.createFromJSON(
data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False))
wflow.view().init({"input": [1, 2, 3]})
assert wflow.view().getRule(
identifier=wflow.rules[0].identifier) == wflow.rules[0]
assert (wflow.view().getRule(name=wflow.rules[0].rule.name,
offset="") == wflow.rules[0])
assert wflow.view().getRule(name="nonexistent") == None
def test_init():
data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/nestedmapreduce')
wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False))
view = wflow.view()
nrules_before = len(wflow.rules)
view.init({'hello':'world'})
nrules_after = len(wflow.rules)
assert nrules_before + 1 == nrules_after
view.rules[-1].apply(wflow)
matches = view.query('init',view.steps)
assert len(matches) == 1
def new_provider(self,name, init_states = None):
new_base_ro = self.base.readwrite + self.base.readonly
new_base_rw = [os.path.join(self.base.readwrite[0],name)]
sub_init = self.sub_inits.get(name,{}).get('_data')
sub_inits = [sub_init] if sub_init else []
init_states = init_states or []
newsinits = {k:v for k,v in self.sub_inits.get(name,{}).items() if k!='_data'}
return LocalFSProvider(LocalFSState(new_base_rw,new_base_ro),
nest = self.nest,
ensure = self.ensure,
init_states = init_states + sub_inits,
sub_init_states = newsinits)
def test_serialize_deserialize():
data = yadage.workflow_loader.workflow(
"workflow.yml", "tests/testspecs/local-helloworld"
)
wflow = YadageWorkflow.createFromJSON(
data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False)
)
wflow.view().init({"hello": "world"})
assert wflow.rules[0].rule.json()
assert wflow.rules[1].rule.json()
assert (
JsonStage.fromJSON(wflow.rules[0].rule.json()).json()
== wflow.rules[0].rule.json()
)
def test_init():
data = yadage.workflow_loader.workflow("workflow.yml",
"tests/testspecs/nestedmapreduce")
wflow = YadageWorkflow.createFromJSON(
data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False))
view = wflow.view()
nrules_before = len(wflow.rules)
view.init({"hello": "world"})
nrules_after = len(wflow.rules)
assert nrules_before + 1 == nrules_after
view.rules[-1].apply(wflow)
matches = view.query("init", view.steps)
assert len(matches) == 1
def new_state(self,name, dependencies, readonly = False):
'''
creates a new context from an existing context.
if subdir is True it declares a new read-write nested under the old
context's read-write and adds all read-write and read-only locations
of the old context as read-only. This is recommended as it makes rolling
back changes to the global state made in this context easy.
else the same readwrite/readonly configuration as the parent context is used
'''
readwrites = []
if not readonly:
readwrites = ['{}/{}'.format(self.base.readwrite[0],name)] if self.nest else self.base.readwrite
log.debug('new state is: rw: %s', readwrites)
state_identifier = name.replace('/','_') # replace in case name is nested path
newstate = LocalFSState(readwrite = readwrites, identifier = state_identifier, dependencies = dependencies)
if not readonly and self.ensure:
newstate.ensure()
return newstate
def basic_localfs_state(tmpdir): return LocalFSState([str(tmpdir)])
def localfs_state_provider(tmpdir):
return LocalFSProvider(LocalFSState([tmpdir.dirname]), ensure=True)
def localfs_state(tmpdir):
return LocalFSState([tmpdir.dirname])
def _merge_states(lhs,rhs):
return LocalFSState(lhs.readwrite + rhs.readwrite,lhs.readonly + rhs.readonly)