def synthesis():
sum = 0
jobs = datasets.source.chain(length=options.chain_length,
stop_ds=datasets.stop)
for src in jobs:
jid = build('dataset_checksum',
options=dict(columns=options.columns, sort=options.sort),
datasets=dict(source=src))
data = blob.load(jobid=jid)
sum ^= data.sum
print("Total: %016x" % (sum, ))
return DotDict(sum=sum,
columns=data.columns,
sort=options.sort,
sources=jobs)
def analysis(sliceno, job):
data = {sliceno}
if options.inner:
if options.file.sliced:
value = options.file.load(sliceno)
assert value == data
value = blob.load(options.file.filename(sliceno))
assert value == data
else:
try:
options.file.load(sliceno)
raise Exception("Allowed sliced load of unsliced file")
except AssertionError:
pass
job.save({'inner': sliceno}, 'inner.pickle', sliceno, temp=False)
job.json_save({'inner': sliceno}, 'inner.json', sliceno, temp=False)
else:
job.save(data, 'data', sliceno, temp=False)
def synthesis(params, job):
data = {'foo'}
if options.inner:
if options.file.sliced:
try:
options.file.load()
raise Exception("Allowed unsliced load of sliced file")
except AssertionError:
pass
else:
value = options.file.load()
assert value == data
value = blob.load(options.file.filename())
assert value == data
job.save({'inner': None}, 'inner.pickle', temp=False)
job.json_save({'inner': None}, 'inner.json', temp=False)
else:
job.save(data, 'data', temp=False)
# construct these JobWithFile the opposite way to the ones inside verify
verify(params, JobWithFile(params.jobid, 'data'))
verify(params, job.withfile('data', True))
def ck(jid, method="dataset_checksum", **kw):
jid = subjobs.build(method, datasets=dict(source=jid), options=kw)
return blob.load(jobid=jid).sum