def get(self, workflow_id, filename):
uuid.UUID(workflow_id)
workflow = Workflow.find_by_id(self.base_path, workflow_id)
zstream = workflow.bag.package_as_zipstream(compression=None)
self.set_status(200)
self.set_header('Content-type', 'application/zip')
self.set_header('Content-length',
str(util.calculate_zipsize(zstream.paths_to_write)))
self.zstream_iter = iter(zstream)
self.send_next_chunk()
def upload_workflow(wf_id,
base_path,
endpoint,
user_config,
start_process=False,
start_output=False):
logger.debug("Uploading workflow to postprocessing server")
workflow = Workflow.find_by_id(base_path, wf_id)
# NOTE: This is kind of nasty.... We temporarily write the user-supplied
# configuration to the bag, update the tag-payload, create the zip, and
# once everything is done, we restore the old version
tmp_cfg = copy.deepcopy(workflow.config)
tmp_cfg.set(user_config)
tmp_cfg_path = workflow.path / 'config.yml'
tmp_cfg.dump(filename=unicode(tmp_cfg_path),
sections=(user_config['plugins'] + ["plugins", "device"]))
workflow.bag.add_tagfiles(unicode(tmp_cfg_path))
# Create a zipstream from the workflow-bag
zstream = workflow.bag.package_as_zipstream(compression=None)
zsize = calculate_zipsize(zstream.paths_to_write)
def zstream_wrapper():
""" Wrapper around our zstream so we can emit a signal when all data
has been streamed to the client.
"""
transferred = 0
progress = "0.00"
for data in zstream:
yield data
transferred += len(data)
# Only update progress if we've progress at least by 0.01
new_progress = "{0:.2f}".format(transferred / zsize)
if new_progress != progress:
progress = new_progress
signals['submit:progressed'].send(
workflow,
progress=float(progress),
status="Uploading workflow...")
# NOTE: This is neccessary since requests makes a chunked upload when
# passed a plain generator, which is not supported by the WSGI
# protocol that receives it. Hence we wrap it inside of a
# GeneratorIO to make it appear as a file-like object with a
# known size.
zstream_fp = GeneratorIO(zstream_wrapper(), zsize)
logger.debug("Projected size for upload: {}".format(zsize))
signals['submit:started'].send(workflow)
resp = requests.post(endpoint,
data=zstream_fp,
headers={'Content-Type': 'application/zip'})
if not resp:
error_msg = "Upload failed: {0}".format(resp.content)
signals['submit:error'].send(workflow,
message=error_msg,
data=resp.content)
logger.error(error_msg)
else:
wfid = resp.json()['id']
if start_process:
requests.post(endpoint + "/{0}/process".format(wfid))
if start_output:
requests.post(endpoint + "/{0}/output".format(wfid))
signals['submit:completed'].send(workflow, remote_id=wfid)
# Restore our old configuration
workflow._save_config()
