async def retrieve_data(): print("retrieving data...") PORTS = node_ports.ports() inputs_path = Path("~/home").expanduser() inputs_path.mkdir(exist_ok=True) values = {} for node_input in PORTS.inputs: if not node_input or node_input.value is None: continue print("getting data from port '{}' with value '{}'...".format( node_input.key, node_input.value)) value = await node_input.get() values[node_input.key] = {"type": node_input.type, "value": value} if "data:" in node_input.type: dest = inputs_path / node_input.key dest.mkdir(exist_ok=True, parents=True) dest = dest / Path(value).name shutil.move(value, dest) values[node_input.key] = { "type": node_input.type, "value": str(dest) } values_file = inputs_path / "values.json" with values_file.open('w') as fp: json.dump(values, fp)
async def upload_data(): logger.info("uploading data to simcore...") PORTS = node_ports.ports() outputs_path = Path(_OUTPUTS_FOLDER).expanduser() for port in PORTS.outputs: logger.debug("uploading data to port '%s' with value '%s'...", port.key, port.value) if _FILE_TYPE_PREFIX in port.type: src_folder = outputs_path / port.key list_files = list(src_folder.glob("*")) if len(list_files) == 1: # special case, direct upload await port.set(list_files[0]) continue # generic case let's create an archive if len(list_files) > 1: temp_file = tempfile.NamedTemporaryFile(suffix=".tgz") temp_file.close() for _file in list_files: with tarfile.open(temp_file.name, mode='w:gz') as tar_ptr: for file_path in list_files: tar_ptr.add(str(file_path), arcname=file_path.name, recursive=False) try: await port.set(temp_file.name) finally: #clean up Path(temp_file.name).unlink() else: values_file = outputs_path / _KEY_VALUE_FILE_NAME if values_file.exists(): values = json.loads(values_file.read_text()) if port.key in values and values[port.key] is not None: await port.set(values[port.key]) logger.info("all data uploaded to simcore")
async def test_port_file_accessors(special_configuration, storage, filemanager_cfg, s3_simcore_location, bucket, item_type, item_value, item_pytype, config_value): # pylint: disable=W0613, W0621 config_dict, project_id, node_uuid = special_configuration( inputs=[("in_1", item_type, config_value)], outputs=[("out_34", item_type, None)]) PORTS = node_ports.ports() check_config_valid(PORTS, config_dict) assert await PORTS.outputs["out_34"].get() is None # check emptyness # with pytest.raises(exceptions.S3InvalidPathError, message="Expecting S3InvalidPathError"): # await PORTS.inputs["in_1"].get() # this triggers an upload to S3 + configuration change await PORTS.outputs["out_34"].set(item_value) # this is the link to S3 storage assert PORTS.outputs["out_34"].value == { "store": s3_simcore_location, "path": Path(str(project_id), str(node_uuid), Path(item_value).name).as_posix() } # this triggers a download from S3 to a location in /tempdir/simcorefiles/item_key assert isinstance(await PORTS.outputs["out_34"].get(), item_pytype) assert (await PORTS.outputs["out_34"].get()).exists() assert str(await PORTS.outputs["out_34"].get()).startswith( str(Path(tempfile.gettempdir(), "simcorefiles", "out_34"))) filecmp.clear_cache() assert filecmp.cmp(item_value, await PORTS.outputs["out_34"].get())
async def retrieve_data(): # get all files in the local system and copy them to the input folder start_time = time.time() PORTS = node_ports.ports() download_tasks = [] for node_input in PORTS.inputs: if not node_input or node_input.value is None: continue # collect coroutines download_tasks.append(node_input.get()) if download_tasks: downloaded_files = await asyncio.gather(*download_tasks) print("downloaded {} files /tmp <br>".format(len(download_tasks))) for local_path in downloaded_files: if local_path is None: continue # log.debug("Completed download of %s in local path %s", node_input.value, local_path) if local_path.exists(): if zipfile.is_zipfile(str(local_path)): zip_ref = zipfile.ZipFile(str(local_path), 'r') zip_ref.extractall(str(_INPUT_PATH)) zip_ref.close() log.debug("Unzipped") print("unzipped {file} to {path}<br>".format(file=str(local_path), path=str(_INPUT_PATH))) else: log.debug("Start moving %s to input path %s", local_path, _INPUT_PATH) shutil.move(str(local_path), str(_INPUT_PATH / local_path.name)) log.debug("Move completed") print("moved {file} to {path}<br>".format(file=str(local_path), path=str(_INPUT_PATH))) end_time = time.time() print("time to download: {} seconds".format(end_time - start_time))
def _process_task_output(self): # pylint: disable=too-many-branches """ There will be some files in the /output - Maybe a output.json (should contain key value for simple things) - other files: should be named by the key in the output port Files will be pushed to S3 with reference in db. output.json will be parsed and the db updated """ PORTS = node_ports.ports() directory = self._executor.out_dir if not os.path.exists(directory): return try: for root, _dirs, files in os.walk(directory): for name in files: filepath = os.path.join(root, name) # the name should match what is in the db! if name == 'output.json': log.debug("POSTRO FOUND output.json") # parse and compare/update with the tasks output ports from db output_ports = dict() with open(filepath) as f: output_ports = json.load(f) task_outputs = PORTS.outputs for to in task_outputs: if to.key in output_ports.keys(): wrap_async_call(to.set(output_ports[to.key])) else: wrap_async_call(PORTS.set_file_by_keymap(Path(filepath))) except (OSError, IOError) as _e: logging.exception("Could not process output")
def test_adding_new_ports(special_configuration, session): config_dict, project_id, node_uuid = special_configuration() PORTS = node_ports.ports() check_config_valid(PORTS, config_dict) # check empty configuration assert not PORTS.inputs assert not PORTS.outputs # replace the configuration now, add an input config_dict["schema"]["inputs"].update({ "in_15": { "label": "additional data", "description": "here some additional data", "displayOrder": 2, "type": "integer" } }) config_dict["inputs"].update({"in_15": 15}) helpers.update_configuration(session, project_id, node_uuid, config_dict) #pylint: disable=E1101 check_config_valid(PORTS, config_dict) # # replace the configuration now, add an output config_dict["schema"]["outputs"].update({ "out_15": { "label": "output data", "description": "a cool output", "displayOrder": 2, "type": "boolean" } }) helpers.update_configuration(session, project_id, node_uuid, config_dict) #pylint: disable=E1101 check_config_valid(PORTS, config_dict)
def _process_task_inputs(self): """ Writes input key-value pairs into a dictionary if the value of any port starts with 'link.' the corresponding output ports a fetched or files dowloaded --> @ jsonld The dictionary is dumped to input.json, files are dumped as port['key']. Both end up in /input/ of the container """ log.debug('Input parsing for %s and node %s from container', self._task.project_id, self._task.internal_id) input_ports = dict() PORTS = node_ports.ports() for port in PORTS.inputs: log.debug(port) self._process_task_input(port, input_ports) log.debug('DUMPING json') #dump json file if input_ports: file_name = os.path.join(self._executor.in_dir, 'input.json') with open(file_name, 'w') as f: json.dump(input_ports, f) log.debug('DUMPING DONE')
async def upload_data(): logger.info("uploading data to simcore...") PORTS = node_ports.ports() outputs_path = Path(_OUTPUTS_FOLDER).expanduser() for port in PORTS.outputs: logger.debug("uploading data to port '%s' with value '%s'...", port.key, port.value) src_folder = outputs_path / port.key list_files = list(src_folder.glob("*")) if len(list_files) == 1: # special case, direct upload await port.set(list_files[0]) continue # generic case let's create an archive if len(list_files) > 1: temp_file = tempfile.NamedTemporaryFile(suffix=".tgz") temp_file.close() for _file in list_files: with tarfile.open(temp_file.name, mode='w:gz') as tar_ptr: for file_path in list_files: tar_ptr.add(file_path, arcname=file_path.name, recursive=False) try: await port.set(temp_file.name) finally: #clean up Path(temp_file.name).unlink() logger.info("all data uploaded to simcore")
async def download_data(): logger.info("retrieving data from simcore...") print("retrieving data from simcore...") # get all files in the local system and copy them to the input folder PORTS = node_ports.ports() for port in PORTS.inputs: if not port or port.value is None: continue local_path = await port.get() dest_path = _INPUTS_FOLDER / port.key dest_path.mkdir(exist_ok=True, parents=True) # clean up destination directory for path in dest_path.iterdir(): if path.is_file(): path.unlink() elif path.is_dir(): shutil.rmtree(path) # check if local_path is a compressed file if tarfile.is_tarfile(local_path): with tarfile.open(local_path) as tar_file: tar_file.extractall(dest_path, members=_no_relative_path_tar(tar_file)) elif zipfile.is_zipfile(local_path): with zipfile.ZipFile(local_path) as zip_file: zip_file.extractall(dest_path, members=_no_relative_path_zip(zip_file)) else: dest_path_name = _INPUTS_FOLDER / (port.key + ":" + Path(local_path).name) shutil.move(local_path, dest_path_name) shutil.rmtree(Path(local_path).parents[0])
def pandas_dataframe_to_output_data(data_frame, title, header=False, port_number=0): title = title.replace(" ", "_") + ".csv" dummy_file_path = Path(title) data_frame.to_csv(dummy_file_path, sep=',', header=header, index=False, encoding='utf-8') ports = node_ports.ports() task = ports.outputs[port_number].set(dummy_file_path) asyncio.get_event_loop().run_until_complete( task )
async def retrieve_data(ports: List[str], cache: Dict) -> int: # get all files in the local system and copy them to the input folder start_time = time.clock() PORTS = node_ports.ports() download_tasks = [] for node_input in PORTS.inputs: # if ports contains some keys only download them log.info("Checking node %s", node_input.key) if ports and node_input.key not in ports: continue # delete the corresponding file(s) if applicable if node_input.key in cache: log.info("Deleting files from %s: %s", node_input.key, cache[node_input.key]) for file_path in cache[node_input.key]: Path(file_path).unlink() del cache[node_input.key] if not node_input or node_input.value is None: continue # collect coroutines download_tasks.append(task(node_input.key, node_input.get)) log.info("retrieving %s data", len(download_tasks)) transfer_bytes = 0 if download_tasks: download_results = await asyncio.gather(*download_tasks) log.info( "completed download, extracting/moving data to final folder...") for node_key, local_path in download_results: if local_path is None: continue if not local_path.exists(): continue transfer_bytes = transfer_bytes + local_path.stat().st_size if zipfile.is_zipfile(str(local_path)): log.info("extracting %s to %s", local_path, input_path()) zip_ref = zipfile.ZipFile(str(local_path), 'r') zip_ref.extractall(str(input_path())) cache[node_key] = \ [str(input_path() / zipped_file) for zipped_file in zip_ref.namelist()] zip_ref.close() log.info("extraction completed") else: log.info("moving %s to input path %s", local_path, input_path()) dest_path = input_path() / local_path.name shutil.move(str(local_path), str(dest_path)) cache[node_key] = [str(dest_path)] log.info("move completed") end_time = time.clock() log.info("retrieval complete: took %.2fseconds for %s bytes", end_time - start_time, transfer_bytes) return transfer_bytes
async def test_get_value_from_previous_node(special_2nodes_configuration, node_link, item_type, item_value, item_pytype): config_dict, _, _ = special_2nodes_configuration( prev_node_outputs=[("output_123", item_type, item_value)], inputs=[("in_15", item_type, node_link("output_123"))]) PORTS = node_ports.ports() check_config_valid(PORTS, config_dict) input_value = await PORTS.inputs["in_15"].get() assert isinstance(input_value, item_pytype) assert await PORTS.inputs["in_15"].get() == item_value
async def upload_data(port_keys: List[str]) -> int: #pylint: disable=too-many-branches logger.info("uploading data to simcore...") start_time = time.perf_counter() PORTS = node_ports.ports() outputs_path = Path(_OUTPUTS_FOLDER).expanduser() # let's gather the tasks temp_files = [] upload_tasks = [] transfer_bytes = 0 for port in PORTS.outputs: logger.info("Checking port %s", port.key) if port_keys and port.key not in port_keys: continue logger.debug("uploading data to port '%s' with value '%s'...", port.key, port.value) if _FILE_TYPE_PREFIX in port.type: src_folder = outputs_path / port.key list_files = list(src_folder.glob("*")) if len(list_files) == 1: # special case, direct upload upload_tasks.append(set_time_wrapped(port, list_files[0])) continue # generic case let's create an archive if len(list_files) > 1: temp_file = tempfile.NamedTemporaryFile(suffix=".zip") temp_file.close() with zipfile.ZipFile(temp_file.name, mode="w") as zip_ptr: for file_path in list_files: zip_ptr.write(str(file_path), arcname=file_path.name) temp_files.append(temp_file.name) upload_tasks.append(set_time_wrapped(port, temp_file.name)) else: data_file = outputs_path / _KEY_VALUE_FILE_NAME if data_file.exists(): data = json.loads(data_file.read_text()) if port.key in data and data[port.key] is not None: upload_tasks.append(set_time_wrapped(port, data[port.key])) if upload_tasks: try: results = await asyncio.gather(*upload_tasks) transfer_bytes = sum(results) finally: # clean up possible compressed files for file_path in temp_files: Path(file_path).unlink() stop_time = time.perf_counter() logger.info("all data uploaded to simcore in %sseconds", stop_time - start_time) return transfer_bytes
async def pandas_dataframe_to_csv(data_frame, title, header=False, port_number=0): title = title.replace(" ", "_") + ".csv" dummy_file_path = Path(title) data_frame.to_csv(dummy_file_path, sep=',', header=header, index=False, encoding='utf-8') PORTS = node_ports.ports() await PORTS.outputs[port_number].set(dummy_file_path)
async def _initialise_platform(port_configuration_path: Path, file_generator, delete_file): with port_configuration_path.open() as file_pointer: configuration = json.load(file_pointer) if not all(k in configuration for k in ("schema", "inputs", "outputs")): raise Exception("invalid port configuration in {}, {}!".format( str(port_configuration_path), configuration)) # init s3 to ensure we have a bucket init_s3() # set up db db = init_db() # create a new pipeline new_Pipeline = ComputationalPipeline(project_id=str(uuid.uuid4())) db.session.add(new_Pipeline) db.session.commit() # create a new node node_uuid = str(uuid.uuid4()) # now create the node in the db with links to S3 new_Node = ComputationalTask(project_id=new_Pipeline.project_id, node_id=node_uuid, schema=configuration["schema"], inputs=configuration["inputs"], outputs=configuration["outputs"]) db.session.add(new_Node) db.session.commit() # set up node_ports node_ports.node_config.NODE_UUID = node_uuid PORTS = node_ports.ports() # push the file to the S3 for each input item file_index = 0 for key, input_item in configuration["schema"]["inputs"].items(): if str(input_item["type"]).startswith("data:"): file_to_upload = file_generator(file_index, input_item["type"]) if file_to_upload is not None: # upload to S3 await PORTS.inputs[key].set(Path(file_to_upload)) file_index += 1 if delete_file: Path(file_to_upload).unlink() # print the node uuid so that it can be set as env variable from outside print("{pipelineid},{nodeuuid}".format(pipelineid=str(new_Node.project_id), nodeuuid=node_uuid))
async def retrieve_data(): print("retrieving data...") PORTS = node_ports.ports() values = {} for node_input in PORTS.inputs: if not node_input or node_input.value is None: continue print("getting data from port '{}' with value '{}'...".format( node_input.key, node_input.value)) value = await node_input.get() values[node_input.key] = {"type": node_input.type, "value": value} print("json={}".format(json.dumps(values))) sys.stdout.flush()
def test_invalid_ports(special_configuration): config_dict, _, _ = special_configuration() PORTS = node_ports.ports() check_config_valid(PORTS, config_dict) assert not PORTS.inputs assert not PORTS.outputs with pytest.raises(exceptions.UnboundPortError, message="Expecting UnboundPortError"): PORTS.inputs[0] with pytest.raises(exceptions.UnboundPortError, message="Expecting UnboundPortError"): PORTS.outputs[0]
def test_removing_ports(special_configuration, session): config_dict, project_id, node_uuid = special_configuration( inputs=[("in_14", "integer", 15), ("in_17", "boolean", False)], outputs=[("out_123", "string", "blahblah"), ("out_2", "number", -12.3)]) #pylint: disable=W0612 PORTS = node_ports.ports() check_config_valid(PORTS, config_dict) # let's remove the first input del config_dict["schema"]["inputs"]["in_14"] del config_dict["inputs"]["in_14"] helpers.update_configuration(session, project_id, node_uuid, config_dict) #pylint: disable=E1101 check_config_valid(PORTS, config_dict) # let's do the same for the second output del config_dict["schema"]["outputs"]["out_2"] del config_dict["outputs"]["out_2"] helpers.update_configuration(session, project_id, node_uuid, config_dict) #pylint: disable=E1101 check_config_valid(PORTS, config_dict)
async def test_get_file_from_previous_node(special_2nodes_configuration, project_id, node_uuid, filemanager_cfg, node_link, store_link, item_type, item_value, item_pytype): config_dict, _, _ = special_2nodes_configuration( prev_node_outputs=[("output_123", item_type, store_link(item_value, project_id, node_uuid))], inputs=[("in_15", item_type, node_link("output_123"))], project_id=project_id, previous_node_id=node_uuid) PORTS = node_ports.ports() check_config_valid(PORTS, config_dict) file_path = await PORTS.inputs["in_15"].get() assert isinstance(file_path, item_pytype) assert file_path == Path(tempfile.gettempdir(), "simcorefiles", "in_15", Path(item_value).name) assert file_path.exists() filecmp.clear_cache() assert filecmp.cmp(file_path, item_value)
def push_output_data(): input_path = OUTPUT_DIR / 'input.csv' cv_path = OUTPUT_DIR / 'CV_plot.csv' t_path = OUTPUT_DIR / 't_plot.csv' ist_path = OUTPUT_DIR / 'Ist_plot.csv' tst_path = OUTPUT_DIR / 'tst_plot.csv' qst_path = OUTPUT_DIR / 'CAP_plot.csv' vpred_path = OUTPUT_DIR / 'V_pred_plot.csv' lpred_path = OUTPUT_DIR / 'Lpred_plot.csv' output_files = [ input_path, cv_path, t_path, ist_path, tst_path, qst_path, vpred_path, lpred_path ] for p in output_files: logger.info('file %s', str(p)) logger.info('exsits %s', p.exists()) ports = node_ports.ports() for idx, path in enumerate(output_files): if path.exists(): task = ports.outputs[idx].set(path) asyncio.get_event_loop().run_until_complete(task)
async def test_port_value_accessors(special_configuration, item_type, item_value, item_pytype): # pylint: disable=W0613, W0621 item_key = "some key" config_dict, _, _ = special_configuration(inputs=[(item_key, item_type, item_value)], outputs=[(item_key, item_type, None)]) PORTS = node_ports.ports() check_config_valid(PORTS, config_dict) assert isinstance(await PORTS.inputs[item_key].get(), item_pytype) assert await PORTS.inputs[item_key].get() == item_value assert await PORTS.outputs[item_key].get() is None assert isinstance(await PORTS.get(item_key), item_pytype) assert await PORTS.get(item_key) == item_value await PORTS.outputs[item_key].set(item_value) assert PORTS.outputs[item_key].value == item_value assert isinstance(await PORTS.outputs[item_key].get(), item_pytype) assert await PORTS.outputs[item_key].get() == item_value
async def test_file_mapping(special_configuration, project_id, node_uuid, filemanager_cfg, s3_simcore_location, bucket, store_link, session, item_type, item_value, item_alias, item_pytype): config_dict, project_id, node_uuid = special_configuration( inputs=[("in_1", item_type, store_link(item_value, project_id, node_uuid))], outputs=[("out_1", item_type, None)], project_id=project_id, node_id=node_uuid) PORTS = node_ports.ports() check_config_valid(PORTS, config_dict) # add a filetokeymap config_dict["schema"]["inputs"]["in_1"]["fileToKeyMap"] = { item_alias: "in_1" } config_dict["schema"]["outputs"]["out_1"]["fileToKeyMap"] = { item_alias: "out_1" } helpers.update_configuration(session, project_id, node_uuid, config_dict) #pylint: disable=E1101 check_config_valid(PORTS, config_dict) file_path = await PORTS.inputs["in_1"].get() assert isinstance(file_path, item_pytype) assert file_path == Path(tempfile.gettempdir(), "simcorefiles", "in_1", item_alias) invalid_alias = Path("invalid_alias.fjfj") with pytest.raises(exceptions.PortNotFound, message="Expecting PortNotFound"): await PORTS.set_file_by_keymap(invalid_alias) await PORTS.set_file_by_keymap(file_path) file_id = helpers.file_uuid(file_path, project_id, node_uuid) assert PORTS.outputs["out_1"].value == { "store": s3_simcore_location, "path": file_id }
async def download_data(): logger.info("retrieving data from simcore...") PORTS = node_ports.ports() inputs_path = Path(_INPUTS_FOLDER).expanduser() values = {} for port in PORTS.inputs: if not port or port.value is None: continue logger.debug("downloading data from port '%s' with value '%s'...", port.key, port.value) value = await port.get() values[port.key] = {"key": port.key, "value": value} if _FILE_TYPE_PREFIX in port.type: dest_path = inputs_path / port.key dest_path.mkdir(exist_ok=True, parents=True) values[port.key] = {"key": port.key, "value": str(dest_path)} # clean up destination directory for path in dest_path.iterdir(): if path.is_file(): path.unlink() elif path.is_dir(): shutil.rmtree(path) # check if value is a compressed file if tarfile.is_tarfile(value): with tarfile.open(value) as tar_file: tar_file.extractall(dest_path, members=list(_no_relative_path_tar(tar_file))) elif zipfile.is_zipfile(value): with zipfile.ZipFile(value) as zip_file: zip_file.extractall(dest_path, members=_no_relative_path_zip(zip_file)) else: dest_path = dest_path / Path(value).name shutil.move(value, dest_path) values_file = inputs_path / _KEY_VALUE_FILE_NAME values_file.write_text(json.dumps(values)) logger.info("all data retrieved from simcore: %s", values)
def download_all_inputs(n_inputs = 1): ports = node_ports.ports() tasks = asyncio.gather(*[ports.inputs[n].get() for n in range(n_inputs)]) paths_to_inputs = asyncio.get_event_loop().run_until_complete( tasks ) return paths_to_inputs
def test_default_configuration(default_configuration): # pylint: disable=W0613, W0621 config_dict = default_configuration check_config_valid(node_ports.ports(), config_dict)
def initialize(self): #pylint: disable=no-self-use PORTS = node_ports.ports() _create_ports_sub_folders(PORTS.inputs, Path(_INPUTS_FOLDER).expanduser()) _create_ports_sub_folders(PORTS.outputs, Path(_OUTPUTS_FOLDER).expanduser())
async def download_data(port_keys: List[str]) -> int: logger.info("retrieving data from simcore...") start_time = time.perf_counter() PORTS = node_ports.ports() inputs_path = Path(_INPUTS_FOLDER).expanduser() data = {} # let's gather all the data download_tasks = [] for node_input in PORTS.inputs: # if port_keys contains some keys only download them logger.info("Checking node %s", node_input.key) if port_keys and node_input.key not in port_keys: continue # collect coroutines download_tasks.append(get_time_wrapped(node_input)) logger.info("retrieving %s data", len(download_tasks)) transfer_bytes = 0 if download_tasks: results = await asyncio.gather(*download_tasks) logger.info("completed download %s", results) for port, value in results: data[port.key] = {"key": port.key, "value": value} if _FILE_TYPE_PREFIX in port.type: # if there are files, move them to the final destination downloaded_file = value dest_path = inputs_path / port.key # first cleanup if dest_path.exists(): logger.info("removing %s", dest_path) shutil.rmtree(dest_path) if not downloaded_file or not downloaded_file.exists(): # the link may be empty continue transfer_bytes = transfer_bytes + downloaded_file.stat( ).st_size # in case of valid file, it is either uncompressed and/or moved to the final directory logger.info("creating directory %s", dest_path) dest_path.mkdir(exist_ok=True, parents=True) data[port.key] = {"key": port.key, "value": str(dest_path)} if zipfile.is_zipfile(downloaded_file): logger.info("unzipping %s", downloaded_file) with zipfile.ZipFile(downloaded_file) as zip_file: zip_file.extractall( dest_path, members=_no_relative_path_zip(zip_file)) logger.info("all unzipped in %s", dest_path) else: logger.info("moving %s", downloaded_file) dest_path = dest_path / Path(downloaded_file).name shutil.move(downloaded_file, dest_path) logger.info("all moved to %s", dest_path) else: transfer_bytes = transfer_bytes + sys.getsizeof(value) # create/update the json file with the new values if data: data_file = inputs_path / _KEY_VALUE_FILE_NAME if data_file.exists(): current_data = json.loads(data_file.read_text()) # merge data data = {**current_data, **data} data_file.write_text(json.dumps(data)) stop_time = time.perf_counter() logger.info("all data retrieved from simcore in %sseconds: %s", stop_time - start_time, data) return transfer_bytes