async def _initialise_platform(port_configuration_path: Path, file_generator, delete_file): with port_configuration_path.open() as file_pointer: configuration = json.load(file_pointer) if not all(k in configuration for k in ("schema", "inputs", "outputs")): raise Exception("invalid port configuration in {}, {}!".format( str(port_configuration_path), configuration)) # init s3 to ensure we have a bucket init_storage() # set up db db = init_db() # create a new pipeline project_id = str(uuid.uuid4()) new_Pipeline = ComputationalPipeline(project_id=project_id) db.session.add(new_Pipeline) db.session.commit() # create a new node node_uuid = str(uuid.uuid4()) # now create the node in the db with links to S3 new_Node = ComputationalTask(project_id=project_id, node_id=node_uuid, schema=configuration["schema"], inputs=configuration["inputs"], outputs=configuration["outputs"]) db.session.add(new_Node) db.session.commit() # set up node_ports node_ports.node_config.NODE_UUID = node_uuid node_ports.node_config.PROJECT_ID = project_id PORTS = await node_ports.ports() # push the file to the S3 for each input item file_index = 0 for key, input_item in configuration["schema"]["inputs"].items(): if str(input_item["type"]).startswith("data:"): file_to_upload = file_generator(file_index, input_item["type"]) if file_to_upload is not None: # upload to S3 await (await PORTS.inputs)[key].set(Path(file_to_upload)) file_index += 1 if delete_file: Path(file_to_upload).unlink() # print the node uuid so that it can be set as env variable from outside print("{pipelineid},{nodeuuid}".format(pipelineid=str(new_Node.project_id), nodeuuid=node_uuid))
def _set_configuration(session, project_id: str, node_id: str, json_configuration: str): node_uuid = node_id json_configuration = json_configuration.replace("SIMCORE_NODE_UUID", str(node_uuid)) configuration = json.loads(json_configuration) new_Node = ComputationalTask(project_id=project_id, node_id=node_uuid, schema=configuration["schema"], inputs=configuration["inputs"], outputs=configuration["outputs"]) session.add(new_Node) session.commit() return node_uuid
async def _create( tasks: Dict[str, Any], dag: Dict[str, List[str]], inputs: Dict[str, Dict[str, Any]], ) -> ComputationalPipeline: # add a pipeline pipeline = ComputationalPipeline(project_id=project_id, dag_adjacency_list=dag) postgres_session.add(pipeline) postgres_session.commit() # create the tasks for each pipeline's node for node_uuid, service in tasks.items(): node_inputs = inputs[node_uuid] comp_task = ComputationalTask( project_id=project_id, node_id=node_uuid, schema=service["schema"], image=service["image"], inputs=node_inputs, outputs={}, ) postgres_session.add(comp_task) postgres_session.commit() # check if file must be uploaded for input_key in node_inputs: if ( isinstance(node_inputs[input_key], dict) and "path" in node_inputs[input_key] ): # update the files in mock_dir to S3 # FIXME: node_ports config shall not global! here making a hack so it works node_ports.node_config.USER_ID = user_id node_ports.node_config.PROJECT_ID = project_id node_ports.node_config.NODE_UUID = node_uuid print("--" * 10) print_module_variables(module=node_ports.node_config) print("--" * 10) PORTS = await node_ports.ports() await (await PORTS.inputs)[input_key].set( mock_dir / node_inputs[input_key]["path"] ) return pipeline
def setup_sleepers(url): db_engine = create_engine(url, client_encoding="utf8", connect_args={"connect_timeout": 30}, pool_pre_ping=True) DatabaseSession = sessionmaker(db_engine) db_session = DatabaseSession() dag_adjacency_list = { "e609a68c-d743-4a12-9745-f31734d1b911": ["3e5103b3-8930-4025-846b-b8995460379e"], "3e5103b3-8930-4025-846b-b8995460379e": [] } pipeline = ComputationalPipeline(dag_adjacency_list=dag_adjacency_list, state=0) db_session.add(pipeline) db_session.flush() project_id = pipeline.project_id node_id_1 = "e609a68c-d743-4a12-9745-f31734d1b911" internal_id_1 = 1 node_schema = { "inputs": { "in_1": { "label": "Number of seconds to sleep", "description": "Number of seconds to sleep", "displayOrder": 0, "type": "data:*/*" }, "in_2": { "label": "Number of seconds to sleep", "description": "Number of seconds to sleep", "displayOrder": 1, "type": "integer", "defaultValue": 4 } }, "outputs": { "out_1": { "label": "Number of seconds to sleep", "description": "Number of seconds to sleep", "displayOrder": 0, "type": "data:*/*" }, "out_2": { "label": "Number of seconds to sleep", "description": "Number of seconds to sleep", "displayOrder": 1, "type": "integer" } } } node_inputs_1 = {} node_outputs_1 = {"out_2": 1} node_key = "simcore/services/comp/itis/sleeper" node_version = "0.0.1" # create the task comp_task_1 = ComputationalTask(project_id=project_id, node_id=node_id_1, internal_id=internal_id_1, schema=node_schema, image={ "name": node_key, "tag": node_version }, inputs=node_inputs_1, outputs=node_outputs_1, submit=datetime.datetime.utcnow()) db_session.add(comp_task_1) db_session.commit() node_id_2 = "3e5103b3-8930-4025-846b-b8995460379e" internal_id_2 = 2 node_inputs_2 = { "in_1": { "nodeUuid": "e609a68c-d743-4a12-9745-f31734d1b911", "output": "out_1" }, "in_2": { "nodeUuid": "e609a68c-d743-4a12-9745-f31734d1b911", "output": "out_2" } } node_outputs_2 = {"out_2": 5} node_key = "simcore/services/comp/itis/sleeper" node_version = "0.0.1" # create the task comp_task_2 = ComputationalTask(project_id=project_id, node_id=node_id_2, internal_id=internal_id_2, schema=node_schema, image={ "name": node_key, "tag": node_version }, inputs=node_inputs_2, outputs=node_outputs_2, submit=datetime.datetime.utcnow()) db_session.add(comp_task_2) db_session.commit() return project_id