async def _initialise_platform(port_configuration_path: Path, file_generator,
delete_file):
with port_configuration_path.open() as file_pointer:
configuration = json.load(file_pointer)
if not all(k in configuration for k in ("schema", "inputs", "outputs")):
raise Exception("invalid port configuration in {}, {}!".format(
str(port_configuration_path), configuration))
# init s3 to ensure we have a bucket
init_storage()
# set up db
db = init_db()
# create a new pipeline
project_id = str(uuid.uuid4())
new_Pipeline = ComputationalPipeline(project_id=project_id)
db.session.add(new_Pipeline)
db.session.commit()
# create a new node
node_uuid = str(uuid.uuid4())
# now create the node in the db with links to S3
new_Node = ComputationalTask(project_id=project_id,
node_id=node_uuid,
schema=configuration["schema"],
inputs=configuration["inputs"],
outputs=configuration["outputs"])
db.session.add(new_Node)
db.session.commit()
# set up node_ports
node_ports.node_config.NODE_UUID = node_uuid
node_ports.node_config.PROJECT_ID = project_id
PORTS = await node_ports.ports()
# push the file to the S3 for each input item
file_index = 0
for key, input_item in configuration["schema"]["inputs"].items():
if str(input_item["type"]).startswith("data:"):
file_to_upload = file_generator(file_index, input_item["type"])
if file_to_upload is not None:
# upload to S3
await (await PORTS.inputs)[key].set(Path(file_to_upload))
file_index += 1
if delete_file:
Path(file_to_upload).unlink()
# print the node uuid so that it can be set as env variable from outside
print("{pipelineid},{nodeuuid}".format(pipelineid=str(new_Node.project_id),
nodeuuid=node_uuid))
def _set_configuration(session, project_id: str, node_id: str,
json_configuration: str):
node_uuid = node_id
json_configuration = json_configuration.replace("SIMCORE_NODE_UUID",
str(node_uuid))
configuration = json.loads(json_configuration)
new_Node = ComputationalTask(project_id=project_id,
node_id=node_uuid,
schema=configuration["schema"],
inputs=configuration["inputs"],
outputs=configuration["outputs"])
session.add(new_Node)
session.commit()
return node_uuid
async def _create(
tasks: Dict[str, Any],
dag: Dict[str, List[str]],
inputs: Dict[str, Dict[str, Any]],
) -> ComputationalPipeline:
# add a pipeline
pipeline = ComputationalPipeline(project_id=project_id, dag_adjacency_list=dag)
postgres_session.add(pipeline)
postgres_session.commit()
# create the tasks for each pipeline's node
for node_uuid, service in tasks.items():
node_inputs = inputs[node_uuid]
comp_task = ComputationalTask(
project_id=project_id,
node_id=node_uuid,
schema=service["schema"],
image=service["image"],
inputs=node_inputs,
outputs={},
)
postgres_session.add(comp_task)
postgres_session.commit()
# check if file must be uploaded
for input_key in node_inputs:
if (
isinstance(node_inputs[input_key], dict)
and "path" in node_inputs[input_key]
):
# update the files in mock_dir to S3
# FIXME: node_ports config shall not global! here making a hack so it works
node_ports.node_config.USER_ID = user_id
node_ports.node_config.PROJECT_ID = project_id
node_ports.node_config.NODE_UUID = node_uuid
print("--" * 10)
print_module_variables(module=node_ports.node_config)
print("--" * 10)
PORTS = await node_ports.ports()
await (await PORTS.inputs)[input_key].set(
mock_dir / node_inputs[input_key]["path"]
)
return pipeline
def setup_sleepers(url):
db_engine = create_engine(url,
client_encoding="utf8",
connect_args={"connect_timeout": 30},
pool_pre_ping=True)
DatabaseSession = sessionmaker(db_engine)
db_session = DatabaseSession()
dag_adjacency_list = {
"e609a68c-d743-4a12-9745-f31734d1b911":
["3e5103b3-8930-4025-846b-b8995460379e"],
"3e5103b3-8930-4025-846b-b8995460379e": []
}
pipeline = ComputationalPipeline(dag_adjacency_list=dag_adjacency_list,
state=0)
db_session.add(pipeline)
db_session.flush()
project_id = pipeline.project_id
node_id_1 = "e609a68c-d743-4a12-9745-f31734d1b911"
internal_id_1 = 1
node_schema = {
"inputs": {
"in_1": {
"label": "Number of seconds to sleep",
"description": "Number of seconds to sleep",
"displayOrder": 0,
"type": "data:*/*"
},
"in_2": {
"label": "Number of seconds to sleep",
"description": "Number of seconds to sleep",
"displayOrder": 1,
"type": "integer",
"defaultValue": 4
}
},
"outputs": {
"out_1": {
"label": "Number of seconds to sleep",
"description": "Number of seconds to sleep",
"displayOrder": 0,
"type": "data:*/*"
},
"out_2": {
"label": "Number of seconds to sleep",
"description": "Number of seconds to sleep",
"displayOrder": 1,
"type": "integer"
}
}
}
node_inputs_1 = {}
node_outputs_1 = {"out_2": 1}
node_key = "simcore/services/comp/itis/sleeper"
node_version = "0.0.1"
# create the task
comp_task_1 = ComputationalTask(project_id=project_id,
node_id=node_id_1,
internal_id=internal_id_1,
schema=node_schema,
image={
"name": node_key,
"tag": node_version
},
inputs=node_inputs_1,
outputs=node_outputs_1,
submit=datetime.datetime.utcnow())
db_session.add(comp_task_1)
db_session.commit()
node_id_2 = "3e5103b3-8930-4025-846b-b8995460379e"
internal_id_2 = 2
node_inputs_2 = {
"in_1": {
"nodeUuid": "e609a68c-d743-4a12-9745-f31734d1b911",
"output": "out_1"
},
"in_2": {
"nodeUuid": "e609a68c-d743-4a12-9745-f31734d1b911",
"output": "out_2"
}
}
node_outputs_2 = {"out_2": 5}
node_key = "simcore/services/comp/itis/sleeper"
node_version = "0.0.1"
# create the task
comp_task_2 = ComputationalTask(project_id=project_id,
node_id=node_id_2,
internal_id=internal_id_2,
schema=node_schema,
image={
"name": node_key,
"tag": node_version
},
inputs=node_inputs_2,
outputs=node_outputs_2,
submit=datetime.datetime.utcnow())
db_session.add(comp_task_2)
db_session.commit()
return project_id