def _build_burst_export_dict(self, burst):
"""
Compute needed export info and return dictionary
"""
burst_info = BurstInformation(burst.to_dict()[1])
workflows = dao.get_workflows_for_burst(burst.id)
for workflow in workflows:
# Get information for each workflow for this burst
workflow_info = WorkflowInformation(workflow.to_dict()[1])
wf_steps, view_steps = self._build_workflow_step_info(workflow)
workflow_info.set_workflow_steps(wf_steps)
workflow_info.set_view_steps(view_steps)
burst_info.add_workflow(workflow_info)
return burst_info.to_dict()
def _build_burst_export_dict(self, burst):
"""
Compute needed export info and return dictionary
"""
burst_info = BurstInformation(burst.to_dict()[1])
workflows = dao.get_workflows_for_burst(burst.id)
for workflow in workflows:
# Get information for each workflow for this burst
workflow_info = WorkflowInformation(workflow.to_dict()[1])
wf_steps, view_steps = self._build_workflow_step_info(workflow)
workflow_info.set_workflow_steps(wf_steps)
workflow_info.set_view_steps(view_steps)
burst_info.add_workflow(workflow_info)
return burst_info.to_dict()
def _build_burst_export_dict(self, burst, bursts_dict):
"""
Compute needed info and add them to burst_dict for export.
"""
burst_info = BurstInformation(burst.to_dict()[1])
workflows = dao.get_workflows_for_burst(burst.id)
for workflow in workflows:
# Get information for each workflow for this burst
workflow_info = WorkflowInformation(workflow.to_dict()[1])
wf_steps, view_steps = self._build_workflow_step_info(workflow)
workflow_info.set_workflow_steps(wf_steps)
workflow_info.set_view_steps(view_steps)
burst_info.add_workflow(workflow_info)
# Save data in dictionary form so we can just save it as a json later on
bursts_dict[burst.id] = burst_info.to_dict()
def _build_burst_export_dict(self, burst, bursts_dict):
"""
Compute needed info and add them to burst_dict for export.
"""
burst_info = BurstInformation(burst.to_dict()[1])
workflows = dao.get_workflows_for_burst(burst.id)
for workflow in workflows:
# Get information for each workflow for this burst
workflow_info = WorkflowInformation(workflow.to_dict()[1])
wf_steps, view_steps = self._build_workflow_step_info(workflow)
workflow_info.set_workflow_steps(wf_steps)
workflow_info.set_view_steps(view_steps)
burst_info.add_workflow(workflow_info)
# Save data in dictionary form so we can just save it as a json later on
bursts_dict[burst.id] = burst_info.to_dict()
def load_burst_entity(self, json_burst, project_id):
"""
Load BurstConfiguration from JSON (possibly exported from a different machine).
Nothing gets persisted in DB or on disk.
:param json_burst: Burst JSON export
:param project_id: Current project ID (it will be used if the user later starts this simulation)
:return: BurstConfiguration filled from JSON
"""
burst_information = BurstInformation.load_from_dict(json_burst)
burst_entity = model.BurstConfiguration(project_id)
burst_entity.from_dict(burst_information.data)
burst_entity.prepare_after_load()
burst_entity.reset_tabs()
workflow_info = burst_information.get_workflows()[0]
workflow_entity = model.Workflow(project_id, None)
workflow_entity.from_dict(workflow_info.data)
view_steps = workflow_info.get_view_steps()
analyze_steps = workflow_info.get_workflow_steps()
for view_step in view_steps:
try:
algorithm = view_step.get_algorithm()
portlet = view_step.get_portlet()
view_step_entity = model.WorkflowStepView(
algorithm.id, portlet_id=portlet.id)
view_step_entity.from_dict(view_step.data)
view_step_entity.workflow = workflow_entity
## For each visualize step, also load all of the analyze steps.
analyzers = []
for an_step in analyze_steps:
if (an_step.data["tab_index"] != view_step_entity.tab_index
or an_step.data["index_in_tab"] !=
view_step_entity.index_in_tab):
continue
algorithm = an_step.get_algorithm()
wf_step_entity = model.WorkflowStep(algorithm.id)
wf_step_entity.from_dict(an_step.data)
wf_step_entity.workflow = workflow_entity
analyzers.append(wf_step_entity)
portlet = PortletConfiguration(portlet.id)
portlet.set_visualizer(view_step_entity)
portlet.set_analyzers(analyzers)
burst_entity.set_portlet(view_step_entity.tab_index,
view_step_entity.index_in_tab,
portlet)
except Exception:
# only log exception and ignore this step from loading
self.logger.exception("Could not restore Workflow Step " +
view_step.get_algorithm().name)
return burst_entity
def load_burst_entity(self, json_burst, project_id):
"""
Load BurstConfiguration from JSON (possibly exported from a different machine).
Nothing gets persisted in DB or on disk.
:param json_burst: Burst JSON export
:param project_id: Current project ID (it will be used if the user later starts this simulation)
:return: BurstConfiguration filled from JSON
"""
burst_information = BurstInformation.load_from_dict(json_burst)
burst_entity = model.BurstConfiguration(project_id)
burst_entity.from_dict(burst_information.data)
burst_entity.prepare_after_load()
burst_entity.reset_tabs()
workflow_info = burst_information.get_workflows()[0]
workflow_entity = model.Workflow(project_id, None)
workflow_entity.from_dict(workflow_info.data)
view_steps = workflow_info.get_view_steps()
analyze_steps = workflow_info.get_workflow_steps()
for view_step in view_steps:
try:
algorithm = view_step.get_algorithm()
portlet = view_step.get_portlet()
view_step_entity = model.WorkflowStepView(algorithm.id, portlet_id=portlet.id)
view_step_entity.from_dict(view_step.data)
view_step_entity.workflow = workflow_entity
## For each visualize step, also load all of the analyze steps.
analyzers = []
for an_step in analyze_steps:
if (an_step.data["tab_index"] != view_step_entity.tab_index
or an_step.data["index_in_tab"] != view_step_entity.index_in_tab):
continue
algorithm = an_step.get_algorithm()
wf_step_entity = model.WorkflowStep(algorithm.id)
wf_step_entity.from_dict(an_step.data)
wf_step_entity.workflow = workflow_entity
analyzers.append(wf_step_entity)
portlet = PortletConfiguration(portlet.id)
portlet.set_visualizer(view_step_entity)
portlet.set_analyzers(analyzers)
burst_entity.set_portlet(view_step_entity.tab_index, view_step_entity.index_in_tab, portlet)
except Exception:
# only log exception and ignore this step from loading
self.logger.exception("Could not restore Workflow Step " + view_step.get_algorithm().name)
return burst_entity
def _import_project_from_folder(self, temp_folder):
"""
Process each project from the uploaded pack, to extract names.
"""
project_roots = []
for root, _, files in os.walk(temp_folder):
if FilesHelper.TVB_PROJECT_FILE in files:
project_roots.append(root)
for project_path in project_roots:
project_entity = self.__populate_project(project_path)
bursts_dict = {}
dt_mappings_dict = {}
bursts_file = os.path.join(project_path, BURST_INFO_FILE)
if os.path.isfile(bursts_file):
bursts_info_dict = json.loads(open(bursts_file, 'r').read())
bursts_dict = bursts_info_dict[BURSTS_DICT_KEY]
dt_mappings_dict = bursts_info_dict[DT_BURST_MAP]
# Compute the path where to store files of the imported project
new_project_path = os.path.join(cfg.TVB_STORAGE,
FilesHelper.PROJECTS_FOLDER,
project_entity.name)
if project_path != new_project_path:
shutil.copytree(project_path, new_project_path)
shutil.rmtree(project_path)
self.created_projects.append(project_entity)
# Re-create old bursts, but keep a mapping between the id it has here and the old-id it had
# in the project where they were exported, so we can re-add the datatypes to them.
burst_ids_mapping = {}
# Keep a list with all burst that were imported since we will want to also add the workflow
# steps after we are finished with importing the operations and datatypes. We need to first
# stored bursts since we need to know which new id's they have for operations parent_burst.
if bursts_dict:
for old_burst_id in bursts_dict:
burst_information = BurstInformation.load_from_dict(
bursts_dict[old_burst_id])
burst_entity = model.BurstConfiguration(project_entity.id)
burst_entity.from_dict(burst_information.data)
burst_entity = dao.store_entity(burst_entity)
burst_ids_mapping[int(old_burst_id)] = burst_entity.id
# We don't need the data in dictionary form anymore, so update it with new BurstInformation object
bursts_dict[old_burst_id] = burst_information
# Now import project operations
self.import_project_operations(project_entity, new_project_path,
dt_mappings_dict, burst_ids_mapping)
# Now we can finally import workflow related entities
self.import_workflows(project_entity, bursts_dict,
burst_ids_mapping)
def _import_bursts(project_entity, bursts_dict):
"""
Re-create old bursts, but keep a mapping between the id it has here and the old-id it had
in the project where they were exported, so we can re-add the datatypes to them.
"""
burst_ids_mapping = {}
for old_burst_id in bursts_dict:
burst_information = BurstInformation.load_from_dict(bursts_dict[old_burst_id])
burst_entity = model.BurstConfiguration(project_entity.id)
burst_entity.from_dict(burst_information.data)
burst_entity = dao.store_entity(burst_entity)
burst_ids_mapping[int(old_burst_id)] = burst_entity.id
# We don't need the data in dictionary form anymore, so update it with new BurstInformation object
bursts_dict[old_burst_id] = burst_information
return burst_ids_mapping
def _import_bursts(project_entity, bursts_dict):
"""
Re-create old bursts, but keep a mapping between the id it has here and the old-id it had
in the project where they were exported, so we can re-add the datatypes to them.
"""
burst_ids_mapping = {}
for old_burst_id in bursts_dict:
burst_information = BurstInformation.load_from_dict(bursts_dict[old_burst_id])
burst_entity = model.BurstConfiguration(project_entity.id)
burst_entity.from_dict(burst_information.data)
burst_entity = dao.store_entity(burst_entity)
burst_ids_mapping[int(old_burst_id)] = burst_entity.id
# We don't need the data in dictionary form anymore, so update it with new BurstInformation object
bursts_dict[old_burst_id] = burst_information
return burst_ids_mapping
def _import_project_from_folder(self, temp_folder):
"""
Process each project from the uploaded pack, to extract names.
"""
project_roots = []
for root, _, files in os.walk(temp_folder):
if FilesHelper.TVB_PROJECT_FILE in files:
project_roots.append(root)
for project_path in project_roots:
project_entity = self.__populate_project(project_path)
bursts_dict = {}
dt_mappings_dict = {}
bursts_file = os.path.join(project_path, BURST_INFO_FILE)
if os.path.isfile(bursts_file):
bursts_info_dict = json.loads(open(bursts_file, 'r').read())
bursts_dict = bursts_info_dict[BURSTS_DICT_KEY]
dt_mappings_dict = bursts_info_dict[DT_BURST_MAP]
# Compute the path where to store files of the imported project
new_project_path = os.path.join(cfg.TVB_STORAGE, FilesHelper.PROJECTS_FOLDER, project_entity.name)
if project_path != new_project_path:
shutil.copytree(project_path, new_project_path)
shutil.rmtree(project_path)
self.created_projects.append(project_entity)
# Re-create old bursts, but keep a mapping between the id it has here and the old-id it had
# in the project where they were exported, so we can re-add the datatypes to them.
burst_ids_mapping = {}
# Keep a list with all burst that were imported since we will want to also add the workflow
# steps after we are finished with importing the operations and datatypes. We need to first
# stored bursts since we need to know which new id's they have for operations parent_burst.
if bursts_dict:
for old_burst_id in bursts_dict:
burst_information = BurstInformation.load_from_dict(bursts_dict[old_burst_id])
burst_entity = model.BurstConfiguration(project_entity.id)
burst_entity.from_dict(burst_information.data)
burst_entity = dao.store_entity(burst_entity)
burst_ids_mapping[int(old_burst_id)] = burst_entity.id
# We don't need the data in dictionary form anymore, so update it with new BurstInformation object
bursts_dict[old_burst_id] = burst_information
# Now import project operations
self.import_project_operations(project_entity, new_project_path, dt_mappings_dict, burst_ids_mapping)
# Now we can finally import workflow related entities
self.import_workflows(project_entity, bursts_dict, burst_ids_mapping)
