def launch_visualization(visualization, frame_width=None, frame_height=None,
method_name=ABCAdapter.LAUNCH_METHOD, is_preview=True):
"""
:param visualization: a visualization workflow step
"""
dynamic_params = visualization.dynamic_param
static_params = visualization.static_param
parameters_dict = static_params
current_project_id = 0
## Current operation id needed for export mechanism. So far just use ##
## the operation of the workflow_step from which the inputs are taken ####
for param in dynamic_params:
step_index = dynamic_params[param][WorkflowStepConfiguration.STEP_INDEX_KEY]
datatype_index = dynamic_params[param][WorkflowStepConfiguration.DATATYPE_INDEX_KEY]
referred_workflow_step = dao.get_workflow_step_by_step_index(visualization.fk_workflow, step_index)
referred_operation_id = referred_workflow_step.fk_operation
referred_operation = dao.get_operation_by_id(referred_operation_id)
current_project_id = referred_operation.fk_launched_in
if type(datatype_index) is IntType:
## Entry is the output of a previous step ##
datatypes = dao.get_results_for_operation(referred_operation_id)
parameters_dict[param] = datatypes[datatype_index].gid
else:
## Entry is the input of a previous step ###
parameters_dict[param] = json.loads(referred_operation.parameters)[datatype_index]
algorithm = dao.get_algorithm_by_id(visualization.fk_algorithm)
adapter_instance = ABCAdapter.build_adapter(algorithm.algo_group)
adapter_instance.current_project_id = current_project_id
prepared_inputs = adapter_instance.prepare_ui_inputs(parameters_dict)
if frame_width is not None:
prepared_inputs[ABCDisplayer.PARAM_FIGURE_SIZE] = (frame_width, frame_height)
if isinstance(adapter_instance, ABCMPLH5Displayer) and is_preview is True:
prepared_inputs[ABCMPLH5Displayer.SHOW_FULL_TOOLBAR] = False
result = eval("adapter_instance." + method_name + "(**prepared_inputs)")
return result, parameters_dict
def build_structure_for_operation(operation):
algo = dao.get_algorithm_by_id(operation.fk_from_algo)
structure = NodeStructure(operation.gid, algo.name)
structure.data = NodeData.build_node_for_operation(operation)
return structure
def build_structure_for_operation(operation):
algo = dao.get_algorithm_by_id(operation.fk_from_algo)
structure = NodeStructure(operation.gid, algo.name)
structure.data = NodeData.build_node_for_operation(operation)
return structure
def group_operation_launch(self, user_id, project, algorithm_id, category_id):
"""
Create and prepare the launch of a group of operations.
"""
algorithm = dao.get_algorithm_by_id(algorithm_id)
ops, _ = self.prepare_operation(user_id, project, algorithm)
for operation in ops:
self.launch_operation(operation.id, True)
def build_structure_for_operation_group(operation_group_gid):
group = dao.get_operationgroup_by_gid(operation_group_gid)
operation = dao.get_operations_in_group(group.id, only_first_operation=True)
algo = dao.get_algorithm_by_id(operation.fk_from_algo)
structure = NodeStructure(operation_group_gid, algo.name)
structure.data = NodeData.build_node_for_operation(operation, group.id)
return structure
def group_operation_launch(self, user_id, project_id, adapter_id, category_id, **kwargs):
"""
Create and prepare the launch of a group of operations.
"""
category = dao.get_category_by_id(category_id)
algorithm = dao.get_algorithm_by_id(adapter_id)
operations, _ = self.prepare_operations(user_id, project_id, algorithm, category, {}, **kwargs)
for operation in operations:
self.launch_operation(operation.id, True)
def group_operation_launch(self, user_id, project_id, adapter_id, category_id, **kwargs):
"""
Create and prepare the launch of a group of operations.
"""
category = dao.get_category_by_id(category_id)
algorithm = dao.get_algorithm_by_id(adapter_id)
operations, _ = self.prepare_operations(user_id, project_id, algorithm, category, {}, **kwargs)
for operation in operations:
self.launch_operation(operation.id, True)
def build_structure_for_operation_group(operation_group_gid):
group = dao.get_operationgroup_by_gid(operation_group_gid)
operation = dao.get_operations_in_group(group.id, only_first_operation=True)
algo = dao.get_algorithm_by_id(operation.fk_from_algo)
structure = NodeStructure(operation_group_gid, algo.displayname)
structure.data = NodeData.build_node_for_operation(operation, group.id)
return structure
def get_algorithm_by_identifier(self, ident):
"""
Retrieve Algorithm entity by ID.
Return None, if ID is not found in DB.
"""
try:
return dao.get_algorithm_by_id(ident)
except Exception, excep:
self.logger.exception(excep)
return None
def get_algorithm_by_identifier(self, ident):
"""
Retrieve Algorithm entity by ID.
Return None, if ID is not found in DB.
"""
try:
return dao.get_algorithm_by_id(ident)
except Exception, excep:
self.logger.exception(excep)
return None
def __init__(self, operation_gid):
NodeStructure.__init__(self, operation_gid, "")
operation = dao.get_operation_by_gid(operation_gid)
algo = dao.get_algorithm_by_id(operation.fk_from_algo)
node_data = NodeData(MAX_SHAPE_SIZE, OPERATION_SHAPE_COLOR, OPERATION_SHAPE,
NODE_OPERATION_TYPE, operation.id, str(operation.start_date))
self.name = algo.name
self.data = node_data
def group_operation_launch(self, user_id, project, algorithm_id, category_id, existing_dt_group=None, **kwargs):
"""
Create and prepare the launch of a group of operations.
"""
category = dao.get_category_by_id(category_id)
algorithm = dao.get_algorithm_by_id(algorithm_id)
ops, _ = self.prepare_operations(user_id, project, algorithm, category,
existing_dt_group=existing_dt_group, **kwargs)
for operation in ops:
self.launch_operation(operation.id, True)
def __init__(self, operation_group_gid):
NodeStructure.__init__(self, operation_group_gid, "")
group = dao.get_operationgroup_by_gid(operation_group_gid)
operation = dao.get_operations_in_group(group.id, only_first_operation=True)
algo = dao.get_algorithm_by_id(operation.fk_from_algo)
node_data = NodeData(MAX_SHAPE_SIZE, OPERATION_GROUP_SHAPE_COLOR, OPERATION_GROUP_SHAPE,
NODE_OPERATION_GROUP_TYPE, group.id, str(operation.start_date))
self.name = algo.name
self.data = node_data
def __init__(self, operation_gid):
NodeStructure.__init__(self, operation_gid, "")
operation = dao.get_operation_by_gid(operation_gid)
algo = dao.get_algorithm_by_id(operation.fk_from_algo)
node_data = NodeData(MAX_SHAPE_SIZE, OPERATION_SHAPE_COLOR,
OPERATION_SHAPE, NODE_OPERATION_TYPE,
operation.id, str(operation.start_date))
self.name = algo.name
self.data = node_data
def _build_workflow_step_info(workflow):
"""
For the input workflow, get all workflow steps and return a list with information
that can be then exported.
"""
wf_steps = []
view_steps = []
for wf_step in dao.get_workflow_steps(workflow.id):
if wf_step.fk_operation is None or wf_step.fk_algorithm is None:
# Avoid exporting old form of View Steps.
LOG.warning("Skipping " + str(workflow) + " " + str(wf_step))
continue
# Get all basic information for this workflow step
wf_step_info = WorkflowStepInformation(wf_step.to_dict()[1])
# We need to store the gid for the operation since the id might be
# different in case of a project export / import
linked_operation = dao.get_operation_by_id(wf_step.fk_operation)
wf_step_info.set_operation(linked_operation)
# We also need to keep info about algorithm in the form of module
# and classname because that id might also be different in case
# of project export / import.
linked_algorithm = dao.get_algorithm_by_id(wf_step.fk_algorithm)
wf_step_info.set_algorithm(linked_algorithm)
wf_steps.append(wf_step_info)
for view_step in dao.get_visualization_steps(workflow.id):
# Get all basic information for this view step
view_step_info = WorkflowViewStepInformation(
view_step.to_dict()[1])
# We need to store portlet identifier, since portlet id might be different
# in project we are importing into.
portlet = dao.get_portlet_by_id(view_step.fk_portlet)
view_step_info.set_portlet(portlet)
# We also need to keep info about algorithm in the form of module
# and classname because that id might also be different in case
# of project export / import.
linked_algorithm = dao.get_algorithm_by_id(view_step.fk_algorithm)
view_step_info.set_algorithm(linked_algorithm)
view_steps.append(view_step_info)
return wf_steps, view_steps
def _create_operation(self, project_id, algorithm_id):
"""
dummy operation
:param project_id: the project in which the operation is created
:param algorithm_id: the algorithm to be run for the operation
:return: a dummy `Operation` with the given specifications
"""
algorithm = dao.get_algorithm_by_id(algorithm_id)
meta = {DataTypeMetaData.KEY_SUBJECT: "John Doe",
DataTypeMetaData.KEY_STATE: "RAW_DATA"}
operation = model.Operation(self.test_user.id, project_id, algorithm.id, 'test params',
meta=json.dumps(meta), status=model.STATUS_FINISHED)
return dao.store_entity(operation)
def _build_workflow_step_info(workflow):
"""
For the input workflow, get all workflow steps and return a list with information
that can be then exported.
"""
wf_steps = []
view_steps = []
for wf_step in dao.get_workflow_steps(workflow.id):
if wf_step.fk_operation is None or wf_step.fk_algorithm is None:
# Avoid exporting old form of View Steps.
LOG.warning("Skipping " + str(workflow) + " " + str(wf_step))
continue
# Get all basic information for this workflow step
wf_step_info = WorkflowStepInformation(wf_step.to_dict()[1])
# We need to store the gid for the operation since the id might be
# different in case of a project export / import
linked_operation = dao.get_operation_by_id(wf_step.fk_operation)
wf_step_info.set_operation(linked_operation)
# We also need to keep info about algorithm in the form of module
# and classname because that id might also be different in case
# of project export / import.
linked_algorithm = dao.get_algorithm_by_id(wf_step.fk_algorithm)
wf_step_info.set_algorithm(linked_algorithm)
wf_steps.append(wf_step_info)
for view_step in dao.get_visualization_steps(workflow.id):
# Get all basic information for this view step
view_step_info = WorkflowViewStepInformation(view_step.to_dict()[1])
# We need to store portlet identifier, since portlet id might be different
# in project we are importing into.
portlet = dao.get_portlet_by_id(view_step.fk_portlet)
view_step_info.set_portlet(portlet)
# We also need to keep info about algorithm in the form of module
# and classname because that id might also be different in case
# of project export / import.
linked_algorithm = dao.get_algorithm_by_id(view_step.fk_algorithm)
view_step_info.set_algorithm(linked_algorithm)
view_steps.append(view_step_info)
return wf_steps, view_steps
def _create_operation(self, project_id, algorithm_id):
"""
dummy operation
:param project_id: the project in which the operation is created
:param algorithm_id: the algorithm to be run for the operation
:return: a dummy `Operation` with the given specifications
"""
algorithm = dao.get_algorithm_by_id(algorithm_id)
meta = {DataTypeMetaData.KEY_SUBJECT: "John Doe",
DataTypeMetaData.KEY_STATE: "RAW_DATA"}
operation = Operation(self.test_user.id, project_id, algorithm.id, 'test params',
meta=json.dumps(meta), status=STATUS_FINISHED)
return dao.store_entity(operation)
def __init__(self, operation_group_gid):
NodeStructure.__init__(self, operation_group_gid, "")
group = dao.get_operationgroup_by_gid(operation_group_gid)
operation = dao.get_operations_in_group(group.id,
only_first_operation=True)
algo = dao.get_algorithm_by_id(operation.fk_from_algo)
node_data = NodeData(MAX_SHAPE_SIZE, OPERATION_GROUP_SHAPE_COLOR,
OPERATION_GROUP_SHAPE, NODE_OPERATION_GROUP_TYPE,
group.id, str(operation.start_date))
self.name = algo.name
self.data = node_data
def prepare_operations_for_workflowsteps(self, workflow_step_list, workflows, user_id, burst_id,
project_id, group, sim_operations):
"""
Create and store Operation entities from a list of Workflow Steps.
Will be generated workflows x workflow_step_list Operations.
For every step in workflow_step_list one OperationGroup and one DataTypeGroup will be created
(in case of PSE).
"""
for step in workflow_step_list:
operation_group = None
if (group is not None) and not isinstance(step, model.WorkflowStepView):
operation_group = model.OperationGroup(project_id=project_id, ranges=group.range_references)
operation_group = dao.store_entity(operation_group)
operation = None
metadata = {DataTypeMetaData.KEY_BURST: burst_id}
algo_category = dao.get_algorithm_by_id(step.fk_algorithm)
if algo_category is not None:
algo_category = algo_category.algorithm_category
for wf_idx, workflow in enumerate(workflows):
cloned_w_step = step.clone()
cloned_w_step.fk_workflow = workflow.id
dynamic_params = cloned_w_step.dynamic_param
op_params = cloned_w_step.static_param
op_params.update(dynamic_params)
range_values = None
group_id = None
if operation_group is not None:
group_id = operation_group.id
range_values = sim_operations[wf_idx].range_values
if not isinstance(step, model.WorkflowStepView):
## For visualization steps, do not create operations, as those are not really needed.
metadata, user_group = self._prepare_metadata(metadata, algo_category, operation_group, op_params)
operation = model.Operation(user_id, project_id, step.fk_algorithm,
json.dumps(op_params, cls=MapAsJson.MapAsJsonEncoder),
meta=json.dumps(metadata),
op_group_id=group_id, range_values=range_values, user_group=user_group)
operation.visible = step.step_visible
operation = dao.store_entity(operation)
cloned_w_step.fk_operation = operation.id
dao.store_entity(cloned_w_step)
if operation_group is not None and operation is not None:
datatype_group = model.DataTypeGroup(operation_group, operation_id=operation.id,
fk_parent_burst=burst_id,
state=metadata[DataTypeMetaData.KEY_STATE])
dao.store_entity(datatype_group)
def prepare_operations_for_workflowsteps(self, workflow_step_list, workflows, user_id, burst_id,
project_id, group, sim_operations):
"""
Create and store Operation entities from a list of Workflow Steps.
Will be generated workflows x workflow_step_list Operations.
For every step in workflow_step_list one OperationGroup and one DataTypeGroup will be created
(in case of PSE).
"""
for step in workflow_step_list:
operation_group = None
if (group is not None) and not isinstance(step, model.WorkflowStepView):
operation_group = model.OperationGroup(project_id=project_id, ranges=group.range_references)
operation_group = dao.store_entity(operation_group)
operation = None
metadata = {DataTypeMetaData.KEY_BURST: burst_id}
algo_category = dao.get_algorithm_by_id(step.fk_algorithm)
if algo_category is not None:
algo_category = algo_category.algo_group.group_category
for wf_idx, workflow in enumerate(workflows):
cloned_w_step = step.clone()
cloned_w_step.fk_workflow = workflow.id
dynamic_params = cloned_w_step.dynamic_param
op_params = cloned_w_step.static_param
op_params.update(dynamic_params)
range_values = None
group_id = None
if operation_group is not None:
group_id = operation_group.id
range_values = sim_operations[wf_idx].range_values
if not isinstance(step, model.WorkflowStepView):
## For visualization steps, do not create operations, as those are not really needed.
metadata, user_group = self._prepare_metadata(metadata, algo_category, operation_group, op_params)
operation = model.Operation(user_id, project_id, step.fk_algorithm,
json.dumps(op_params, cls=MapAsJson.MapAsJsonEncoder),
meta=json.dumps(metadata), method_name=ABCAdapter.LAUNCH_METHOD,
op_group_id=group_id, range_values=range_values, user_group=user_group)
operation.visible = step.step_visible
operation = dao.store_entity(operation)
cloned_w_step.fk_operation = operation.id
dao.store_entity(cloned_w_step)
if operation_group is not None and operation is not None:
datatype_group = model.DataTypeGroup(operation_group, operation_id=operation.id,
fk_parent_burst=burst_id,
state=metadata[DataTypeMetaData.KEY_STATE])
dao.store_entity(datatype_group)
def _create_operation(self, project_id, algorithm_id):
"""
dummy operation
:param project_id: the project in which the operation is created
:param algorithm_id: the algorithm to be run for the operation
:return: a dummy `Operation` with the given specifications
"""
algorithm = dao.get_algorithm_by_id(algorithm_id)
operation = Operation(self.test_user.id,
project_id,
algorithm.id,
'test params',
status=STATUS_FINISHED)
return dao.store_entity(operation)
def _create_operation(self, project_id, algorithm_id):
algorithm = dao.get_algorithm_by_id(algorithm_id)
meta = {
DataTypeMetaData.KEY_SUBJECT: "John Doe",
DataTypeMetaData.KEY_STATE: "RAW"
}
operation = model.Operation(self.test_user.id,
project_id,
algorithm.id,
'test params',
meta=json.dumps(meta),
status="FINISHED",
method_name=ABCAdapter.LAUNCH_METHOD)
return dao.store_entity(operation)
def launch_visualization(visualization,
frame_width=None,
frame_height=None,
is_preview=True):
"""
:param visualization: a visualization workflow step
"""
dynamic_params = visualization.dynamic_param
static_params = visualization.static_param
parameters_dict = static_params
current_project_id = 0
# Current operation id needed for export mechanism. So far just use ##
# the operation of the workflow_step from which the inputs are taken ####
for param in dynamic_params:
step_index = dynamic_params[param][
WorkflowStepConfiguration.STEP_INDEX_KEY]
datatype_index = dynamic_params[param][
WorkflowStepConfiguration.DATATYPE_INDEX_KEY]
referred_workflow_step = dao.get_workflow_step_by_step_index(
visualization.fk_workflow, step_index)
referred_operation_id = referred_workflow_step.fk_operation
referred_operation = dao.get_operation_by_id(referred_operation_id)
current_project_id = referred_operation.fk_launched_in
if type(datatype_index) is IntType:
# Entry is the output of a previous step ##
datatypes = dao.get_results_for_operation(
referred_operation_id)
parameters_dict[param] = datatypes[datatype_index].gid
else:
# Entry is the input of a previous step ###
parameters_dict[param] = json.loads(
referred_operation.parameters)[datatype_index]
algorithm = dao.get_algorithm_by_id(visualization.fk_algorithm)
adapter_instance = ABCAdapter.build_adapter(algorithm.algo_group)
adapter_instance.current_project_id = current_project_id
prepared_inputs = adapter_instance.prepare_ui_inputs(parameters_dict)
if frame_width is not None:
prepared_inputs[ABCDisplayer.PARAM_FIGURE_SIZE] = (frame_width,
frame_height)
if isinstance(adapter_instance,
ABCMPLH5Displayer) and is_preview is True:
prepared_inputs[ABCMPLH5Displayer.SHOW_FULL_TOOLBAR] = False
if is_preview:
result = eval(
"adapter_instance.generate_preview(**prepared_inputs)")
else:
result = eval("adapter_instance.launch(**prepared_inputs)")
return result, parameters_dict
def _get_new_form_view_model(operation, xml_parameters):
# type (Operation) -> ViewModel
algo = dao.get_algorithm_by_id(operation.fk_from_algo)
ad = ABCAdapter.build_adapter(algo)
view_model = ad.get_view_model_class()()
if xml_parameters:
declarative_attrs = type(view_model).declarative_attrs
if isinstance(xml_parameters, str):
xml_parameters = json.loads(xml_parameters)
for param in xml_parameters:
new_param_name = param
if param != '' and param[0] == "_":
new_param_name = param[1:]
new_param_name = new_param_name.lower()
if new_param_name in declarative_attrs:
try:
setattr(view_model, new_param_name,
xml_parameters[param])
except (TraitTypeError, TraitAttributeError):
pass
return view_model
def retrieve_project_full(self, project_id, applied_filters=None, current_page=1):
"""
Return a Tuple with Project entity and Operations for current Project.
:param project_id: Current Project Identifier
:param applied_filters: Filters to apply on Operations
:param current_page: Number for current page in operations
"""
selected_project = self.find_project(project_id)
total_filtered = self.count_filtered_operations(project_id, applied_filters)
pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (1 if total_filtered % OPERATIONS_PAGE_SIZE else 0)
total_ops_nr = self.count_filtered_operations(project_id)
start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1)
current_ops = dao.get_filtered_operations(project_id, applied_filters, start_idx, OPERATIONS_PAGE_SIZE)
if current_ops is None:
return selected_project, 0, [], 0
operations = []
view_categ_id = dao.get_visualisers_categories()[0].id
for one_op in current_ops:
try:
result = {}
if one_op[0] != one_op[1]:
result["id"] = str(one_op[0]) + "-" + str(one_op[1])
else:
result["id"] = str(one_op[0])
burst = dao.get_burst_for_operation_id(one_op[0])
result["burst_name"] = burst.name if burst else '-'
result["count"] = one_op[2]
result["gid"] = one_op[13]
if one_op[3] is not None and one_op[3]:
try:
operation_group = dao.get_generic_entity(OperationGroup, one_op[3])[0]
result["group"] = operation_group.name
result["group"] = result["group"].replace("_", " ")
result["operation_group_id"] = operation_group.id
datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3])
result["datatype_group_gid"] = datatype_group.gid
result["gid"] = operation_group.gid
## Filter only viewers for current DataTypeGroup entity:
result["view_groups"] = FlowService().get_visualizers_for_group(datatype_group.gid)
except Exception:
self.logger.exception("We will ignore group on entity:" + str(one_op))
result["datatype_group_gid"] = None
else:
result['group'] = None
result['datatype_group_gid'] = None
result["algorithm"] = dao.get_algorithm_by_id(one_op[4])
result["user"] = dao.get_user_by_id(one_op[5])
if type(one_op[6]) is str:
result["create"] = string2date(str(one_op[6]))
else:
result["create"] = one_op[6]
if type(one_op[7]) is str:
result["start"] = string2date(str(one_op[7]))
else:
result["start"] = one_op[7]
if type(one_op[8]) is str:
result["complete"] = string2date(str(one_op[8]))
else:
result["complete"] = one_op[8]
if result["complete"] is not None and result["start"] is not None:
result["duration"] = format_timedelta(result["complete"] - result["start"])
result["status"] = one_op[9]
result["additional"] = one_op[10]
result["visible"] = True if one_op[11] > 0 else False
result['operation_tag'] = one_op[12]
result['figures'] = None
if not result['group']:
datatype_results = dao.get_results_for_operation(result['id'])
result['results'] = []
for dt in datatype_results:
dt_loaded = ABCAdapter.load_entity_by_gid(dt.gid)
if dt_loaded:
result['results'].append(dt_loaded)
else:
self.logger.warning("Could not retrieve datatype %s" % str(dt))
operation_figures = dao.get_figures_for_operation(result['id'])
# Compute the full path to the figure / image on disk
for figure in operation_figures:
figures_folder = self.structure_helper.get_images_folder(figure.project.name)
figure_full_path = os.path.join(figures_folder, figure.file_path)
# Compute the path available from browser
figure.figure_path = utils.path2url_part(figure_full_path)
result['figures'] = operation_figures
else:
result['results'] = None
operations.append(result)
except Exception:
## We got an exception when processing one Operation Row. We will continue with the rest of the rows.
self.logger.exception("Could not prepare operation for display:" + str(one_op))
return selected_project, total_ops_nr, operations, pages_no
def retrieve_project_full(self, project_id, applied_filters=None, current_page=1):
"""
Return a Tuple with Project entity and Operations for current Project.
:param project_id: Current Project Identifier
:param applied_filters: Filters to apply on Operations
:param current_page: Number for current page in operations
"""
selected_project = self.find_project(project_id)
total_filtered = self.count_filtered_operations(project_id, applied_filters)
start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1)
if total_filtered >= start_idx + OPERATIONS_PAGE_SIZE:
end_idx = OPERATIONS_PAGE_SIZE
else:
end_idx = total_filtered - start_idx
pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (1 if total_filtered % OPERATIONS_PAGE_SIZE else 0)
total_ops_nr = self.count_filtered_operations(project_id)
current_ops = dao.get_filtered_operations(project_id, applied_filters, start_idx, end_idx)
started_ops = 0
if current_ops is None:
return selected_project, [], 0
operations = []
for one_op in current_ops:
try:
result = {}
if one_op[0] != one_op[1]:
result["id"] = str(one_op[0]) + "-" + str(one_op[1])
else:
result["id"] = str(one_op[0])
burst = dao.get_burst_for_operation_id(one_op[0])
result["burst_name"] = burst.name if burst else '-'
result["count"] = one_op[2]
result["gid"] = one_op[14]
if one_op[3] is not None and one_op[3]:
try:
operation_group = dao.get_generic_entity(model.OperationGroup, one_op[3])[0]
result["group"] = operation_group.name
result["group"] = result["group"].replace("_", " ")
result["operation_group_id"] = operation_group.id
datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3])
datatype = dao.get_datatype_by_id(datatype_group.id)
result["datatype_group_gid"] = datatype.gid
result["gid"] = operation_group.gid
all_categs = dao.get_algorithm_categories()
view_categ = dao.get_visualisers_categories()[0]
excludes = [categ.id for categ in all_categs if categ.id != view_categ.id]
algo = self.retrieve_launchers("DataTypeGroup", datatype.gid,
exclude_categories=excludes).values()[0]
view_groups = []
for algo in algo.values():
url = '/flow/' + str(algo['category']) + '/' + str(algo['id'])
if algo['part_of_group']:
url = '/flow/prepare_group_launch/' + datatype.gid + '/' + \
str(algo['category']) + '/' + str(algo['id'])
view_groups.append(dict(name=algo["displayName"],
url=url,
param_name=algo['children'][0]['param_name'],
part_of_group=algo['part_of_group']))
result["view_groups"] = view_groups
except Exception, excep:
self.logger.error(excep)
self.logger.warning("Will ignore group on entity:" + str(one_op))
result["datatype_group_gid"] = None
else:
result['group'] = None
result['datatype_group_gid'] = None
result["algorithm"] = dao.get_algorithm_by_id(one_op[4])
result["method"] = one_op[5]
result["user"] = dao.get_user_by_id(one_op[6])
if type(one_op[7]) in (str, unicode):
result["create"] = string2date(str(one_op[7]))
else:
result["create"] = one_op[7]
if type(one_op[8]) in (str, unicode):
result["start"] = string2date(str(one_op[8]))
else:
result["start"] = one_op[8]
if type(one_op[9]) in (str, unicode):
result["complete"] = string2date(str(one_op[9]))
else:
result["complete"] = one_op[9]
if result["complete"] is not None and result["start"] is not None:
result["duration"] = timedelta2string(result["complete"] - result["start"])
result["status"] = one_op[10]
if result["status"] == model.STATUS_STARTED:
started_ops += 1
result["additional"] = one_op[11]
result["visible"] = True if one_op[12] > 0 else False
result['operation_tag'] = one_op[13]
result['figures'] = None
if not result['group']:
datatype_results = dao.get_results_for_operation(result['id'])
result['results'] = [dao.get_generic_entity(dt.module + '.' + dt.type,
dt.gid, 'gid')[0] for dt in datatype_results]
operation_figures = dao.get_figures_for_operation(result['id'])
# Compute the full path to the figure / image on disk
for figure in operation_figures:
figures_folder = self.structure_helper.get_images_folder(figure.project.name,
figure.operation.id)
figure_full_path = os.path.join(figures_folder, figure.file_path)
# Compute the path available from browser
figure.figure_path = utils.path2url_part(figure_full_path)
result['figures'] = operation_figures
else:
result['results'] = None
operations.append(result)
def get_algorithm_by_identifier(ident):
"""
Retrieve Algorithm entity by ID.
Return None, if ID is not found in DB.
"""
return dao.get_algorithm_by_id(ident)
def retrieve_project_full(self,
project_id,
applied_filters=None,
current_page=1):
"""
Return a Tuple with Project entity and Operations for current Project.
:param project_id: Current Project Identifier
:param applied_filters: Filters to apply on Operations
:param current_page: Number for current page in operations
"""
selected_project = self.find_project(project_id)
total_filtered = self.count_filtered_operations(
project_id, applied_filters)
pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (
1 if total_filtered % OPERATIONS_PAGE_SIZE else 0)
total_ops_nr = self.count_filtered_operations(project_id)
start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1)
current_ops = dao.get_filtered_operations(project_id, applied_filters,
start_idx,
OPERATIONS_PAGE_SIZE)
if current_ops is None:
return selected_project, 0, [], 0
operations = []
for one_op in current_ops:
try:
result = {}
if one_op[0] != one_op[1]:
result["id"] = str(one_op[0]) + "-" + str(one_op[1])
else:
result["id"] = str(one_op[0])
burst = dao.get_burst_for_operation_id(one_op[0])
result["burst_name"] = burst.name if burst else '-'
result["count"] = one_op[2]
result["gid"] = one_op[13]
operation_group_id = one_op[3]
if operation_group_id is not None and operation_group_id:
try:
operation_group = dao.get_generic_entity(
OperationGroup, operation_group_id)[0]
result["group"] = operation_group.name
result["group"] = result["group"].replace("_", " ")
result["operation_group_id"] = operation_group.id
datatype_group = dao.get_datatypegroup_by_op_group_id(
operation_group_id)
result[
"datatype_group_gid"] = datatype_group.gid if datatype_group is not None else None
result["gid"] = operation_group.gid
# Filter only viewers for current DataTypeGroup entity:
if datatype_group is None:
view_groups = None
else:
view_groups = AlgorithmService(
).get_visualizers_for_group(datatype_group.gid)
result["view_groups"] = view_groups
except Exception:
self.logger.exception(
"We will ignore group on entity:" + str(one_op))
result["datatype_group_gid"] = None
else:
result['group'] = None
result['datatype_group_gid'] = None
result["algorithm"] = dao.get_algorithm_by_id(one_op[4])
result["user"] = dao.get_user_by_id(one_op[5])
if type(one_op[6]) is str:
result["create"] = string2date(str(one_op[6]))
else:
result["create"] = one_op[6]
if type(one_op[7]) is str:
result["start"] = string2date(str(one_op[7]))
else:
result["start"] = one_op[7]
if type(one_op[8]) is str:
result["complete"] = string2date(str(one_op[8]))
else:
result["complete"] = one_op[8]
if result["complete"] is not None and result[
"start"] is not None:
result["duration"] = format_timedelta(result["complete"] -
result["start"])
result["status"] = one_op[9]
result["additional"] = one_op[10]
result["visible"] = True if one_op[11] > 0 else False
result['operation_tag'] = one_op[12]
if not result['group']:
result['results'] = dao.get_results_for_operation(
result['id'])
else:
result['results'] = None
operations.append(result)
except Exception:
# We got an exception when processing one Operation Row. We will continue with the rest of the rows.
self.logger.exception(
"Could not prepare operation for display:" + str(one_op))
return selected_project, total_ops_nr, operations, pages_no
def retrieve_project_full(self, project_id, applied_filters=None, current_page=1):
"""
Return a Tuple with Project entity and Operations for current Project.
:param project_id: Current Project Identifier
:param applied_filters: Filters to apply on Operations
:param current_page: Number for current page in operations
"""
selected_project = self.find_project(project_id)
total_filtered = self.count_filtered_operations(project_id, applied_filters)
pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (1 if total_filtered % OPERATIONS_PAGE_SIZE else 0)
total_ops_nr = self.count_filtered_operations(project_id)
start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1)
current_ops = dao.get_filtered_operations(project_id, applied_filters, start_idx, OPERATIONS_PAGE_SIZE)
if current_ops is None:
return selected_project, 0, [], 0
operations = []
view_categ_id = dao.get_visualisers_categories()[0].id
for one_op in current_ops:
try:
result = {}
if one_op[0] != one_op[1]:
result["id"] = str(one_op[0]) + "-" + str(one_op[1])
else:
result["id"] = str(one_op[0])
burst = dao.get_burst_for_operation_id(one_op[0])
result["burst_name"] = burst.name if burst else '-'
result["count"] = one_op[2]
result["gid"] = one_op[14]
if one_op[3] is not None and one_op[3]:
try:
operation_group = dao.get_generic_entity(model.OperationGroup, one_op[3])[0]
result["group"] = operation_group.name
result["group"] = result["group"].replace("_", " ")
result["operation_group_id"] = operation_group.id
datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3])
result["datatype_group_gid"] = datatype_group.gid
result["gid"] = operation_group.gid
## Filter only viewers for current DataTypeGroup entity:
launcher = self.retrieve_launchers(datatype_group.gid,
include_categories=[view_categ_id]).values()[0]
view_groups = []
for launcher in launcher.values():
url = '/flow/' + str(launcher['category']) + '/' + str(launcher['id'])
if launcher['part_of_group']:
url = '/flow/prepare_group_launch/' + datatype_group.gid + '/' + \
str(launcher['category']) + '/' + str(launcher['id'])
view_groups.append(dict(name=launcher["displayName"],
url=url,
param_name=launcher['children'][0]['param_name'],
part_of_group=launcher['part_of_group']))
result["view_groups"] = view_groups
except Exception:
self.logger.exception("We will ignore group on entity:" + str(one_op))
result["datatype_group_gid"] = None
else:
result['group'] = None
result['datatype_group_gid'] = None
result["algorithm"] = dao.get_algorithm_by_id(one_op[4])
result["method"] = one_op[5]
result["user"] = dao.get_user_by_id(one_op[6])
if type(one_op[7]) in (str, unicode):
result["create"] = string2date(str(one_op[7]))
else:
result["create"] = one_op[7]
if type(one_op[8]) in (str, unicode):
result["start"] = string2date(str(one_op[8]))
else:
result["start"] = one_op[8]
if type(one_op[9]) in (str, unicode):
result["complete"] = string2date(str(one_op[9]))
else:
result["complete"] = one_op[9]
if result["complete"] is not None and result["start"] is not None:
result["duration"] = format_timedelta(result["complete"] - result["start"])
result["status"] = one_op[10]
result["additional"] = one_op[11]
result["visible"] = True if one_op[12] > 0 else False
result['operation_tag'] = one_op[13]
result['figures'] = None
if not result['group']:
datatype_results = dao.get_results_for_operation(result['id'])
result['results'] = [dao.get_generic_entity(dt.module + '.' + dt.type,
dt.gid, 'gid')[0] for dt in datatype_results]
operation_figures = dao.get_figures_for_operation(result['id'])
# Compute the full path to the figure / image on disk
for figure in operation_figures:
figures_folder = self.structure_helper.get_images_folder(figure.project.name)
figure_full_path = os.path.join(figures_folder, figure.file_path)
# Compute the path available from browser
figure.figure_path = utils.path2url_part(figure_full_path)
result['figures'] = operation_figures
else:
result['results'] = None
operations.append(result)
except Exception:
## We got an exception when processing one Operation Row. We will continue with the rest of the rows.
self.logger.exception("Could not prepare operation for display:" + str(one_op))
return selected_project, total_ops_nr, operations, pages_no
def get_algorithm_by_identifier(ident):
"""
Retrieve Algorithm entity by ID.
Return None, if ID is not found in DB.
"""
return dao.get_algorithm_by_id(ident)
