def retrieve_project_full(self, project_id, applied_filters=None, current_page=1):
"""
Return a Tuple with Project entity and Operations for current Project.
:param project_id: Current Project Identifier
:param applied_filters: Filters to apply on Operations
:param current_page: Number for current page in operations
"""
selected_project = self.find_project(project_id)
total_filtered = self.count_filtered_operations(project_id, applied_filters)
pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (1 if total_filtered % OPERATIONS_PAGE_SIZE else 0)
total_ops_nr = self.count_filtered_operations(project_id)
start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1)
current_ops = dao.get_filtered_operations(project_id, applied_filters, start_idx, OPERATIONS_PAGE_SIZE)
if current_ops is None:
return selected_project, 0, [], 0
operations = []
view_categ_id = dao.get_visualisers_categories()[0].id
for one_op in current_ops:
try:
result = {}
if one_op[0] != one_op[1]:
result["id"] = str(one_op[0]) + "-" + str(one_op[1])
else:
result["id"] = str(one_op[0])
burst = dao.get_burst_for_operation_id(one_op[0])
result["burst_name"] = burst.name if burst else '-'
result["count"] = one_op[2]
result["gid"] = one_op[13]
if one_op[3] is not None and one_op[3]:
try:
operation_group = dao.get_generic_entity(OperationGroup, one_op[3])[0]
result["group"] = operation_group.name
result["group"] = result["group"].replace("_", " ")
result["operation_group_id"] = operation_group.id
datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3])
result["datatype_group_gid"] = datatype_group.gid
result["gid"] = operation_group.gid
## Filter only viewers for current DataTypeGroup entity:
result["view_groups"] = FlowService().get_visualizers_for_group(datatype_group.gid)
except Exception:
self.logger.exception("We will ignore group on entity:" + str(one_op))
result["datatype_group_gid"] = None
else:
result['group'] = None
result['datatype_group_gid'] = None
result["algorithm"] = dao.get_algorithm_by_id(one_op[4])
result["user"] = dao.get_user_by_id(one_op[5])
if type(one_op[6]) is str:
result["create"] = string2date(str(one_op[6]))
else:
result["create"] = one_op[6]
if type(one_op[7]) is str:
result["start"] = string2date(str(one_op[7]))
else:
result["start"] = one_op[7]
if type(one_op[8]) is str:
result["complete"] = string2date(str(one_op[8]))
else:
result["complete"] = one_op[8]
if result["complete"] is not None and result["start"] is not None:
result["duration"] = format_timedelta(result["complete"] - result["start"])
result["status"] = one_op[9]
result["additional"] = one_op[10]
result["visible"] = True if one_op[11] > 0 else False
result['operation_tag'] = one_op[12]
result['figures'] = None
if not result['group']:
datatype_results = dao.get_results_for_operation(result['id'])
result['results'] = []
for dt in datatype_results:
dt_loaded = ABCAdapter.load_entity_by_gid(dt.gid)
if dt_loaded:
result['results'].append(dt_loaded)
else:
self.logger.warning("Could not retrieve datatype %s" % str(dt))
operation_figures = dao.get_figures_for_operation(result['id'])
# Compute the full path to the figure / image on disk
for figure in operation_figures:
figures_folder = self.structure_helper.get_images_folder(figure.project.name)
figure_full_path = os.path.join(figures_folder, figure.file_path)
# Compute the path available from browser
figure.figure_path = utils.path2url_part(figure_full_path)
result['figures'] = operation_figures
else:
result['results'] = None
operations.append(result)
except Exception:
## We got an exception when processing one Operation Row. We will continue with the rest of the rows.
self.logger.exception("Could not prepare operation for display:" + str(one_op))
return selected_project, total_ops_nr, operations, pages_no
def retrieve_launchers(self, datatype_gid, inspect_group=False, include_categories=None):
"""
Returns all the available launch-able algorithms from the database.
Filter the ones accepting as required input a specific DataType.
:param datatype_gid: GID, to filter algorithms for this particular entity.
:param inspect_group: TRUE if we are now in the inspection of sub-entities in a DataTypeGroup
:param include_categories: List of categories to be included in the result.
When None, all lanchable categories are included
"""
try:
all_launch_categ = dao.get_launchable_categories()
launch_categ = dict((categ.id, categ.displayname) for categ in all_launch_categ
if include_categories is None or categ.id in include_categories)
datatype_instance = dao.get_datatype_by_gid(datatype_gid)
data_class = datatype_instance.__class__
all_compatible_classes = [data_class.__name__]
for one_class in getmro(data_class):
if issubclass(one_class, MappedType) and one_class.__name__ not in all_compatible_classes:
all_compatible_classes.append(one_class.__name__)
self.logger.debug("Searching in categories: " + str(len(launch_categ)) + " - " +
str(launch_categ.keys()) + "-" + str(include_categories))
launchable_groups = dao.get_apliable_algo_groups(all_compatible_classes, launch_categ.keys())
to_remove = []
for one_group in launchable_groups:
compatible_algorithms = []
for one_algo in one_group.children:
filter_chain = FilterChain.from_json(one_algo.datatype_filter)
if not filter_chain or filter_chain.get_python_filter_equivalent(datatype_instance):
compatible_algorithms.append(one_algo)
if len(compatible_algorithms) > 0:
one_group.children = copy.deepcopy(compatible_algorithms)
else:
to_remove.append(one_group)
for one_group in to_remove:
launchable_groups.remove(one_group)
del one_group
launchers = ProjectService.__prepare_group_result(launchable_groups, launch_categ, inspect_group)
if data_class.__name__ == model.DataTypeGroup.__name__:
# If part of a group, update also with specific launchers of the child datatype
dt_group = dao.get_datatype_group_by_gid(datatype_gid)
datatypes = dao.get_datatypes_from_datatype_group(dt_group.id)
if len(datatypes):
datatype = datatypes[-1]
datatype = dao.get_datatype_by_gid(datatype.gid)
views_categ_id = dao.get_visualisers_categories()[0].id
categories_for_small_type = [categ.id for categ in all_launch_categ
if categ.id != views_categ_id and (include_categories is None or
categ.id in include_categories)]
if categories_for_small_type:
specific_launchers = self.retrieve_launchers(datatype.gid, True, categories_for_small_type)
for key in specific_launchers:
if key in launchers:
launchers[key].update(specific_launchers[key])
else:
launchers[key] = specific_launchers[key]
return launchers
except Exception, excep:
ProjectService().logger.exception(excep)
ProjectService().logger.warning("Attempting to filter launcher for group despite exception!")
return ProjectService.__prepare_group_result([], [], inspect_group)
def get_visualisers_category():
"""Retrieve all Algorithm categories, with display capability"""
result = dao.get_visualisers_categories()
if result is None or len(result) < 1:
raise Exception("View Category not found!!!")
return result[0]
def get_visualisers_category():
"""Retrieve all Algorithm categories, with display capability"""
result = dao.get_visualisers_categories()
if not result:
raise ValueError("View Category not found!!!")
return result[0]
def retrieve_project_full(self, project_id, applied_filters=None, current_page=1):
"""
Return a Tuple with Project entity and Operations for current Project.
:param project_id: Current Project Identifier
:param applied_filters: Filters to apply on Operations
:param current_page: Number for current page in operations
"""
selected_project = self.find_project(project_id)
total_filtered = self.count_filtered_operations(project_id, applied_filters)
pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (1 if total_filtered % OPERATIONS_PAGE_SIZE else 0)
total_ops_nr = self.count_filtered_operations(project_id)
start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1)
current_ops = dao.get_filtered_operations(project_id, applied_filters, start_idx, OPERATIONS_PAGE_SIZE)
if current_ops is None:
return selected_project, 0, [], 0
operations = []
view_categ_id = dao.get_visualisers_categories()[0].id
for one_op in current_ops:
try:
result = {}
if one_op[0] != one_op[1]:
result["id"] = str(one_op[0]) + "-" + str(one_op[1])
else:
result["id"] = str(one_op[0])
burst = dao.get_burst_for_operation_id(one_op[0])
result["burst_name"] = burst.name if burst else '-'
result["count"] = one_op[2]
result["gid"] = one_op[14]
if one_op[3] is not None and one_op[3]:
try:
operation_group = dao.get_generic_entity(model.OperationGroup, one_op[3])[0]
result["group"] = operation_group.name
result["group"] = result["group"].replace("_", " ")
result["operation_group_id"] = operation_group.id
datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3])
result["datatype_group_gid"] = datatype_group.gid
result["gid"] = operation_group.gid
## Filter only viewers for current DataTypeGroup entity:
launcher = self.retrieve_launchers(datatype_group.gid,
include_categories=[view_categ_id]).values()[0]
view_groups = []
for launcher in launcher.values():
url = '/flow/' + str(launcher['category']) + '/' + str(launcher['id'])
if launcher['part_of_group']:
url = '/flow/prepare_group_launch/' + datatype_group.gid + '/' + \
str(launcher['category']) + '/' + str(launcher['id'])
view_groups.append(dict(name=launcher["displayName"],
url=url,
param_name=launcher['children'][0]['param_name'],
part_of_group=launcher['part_of_group']))
result["view_groups"] = view_groups
except Exception:
self.logger.exception("We will ignore group on entity:" + str(one_op))
result["datatype_group_gid"] = None
else:
result['group'] = None
result['datatype_group_gid'] = None
result["algorithm"] = dao.get_algorithm_by_id(one_op[4])
result["method"] = one_op[5]
result["user"] = dao.get_user_by_id(one_op[6])
if type(one_op[7]) in (str, unicode):
result["create"] = string2date(str(one_op[7]))
else:
result["create"] = one_op[7]
if type(one_op[8]) in (str, unicode):
result["start"] = string2date(str(one_op[8]))
else:
result["start"] = one_op[8]
if type(one_op[9]) in (str, unicode):
result["complete"] = string2date(str(one_op[9]))
else:
result["complete"] = one_op[9]
if result["complete"] is not None and result["start"] is not None:
result["duration"] = format_timedelta(result["complete"] - result["start"])
result["status"] = one_op[10]
result["additional"] = one_op[11]
result["visible"] = True if one_op[12] > 0 else False
result['operation_tag'] = one_op[13]
result['figures'] = None
if not result['group']:
datatype_results = dao.get_results_for_operation(result['id'])
result['results'] = [dao.get_generic_entity(dt.module + '.' + dt.type,
dt.gid, 'gid')[0] for dt in datatype_results]
operation_figures = dao.get_figures_for_operation(result['id'])
# Compute the full path to the figure / image on disk
for figure in operation_figures:
figures_folder = self.structure_helper.get_images_folder(figure.project.name)
figure_full_path = os.path.join(figures_folder, figure.file_path)
# Compute the path available from browser
figure.figure_path = utils.path2url_part(figure_full_path)
result['figures'] = operation_figures
else:
result['results'] = None
operations.append(result)
except Exception:
## We got an exception when processing one Operation Row. We will continue with the rest of the rows.
self.logger.exception("Could not prepare operation for display:" + str(one_op))
return selected_project, total_ops_nr, operations, pages_no
def get_visualisers_category():
"""Retrieve all Algorithm categories, with display capability"""
result = dao.get_visualisers_categories()
if result is None or len(result) < 1:
raise Exception("View Category not found!!!")
return result[0]
def get_visualizers_for_group(self, dt_group_gid):
categories = dao.get_visualisers_categories()
return self._get_launchable_algorithms(dt_group_gid, categories)[1]
def get_visualisers_category():
"""Retrieve all Algorithm categories, with display capability"""
result = dao.get_visualisers_categories()
if not result:
raise ValueError("View Category not found!!!")
return result[0]
def get_visualizers_for_group(self, dt_group_gid):
categories = dao.get_visualisers_categories()
return self._get_launchable_algorithms(dt_group_gid, categories)[1]
def retrieve_launchers(self, dataname, datatype_gid=None, inspect_group=False, exclude_categories=None):
"""
Returns all the available launch-able algorithms from the database.
Filter the ones accepting as required input a specific DataType.
:param dataname: String or class representing DataType to retrieve filters for it.
:param datatype_gid: Optional GID, to filter algorithms for this particular entity.
:param inspect_group: TRUE if we are now in the inspection of sub-entities in a DataTypeGroup
:param exclude_categories: List of categories to be excluded from the result.
"""
if exclude_categories is None:
exclude_categories = []
launch_categ = dao.get_launchable_categories()
launch_categ = dict((categ.id, categ.displayname) for categ in launch_categ
if categ.id not in exclude_categories)
launch_groups = dao.get_apliable_algo_groups(dataname, launch_categ.keys())
if datatype_gid is None:
return ProjectService.__prepare_group_result(launch_groups, launch_categ, inspect_group)
try:
datatype_instance = dao.get_datatype_by_gid(datatype_gid)
data_class = datatype_instance.__class__
for one_class in data_class.__bases__:
launch_groups.extend(dao.get_apliable_algo_groups(one_class.__name__, launch_categ.keys()))
specific_datatype = dao.get_generic_entity(data_class, datatype_gid, "gid")
to_remove = []
for one_group in launch_groups:
valid_algorithms = []
for one_algo in one_group.children:
filter_chain = FilterChain.from_json(one_algo.datatype_filter)
if not filter_chain or filter_chain.get_python_filter_equivalent(specific_datatype[0]):
valid_algorithms.append(one_algo)
if len(valid_algorithms) > 0:
one_group.children = copy.deepcopy(valid_algorithms)
else:
to_remove.append(one_group)
for one_group in to_remove:
launch_groups.remove(one_group)
del one_group
launchers = ProjectService.__prepare_group_result(launch_groups, launch_categ, inspect_group)
if dataname == model.DataTypeGroup.__name__:
# If part of a group, update also with specific launchers of that datatype
dt_group = dao.get_datatype_group_by_gid(datatype_gid)
datatypes = dao.get_datatypes_from_datatype_group(dt_group.id)
if len(datatypes):
datatype = datatypes[-1]
datatype = dao.get_datatype_by_gid(datatype.gid)
views_categ_id = dao.get_visualisers_categories()[0].id
specific_launchers = self.retrieve_launchers(datatype.__class__.__name__, datatype.gid,
True, [views_categ_id] + exclude_categories)
for key in specific_launchers:
if key in launchers:
launchers[key].update(specific_launchers[key])
else:
launchers[key] = specific_launchers[key]
return launchers
except Exception, excep:
ProjectService().logger.exception(excep)
ProjectService().logger.warning("Attempting to filter launcher for group despite exception!")
return ProjectService.__prepare_group_result(launch_groups, launch_categ, inspect_group)
def retrieve_project_full(self, project_id, applied_filters=None, current_page=1):
"""
Return a Tuple with Project entity and Operations for current Project.
:param project_id: Current Project Identifier
:param applied_filters: Filters to apply on Operations
:param current_page: Number for current page in operations
"""
selected_project = self.find_project(project_id)
total_filtered = self.count_filtered_operations(project_id, applied_filters)
start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1)
if total_filtered >= start_idx + OPERATIONS_PAGE_SIZE:
end_idx = OPERATIONS_PAGE_SIZE
else:
end_idx = total_filtered - start_idx
pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (1 if total_filtered % OPERATIONS_PAGE_SIZE else 0)
total_ops_nr = self.count_filtered_operations(project_id)
current_ops = dao.get_filtered_operations(project_id, applied_filters, start_idx, end_idx)
started_ops = 0
if current_ops is None:
return selected_project, [], 0
operations = []
for one_op in current_ops:
try:
result = {}
if one_op[0] != one_op[1]:
result["id"] = str(one_op[0]) + "-" + str(one_op[1])
else:
result["id"] = str(one_op[0])
burst = dao.get_burst_for_operation_id(one_op[0])
result["burst_name"] = burst.name if burst else '-'
result["count"] = one_op[2]
result["gid"] = one_op[14]
if one_op[3] is not None and one_op[3]:
try:
operation_group = dao.get_generic_entity(model.OperationGroup, one_op[3])[0]
result["group"] = operation_group.name
result["group"] = result["group"].replace("_", " ")
result["operation_group_id"] = operation_group.id
datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3])
datatype = dao.get_datatype_by_id(datatype_group.id)
result["datatype_group_gid"] = datatype.gid
result["gid"] = operation_group.gid
all_categs = dao.get_algorithm_categories()
view_categ = dao.get_visualisers_categories()[0]
excludes = [categ.id for categ in all_categs if categ.id != view_categ.id]
algo = self.retrieve_launchers("DataTypeGroup", datatype.gid,
exclude_categories=excludes).values()[0]
view_groups = []
for algo in algo.values():
url = '/flow/' + str(algo['category']) + '/' + str(algo['id'])
if algo['part_of_group']:
url = '/flow/prepare_group_launch/' + datatype.gid + '/' + \
str(algo['category']) + '/' + str(algo['id'])
view_groups.append(dict(name=algo["displayName"],
url=url,
param_name=algo['children'][0]['param_name'],
part_of_group=algo['part_of_group']))
result["view_groups"] = view_groups
except Exception, excep:
self.logger.error(excep)
self.logger.warning("Will ignore group on entity:" + str(one_op))
result["datatype_group_gid"] = None
else:
result['group'] = None
result['datatype_group_gid'] = None
result["algorithm"] = dao.get_algorithm_by_id(one_op[4])
result["method"] = one_op[5]
result["user"] = dao.get_user_by_id(one_op[6])
if type(one_op[7]) in (str, unicode):
result["create"] = string2date(str(one_op[7]))
else:
result["create"] = one_op[7]
if type(one_op[8]) in (str, unicode):
result["start"] = string2date(str(one_op[8]))
else:
result["start"] = one_op[8]
if type(one_op[9]) in (str, unicode):
result["complete"] = string2date(str(one_op[9]))
else:
result["complete"] = one_op[9]
if result["complete"] is not None and result["start"] is not None:
result["duration"] = timedelta2string(result["complete"] - result["start"])
result["status"] = one_op[10]
if result["status"] == model.STATUS_STARTED:
started_ops += 1
result["additional"] = one_op[11]
result["visible"] = True if one_op[12] > 0 else False
result['operation_tag'] = one_op[13]
result['figures'] = None
if not result['group']:
datatype_results = dao.get_results_for_operation(result['id'])
result['results'] = [dao.get_generic_entity(dt.module + '.' + dt.type,
dt.gid, 'gid')[0] for dt in datatype_results]
operation_figures = dao.get_figures_for_operation(result['id'])
# Compute the full path to the figure / image on disk
for figure in operation_figures:
figures_folder = self.structure_helper.get_images_folder(figure.project.name,
figure.operation.id)
figure_full_path = os.path.join(figures_folder, figure.file_path)
# Compute the path available from browser
figure.figure_path = utils.path2url_part(figure_full_path)
result['figures'] = operation_figures
else:
result['results'] = None
operations.append(result)
def retrieve_launchers(self, datatype_gid, inspect_group=False, include_categories=None):
"""
Returns all the available launch-able algorithms from the database.
Filter the ones accepting as required input a specific DataType.
:param datatype_gid: GID, to filter algorithms for this particular entity.
:param inspect_group: TRUE if we are now in the inspection of sub-entities in a DataTypeGroup
:param include_categories: List of categories to be included in the result.
When None, all lanchable categories are included
"""
try:
all_launch_categ = dao.get_launchable_categories()
launch_categ = dict((categ.id, categ.displayname) for categ in all_launch_categ
if include_categories is None or categ.id in include_categories)
datatype_instance = dao.get_datatype_by_gid(datatype_gid)
data_class = datatype_instance.__class__
all_compatible_classes = [data_class.__name__]
for one_class in getmro(data_class):
if issubclass(one_class, MappedType) and one_class.__name__ not in all_compatible_classes:
all_compatible_classes.append(one_class.__name__)
self.logger.debug("Searching in categories: " + str(len(launch_categ)) + " - " +
str(launch_categ.keys()) + "-" + str(include_categories))
launchable_groups = dao.get_apliable_algo_groups(all_compatible_classes, launch_categ.keys())
to_remove = []
for one_group in launchable_groups:
compatible_algorithms = []
for one_algo in one_group.children:
filter_chain = FilterChain.from_json(one_algo.datatype_filter)
if not filter_chain or filter_chain.get_python_filter_equivalent(datatype_instance):
compatible_algorithms.append(one_algo)
if len(compatible_algorithms) > 0:
one_group.children = copy.deepcopy(compatible_algorithms)
else:
to_remove.append(one_group)
for one_group in to_remove:
launchable_groups.remove(one_group)
del one_group
launchers = ProjectService.__prepare_group_result(launchable_groups, launch_categ, inspect_group)
if data_class.__name__ == model.DataTypeGroup.__name__:
# If part of a group, update also with specific launchers of the child datatype
dt_group = dao.get_datatype_group_by_gid(datatype_gid)
datatypes = dao.get_datatypes_from_datatype_group(dt_group.id)
if len(datatypes):
datatype = datatypes[-1]
datatype = dao.get_datatype_by_gid(datatype.gid)
views_categ_id = dao.get_visualisers_categories()[0].id
categories_for_small_type = [categ.id for categ in all_launch_categ
if categ.id != views_categ_id and (include_categories is None or
categ.id in include_categories)]
if categories_for_small_type:
specific_launchers = self.retrieve_launchers(datatype.gid, True, categories_for_small_type)
for key in specific_launchers:
if key in launchers:
launchers[key].update(specific_launchers[key])
else:
launchers[key] = specific_launchers[key]
return launchers
except Exception, excep:
ProjectService().logger.exception(excep)
ProjectService().logger.warning("Attempting to filter launcher for group despite exception!")
return ProjectService.__prepare_group_result([], [], inspect_group)
