def generate_table_columns(component=str()):
da_object = DAComponent(component=component)
# get and filter schema elements based on displayable columns
schema = [x for x in da_object.get_schema().get("schema_dict") if x.get("show_in_table", True)]
columns = list()
columns.append(dict(data="record_id", visible=False))
detail_dict = dict(className='summary-details-control detail-hover-message', orderable=False, data=None,
title='', defaultContent='', width="5%")
columns.insert(0, detail_dict)
# get indexed fields - only fields that are indexed can be ordered when using server-side processing
indexed_fields = list()
for k, v in da_object.get_collection_handle().index_information().items():
indexed_fields.append(v['key'][0][0])
for x in schema:
x["id"] = x["id"].split(".")[-1]
orderable = False
if x["id"] in indexed_fields:
orderable = True
columns.append(dict(data=x["id"], title=x["label"], orderable=orderable))
# add column for annotation control
if component == "datafile":
special_dict = dict(className='annotate-datafile', orderable=False, data=None,
title='', width="1%",
defaultContent='<span title="Annotate datafile" style="cursor: '
'pointer;" class="copo-tooltip">'
'<i class="ui icon violet write" aria-hidden="true"></i></span>')
columns.append(special_dict)
return columns
class BrokerDA:
def __init__(self, **kwargs):
self.param_dict = kwargs
self.context = self.param_dict.get("context", dict())
self.component = self.param_dict.get("component", str())
self.visualize = self.param_dict.get("visualize", str())
self.profile_id = self.param_dict.get("profile_id", str())
self.auto_fields = self.param_dict.get("auto_fields", dict())
if self.auto_fields and isinstance(self.auto_fields, str):
self.auto_fields = ast.literal_eval(self.auto_fields)
self.broker_visuals = BrokerVisuals(**kwargs)
self.da_object = DAComponent(self.profile_id, self.component)
da_dict = dict(
publication=Publication,
person=Person,
sample=Sample,
source=Source,
profile=Profile,
datafile=DataFile,
submission=Submission,
annotation=Annotation
)
if da_dict.get(self.component):
self.da_object = da_dict[self.component](self.profile_id)
def set_extra_params(self, extra_param):
for k, v in extra_param.items():
self.param_dict[k] = v
def do_copo_schemas(self):
copo_schemas = dict(
ontology_schema=d_utils.get_copo_schema("ontology_annotation"),
comment_schema=d_utils.get_copo_schema("comment"),
characteristics_schema=d_utils.get_copo_schema("material_attribute_value"),
source_schema=d_utils.get_copo_schema("source")
)
self.context["copo_schemas"] = copo_schemas
return self.context
def do_save_edit(self):
kwargs = dict()
kwargs["target_id"] = self.param_dict.get("target_id", str())
record_object = self.da_object.save_record(self.auto_fields, **kwargs)
# process visualisation context
self.broker_visuals.set_extra_params(dict(record_object=record_object))
visualize_dict = dict(profiles_counts=self.broker_visuals.do_profiles_counts,
sources_json=self.broker_visuals.get_sources_json,
sources_json_and_last_record_id=self.broker_visuals.get_sources_json_last_record_id,
last_record=self.broker_visuals.get_last_record,
get_profile_count=self.broker_visuals.get_profile_count
)
if self.visualize in visualize_dict:
self.context = visualize_dict[self.visualize]()
elif self.param_dict.get("target_id", str()):
self.context = self.broker_visuals.do_table_data()
else:
self.context = self.broker_visuals.do_row_data()
return self.context
def do_delete(self):
target_ids = [ObjectId(i) for i in self.param_dict.get("target_ids")]
self.da_object.get_collection_handle().update_many(
{"_id": {"$in": target_ids}}, {"$set": {"deleted": d_utils.get_deleted_flag()}}
)
self.context = self.broker_visuals.do_table_data()
return self.context
def do_form(self):
target_id = self.param_dict.get("target_id")
component_dict = self.param_dict.get("component_dict", dict())
message_dict = self.param_dict.get("message_dict", dict())
self.context["form"] = htags.generate_copo_form(self.component, target_id, component_dict, message_dict,
self.profile_id)
self.context["form"]["visualize"] = self.param_dict.get("visualize")
return self.context
def do_doi(self):
id_handle = self.param_dict.get("id_handle")
id_type = self.param_dict.get("id_type")
doi_resolve = DOI2Metadata(id_handle, id_type).get_resolve(self.component)
self.set_extra_params(dict(target_id=str(),
component_dict=doi_resolve.get("component_dict", dict()),
message_dict=doi_resolve.get("message_dict", dict()))
)
return self.do_form()
def do_initiate_submission(self):
kwarg = dict(datafile_ids=self.param_dict.get("datafile_ids", list()))
self.context["submission_token"] = str(self.da_object.save_record(dict(), **kwarg).get("_id", str()))
return self.context
def do_user_email(self):
user_id = self.param_dict.get("user_id", str())
user_email = self.param_dict.get("user_email", str())
user = User.objects.get(pk=int(user_id))
user.email = user_email
user.save()
return self.context
class BrokerDA:
def __init__(self, **kwargs):
self.param_dict = kwargs
self.context = self.param_dict.get("context", dict())
self.component = self.param_dict.get("component", str())
self.visualize = self.param_dict.get("visualize", str())
self.profile_id = self.param_dict.get("profile_id", str())
self.auto_fields = self.param_dict.get("auto_fields", dict())
if self.auto_fields and isinstance(self.auto_fields, str):
self.auto_fields = json.loads(self.auto_fields)
self.broker_visuals = BrokerVisuals(**kwargs)
self.da_object = DAComponent(self.profile_id, self.component)
da_dict = dict(publication=Publication,
person=Person,
sample=Sample,
source=Source,
profile=Profile,
datafile=DataFile,
submission=Submission,
annotation=Annotation,
cgcore=CGCore,
metadata_template=MetadataTemplate)
if self.component in da_dict:
self.da_object = da_dict[self.component](self.profile_id)
def set_extra_params(self, extra_param):
for k, v in extra_param.items():
self.param_dict[k] = v
def do_form_control_schemas(self):
"""
function returns object type control schemas used in building form controls
:return:
"""
copo_schemas = dict()
for k, v in d_utils.object_type_control_map().items():
copo_schemas[k] = d_utils.get_copo_schema(v)
self.context["copo_schemas"] = copo_schemas
return self.context
def do_save_edit(self):
kwargs = dict()
kwargs["target_id"] = self.param_dict.get("target_id", str())
# set report parameter
status = "success" # 'success', 'warning', 'info', 'danger' - modelled after bootstrap alert classes
action_type = "add"
report_metadata = dict()
if self.param_dict.get("target_id", str()):
action_type = "edit"
record_object = self.da_object.save_record(self.auto_fields, **kwargs)
if not record_object:
status = "danger"
if action_type == "add":
report_metadata[
"message"] = "New " + self.component + " record created!"
if status != "success":
report_metadata[
"message"] = "There was a problem creating the " + self.component + " record!"
elif action_type == "edit":
report_metadata["message"] = "Record updated!"
if status != "success":
report_metadata[
"message"] = "There was a problem updating the " + self.component + " record!"
report_metadata["status"] = status
self.context["action_feedback"] = report_metadata
# process visualisation context,
# set extra parameters which will be passed along to the visualize object
self.broker_visuals.set_extra_params(
dict(record_object=record_object,
data_source=self.param_dict.get("data_source", str())))
# build dictionary of executable tasks/functions
visualize_dict = dict(
profiles_counts=self.broker_visuals.do_profiles_counts,
created_component_json=self.broker_visuals.
get_created_component_json,
last_record=self.broker_visuals.get_last_record,
get_profile_count=self.broker_visuals.get_profile_count)
if self.visualize in visualize_dict:
self.context = visualize_dict[self.visualize]()
elif self.param_dict.get("target_id", str()):
self.context = self.broker_visuals.do_table_data()
else:
self.context = self.broker_visuals.do_row_data()
return self.context
def do_delete(self):
target_ids = [ObjectId(i) for i in self.param_dict.get("target_ids")]
# if ever it was needed to re-implement 'soft' delete uncomment the following lines and
# comment out the 'hard' delete query
# soft delete
# self.da_object.get_collection_handle().update_many(
# {"_id": {"$in": target_ids}}, {"$set": {"deleted": d_utils.get_deleted_flag()}}
# )
# hard delete
self.da_object.get_collection_handle().remove(
{'_id': {
'$in': target_ids
}})
self.context = self.broker_visuals.do_table_data()
return self.context
def do_form(self):
target_id = self.param_dict.get("target_id")
component_dict = self.param_dict.get("component_dict", dict())
message_dict = self.param_dict.get("message_dict", dict())
kwargs = dict()
kwargs["referenced_field"] = self.param_dict.get(
"referenced_field", str())
kwargs["referenced_type"] = self.param_dict.get(
"referenced_type", str())
self.context["form"] = htags.generate_copo_form(
self.component, target_id, component_dict, message_dict,
self.profile_id, **kwargs)
self.context["form"]["visualize"] = self.param_dict.get("visualize")
return self.context
def do_form_and_component_records(self):
# generates form, and in addition returns records of the form component, this could, for instance, be
# used for cloning of a record
kwargs = dict()
kwargs["referenced_field"] = self.param_dict.get(
"referenced_field", str())
kwargs["referenced_type"] = self.param_dict.get(
"referenced_type", str())
self.context = self.do_form()
self.context["component_records"] = htags.generate_component_records(
self.component, self.profile_id, **kwargs)
return self.context
def do_doi(self):
id_handle = self.param_dict.get("id_handle")
id_type = self.param_dict.get("id_type")
doi_resolve = DOI2Metadata(id_handle,
id_type).get_resolve(self.component)
self.set_extra_params(
dict(target_id=str(),
component_dict=doi_resolve.get("component_dict", dict()),
message_dict=doi_resolve.get("message_dict", dict())))
return self.do_form()
def do_initiate_submission(self):
kwarg = dict(datafile_ids=self.param_dict.get("datafile_ids", list()))
self.context["submission_token"] = str(
self.da_object.save_record(dict(), **kwarg).get("_id", str()))
return self.context
def do_user_email(self):
user_id = self.param_dict.get("user_id", str())
user_email = self.param_dict.get("user_email", str())
user = User.objects.get(pk=int(user_id))
user.email = user_email
user.save()
return self.context
def do_component_record(self):
self.context["component_record"] = self.da_object.get_record(
self.param_dict.get("target_id"))
return self.context
def component_form_record(self):
target_id = self.param_dict.get("target_id")
component_dict = self.param_dict.get("component_dict", dict())
message_dict = self.param_dict.get("message_dict", dict())
kwargs = dict()
kwargs["referenced_field"] = self.param_dict.get(
"referenced_field", str())
kwargs["referenced_type"] = self.param_dict.get(
"referenced_type", str())
kwargs["action_type"] = self.param_dict.get("action_type", str())
form_value = htags.generate_copo_form(self.component, target_id,
component_dict, message_dict,
self.profile_id, **kwargs)
self.context["component_record"] = form_value["form_value"]
self.context["component_schema"] = form_value["form_schema"]
return self.context
def do_sanitise_submissions(self):
records = self.da_object.get_all_records()
for submission in records:
if "bundle_meta" not in submission:
bundle_meta = list()
for file_id in submission.get("bundle", list()):
datafile = DataFile().get_record(file_id)
if datafile:
upload_status = False
if str(submission.get("complete",
False)).lower() == 'true':
upload_status = True
bundle_meta.append(
dict(file_id=file_id,
file_path=datafile.get(
"file_location", str()),
upload_status=upload_status))
submission["bundle_meta"] = bundle_meta
submission['target_id'] = str(submission.pop('_id'))
self.da_object.save_record(dict(), **submission)
self.context["sanitise_status"] = True
return self.context
def do_clone_description_bundle(self):
"""
function creates a new description by cloning an existing (specified) bundle
:return:
"""
target_id = self.param_dict.get("target_id", str())
bundle_name = self.param_dict.get("bundle_name", str())
result = dict(status="success", message="")
if Description().get_description_handle().find({
"name": {
'$regex': "^" + bundle_name + "$",
"$options": 'i'
}
}).count() >= 1:
result["status"] = "error"
result["message"] = "Bundle name must be unique"
self.context["result"] = result
return self.context
# retrieve clone target
description = Description().GET(target_id)
# new bundle being created
try:
bundle = Description().create_description(
profile_id=self.profile_id,
component=self.component,
name=bundle_name,
stages=description.get('stages', list()),
attributes=description.get('attributes', dict()),
meta=description.get('meta', dict()))
result["data"] = dict(id=str(bundle["_id"]), name=bundle["name"])
except Exception as e:
message = "Couldn't create bundle: " + bundle_name + " " + str(e)
result["status"] = "error"
result["message"] = message
self.context["result"] = result
return self.context
def create_rename_description_bundle(self):
"""
function creates a new description bundle or renames an existing one
:return:
"""
target_id = self.param_dict.get("target_id", str())
bundle_name = self.param_dict.get("bundle_name", str())
result = dict(status="success", message="")
if Description().get_description_handle().find({
"name": {
'$regex': "^" + bundle_name + "$",
"$options": 'i'
}
}).count() >= 1:
result["status"] = "error"
result["message"] = "Bundle name must be unique"
elif target_id:
# updating existing bundle
Description().edit_description(target_id, {"name": bundle_name})
try:
Description().edit_description(target_id,
{"name": bundle_name})
except Exception as e:
message = "Couldn't update bundle: " + bundle_name + " " + str(
e)
result["status"] = "error"
result["message"] = message
else:
# new bundle being created
try:
bundle = Description().create_description(
profile_id=self.profile_id,
component=self.component,
name=bundle_name)
result["data"] = dict(id=str(bundle["_id"]),
name=bundle["name"])
except Exception as e:
message = "Couldn't create bundle: " + bundle_name + " " + str(
e)
result["status"] = "error"
result["message"] = message
self.context["result"] = result
return self.context
def generate_server_side_table_records(profile_id=str(), component=str(), request=dict()):
# function generates component records for building an UI table using server-side processing
# - please note that for effective data display,
# all array and object-type fields (e.g., characteristics) are deferred to sub-table display.
# please define such in the schema as "show_in_table": false and "show_as_attribute": true
data_set = list()
n_size = int(request.get("length", 10)) # assumes 10 records per page if length not set
draw = int(request.get("draw", 1))
start = int(request.get("start", 0))
# instantiate data access object
da_object = DAComponent(profile_id, component)
return_dict = dict()
records_total = da_object.get_collection_handle().count(
{'profile_id': profile_id, 'deleted': data_utils.get_not_deleted_flag()})
# retrieve and process records
filter_by = dict()
if component == "datafile":
# get all active bundles in the profile
existing_bundles = Description().get_all_records_columns(projection=dict(_id=1),
filter_by=dict(profile_id=profile_id,
component=component))
existing_bundles = [str(x["_id"]) for x in existing_bundles]
records_total = da_object.get_collection_handle().count({"$and": [
{"profile_id": profile_id, 'deleted': data_utils.get_not_deleted_flag()},
{"$or": [
{"description_token": {"$in": [None, False, ""]}},
{"description_token": {"$nin": existing_bundles}}]}
]})
filter_by = {"$or": [
{"description_token": {"$in": [None, False, ""]}},
{"description_token": {"$nin": existing_bundles}}]}
# get and filter schema elements based on displayable columns
schema = [x for x in da_object.get_schema().get("schema_dict") if x.get("show_in_table", True)]
# build db column projection
projection = [(x["id"].split(".")[-1], 1) for x in schema]
# order by
sort_by = request.get('order[0][column]', '0')
sort_by = request.get('columns[' + sort_by + '][data]', '')
sort_direction = request.get('order[0][dir]', 'asc')
sort_by = '_id' if not sort_by else sort_by
sort_direction = 1 if sort_direction == 'asc' else -1
# search
search_term = request.get('search[value]', '').strip()
records = da_object.get_all_records_columns_server(sort_by=sort_by, sort_direction=sort_direction,
search_term=search_term, projection=dict(projection),
limit=n_size, skip=start, filter_by=filter_by)
records_filtered = records_total
if search_term:
records_filtered = da_object.get_collection_handle().count(
{'profile_id': profile_id, 'deleted': data_utils.get_not_deleted_flag(),
'name': {'$regex': search_term, "$options": 'i'}})
if records:
df = pd.DataFrame(records)
df['record_id'] = df._id.astype(str)
df["DT_RowId"] = df.record_id
df.DT_RowId = 'row_' + df.DT_RowId
df = df.drop('_id', axis='columns')
for x in schema:
x["id"] = x["id"].split(".")[-1]
df[x["id"]] = df[x["id"]].apply(resolve_control_output_apply, args=(x,)).astype(str)
data_set = df.to_dict('records')
return_dict["records_total"] = records_total
return_dict["records_filtered"] = records_filtered
return_dict["data_set"] = data_set
return_dict["draw"] = draw
return return_dict
