def test_data_processing(app):
base_dir = os.path.dirname(os.path.realpath(__file__))
data = yaml.safe_load(file(os.path.join(base_dir, 'test_data/data_table.yaml')))
assert ('independent_variables' in data)
assert ('dependent_variables' in data)
assert (len(data['independent_variables']) == 1)
assert (len(data['independent_variables'][0]['values']) == 3)
assert (len(data['dependent_variables']) == 1)
assert (len(data['dependent_variables'][0]['values']) == 3)
data["name"] = 'test'
data["title"] = 'test'
data["keywords"] = None
data["doi"] = 'doi/10.2342'
data["review"] = []
data["associated_files"] = []
table_structure = generate_table_structure(data)
assert(table_structure["x_count"] == 1)
assert(len(table_structure["headers"]) == 2)
assert(len(table_structure["qualifiers"]) == 2)
def test_data_processing(app):
base_dir = os.path.dirname(os.path.realpath(__file__))
data = yaml.load(open(os.path.join(base_dir, 'test_data/data_table.yaml'),
'rt'),
Loader=yaml.CSafeLoader)
assert ('independent_variables' in data)
assert ('dependent_variables' in data)
assert (len(data['independent_variables']) == 1)
assert (len(data['independent_variables'][0]['values']) == 3)
assert (len(data['dependent_variables']) == 1)
assert (len(data['dependent_variables'][0]['values']) == 3)
data["name"] = 'test'
data["title"] = 'test'
data["keywords"] = None
data["doi"] = 'doi/10.2342'
data["location"] = 'Data from Figure 2 of preprint'
data["review"] = []
data["associated_files"] = []
table_structure = generate_table_structure(data)
assert (table_structure["x_count"] == 1)
assert (len(table_structure["headers"]) == 2)
assert (len(table_structure["qualifiers"]) == 2)
def get_table_details(recid, data_recid, version):
"""
:param recid:
:param data_recid:
:param version:
:return:
"""
datasub_query = DataSubmission.query.filter_by(id=data_recid, version=version)
table_contents = {}
if datasub_query.count() > 0:
datasub_record = datasub_query.one()
data_query = db.session.query(DataResource).filter(DataResource.id == datasub_record.data_file)
if data_query.count() > 0:
data_record = data_query.one()
file_location = data_record.file_location
try:
table_contents = yaml.load(file(file_location), Loader=yaml.CSafeLoader)
except:
table_contents = yaml.load(file(file_location))
table_contents["name"] = datasub_record.name
table_contents["title"] = datasub_record.description
table_contents["keywords"] = datasub_record.keywords
table_contents["doi"] = datasub_record.doi
# we create a map of files mainly to accommodate the use of thumbnails for images where possible.
tmp_assoc_files = {}
for associated_data_file in datasub_record.resources:
alt_location = associated_data_file.file_location
location_parts = alt_location.split("/")
key = location_parts[-1].replace("thumb_", "")
if key not in tmp_assoc_files:
tmp_assoc_files[key] = {}
if "thumb_" in alt_location:
tmp_assoc_files[key]["preview_location"] = "/record/resource/{0}?view=true".format(
associated_data_file.id
)
else:
tmp_assoc_files[key].update(
{
"description": associated_data_file.file_description,
"type": associated_data_file.file_type,
"id": associated_data_file.id,
"alt_location": alt_location,
}
)
# add associated files to the table contents
table_contents["associated_files"] = tmp_assoc_files.values()
table_contents["review"] = {}
data_review_record = create_data_review(data_recid, recid, version)
table_contents["review"]["review_flag"] = data_review_record.status
table_contents["review"]["messages"] = len(data_review_record.messages) > 0
# translate the table_contents to an easy to render format of the qualifiers (with colspan),
# x and y headers (should not require a colspan)
# values, that also encompass the errors
return jsonify(generate_table_structure(table_contents))
def get_table_details(recid, data_recid, version):
"""
Get the table details.
:param recid:
:param data_recid:
:param version:
:return:
"""
datasub_query = DataSubmission.query.filter_by(id=data_recid,
version=version)
table_contents = {}
if datasub_query.count() > 0:
datasub_record = datasub_query.one()
data_query = db.session.query(DataResource).filter(
DataResource.id == datasub_record.data_file)
if data_query.count() > 0:
data_record = data_query.one()
file_location = data_record.file_location
attempts = 0
while True:
try:
with open(file_location, 'r') as table_file:
table_contents = yaml.load(table_file, Loader=Loader)
except:
attempts += 1
# allow multiple attempts to read file in case of temporary disk problems
if (table_contents and table_contents is not None) or attempts > 5:
break
table_contents["name"] = datasub_record.name
table_contents["title"] = datasub_record.description
table_contents["keywords"] = datasub_record.keywords
table_contents["doi"] = datasub_record.doi
table_contents["location"] = datasub_record.location_in_publication
# we create a map of files mainly to accommodate the use of thumbnails for images where possible.
tmp_assoc_files = {}
for associated_data_file in datasub_record.resources:
alt_location = associated_data_file.file_location
location_parts = alt_location.split('/')
key = location_parts[-1].replace("thumb_", "")
if key not in tmp_assoc_files:
tmp_assoc_files[key] = {}
if "thumb_" in alt_location and associated_data_file.file_type.lower() in IMAGE_TYPES:
tmp_assoc_files[key]['preview_location'] = '/record/resource/{0}?view=true'.format(
associated_data_file.id)
else:
tmp_assoc_files[key].update({'description': associated_data_file.file_description,
'type': associated_data_file.file_type,
'id': associated_data_file.id,
'alt_location': alt_location})
# add associated files to the table contents
table_contents['associated_files'] = list(tmp_assoc_files.values())
table_contents["review"] = {}
data_review_record = create_data_review(data_recid, recid, version)
table_contents["review"]["review_flag"] = data_review_record.status if data_review_record else "todo"
table_contents["review"]["messages"] = len(data_review_record.messages) > 0 if data_review_record else False
# translate the table_contents to an easy to render format of the qualifiers (with colspan),
# x and y headers (should not require a colspan)
# values, that also encompass the errors
return jsonify(generate_table_structure(table_contents))
