def test_data_processing(app): base_dir = os.path.dirname(os.path.realpath(__file__)) data = yaml.safe_load(file(os.path.join(base_dir, 'test_data/data_table.yaml'))) assert ('independent_variables' in data) assert ('dependent_variables' in data) assert (len(data['independent_variables']) == 1) assert (len(data['independent_variables'][0]['values']) == 3) assert (len(data['dependent_variables']) == 1) assert (len(data['dependent_variables'][0]['values']) == 3) data["name"] = 'test' data["title"] = 'test' data["keywords"] = None data["doi"] = 'doi/10.2342' data["review"] = [] data["associated_files"] = [] table_structure = generate_table_structure(data) assert(table_structure["x_count"] == 1) assert(len(table_structure["headers"]) == 2) assert(len(table_structure["qualifiers"]) == 2)
def test_data_processing(app): base_dir = os.path.dirname(os.path.realpath(__file__)) data = yaml.load(open(os.path.join(base_dir, 'test_data/data_table.yaml'), 'rt'), Loader=yaml.CSafeLoader) assert ('independent_variables' in data) assert ('dependent_variables' in data) assert (len(data['independent_variables']) == 1) assert (len(data['independent_variables'][0]['values']) == 3) assert (len(data['dependent_variables']) == 1) assert (len(data['dependent_variables'][0]['values']) == 3) data["name"] = 'test' data["title"] = 'test' data["keywords"] = None data["doi"] = 'doi/10.2342' data["location"] = 'Data from Figure 2 of preprint' data["review"] = [] data["associated_files"] = [] table_structure = generate_table_structure(data) assert (table_structure["x_count"] == 1) assert (len(table_structure["headers"]) == 2) assert (len(table_structure["qualifiers"]) == 2)
def get_table_details(recid, data_recid, version): """ :param recid: :param data_recid: :param version: :return: """ datasub_query = DataSubmission.query.filter_by(id=data_recid, version=version) table_contents = {} if datasub_query.count() > 0: datasub_record = datasub_query.one() data_query = db.session.query(DataResource).filter(DataResource.id == datasub_record.data_file) if data_query.count() > 0: data_record = data_query.one() file_location = data_record.file_location try: table_contents = yaml.load(file(file_location), Loader=yaml.CSafeLoader) except: table_contents = yaml.load(file(file_location)) table_contents["name"] = datasub_record.name table_contents["title"] = datasub_record.description table_contents["keywords"] = datasub_record.keywords table_contents["doi"] = datasub_record.doi # we create a map of files mainly to accommodate the use of thumbnails for images where possible. tmp_assoc_files = {} for associated_data_file in datasub_record.resources: alt_location = associated_data_file.file_location location_parts = alt_location.split("/") key = location_parts[-1].replace("thumb_", "") if key not in tmp_assoc_files: tmp_assoc_files[key] = {} if "thumb_" in alt_location: tmp_assoc_files[key]["preview_location"] = "/record/resource/{0}?view=true".format( associated_data_file.id ) else: tmp_assoc_files[key].update( { "description": associated_data_file.file_description, "type": associated_data_file.file_type, "id": associated_data_file.id, "alt_location": alt_location, } ) # add associated files to the table contents table_contents["associated_files"] = tmp_assoc_files.values() table_contents["review"] = {} data_review_record = create_data_review(data_recid, recid, version) table_contents["review"]["review_flag"] = data_review_record.status table_contents["review"]["messages"] = len(data_review_record.messages) > 0 # translate the table_contents to an easy to render format of the qualifiers (with colspan), # x and y headers (should not require a colspan) # values, that also encompass the errors return jsonify(generate_table_structure(table_contents))
def get_table_details(recid, data_recid, version): """ Get the table details. :param recid: :param data_recid: :param version: :return: """ datasub_query = DataSubmission.query.filter_by(id=data_recid, version=version) table_contents = {} if datasub_query.count() > 0: datasub_record = datasub_query.one() data_query = db.session.query(DataResource).filter( DataResource.id == datasub_record.data_file) if data_query.count() > 0: data_record = data_query.one() file_location = data_record.file_location attempts = 0 while True: try: with open(file_location, 'r') as table_file: table_contents = yaml.load(table_file, Loader=Loader) except: attempts += 1 # allow multiple attempts to read file in case of temporary disk problems if (table_contents and table_contents is not None) or attempts > 5: break table_contents["name"] = datasub_record.name table_contents["title"] = datasub_record.description table_contents["keywords"] = datasub_record.keywords table_contents["doi"] = datasub_record.doi table_contents["location"] = datasub_record.location_in_publication # we create a map of files mainly to accommodate the use of thumbnails for images where possible. tmp_assoc_files = {} for associated_data_file in datasub_record.resources: alt_location = associated_data_file.file_location location_parts = alt_location.split('/') key = location_parts[-1].replace("thumb_", "") if key not in tmp_assoc_files: tmp_assoc_files[key] = {} if "thumb_" in alt_location and associated_data_file.file_type.lower() in IMAGE_TYPES: tmp_assoc_files[key]['preview_location'] = '/record/resource/{0}?view=true'.format( associated_data_file.id) else: tmp_assoc_files[key].update({'description': associated_data_file.file_description, 'type': associated_data_file.file_type, 'id': associated_data_file.id, 'alt_location': alt_location}) # add associated files to the table contents table_contents['associated_files'] = list(tmp_assoc_files.values()) table_contents["review"] = {} data_review_record = create_data_review(data_recid, recid, version) table_contents["review"]["review_flag"] = data_review_record.status if data_review_record else "todo" table_contents["review"]["messages"] = len(data_review_record.messages) > 0 if data_review_record else False # translate the table_contents to an easy to render format of the qualifiers (with colspan), # x and y headers (should not require a colspan) # values, that also encompass the errors return jsonify(generate_table_structure(table_contents))