def process_metadata_import(dataset_accession, dryrun=False, massive_host=None):
dataset_metadatum = find_dataset_metadata(dataset_accession, useftp=True, massive_host=massive_host)
if dataset_metadatum == None:
print("Not Importing %s, no metadata" % dataset_accession)
return -2, -2
else:
print("Importing %s" % dataset_accession, dataset_metadatum)
#Save files Locally
local_metadata_path = os.path.join("tempuploads", dataset_accession + ".tsv")
try:
massive_host.download(dataset_metadatum["path"], local_metadata_path)
except:
print("CANT DOWNLOAD", dataset_metadatum["path"])
raise
metadata_validator.rewrite_metadata(local_metadata_path)
#Validate
pass_validation, failures, errors_list, valid_rows, total_rows_count = metadata_validator.perform_validation(local_metadata_path)
#Filtering out lines that are not valid
local_filtered_metadata_path = os.path.join("tempuploads", "filtered_" + dataset_accession + ".tsv")
if len([error for error in errors_list if error["error_string"].find("Missing column") != -1]) > 0:
print("Missing Columns, Rejected")
return -1, -1
#Filtering out lines that do not match the dataset accession
metadata_df = pd.DataFrame(valid_rows)
try:
metadata_df = metadata_df[metadata_df['MassiveID'] == dataset_accession]
except:
metadata_df = pd.DataFrame(valid_rows)
metadata_df.to_csv(local_filtered_metadata_path, sep="\t", index=False)
try:
pass_validation, failures, errors_list, valid_rows, total_rows_count = metadata_validator.perform_validation(local_filtered_metadata_path)
except:
pass_validation = False
added_files_count = 0
if pass_validation:
print("Importing Data")
if not dryrun:
added_files_count = populate_metadata.populate_dataset_metadata(local_filtered_metadata_path, massive_host=massive_host)
else:
print("Filtered File is not valid")
return len(metadata_df), added_files_count
def main():
parser = argparse.ArgumentParser(description='Importing Database')
parser.add_argument('--importmetadata', default=None, help='Imports metadata, options are all, dataset, file')
parser.add_argument('--metadatafile', help='Imports metadata filename')
parser.add_argument('--metadataaccession', help='Imports metadata accession')
parser.add_argument('--importidentifications', default=None, help='Imports identifications, from task file')
parser.add_argument('--identifications_output', help='identifications_output')
massive_host = ftputil.FTPHost("massive.ucsd.edu", "anonymous", "")
args = parser.parse_args()
# Importing Metadata First
if args.importmetadata == "all":
summary_list = []
all_datasets = ming_proteosafe_library.get_all_datasets()
for dataset in all_datasets:
if not "GNPS" in dataset["title"].upper():
continue
# Checking the FTP host
try:
list_names = massive_host.listdir("/")
except Exception as e:
print("MassIVE connection broken, reconnecting", e)
massive_host = ftputil.FTPHost("massive.ucsd.edu", "anonymous", "")
try:
print("Importing, ", dataset["dataset"])
total_valid_metadata_entries, files_added = process_metadata_import(dataset["dataset"], dryrun=False, massive_host=massive_host)
except KeyboardInterrupt:
raise
except:
total_valid_metadata_entries = -1
files_added = -1
summary_dict = {}
summary_dict["total_valid_metadata_entries"] = total_valid_metadata_entries
summary_dict["files_added"] = files_added
summary_dict["accession"] = dataset["dataset"]
summary_list.append(summary_dict)
try:
pd.DataFrame(summary_list).to_csv("/app/database/add_metadata_summary.tsv", sep="\t", index=False)
except:
continue
elif args.importmetadata == "dataset":
total_valid_metadata_entries, files_added = process_metadata_import(args.metadataaccession, massive_host=massive_host)
print(total_valid_metadata_entries, files_added)
elif args.importmetadata == "file":
populate_metadata.populate_dataset_metadata(args.metadatafile, massive_host=massive_host)
# Import Library Identifications
if args.importidentifications is not None:
import_identification(args.importidentifications, args.identifications_output, force=True)
def main():
parser = argparse.ArgumentParser(description='Importing Database')
parser.add_argument(
'--importmetadata',
default=None,
help='Imports metadata, options are all, dataset, file')
parser.add_argument('--metadatafile', help='Imports metadata filename')
parser.add_argument('--metadataaccession',
help='Imports metadata accession')
parser.add_argument('--importidentifications',
default=None,
help='Imports identifications, from task file')
parser.add_argument('--identifications_output',
help='identifications_output')
args = parser.parse_args()
# Importing Metadata First
if args.importmetadata == "all":
summary_list = []
all_datasets = ming_proteosafe_library.get_all_datasets()
for dataset in all_datasets:
if not "GNPS" in dataset["title"].upper():
continue
try:
total_valid_metadata_entries, files_added = process_metadata_import(
dataset["dataset"], dryrun=False)
except KeyboardInterrupt:
raise
except:
total_valid_metadata_entries = -1
files_added = -1
summary_dict = {}
summary_dict[
"total_valid_metadata_entries"] = total_valid_metadata_entries
summary_dict["files_added"] = files_added
summary_dict["accession"] = dataset["dataset"]
summary_list.append(summary_dict)
try:
pd.DataFrame(summary_list).to_csv(
"/app/database/add_metadata_summary.tsv",
sep="\t",
index=False)
except:
continue
elif args.importmetadata == "dataset":
total_valid_metadata_entries, files_added = process_metadata_import(
args.metadataaccession)
print(total_valid_metadata_entries, files_added)
elif args.importmetadata == "file":
populate_metadata.populate_dataset_metadata(args.metadatafile)
# Import Library Identifications
if args.importidentifications is not None:
import_identification(args.importidentifications,
args.identifications_output)