def main(unparsed_args_list):
"""Run main get_gb_records pipeline."""
# Parse command line arguments
args = parse_args(unparsed_args_list)
date = time.strftime("%Y%m%d")
args.output_folder = basic.set_path(args.output_folder,
kind="dir",
expect=True)
working_dir = pathlib.Path(f"{date}_get_gb_records")
working_path = basic.make_new_dir(args.output_folder,
working_dir,
attempt=10)
if working_path is None:
print(f"Invalid working directory '{working_dir}'")
sys.exit(1)
ncbi_cred_dict = ncbi.get_ncbi_creds(args.ncbi_credentials_file)
# Verify database connection and schema compatibility.
print("Connecting to the MySQL database...")
engine = mysqldb.connect_to_db(args.database)
mysqldb.check_schema_compatibility(engine, "the get_gb_records pipeline")
# Create data sets
print("Retrieving accessions from the database...")
accessions = mysqldb.get_distinct_data(engine, "phage", "Accession")
engine.dispose()
if "" in accessions:
accessions.remove("")
if None in accessions:
accessions.remove(None)
get_genbank_data(working_path, accessions, ncbi_cred_dict)
def main(unparsed_args):
"""Runs the complete update pipeline."""
args = parse_args(unparsed_args)
# Verify database connection and schema compatibility.
print("Connecting to the MySQL database...")
engine = mysqldb.connect_to_db(args.database)
mysqldb.check_schema_compatibility(engine, "the update pipeline")
if args.version == True:
mysqldb.change_version(engine)
print("Database version updated.")
if args.ticket_table is not None:
update_table_path = basic.set_path(args.ticket_table,
kind="file",
expect=True)
# Iterate through the tickets and process them sequentially.
list_of_update_tickets = []
with update_table_path.open(mode='r') as f:
file_reader = csv.DictReader(f)
for dict in file_reader:
list_of_update_tickets.append(dict)
# Variables to be used for end summary
processed = 0
succeeded = 0
failed = 0
for dict in list_of_update_tickets:
conn = engine.connect()
# Pass the raw db_api connection
handler = RandomFieldUpdateHandler(conn.connection)
handler.table = dict["table"] # Which table will be updated?
handler.field = dict["field"] # Which field will be updated?
handler.value = dict[
"value"] # What value will be put in that field?
handler.key_name = dict[
"key_name"] # How will we know which row is the right one?
handler.key_value = dict[
"key_value"] # How will we know which row is the right one?
handler.validate_ticket(
) # Make sure all handler attributes are valid
status = handler.execute_ticket() # Do what was requested
if status == 1:
processed += 1
succeeded += 1
else:
processed += 1
failed += 1
engine.dispose()
print("\nDone iterating through tickets.")
if succeeded > 0:
print(f"{succeeded} / {processed} tickets successfully handled.")
if failed > 0:
print(f"{failed} / {processed} tickets failed to be handled.")
def main(unparsed_args_list):
"""Run main get_gb_records pipeline."""
# Parse command line arguments
args = parse_args(unparsed_args_list)
# Filters input: phage.Status=draft AND phage.HostGenus=Mycobacterium
# Args structure: [['phage.Status=draft'], ['phage.HostGenus=Mycobacterium']]
filters = args.filters
ncbi_cred_dict = ncbi.get_ncbi_creds(args.ncbi_credentials_file)
output_folder = basic.set_path(args.output_folder, kind="dir", expect=True)
working_dir = pathlib.Path(RESULTS_FOLDER)
working_path = basic.make_new_dir(output_folder, working_dir, attempt=50)
if working_path is None:
print(f"Invalid working directory '{working_dir}'")
sys.exit(1)
# Verify database connection and schema compatibility.
print("Connecting to the MySQL database...")
alchemist = AlchemyHandler(database=args.database)
alchemist.connect(pipeline=True)
engine = alchemist.engine
mysqldb.check_schema_compatibility(engine, "the get_gb_records pipeline")
# Get SQLAlchemy metadata Table object
# table_obj.primary_key.columns is a
# SQLAlchemy ColumnCollection iterable object
# Set primary key = 'phage.PhageID'
alchemist.build_metadata()
table = querying.get_table(alchemist.metadata, TARGET_TABLE)
for column in table.primary_key.columns:
primary_key = column
# Create filter object and then add command line filter strings
db_filter = Filter(alchemist=alchemist, key=primary_key)
db_filter.values = []
# Attempt to add filters and exit if needed.
add_filters(db_filter, filters)
# Performs the query
db_filter.update()
# db_filter.values now contains list of PhageIDs that pass the filters.
# Get the accessions associated with these PhageIDs.
keep_set = set(db_filter.values)
# Create data sets
print("Retrieving accessions from the database...")
query = construct_accession_query(keep_set)
list_of_dicts = mysqldb_basic.query_dict_list(engine, query)
id_acc_dict = get_id_acc_dict(list_of_dicts)
acc_id_dict = get_acc_id_dict(id_acc_dict)
engine.dispose()
if len(acc_id_dict.keys()) > 0:
get_data(working_path, acc_id_dict, ncbi_cred_dict)
else:
print("There are no records to retrieve.")
def main(unparsed_args):
"""Runs the complete update pipeline."""
args = parse_args(unparsed_args[2:])
# Verify database connection and schema compatibility.
print("Connecting to the MySQL database...")
# Create config object with data obtained from file and/or defaults.
config = configfile.build_complete_config(args.config_file)
mysql_creds = config["mysql"]
alchemist = AlchemyHandler(database=args.database,
username=mysql_creds["user"],
password=mysql_creds["password"])
alchemist.connect(pipeline=True)
engine = alchemist.engine
mysqldb.check_schema_compatibility(engine, "the update pipeline")
if args.version is True:
mysqldb.change_version(engine)
print("Database version updated.")
if args.ticket_table is not None:
update_table_path = basic.set_path(args.ticket_table,
kind="file",
expect=True)
# Iterate through the tickets and process them sequentially.
list_of_update_tickets = []
with update_table_path.open(mode='r') as f:
file_reader = csv.DictReader(f)
for dict in file_reader:
list_of_update_tickets.append(dict)
# Variables to be used for end summary
processed = 0
succeeded = 0
failed = 0
for dict in list_of_update_tickets:
status = update_field(alchemist, dict)
if status == 1:
processed += 1
succeeded += 1
else:
processed += 1
failed += 1
engine.dispose()
print("\nDone iterating through tickets.")
if succeeded > 0:
print(f"{succeeded} / {processed} tickets successfully handled.")
if failed > 0:
print(f"{failed} / {processed} tickets failed to be handled.")
def main(unparsed_args_list):
"""Uses parsed args to run the entirety of the file export pipeline.
:param unparsed_args_list: Input a list of command line args.
:type unparsed_args_list: list[str]
"""
#Returns after printing appropriate error message from parsing/connecting.
args = parse_export(unparsed_args_list)
alchemist = AlchemyHandler(database=args.database)
alchemist.connect(ask_database=True, pipeline=True)
alchemist.build_graph()
# Exporting as a SQL file is not constricted by schema version.
if args.pipeline != "sql":
mysqldb.check_schema_compatibility(alchemist.engine, "export")
values = []
if args.pipeline in FILTERABLE_PIPELINES:
values = parse_value_input(args.input)
if not args.pipeline in PIPELINES:
print("ABORTED EXPORT: Unknown pipeline option discrepency.\n"
"Pipeline parsed from command line args is not supported")
sys.exit(1)
if args.pipeline != "I":
execute_export(alchemist,
args.folder_path,
args.folder_name,
args.pipeline,
table=args.table,
values=values,
filters=args.filters,
groups=args.groups,
sort=args.sort,
include_columns=args.include_columns,
exclude_columns=args.exclude_columns,
sequence_columns=args.sequence_columns,
raw_bytes=args.raw_bytes,
concatenate=args.concatenate,
verbose=args.verbose)
else:
pass
def main(unparsed_args_list):
"""Run main freeze database pipeline."""
args = parse_args(unparsed_args_list)
# engine1, msg = mysqldb.get_engine(database=args.database, echo=False)
# Verify database connection and schema compatibility.
print("Connecting to the MySQL database...")
engine1 = mysqldb.connect_to_db(args.database)
mysqldb.check_schema_compatibility(engine1, "the freeze pipeline")
# Get the number of draft genomes.
query = "SELECT count(*) as count FROM phage WHERE Status != 'draft'"
result = engine1.execute(query).fetchall()
phage_count = result[0]["count"]
# Create the new frozen database folder e.g. Actinobacteriophage_XYZ.
prefix = get_prefix()
new_database = f"{prefix}_{phage_count}"
# Create the new database
result = mysqldb.drop_create_db(engine1, new_database)
# Copy database.
if result == 0:
result = mysqldb.copy_db(engine1, new_database)
if result == 0:
print(f"Deleting 'draft' genomes...")
engine2, msg = mysqldb.get_engine(username=engine1.url.username,
password=engine1.url.password,
database=new_database)
statement3 = "DELETE FROM phage WHERE Status = 'draft'"
engine2.execute(statement3)
statement4 = "UPDATE version SET Version = 0"
engine2.execute(statement4)
# Close up all connections in the connection pool.
engine2.dispose()
else:
print("Unable to copy the database.")
# Close up all connections in the connection pool.
engine1.dispose()
else:
print(f"Error creating new database: {new_database}.")
print("Freeze database script completed.")
def main(unparsed_args_list):
"""Run main get_gb_records pipeline."""
# Parse command line arguments
args = parse_args(unparsed_args_list)
# Create config object with data obtained from file and/or defaults.
config = configfile.build_complete_config(args.config_file)
alchemist = pipelines_basic.build_alchemist(args.database, config=config)
mysqldb.check_schema_compatibility(alchemist.engine,
"the get_gb_records pipeline")
values = pipelines_basic.parse_value_input(args.input)
execute_get_gb_records(alchemist, args.file_type,
folder_path=args.folder_path,
folder_name=args.folder_name,
config=config,
values=values, verbose=args.verbose,
filters=args.filters, groups=args.groups)
def main(unparsed_args_list):
"""Uses parsed args to run the entirety of the file export pipeline.
:param unparsed_args_list: Input a list of command line args.
:type unparsed_args_list: list[str]
"""
# Returns after printing appropriate error message from parsing/connecting.
args = parse_export(unparsed_args_list)
config = configfile.build_complete_config(args.config_file)
alchemist = pipelines_basic.build_alchemist(args.database, config=config)
# Exporting as a SQL file is not constricted by schema version.
if args.pipeline != "sql":
mysqldb.check_schema_compatibility(alchemist.engine, "export")
values = None
if args.pipeline in FILTERABLE_PIPELINES:
values = pipelines_basic.parse_value_input(args.input)
if not values:
values = None
if args.pipeline not in PIPELINES:
print("ABORTED EXPORT: Unknown pipeline option discrepency.\n"
"Pipeline parsed from command line args is not supported")
sys.exit(1)
if args.pipeline != "I":
execute_export(alchemist, args.pipeline, folder_path=args.folder_path,
folder_name=args.folder_name, table=args.table,
values=values, filters=args.filters, groups=args.groups,
sort=args.sort, include_columns=args.include_columns,
exclude_columns=args.exclude_columns,
sequence_columns=args.sequence_columns,
raw_bytes=args.raw_bytes,
concatenate=args.concatenate, db_name=args.db_name,
verbose=args.verbose, dump=args.dump, force=args.force,
threads=args.number_processes, phams_out=args.phams_out)
else:
pass
def main(unparsed_args_list):
"""Run main retrieve_updates pipeline."""
# Parse command line arguments
args = parse_args(unparsed_args_list)
force = args.force_download
args.output_folder = basic.set_path(args.output_folder,
kind="dir",
expect=True)
working_dir = pathlib.Path(RESULTS_FOLDER)
working_path = basic.make_new_dir(args.output_folder,
working_dir,
attempt=50)
if working_path is None:
print(f"Invalid working directory '{working_dir}'")
sys.exit(1)
# Create config object with data obtained from file and/or defaults.
config = configfile.build_complete_config(args.config_file)
mysql_creds = config["mysql"]
ncbi_creds = config["ncbi"]
# Verify database connection and schema compatibility.
print("Preparing genome data sets from the MySQL database...")
alchemist = AlchemyHandler(database=args.database,
username=mysql_creds["user"],
password=mysql_creds["password"])
alchemist.connect(pipeline=True)
engine = alchemist.engine
mysqldb.check_schema_compatibility(engine, "the get_data pipeline")
# Get existing data from MySQL to determine what needs to be updated.
query = ("SELECT PhageID, Name, HostGenus, Status, Cluster, "
"DateLastModified, Accession, RetrieveRecord, Subcluster, "
"AnnotationAuthor FROM phage")
mysqldb_genome_list = mysqldb.parse_genome_data(engine=engine,
phage_query=query,
gnm_type="mysqldb")
engine.dispose()
mysqldb_genome_dict = {}
for gnm in mysqldb_genome_list:
# With default date, the date of all records retrieved will be newer.
if force:
gnm.date = constants.EMPTY_DATE
mysqldb_genome_dict[gnm.id] = gnm
# Get data from PhagesDB
if (args.updates or args.final or args.draft) is True:
print("Retrieving data from PhagesDB...")
phagesdb_phages = phagesdb.get_phagesdb_data(constants.API_SEQUENCED)
phagesdb_phages_dict = basic.convert_list_to_dict(
phagesdb_phages, "phage_name")
phagesdb_genome_dict = phagesdb.parse_genomes_dict(
phagesdb_phages_dict, gnm_type="phagesdb", seq=False)
# Exit if all phage data wasn't retrieved.
if len(phagesdb_genome_dict) == 0:
sys.exit(1)
# Returns a list of tuples.
tup = match_genomes(mysqldb_genome_dict, phagesdb_genome_dict)
matched_genomes = tup[0]
unmatched_phagesdb_ids = tup[1]
if args.updates is True:
get_update_data(working_path, matched_genomes)
if args.final is True:
get_final_data(working_path, matched_genomes)
if args.genbank is True:
get_genbank_data(working_path,
mysqldb_genome_dict,
ncbi_creds,
args.genbank_results,
force=force)
if args.draft is True:
if force:
# Add all draft genomes currently in database to the list of
# draft genomes to be downloaded.
drafts = get_matched_drafts(matched_genomes)
unmatched_phagesdb_ids |= drafts
get_draft_data(working_path, unmatched_phagesdb_ids)
def main(argument_list):
"""
:param argument_list:
:return:
"""
# Setup argument parser
cdd_parser = setup_argparser()
# Use argument parser to parse argument_list
args = cdd_parser.parse_args(argument_list)
# Store arguments in more easily accessible variables
database = args.db
cdd_dir = expand_path(args.dir)
cdd_name = learn_cdd_name(cdd_dir)
threads = args.threads
evalue = args.evalue
rpsblast = args.rpsblast
tmp_dir = args.tmp_dir
output_folder = args.output_folder
log_file = args.log_file
reset = args.reset
# Set up directory.
output_folder = basic.set_path(output_folder, kind="dir", expect=True)
results_folder = pathlib.Path(RESULTS_FOLDER)
results_path = basic.make_new_dir(output_folder,
results_folder,
attempt=10)
if results_path is None:
print("Unable to create output_folder.")
sys.exit(1)
log_file = pathlib.Path(results_path, log_file)
# Set up root logger.
logging.basicConfig(
filename=log_file,
filemode="w",
level=logging.DEBUG,
format="pdm_utils find_domains: %(levelname)s: %(message)s")
logger.info(f"pdm_utils version: {VERSION}")
logger.info(f"CDD run date: {constants.CURRENT_DATE}")
logger.info(f"Command line arguments: {' '.join(argument_list)}")
logger.info(f"Results directory: {results_path}")
# Early exit if either 1) cdd_name == "" or 2) no rpsblast given and we are
# unable to find one
if cdd_name == "":
msg = (f"Unable to learn CDD database name. Make sure the files in "
f"{cdd_dir} all have the same basename.")
logger.error(msg)
print(msg)
return
# Get the rpsblast command and path.
if rpsblast == "":
command = get_rpsblast_command()
rpsblast = get_rpsblast_path(command)
# Verify database connection and schema compatibility.
engine = mysqldb.connect_to_db(database)
logger.info(f"Connected to database: {database}.")
mysqldb.check_schema_compatibility(engine, "the find_domains pipeline")
logger.info(f"Schema version is compatible.")
logger.info("Command line arguments verified.")
if reset:
logger.info("Clearing all domain data currently in the database.")
clear_domain_data(engine)
# Get gene data that needs to be processed
# in dict format where key = column name, value = stored value.
# result = engine.execute(GET_GENES_FOR_CDD)
cdd_genes = mysqldb.query_dict_list(engine, GET_GENES_FOR_CDD)
msg = f"{len(cdd_genes)} genes to search for conserved domains..."
logger.info(msg)
print(msg)
# Only run the pipeline if there are genes returned that need it
if len(cdd_genes) > 0:
log_gene_ids(cdd_genes)
# Create temp_dir
make_tempdir(tmp_dir)
# TODO dev
# translations = get_unique_translations(cdd_genes)
# Build jobs list
jobs = []
# TODO dev
# translation_id = 0
# for translation in translations:
# translation_id += 1
# jobs.append((rpsblast, cdd_name, tmp_dir, evalue,
# translation_id, translation))
for cdd_gene in cdd_genes:
jobs.append((rpsblast, cdd_name, tmp_dir, evalue,
cdd_gene["GeneID"], cdd_gene["Translation"]))
results = parallelize(jobs, threads, search_and_process)
print("\n")
# TODO dev
# results_dict = create_results_dict(results)
# map_results_to_genes(cdd_genes, results_dict)
insert_domain_data(engine, results)
engine.dispose()
return
def main(argument_list):
"""
:param argument_list:
:return:
"""
# Setup argument parser
cdd_parser = setup_argparser()
# Use argument parser to parse argument_list
args = cdd_parser.parse_args(argument_list)
# Store arguments in more easily accessible variables
database = args.database
cdd_dir = expand_path(args.cdd)
cdd_name = learn_cdd_name(cdd_dir)
threads = args.threads
evalue = args.evalue
rpsblast = args.rpsblast
tmp_dir = args.tmp_dir
output_folder = args.output_folder
reset = args.reset
batch_size = args.batch_size
# Set up directory.
output_folder = basic.set_path(output_folder, kind="dir", expect=True)
results_folder = pathlib.Path(RESULTS_FOLDER)
results_path = basic.make_new_dir(output_folder, results_folder,
attempt=50)
if results_path is None:
print("Unable to create output_folder.")
sys.exit(1)
log_file = pathlib.Path(results_path, MAIN_LOG_FILE)
# Set up root logger.
logging.basicConfig(filename=log_file, filemode="w", level=logging.DEBUG,
format="pdm_utils find_domains: %(levelname)s: %(message)s")
logger.info(f"pdm_utils version: {VERSION}")
logger.info(f"CDD run date: {constants.CURRENT_DATE}")
logger.info(f"Command line arguments: {' '.join(argument_list)}")
logger.info(f"Results directory: {results_path}")
# Early exit if either 1) cdd_name == "" or 2) no rpsblast given and we are
# unable to find one
if cdd_name == "":
msg = (f"Unable to learn CDD database name. Make sure the files in "
f"{cdd_dir} all have the same basename.")
logger.error(msg)
print(msg)
return
# Get the rpsblast command and path.
if rpsblast == "":
command = get_rpsblast_command()
rpsblast = get_rpsblast_path(command)
# Verify database connection and schema compatibility.
alchemist = AlchemyHandler(database=database)
alchemist.connect(pipeline=True)
engine = alchemist.engine
logger.info(f"Connected to database: {database}.")
mysqldb.check_schema_compatibility(engine, "the find_domains pipeline")
logger.info(f"Schema version is compatible.")
logger.info("Command line arguments verified.")
if reset:
logger.info("Clearing all domain data currently in the database.")
clear_domain_data(engine)
# Get gene data that needs to be processed
# in dict format where key = column name, value = stored value.
cdd_genes = mysqldb_basic.query_dict_list(engine, GET_GENES_FOR_CDD)
msg = f"{len(cdd_genes)} genes to search for conserved domains..."
logger.info(msg)
print(msg)
# Only run the pipeline if there are genes returned that need it
if len(cdd_genes) > 0:
log_gene_ids(cdd_genes)
make_tempdir(tmp_dir)
# Identify unique translations to process mapped to GeneIDs.
cds_trans_dict = create_cds_translation_dict(cdd_genes)
unique_trans = list(cds_trans_dict.keys())
msg = (f"{len(unique_trans)} unique translations "
"to search for conserved domains...")
logger.info(msg)
print(msg)
# Process translations in batches. Otherwise, searching could take
# so long that MySQL connection closes resulting in 1 or more
# transaction errors.
batch_indices = basic.create_indices(unique_trans, batch_size)
total_rolled_back = 0
for indices in batch_indices:
start = indices[0]
stop = indices[1]
msg = f"Processing translations {start + 1} to {stop}..."
logger.info(msg)
print(msg)
sublist = unique_trans[start:stop]
batch_rolled_back = search_translations(
rpsblast, cdd_name, tmp_dir, evalue,
threads, engine, sublist, cds_trans_dict)
total_rolled_back += batch_rolled_back
search_summary(total_rolled_back)
engine.dispose()
return
def main(unparsed_args_list):
"""Run main freeze database pipeline."""
args = parse_args(unparsed_args_list)
ref_database = args.database
reset = args.reset
new_database = args.new_database_name
prefix = args.prefix
# Filters input: phage.Status=draft AND phage.HostGenus=Mycobacterium
# Args structure: [['phage.Status=draft'], ['phage.HostGenus=Mycobacterium']]
filters = args.filters
# Create config object with data obtained from file and/or defaults.
config = configfile.build_complete_config(args.config_file)
mysql_creds = config["mysql"]
# Verify database connection and schema compatibility.
print("Connecting to the MySQL database...")
alchemist1 = AlchemyHandler(database=ref_database,
username=mysql_creds["user"],
password=mysql_creds["password"])
alchemist1.connect(pipeline=True)
engine1 = alchemist1.engine
mysqldb.check_schema_compatibility(engine1, "the freeze pipeline")
# Get SQLAlchemy metadata Table object
# table_obj.primary_key.columns is a
# SQLAlchemy ColumnCollection iterable object
# Set primary key = 'phage.PhageID'
alchemist1.build_metadata()
table = querying.get_table(alchemist1.metadata, TARGET_TABLE)
for column in table.primary_key.columns:
primary_key = column
# Create filter object and then add command line filter strings
db_filter = Filter(alchemist=alchemist1, key=primary_key)
db_filter.values = []
# Attempt to add filters and exit if needed.
add_filters(db_filter, filters)
# Performs the query
db_filter.update()
# db_filter.values now contains list of PhageIDs that pass the filters.
# Get the number of genomes that will be retained and build the
# MYSQL DELETE statement.
keep_set = set(db_filter.values)
delete_stmt = construct_delete_stmt(TARGET_TABLE, primary_key, keep_set)
count_query = construct_count_query(TARGET_TABLE, primary_key, keep_set)
phage_count = mysqldb_basic.scalar(alchemist1.engine, count_query)
# Determine the name of the new database.
if new_database is None:
if prefix is None:
prefix = get_prefix()
new_database = f"{prefix}_{phage_count}"
# Create the new database, but prevent overwriting of current database.
if engine1.url.database != new_database:
result = mysqldb_basic.drop_create_db(engine1, new_database)
else:
print(
"Error: names of the reference and frozen databases are the same.")
print("No database will be created.")
result = 1
# Copy database.
if result == 0:
print(f"Reference database: {ref_database}")
print(f"New database: {new_database}")
result = mysqldb_basic.copy_db(engine1, new_database)
if result == 0:
print(f"Deleting genomes...")
alchemist2 = AlchemyHandler(database=new_database,
username=engine1.url.username,
password=engine1.url.password)
alchemist2.connect(pipeline=True)
engine2 = alchemist2.engine
engine2.execute(delete_stmt)
if reset:
engine2.execute(RESET_VERSION)
# Close up all connections in the connection pool.
engine2.dispose()
else:
print("Unable to copy the database.")
# Close up all connections in the connection pool.
engine1.dispose()
else:
print(f"Error creating new database: {new_database}.")
print("Freeze database script completed.")
def main(unparsed_args_list):
"""Run main retrieve_updates pipeline."""
# Parse command line arguments
args = parse_args(unparsed_args_list)
date = time.strftime("%Y%m%d")
args.output_folder = basic.set_path(args.output_folder,
kind="dir",
expect=True)
working_dir = pathlib.Path(f"{date}_get_data")
working_path = basic.make_new_dir(args.output_folder,
working_dir,
attempt=10)
if working_path is None:
print(f"Invalid working directory '{working_dir}'")
sys.exit(1)
ncbi_cred_dict = ncbi.get_ncbi_creds(args.ncbi_credentials_file)
# Verify database connection and schema compatibility.
print("Preparing genome data sets from the MySQL database...")
engine = mysqldb.connect_to_db(args.database)
mysqldb.check_schema_compatibility(engine, "the get_data pipeline")
# Get existing data from MySQL to determine what needs to be updated.
query = ("SELECT PhageID, Name, HostGenus, Status, Cluster, "
"DateLastModified, Accession, RetrieveRecord, Subcluster, "
"AnnotationAuthor FROM phage")
mysqldb_genome_list = mysqldb.parse_genome_data(engine=engine,
phage_query=query,
gnm_type="mysqldb")
engine.dispose()
mysqldb_genome_dict = {}
for gnm in mysqldb_genome_list:
mysqldb_genome_dict[gnm.id] = gnm
# Get data from PhagesDB
if (args.updates or args.final or args.draft) is True:
print("Retrieving data from PhagesDB...")
phagesdb_phages = phagesdb.get_phagesdb_data(constants.API_SEQUENCED)
phagesdb_phages_dict = basic.convert_list_to_dict(
phagesdb_phages, "phage_name")
phagesdb_genome_dict = phagesdb.parse_genomes_dict(
phagesdb_phages_dict, gnm_type="phagesdb", seq=False)
# Exit if all phage data wasn't retrieved.
if len(phagesdb_genome_dict) == 0:
sys.exit(1)
# Returns a list of tuples.
match_output = match_genomes(mysqldb_genome_dict, phagesdb_genome_dict)
matched_genomes = match_output[0]
unmatched_phagesdb_ids = match_output[1]
if args.updates is True:
get_update_data(working_path, matched_genomes)
if args.final is True:
get_final_data(working_path, matched_genomes)
if args.genbank is True:
get_genbank_data(working_path, mysqldb_genome_dict, ncbi_cred_dict,
args.genbank_results)
if args.draft is True:
get_draft_data(working_path, unmatched_phagesdb_ids)
print("\n\n\nRetrieve updates script completed.")