def output_genbank_summary(output_folder, results):
"""Save summary of GenBank retrieval results to file."""
filepath = pathlib.Path(output_folder, GENBANK_RESULTS_TABLE)
fileio.export_data_dict(results,
filepath,
NCBI_RESULTS_COLUMNS,
include_headers=True)
def write_clustering_update_ticket(working_dir, scheme_alterations,
field="Cluster", filename=None):
if filename is None:
filename = working_dir.with_suffix(".csv").name
update_dicts = []
for cluster, diff_data in scheme_alterations.items():
for data_dict in diff_data:
update_dict = {}
if cluster is None:
cluster = "NULL"
update_data = ("phage", field, cluster, "PhageID", data_dict["id"])
for i in range(len(TICKET_HEADER)):
update_dict[TICKET_HEADER[i]] = update_data[i]
update_dicts.append(update_dict)
if not update_dicts:
return False
filepath = working_dir.joinpath(filename)
pdm_fileio.export_data_dict(update_dicts, filepath, TICKET_HEADER,
include_headers=True)
return True
def create_ticket_table(tickets, output_folder):
"""Save tickets associated with retrieved from GenBank files."""
filepath = pathlib.Path(output_folder, IMPORT_TABLE)
tickets = convert_tickets_to_dict(tickets)
fileio.export_data_dict(tickets,
filepath,
IMPORT_COLUMNS,
include_headers=True)
def execute_csv_export(db_filter, export_path, folder_path, columns, csv_name,
data_cache=None, sort=[], raw_bytes=False,
verbose=False, dump=False):
"""Executes csv export of a MySQL database table with select columns.
:param db_filter: A connected and fully built Filter object.
:type db_filter: Filter
:param export_path: Path to a dir for file creation.
:type export_path: Path
:param folder_path: Path to a top-level dir.
:type folder_path: Path
:param table: MySQL table name.
:type table: str
:param conditionals: MySQL WHERE clause-related SQLAlchemy objects.
:type conditionals: list[BinaryExpression]
:param sort: A list of SQLAlchemy Columns to sort by.
:type sort: list[Column]
:param values: List of values to fitler database results.
:type values: list[str]
:param verbose: A boolean value to toggle progress print statements.
:type verbose: bool
:param dump: A boolean value to toggle dump in current working dir.
:type dump: bool
"""
if data_cache is None:
data_cache = {}
if verbose:
relative_path = str(export_path.relative_to(folder_path))
print(f"Preparing {csv_name} export for '{relative_path}'...")
headers = [db_filter._key.name]
for column in columns:
if column.name != db_filter._key.name:
headers.append(column.name)
results = db_filter.select(columns)
if not raw_bytes:
decode_results(results, columns, verbose=verbose)
if len(results) == 0:
print(f"No database entries received for {csv_name}.")
if not dump:
export_path.rmdir()
else:
if dump:
if export_path != folder_path:
export_path.rmdir()
export_path = export_path.parent
if verbose:
print(f"...Writing csv {csv_name}.csv in '{export_path.name}'...")
file_path = export_path.joinpath(f"{csv_name}.csv")
fileio.export_data_dict(results, file_path, headers,
include_headers=True)
def write_report(data, export_path, header, csv_name="Report", verbose=False):
"""Outputs a csv file
"""
if not export_path.is_dir():
print("Passed in path is not a directory.")
sys.exit(1)
file_path = export_path.joinpath(f"{csv_name}.csv")
if verbose:
print(f"Writing {file_path.name} in {export_path.name}...")
fileio.export_data_dict(data, file_path, header, include_headers=True)
def setUpClass(self):
test_db_utils.create_filled_test_db()
self.test_dir = Path(TEST_DIR)
if self.test_dir.is_dir():
shutil.rmtree(TEST_DIR)
self.test_dir.mkdir()
self.revise_form = self.test_dir.joinpath("revise_form.txt")
fileio.export_data_dict(TEST_FR_DATA,
self.revise_form,
REVIEW_HEADER,
include_headers=True)
def write_cluster_analysis(intracluster_edges, working_dir, file_name=None):
data_dicts = []
for edge in intracluster_edges:
data_dict = {}
for i in range(len(edge)):
data_dict[CLUSTER_ANALYSIS_HEADER[i]] = edge[i]
data_dicts.append(data_dict)
if file_name is None:
file_name = working_dir.name
filepath = working_dir.joinpath(file_name).with_suffix(".csv")
pdm_fileio.export_data_dict(data_dicts, filepath, CLUSTER_ANALYSIS_HEADER,
include_headers=True)
def setUp(self):
self.alchemist = AlchemyHandler()
self.alchemist.username = USER
self.alchemist.password = PWD
self.alchemist.database = DB
self.alchemist.connect(ask_database=True, login_attempts=0)
self.revise_test_dir = self.test_dir.joinpath("revise_test_dir")
self.fr_input_file_path = self.test_dir.joinpath("FunctionReport.csv")
self.csv_input_file_path = self.revise_test_dir.joinpath("gene.csv")
fileio.export_data_dict(TEST_FR_DATA,
self.fr_input_file_path,
REVIEW_HEADER,
include_headers=True)
self.assertTrue(self.fr_input_file_path.is_file())
def write_revise_file(data_dicts,
output_path,
file_format="p_curation",
file_name=CURATION_NAME,
verbose=False):
"""Writes a revision csv in the desired file format with necessary changes.
:param data_dicts: List of data dictionaries to convert to curation format.
:type data_dicts: list[dict]
:param output_path: Path to a dir for file creation.
:type output_path: Path
:param file_format: Format of the csv to be written.
:type file_format: str
:param file_name: Name of the file to write curation data to.
:type file_name: str
:param verbose: A boolean value to toggle progress print statements.
:type verbose: bool
"""
data_dicts = sorted(data_dicts, key=lambda data_dict: data_dict["PhageID"])
if file_format == "p_curation":
format_data = format_curation_data
elif file_format == "ticket":
format_data = format_update_ticket_data
else:
raise ValueError(f"File format '{file_format}' is not "
"recognized for revise output.")
for d in data_dicts:
format_data(d)
if verbose:
print(f"Writing {file_name} in {output_path.name}...")
file_path = output_path.joinpath(file_name)
if file_format == "p_curation":
header = CURATION_HEADER
elif file_format == "ticket":
header = TICKET_HEADER
fileio.export_data_dict(data_dicts,
file_path,
header,
include_headers=True)
def test_export_data_dict_1(self):
"""Verify data is exported correctly."""
list_of_data = [self.tkt_dict1, self.tkt_dict2]
headers = ["type", "phage_id", "host_genus", "cluster"]
fileio.export_data_dict(list_of_data,
self.data_dict_file,
headers,
include_headers=True)
exp_success_tkts = []
with open(self.data_dict_file, 'r') as file:
file_reader = csv.DictReader(file)
for dict in file_reader:
exp_success_tkts.append(dict)
with self.subTest():
self.assertEqual(len(exp_success_tkts), 2)
with self.subTest():
self.assertEqual(set(exp_success_tkts[0].keys()), set(headers))
def get_update_data(output_folder, matched_genomes):
"""Run sub-pipeline to retrieve field updates from PhagesDB."""
updates_folder = pathlib.Path(output_folder, UPDATES_FOLDER)
updates_folder.mkdir()
update_tickets = []
for gnm_pair in matched_genomes:
tkt_list = compare_data(gnm_pair)
update_tickets.extend(tkt_list)
# Field updates
if len(update_tickets) > 0:
print(f"\n\n{len(update_tickets)} field updates are available.")
filepath = pathlib.Path(updates_folder, UPDATE_TABLE)
fileio.export_data_dict(update_tickets,
filepath,
UPDATE_COLUMNS,
include_headers=True)
else:
print("\n\nNo field updates.")
# Now remove empty folders.
if len(basic.identify_contents(updates_folder, kind=None)) == 0:
updates_folder.rmdir()
def execute_remote_revise(alchemist,
folder_path=None,
folder_name=DEFAULT_FOLDER_NAME,
config=None,
output_type="p_curation",
values=None,
filters="",
groups=[],
verbose=False,
force=False):
ncbi_creds = {}
if config is not None:
ncbi_creds = config["ncbi"]
db_filter = pipelines_basic.build_filter(alchemist,
"phage",
filters,
values=values,
verbose=verbose)
db_filter.add(BASE_CONDITIONALS)
revise_path = pipelines_basic.create_working_path(folder_path,
folder_name,
force=force)
conditionals_map = pipelines_basic.build_groups_map(db_filter,
revise_path,
groups=groups,
verbose=verbose,
force=force)
values = db_filter.values
for mapped_path in conditionals_map.keys():
db_filter.reset()
db_filter.values = values
conditionals = conditionals_map[mapped_path]
db_filter.values = db_filter.build_values(where=conditionals)
if db_filter.hits() == 0:
print(f"No database entries received for '{mapped_path}'.")
pipelines_basic.create_working_dir(mapped_path, force=force)
build_revise_log_file(mapped_path)
logger.info(f"pdm_utils version: {VERSION}")
logger.info(f"Revise run date: {CURRENT_DATE}")
logger.info(f"Connected to database: {alchemist.database}")
accession_data = db_filter.select(["phage.PhageID", "phage.Accession"])
acc_id_dict = {}
for data_dict in accession_data:
accession = data_dict["Accession"]
if not (accession is None or accession == ""):
acc_id_dict[accession] = data_dict["PhageID"]
tbl_records = get_tbl_records(acc_id_dict, ncbi_cred_dict=ncbi_creds)
validated_phages = []
for tbl_record in tbl_records:
validated_phages.append(tbl_record.name)
id_record_map = build_id_record_map(alchemist, validated_phages)
if output_type == "tbl":
revised_records = revise_seqrecords(id_record_map,
tbl_records,
verbose=verbose)
if not revised_records:
print("No discrepancies detected between "
f"local data and GenBank data for '{mapped_path}'.")
continue
elif output_type == "p_curation":
curation_data_dicts = find_product_discrepancies(id_record_map,
tbl_records,
verbose=verbose)
if not curation_data_dicts:
print("No discrepancies detected between "
f"local data and GenBank data for '{mapped_path}'.")
continue
if output_type == "tbl":
fileio.write_feature_table(revised_records,
mapped_path,
verbose=verbose)
elif output_type == "p_curation":
file_path = mapped_path.joinpath("revise.csv")
fileio.export_data_dict(curation_data_dicts,
file_path,
CURATION_HEADER,
include_headers=True)
def execute_pham_finder(alchemist, folder_path, folder_name,
adatabase, bdatabase, values=None,
filters="", groups=[], sort=[],
show_per=False, use_locus=False, verbose=False):
"""Executes the entirety of the file export pipeline.
:param alchemist: A connected and fully build AlchemyHandler object.
:type alchemist: AlchemyHandler
:param folder_path: Path to a valid dir for new dir creation.
:type folder_path: Path
:param folder_name: A name for the export folder.
:type folder_name: str
:param adatabase: Name of reference database to source phams from.
:type adatabase: str
:param bdatabase: Name of database to find corresponding phams for.
:type bdatabase: str
:param values: List of values to filter database results:
:type values: list[str]
:param verbose: A boolean value to toggle progress print statements.
:type verbose: bool
:param table: MySQL table name.
:type table: str
:param filters: A list of lists with filter values, grouped by ORs.
:type filters: str
:param groups: A list of supported MySQL column names to group by.
:type groups: A list of supported MySQL column names to group by.
:type groups: list[str]
:param sort: A list of supported MySQL column names to sort by.
:type sort: list[str]
:param show_per: Enables display gene coverage of the corresponding phams.
:type show_per: bool
:param use_locus: Toggles conversion between phams using LocusTag instead
:type use_locus: bool
"""
if not (adatabase in alchemist.databases and \
bdatabase in alchemist.databases):
print("User credentials does not have access to both "
f"databases {adatabase} and {bdatabase}.\n"
"Please check your database access and try again.")
sys.exit(1)
alchemist.database = adatabase
alchemist.connect()
a_filter = pipelines_basic.build_filter(alchemist, "gene.PhamID", filters,
values=values, verbose=verbose)
alchemist.database = bdatabase
alchemist.connect()
if use_locus:
b_filter = pipelines_basic.build_filter(alchemist, "gene.LocusTag", "")
else:
b_filter = pipelines_basic.build_filter(alchemist, "gene", "")
if sort:
try:
a_filter.sort(sort)
except:
print("Please check your syntax for sorting columns:\n"
f"{', '.join(sort)}")
sys.exit(1)
if verbose:
print("Creating pham_finder folder...")
export_path = folder_path.joinpath(folder_name)
export_path = basic.make_new_dir(folder_path, export_path, attempt=50)
conditionals_map = {}
pipelines_basic.build_groups_map(a_filter, export_path,
conditionals_map,
groups=groups,
verbose=verbose)
if verbose:
print("Prepared query and path structure, beginning export...")
values = a_filter.values
for mapped_path in conditionals_map.keys():
a_filter.reset()
a_filter.values = values
conditionals = conditionals_map[mapped_path]
a_filter.values = a_filter.build_values(where=conditionals)
if a_filter.hits() == 0:
print("No database entries received from gene.PhamID "
f"for '{mapped_path}'.")
shutil.rmtree(mapped_path)
continue
if sort:
sort_columns = get_sort_columns(alchemist, sort)
a_filter.sort(sort_columns)
mapped_phams = find_phams(a_filter, b_filter, show_per=show_per)
if not mapped_phams:
print("Phams are consistent between the two databases "
f"for '{mapped_path}'.")
shutil.rmtree(mapped_path)
continue
out_data_dicts = []
for ref_pham, corr_phams in mapped_phams.items():
data_dict = {}
data_dict[PHAM_FINDER_HEADER[0]] = ref_pham
data_dict[PHAM_FINDER_HEADER[1]] = corr_phams
out_data_dicts.append(data_dict)
file_path = mapped_path.joinpath("PhamMap.csv")
fileio.export_data_dict(out_data_dicts, file_path, PHAM_FINDER_HEADER,
include_headers=True)