def __init__ (self, *arg, backup_dir = None, backup_subdir = '', log_file = '', file_limit = 10, day_cutoff = 120, **kw): super(Backups, self).__init__(*arg, **kw) self.backup_dir = os.path.join(backup_dir, backup_subdir) self.log_file = log_file self.file_limit = int(file_limit) self.str_format = '%Y-%m-%d' self.date = datetime.datetime.now() self.date_cutoff = self.date - datetime.timedelta(days = day_cutoff) self.oldest_backup = None self.most_recent_backup = None # Start the logger startLogger(self.log_file, filemode = 'a') # Check if data was not assigned if not self: self.assignBackups()
def main():
# Assign the demultiplex args
demultiplex_args = deMultiplexParser()
# Assign the i7 map path from the package
if not demultiplex_args.i7_map:
i7_map_path = pkg_resources.resource_filename('kocher_tools',
'data/i7_map.txt')
if not os.path.exists(i7_map_path):
raise IOError('Cannot assign i7 map from package')
demultiplex_args.i7_map = i7_map_path
# Create the log file and log the args
startLogger(demultiplex_args.out_log)
logArgs(demultiplex_args)
# Check for previous output
if os.path.exists(demultiplex_args.out_dir):
if demultiplex_args.overwrite: shutil.rmtree(demultiplex_args.out_dir)
else:
raise Exception(
f'{demultiplex_args.out_dir} already exists. Please alter --out-dir or use --overwrite'
)
# Create the multiplex job
demultiplex_job = Multiplex()
# Assign the output path for all multiplex files
demultiplex_job.assignOutputPath(demultiplex_args.out_dir)
# Assign the read file for the multiplex job
demultiplex_job.assignFiles(demultiplex_args.i5_read_file,
demultiplex_args.i7_read_file,
demultiplex_args.R1_read_file,
demultiplex_args.R2_read_file)
# Assign the plate using the i5 map
demultiplex_job.assignPlates(demultiplex_args.i5_map)
logging.info('Starting i5 deMultiplex')
# Run the i5 barcode job using the i5 map
demultiplex_job.deMultiplex(demultiplex_args.i5_map)
logging.info('Finished i5 deMultiplex')
# Move the plates into directories of their plate and locus names
demultiplex_job.movePlates()
# Remove unmatched files (this should be an option in beta)
demultiplex_job.removeUnmatched()
# Loop the plates in the multiplex job
for plate in demultiplex_job:
# Assign the well of the current plate
plate.assignWells()
logging.info(f'Starting {plate.name} i7 deMultiplex')
# Run the i7 barcode job using the i7 map
plate.deMultiplexPlate(demultiplex_args.i7_map)
logging.info(f'Finished {plate.name} i7 deMultiplex')
# Move the wells into the Wells directory. Should this be user specified?
plate.moveWells()
# Remove any unmatched files for the current plate
plate.removeUnmatchedPlate()
# Define args
db_parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
db_parser.add_argument('--yaml',
help="Database YAML config file",
type=str,
required=True,
action=confirmFile())
db_parser.add_argument('--out-log',
help='Filename of the log file',
type=str,
default='create_database.log')
db_args = db_parser.parse_args()
# Start a log file for this run
startLogger(log_filename=db_args.out_log)
# Log the arguments used
logArgs(db_args)
# Read in the YAML config file
db_config_data = ConfigDB.readConfig(db_args.yaml)
# Get the password, if not found
if db_config_data.passwd_required and not db_config_data.passwd:
db_config_data.passwd = getpass.getpass("Password: ")
# Create the tables
sql_engine = createEngineFromConfig(db_config_data)
createAllFromConfig(db_config_data, sql_engine)
def main():
# Assign the barcode args
barcode_args = barcodePipelineParser()
# Check if no i7 map was assigned
if not barcode_args.i7_map:
# Assign the i7 map path from the package
i7_map_path = pkg_resources.resource_filename('kocher_tools',
'data/i7_map.txt')
# Check if i7 map does not exists
if not os.path.exists(i7_map_path):
# Return an error
raise IOError('Cannot assign i7 map from package')
# Assign the i7 map
barcode_args.i7_map = i7_map_path
# Create the log file
startLogger(barcode_args.out_log)
# Log the arguments used
logArgs(barcode_args)
# Check for previous output
if os.path.exists(barcode_args.out_dir):
# Check if previous output should be overwritten
if barcode_args.overwrite:
# Remove the previous output
shutil.rmtree(barcode_args.out_dir)
# Check if previous output shouldn't be overwritten
else:
# Raise an exception
raise Exception(
'%s already exists. Please alter --out-dir or use --overwrite'
% barcode_args.out_dir)
# Create the multiplex job
demultiplex_job = Multiplex()
# Assign the output path for all multiplex files
demultiplex_job.assignOutputPath(barcode_args.out_dir)
# Assign the read file for the multiplex job
demultiplex_job.assignFiles(barcode_args.i5_read_file,
barcode_args.i7_read_file,
barcode_args.R1_read_file,
barcode_args.R2_read_file)
# Assign the plate using the i5 map
demultiplex_job.assignPlates(barcode_args.i5_map)
logging.info('Starting i5 deMultiplex')
# Run the i5 barcode job using the i5 map
demultiplex_job.deMultiplex(barcode_args.i5_map)
logging.info('Finished i5 deMultiplex')
# Move the plates into directories of their plate and locus names
demultiplex_job.movePlates()
# Remove unmatched files (this should be an option in beta)
demultiplex_job.removeUnmatched()
# Loop the plates in the multiplex job
for plate in demultiplex_job:
# Assign the well of the current plate
plate.assignWells()
logging.info('Starting %s i7 deMultiplex' % plate.name)
# Run the i7 barcode job using the i7 map
plate.deMultiplexPlate(barcode_args.i7_map)
logging.info('Finished %s i7 deMultiplex' % plate.name)
# Move the wells into the Wells directory. Should this be user specified?
plate.moveWells()
# Remove any unmatched files for the current plate
plate.removeUnmatchedPlate()
logging.info('Starting %s abundant reads identification' % plate.name)
# Loop each well
for well in plate:
# Merge the R1/R2 files for the current well
well.mergeWell()
# Truncate the merged file for the current well
well.truncateWell()
# Filter the truncated file for the current well
well.filterWell()
# Dereplicate the filtered file for the current well
well.dereplicateWell()
# Cluster the dereplicated file for the current well
well.clusterWell()
# Identify the most abundant reads
well.mostAbundantWell()
logging.info('Finished %s abundant reads identification' % plate.name)
# Assign the compiled file path
compiled_file_path = os.path.join(barcode_args.out_dir,
barcode_args.out_compiled)
# Assign the blast output file path
blast_file_path = os.path.join(barcode_args.out_dir,
barcode_args.out_blast)
# Compile the most abundant reads into a single file
demultiplex_job.compileMostAbundant(compiled_file_path)
logging.info('Starting BLAST')
# Get the top BLAST hits for each sequence in the compiled file
blastTopHits(compiled_file_path, blast_file_path,
barcode_args.blast_database, barcode_args.threads)
logging.info('Finished BLAST')
def main():
# Assign arguments
upload_args = upload_sample_parser()
# Check if log should be sent to stdout
if upload_args.log_stdout:
# Start logging to stdout for this run
startLogger()
else:
# Start a log file for this run
startLogger(log_filename=upload_args.out_log)
# Read in the database config file
db_config_data = readConfig(upload_args.yaml)
# Log the arguments used
logArgs(upload_args)
# Check if the backup process should be skipped
if upload_args.no_backup:
logging.info('Skipping backup procedure')
else:
# Load the current backups
current_backups = Backups(
out_dir=db_config_data.backup_out_dir,
limit=db_config_data.backup_limit,
update_freq=db_config_data.backup_update_freq)
# Check if a backup is required
if current_backups.backupNeeded() or upload_args.backup:
# Backup the database
current_backups.newBackup(upload_args.sqlite_db)
# Connect to the sqlite database
sqlite_connection = sqlite3.connect(upload_args.sqlite_db)
# Setup SQLite to reture the rows as dict with columns
sqlite_connection.row_factory = sqlite3.Row
# Create the cursor
cursor = sqlite_connection.cursor()
# Check if a collection app file has been specified
if upload_args.app_files:
# Loop the collection app files
for app_file in upload_args.app_files:
# Check if a selection key has been specified
if upload_args.update_with_file:
# Check if the Collection should be updated
if upload_args.app_upload_method in ['Collection', 'Both']:
# Update the database with the file
updateAppCollectionToDatabase(cursor, 'Collection',
'Unique ID', app_file)
# Check if the Collection should be updated
if upload_args.app_upload_method in ['Location', 'Both']:
# Update the database with the file
updateAppLocationsToDatabase(cursor, 'Locations',
'Site Code', app_file)
else:
# Check if the Collection should be added
if upload_args.app_upload_method in ['Collection', 'Both']:
# Add file to the database
addAppCollectionToDatabase(cursor, 'Collection', app_file)
# Check if the Collection should be updated
if upload_args.app_upload_method in ['Location', 'Both']:
# Add file to the database
addAppLocationsToDatabase(cursor, 'Locations', app_file)
# Check if barcode files have been specified
if upload_args.barcode_files:
# Loop the collection app files
for barcode_files in upload_args.barcode_files:
# Check if only a pair of files was passed
if len(barcode_files) == 2:
# Assign the blast and fasta file
blast_file, fasta_file = barcode_files
# Set the failed file to None
failed_file = None
# Check if all files was passed
elif len(barcode_files) == 3:
# Assign the blast and fasta file
blast_file, fasta_file, failed_file = barcode_files
# Check if a selection key has been specified
if upload_args.update_with_file:
# Update the database with the file
updateSeqFilesToDatabase(cursor, 'Sequencing', 'Sequence ID',
blast_file, fasta_file, failed_file,
'Storage', 'Storage ID')
else:
# Add file to the database
addSeqFilesToDatabase(cursor, 'Sequencing', blast_file,
fasta_file, failed_file, 'Storage',
'Storage ID')
# Check if a location file has been specified
if upload_args.loc_files:
# Loop the location files
for loc_file in upload_args.loc_files:
# Check if a selection key has been specified
if upload_args.update_with_file:
# Update the database with the file
updateLocFileToDatabase(cursor, 'Locations', 'Site Code',
loc_file)
else:
# Add file to the database
addLocFileToDatabase(cursor, 'Locations', loc_file)
# Check if a storage file has been specified
if upload_args.storage_files:
# Loop the storage files
for storage_file in upload_args.storage_files:
# Check if a selection key has been specified
if upload_args.update_with_file:
# Update the database with the file
updateStorageFileToDatabase(cursor, 'Storage', 'Storage ID',
storage_file)
else:
# Add file to the database
addStorageFileToDatabase(cursor, 'Storage', storage_file)
# Check if update data has been specified
if upload_args.update:
# Check if the update is impossible
if not canUpdate(upload_args.update):
raise Exception('Not allowed to alter: %s' % column)
# Assign the tables that need to be updated
selection_tables = assignTables(db_config_data, **vars(upload_args))
# Assign a defaultdict with all the selection information
selection_dict = assignSelectionDict(db_config_data,
**vars(upload_args))
# Create a list of the tables to update
tables_to_update = db_config_data.returnTables(
list(upload_args.update.keys()))
# Loop the tables that need to be updated
for table_to_update in tables_to_update:
# Create the update statement dict for the current table
update_statement_dict = db_config_data.returnColumnDict(
upload_args.update, table_to_update)
# Check if the current table is within the tables to join
if table_to_update in selection_tables and len(
selection_tables) == 1:
# Update the specified data
updateValues(cursor, table_to_update, selection_dict,
update_statement_dict)
# Run the expanded command if there are tables to join
else:
# Assign the key for the table to update
join_by_column = db_config_data[table_to_update].join_by_key
# Create a copy of the selection tables
tables_to_join = copy.copy(selection_tables)
# Add the table that is going to be updated
tables_to_join.append(table_to_update)
# Remove any duplicates
tables_to_join = list(set(tables_to_join))
# Assign the tables and keys that need to be joined
join_by_names, join_by_columns = db_config_data.returnJoinLists(
tables_to_join)
# Update the specified data
updateValues(cursor,
table_to_update,
selection_dict,
update_statement_dict,
update_table_column=join_by_column,
tables_to_join=join_by_names,
join_table_columns=join_by_columns)
# Commit any changes
sqlite_connection.commit()
# Close the connection
cursor.close()
def main():
# Assign arguments
retrieve_args = retrieveSampleParser()
# Create string to hold the output filename
out_filename = ''
# Check if an output filename was specified
if retrieve_args.out:
# Assign the out filename
out_filename = retrieve_args.out
# Assign the filename with --out-prefix otherwise
else:
# Assign the out filename
out_filename = retrieve_args.out_prefix + '.' + retrieve_args.out_format
# Check that the output mode isn't stdout
if not retrieve_args.stdout:
# Check if previous output should be overwritten
if retrieve_args.overwrite:
# Remove the previous output, if it exists
if os.path.exists(out_filename):
os.remove(out_filename)
# Check if previous output shouldn't be overwritten
else:
# Check if previous output exists
if os.path.exists(out_filename):
# Raise an exception
raise Exception(
'Output already exists. Please alter the output arguments or use --overwrite'
)
# Check if log should be sent to stdout
if retrieve_args.log_stdout:
# Start logging to stdout for this run
startLogger()
else:
# Start a log file for this run
startLogger(log_filename=retrieve_args.out_log)
# Read in the database config file
db_config_data = readConfig(retrieve_args.yaml)
# Log the arguments used
logArgs(retrieve_args)
# List to hold columns to be printed
columns_assignment_list = []
# List to hold table to be printed, required for printing single columns
table_assignment_list = []
# Check if the user specified a table
if retrieve_args.table:
# Assign the tables
table_assignment_list.extend(retrieve_args.table)
# Loop the list of specified tables
for table_str in table_assignment_list:
# Check if the database has the specified table
if table_str not in db_config_data:
# Print an error message
raise Exception(
'Table (%s) not found. Please select from: %s' %
(table_str, ', '.join(db_config_data.tables)))
# Get the columns from the table and add to the column assignment list
columns_assignment_list.extend(
db_config_data[table_str].retieveColumnPaths(
keep_db_specific_columns=True))
# Update log
logging.info('Successfully assigned table(s) from command-line')
# Check if the user specified a column
elif retrieve_args.column:
# Assign the tables
table_assignment_list.extend(
db_config_data.returnTables(retrieve_args.column))
# Loop the list of specified column
for column_str in retrieve_args.column:
# Assign the column path
columns_assignment_list.append(
db_config_data.returnColumnPath(column_str))
# Update log
logging.info('Successfully assigned column(s) from command-line')
# Assign a defaultdict with all the selection information
selection_dict = assignSelectionDict(db_config_data, **vars(retrieve_args))
# Assign the tables that need to be updated
selection_tables = assignTables(db_config_data, **vars(retrieve_args))
# Assign the tables for retrieval
tables = table_assignment_list + selection_tables
# Remove potential duplicates
tables = list(set(tables))
# Connect to the sqlite database
sqlite_connection = sqlite3.connect(retrieve_args.sqlite_db)
# Setup SQLite to reture the rows as dict with columns
sqlite_connection.row_factory = sqlite3.Row
# Create the cursor
cursor = sqlite_connection.cursor()
# Check if only a single table is required
if len(tables) == 1:
# Retrieve the selected entries from the database
retrieved_entries = retrieveValues(cursor, tables, selection_dict,
columns_assignment_list)
# Otherwise, run the process for multiple tables
else:
# Assign the tables and keys that need to be joined
tables, join_by_columns = db_config_data.returnJoinLists(tables)
# Retrieve the selected entries from the database
retrieved_entries = retrieveValues(cursor,
tables,
selection_dict,
columns_assignment_list,
join_table_columns=join_by_columns)
# Commit any changes
sqlite_connection.commit()
# Close the connection
cursor.close()
# Set the deafult delimiter
delimiter = '\t'
# Check if the csv format was requested
if retrieve_args.out_format == 'csv':
# Update the delimiter
delimiter = ','
# Check if stdout was requested
if retrieve_args.stdout:
# Print the entries to stdout
entriesToScreen(retrieved_entries, delimiter)
else:
# Write retrieved entries to a file
entriesToFile(retrieved_entries, out_filename, delimiter)
def main():
# Assign the barcode args
barcode_args = barcodeFilterParser()
# Create the log file
startLogger(barcode_args.out_log)
# Check if a BLAST output filename was defined
if barcode_args.out_blast:
# Define the BLAST output filename
blast_out_filename = barcode_args.out_blast
# Get the output dirname
out_dirname = os.path.dirname(blast_out_filename)
# Define the JSON output filename
failed_out_filename = os.path.join(out_dirname,
barcode_args.out_failed)
# If not, define using a prefix
else:
# Get the BLAST basename
blast_basename = os.path.basename(barcode_args.blast_file)
# Get the BLAST dirname
blast_dirname = os.path.dirname(barcode_args.blast_file)
# Define the BLAST output filename
blast_out_filename = os.path.join(
blast_dirname, barcode_args.out_prefix + blast_basename)
# Define the JSON output filename
failed_out_filename = os.path.join(blast_dirname,
barcode_args.out_failed)
# Check if previous output should be overwritten
if barcode_args.overwrite:
# Remove the previous output, if it exists
if os.path.exists(blast_out_filename):
os.remove(blast_out_filename)
if os.path.exists(failed_out_filename):
os.remove(failed_out_filename)
# Check if previous output shouldn't be overwritten
else:
# Check if previous output exists
if os.path.exists(blast_out_filename) or os.path.exists(
failed_out_filename):
# Raise an exception
raise Exception(
'Output already exists. Please alter the output arguments or use --overwrite'
)
# Create a list of negative control IDs
negative_control_list = []
# Check if negative control IDs were given on the command line
if barcode_args.negative_control:
# Assign the negative control IDs
negative_control_list.extend(barcode_args.negative_control)
# Check if a file of negative control IDs was given
if barcode_args.negative_control_file:
# Open the file
with open(barcode_args.negative_control_file,
'r') as negative_control_file:
# Read the file, line by line
for negative_control_line in negative_control_file:
# Assign the negative control ID
negative_control_list.append(negative_control_line.strip())
# Create a list to store failed sample dictonaries
failed_samples_list = []
# Open the BLAST input file
with open(barcode_args.blast_file) as blast_file:
# Read the blast using DictReader
blast_reader = csv.DictReader(blast_file, delimiter='\t')
# Create the blast output file
blast_out_file = open(blast_out_filename, 'w')
# Create the blast writer using DictReader
blast_writer = csv.DictWriter(blast_out_file,
fieldnames=blast_reader.fieldnames,
delimiter='\t')
# Write the header for the output file
blast_writer.writeheader()
# Loop each query with its best hits
for current_query, best_blast_hits in yieldBestHits(blast_reader):
# Create a list to store the filtered blast hits
filtered_blast_hits = []
# Save the current ID
current_ID = current_query.rsplit('_', 1)[0]
# Save the current abundance
current_abundance = int(current_query.split('=')[1])
# Check if an abundance cutoff was specified
if barcode_args.abundance_cutoff:
# Check if the current abundance is below the cutoff
if current_abundance < barcode_args.abundance_cutoff:
# Save warning message
warning_message = 'Insufficent read abundance'
# Create dict to hold all relevant information for the failed sample
failed_sample_dict = {
'Query ID': current_query,
'Status': 'Insufficent reads'
}
# Add the dict to the list
failed_samples_list.append(failed_sample_dict)
# Log the failure
logging.warning(
'%s: %s' %
(current_query.split(';')[0], warning_message))
continue
# Loop each best hit
for blast_hit in best_blast_hits:
# Save the current evalue
current_evalue = float(blast_hit['E-Value'])
# Save the current query length
current_query_length = float(blast_hit['Query Length'])
# Save the current alignment length
current_alignment_length = float(blast_hit['Alignment Length'])
# Save the current identity
current_identity = float(blast_hit['Percent Identity'])
# Check if an E-Value cutoff was specified
if barcode_args.evalue_cutoff:
# Check if the current evalue is larger than the cutoff
if current_evalue > barcode_args.evalue_cutoff:
continue
# Check if a coverage cutoff was specified
if barcode_args.coverage_cutoff:
# Save the percent coverage
current_coverage = current_alignment_length / current_query_length
# Check if the current coverage is samller than the cutoff
if current_coverage < barcode_args.coverage_cutoff:
continue
# Check if an identity cutoff was specified
if barcode_args.identity_cutoff:
# Check if the current identity is smaller than the cutoff
if current_identity < barcode_args.identity_cutoff:
continue
# Add the blast hit to the filtered list, if passed
filtered_blast_hits.append(blast_hit)
# Check if no blast hits passed the filter
if len(filtered_blast_hits) == 0:
# Save warning message
warning_message = 'Filtered resulted in the removal of all data'
# Create dict to hold all relevant information for the failed sample
failed_sample_dict = {
'Query ID': current_query,
'Status': 'No Hits'
}
# Add the dict to the list
failed_samples_list.append(failed_sample_dict)
# Log the failure
logging.warning('%s: %s' %
(current_query.split(';')[0], warning_message))
# Copy the filtered list before sorting
sorted_best_hits = copy.deepcopy(filtered_blast_hits)
# Loop the sort methods
for sort_method in barcode_args.sort_best_hits_by:
# Sort the best hits
sorted_best_hits = sortBestHits(sorted_best_hits, sort_method)
# Check if there is more than one sorted best hit
if len(sorted_best_hits) > 1:
# Create a list for checking the number of species
sorted_species_list = []
# Create a list for checking the number of bins
sorted_bin_list = []
# Loop the sorted best hits
for sorted_best_hit in sorted_best_hits:
# Assign the species of the hit
sorted_species = sorted_best_hit['Subject ID'].split(
'|')[1].replace('_', ' ')
# Append the species to the list
sorted_species_list.append(sorted_species)
# Assign the bin of the hit
sorted_bin = sorted_best_hit['Subject ID'].split('|')[2]
# Append the bin to the list
sorted_bin_list.append(sorted_bin)
# Remove duplicate species
sorted_species_list = list(set(sorted_species_list))
# Remove duplicate bins
sorted_bin_list = list(set(sorted_bin_list))
# Check if a BOLD:N/A was found
if 'BOLD:N/A' in sorted_bin_list and len(sorted_bin_list) > 1:
# Move the BOLD:N/A to the end of the list
sorted_bin_list.append(
sorted_bin_list.pop(sorted_bin_list.index('BOLD:N/A')))
# Check if more than a single species among the best hits
if len(sorted_species_list) > 1:
# Save warning message
warning_message = 'Multiple species identified'
# Create dict to hold all relevant information for the failed sample
failed_sample_dict = {
'Query ID': current_query,
'Status': 'Ambiguous Hits',
'Species': sorted_species_list,
'Bins': sorted_bin_list
}
# Add the dict to the list
failed_samples_list.append(failed_sample_dict)
# Log the failure
logging.warning(
'%s: %s' %
(current_query.split(';')[0], warning_message))
continue
# Check if merging should occur, and if there is only a single species among the best hits
elif not barcode_args.dont_merge_species and len(
sorted_species_list) == 1:
# Take the first sorted hit, update this later with preferences
sorted_best_hits = [sorted_best_hits[0]]
# Check if any negative control IDs are assigned
if sorted_best_hits and negative_control_list and current_ID in negative_control_list:
logging.warning(
'Negative control (%s) passed filters. Please consider stricter cutoffs'
% current_ID)
continue
# Loop the sorted best hits
for sorted_best_hit in sorted_best_hits:
# Write the passed blast entry to the output
blast_writer.writerow(sorted_best_hit)
# Close the output file
blast_out_file.close()
# Check if any samples failed to find best hits
if failed_samples_list:
logging.warning(
'Samples (%s) failed filters. Samples may be found in: %s' %
(len(failed_samples_list), failed_out_filename))
# Open the json failure file
with open(failed_out_filename, 'w') as json_failed_file:
# Dump the failed sample list to the JSON file
json.dump(failed_samples_list, json_failed_file, indent=4)