def set_logger(settings):
'''
This functions creates a logger object that always logs to the command
line or optionally to a log file. Refer to the documentation of the
standard logging module for more information. The logger object is
stored in self.logger.
the following values should be present in self.settings:
string self.settings.logfile: the file to which messages are logged. If
logging to a file is not required, this value should be None.
int self.settings.loglevel: the threshold level for logging.
See the built-in logging module for details.
'''
# Check log level for validity
numeric_level = getattr(logging, settings.loglevel.upper())
if not isinstance(numeric_level, int):
raise InvalidOptionException('Invalid log level: %s' %
settings.loglevel)
# Root logger, stdout handler will be removed
logger = logging.getLogger()
lh_stdout = logger.handlers[0]
logger.setLevel(numeric_level)
# Configure logging to console
if settings.logfile is None:
# Only import when printing to terminal otherwise the ASCI escapes end up in a (log) file
from pyPaSWAS.Core.cfg import Colorer
console_format = logging.Formatter('%(levelname)s - %(message)s')
console_format.propagate = False
console_handler = logging.StreamHandler()
console_handler.setLevel(numeric_level)
console_handler.setFormatter(console_format)
logger.addHandler(console_handler)
elif settings.logfile is not None:
# Check log file for validity. For uniformity a ValueError may be raised
try:
logfile = open(settings.logfile, 'a')
_log_settings_to_file(logfile, settings)
logfile.close()
except (IOError):
raise InvalidOptionException(
'Invalid log file or writing forbidden: %s' % settings.logfile)
file_format = logging.Formatter(
'%(asctime)s - %(levelname)s - %(message)s')
file_format.propagate = False
file_handler = logging.FileHandler(settings.logfile)
file_handler.setLevel(numeric_level)
file_handler.setFormatter(file_format)
logger.addHandler(file_handler)
# Disable root logger (stdout)
logger.removeHandler(lh_stdout)
return logger
def _set_device(self, device):
'''Sets the device number'''
try:
self.device = int(device) if device else self.device
except ValueError:
raise InvalidOptionException(
'device should be an int but is {0}'.format(device))
def __init__(self, logger, settings):
'''
Constructor
Parameters:
matrix: the custom scoring matrix. Due to the current implementation of pyPaSWAS,
this should be a Python list consisting of 26 lists. Each of these inner
lists should contain 26 integers. Each integer represents the score for
matching nucleotides or amino acids represented by a certain character
in subject and query. E.g. the integer at matrix[0][0] contains the score
when both the subject and the query contain the character 'A' at the
current position, matrix[0][1] when at the current position an 'A' is
aligned with a 'B' etc.
gap_score: score for a missing nucleotide or amino acid in either subject or query
'''
Score.__init__(self, logger, settings)
self.logger.debug('Initializing CustomScore...')
#check/set defaults for arguments
matrix = settings.custom_matrix
if matrix is None:
raise InvalidOptionException(
'Required argument {0} is missing'.format('matrix'))
self.char_offset = 'A'
self.set_dimensions(26)
self._create_matrix(matrix)
self.logger.debug('Initializing CustomScore finished.')
def _set_internal_limit(self, limit):
'''sets the internal limit'''
try:
self.internal_limit = int(limit) if limit else self.internal_limit
except ValueError:
raise InvalidOptionException(
'internal_limit should be a int but is {0}'.format(limit))
def _create_matrix(self, matrix):
self.logger.debug('Creating matrix with parameters:\n\t\tmatrix: '
'{0},\n\t\tgap_score: {1}'.format(matrix, self.gap_score))
if len(matrix) != self.dimensions or (len(matrix) > 0 and len(matrix[0]) != self.dimensions):
raise InvalidOptionException('A matrix should consist of a list containing 26 lists of '
'26 integers each.')
else:
self.matrix = matrix
self.highest_score = max(max(self.matrix))
def _set_program(self):
'''Determines what program from the suite should be used and instantiates it'''
if self.settings.program == 'aligner':
self.program = Aligner(self.logger, self.score, self.settings)
elif self.settings.program == 'trimmer':
self.program = Trimmer(self.logger, self.score, self.settings)
else:
raise InvalidOptionException('Invalid program selected {0}'.format(
self.settings.program))
def _set_output_format(self):
'''Determines the output format.
Currently only TXT for text and SAM for SAM are supported.
'''
if self.settings.out_format.upper() == 'FASTA':
self.output_format = 'FASTA'
else:
raise InvalidOptionException('Invalid output format {0}.'.format(
self.settings.out_format))
def _set_filter_factor(self, filterfactor):
'''sets the filter factor'''
try:
if filterfactor:
self.filter_factor = float(
filterfactor) if filterfactor else self.filter_factor
except ValueError:
raise InvalidOptionException(
'Filterfactor should be a float but is {0}'.format(
filterfactor))
def _set_max_genome_length(self, max_genome_length):
'''sets the maximum length of the genome'''
try:
self.max_genome_length = int(
max_genome_length
) if max_genome_length else self.max_genome_length
except ValueError:
raise InvalidOptionException(
'maxGenomeLength should be a int but is {0}'.format(
max_genome_length))
def _get_query_sequences(self, queryfile, start=0, end=None):
queryfile = normalize_file_path(queryfile)
if not os.path.exists(queryfile):
raise InvalidOptionException(
'File {0} does not exist.'.format(queryfile))
else:
reader = BioPythonReader(self.logger, queryfile,
self.settings.filetype1,
self.settings.limit_length)
reader.read_records(start, end)
reader.sort_records()
return reader.get_records()
def _set_program(self):
'''Determines what program from the suite should be used and instantiates it'''
if self.settings.program == "palindrome":
self.program = Palindrome(self.logger, self.score, self.settings)
self.logger.warning("Forcing output to FASTA")
self.output_format = "FASTA"
self.logger.warning("Forcing query step to 1")
self.settings.query_step = "1"
self.logger.warning("Forcing sequence step to 1")
self.settings.sequence_step = "1"
self.logger.warning("Forcing Matrix to PALINDROME")
self.settings.matrix_name = "PALINDROME"
self.score = PalindromeScore(self.logger, self.settings)
else:
raise InvalidOptionException('Invalid program selected {0}'.format(
self.settings.program))
def _set_scoring_matrix(self):
''' Instantiate the scoring matrix. For more information refer to the
documentation of the different score types. '''
matrix_name = self.settings.matrix_name.upper()
score = None
if matrix_name == 'DNA-RNA':
score = DnaRnaScore(self.logger, self.settings)
elif matrix_name == 'PALINDROME':
score = PalindromeScore(self.logger, self.settings)
elif matrix_name == 'BASIC':
score = BasicScore(self.logger, self.settings)
elif matrix_name == 'CUSTOM':
score = CustomScore(self.logger, self.settings)
else:
raise InvalidOptionException(matrix_name +
' is not a valid substitution matrix')
self.score = score
def _set_output_format(self):
'''Determines the output format.
Currently only TXT for text and SAM for SAM are supported.
'''
if self.settings.out_format.upper() == 'SAM':
self.output_format = 'SAM'
elif self.settings.out_format.upper() == 'TXT':
self.output_format = 'TXT'
elif self.settings.out_format.upper() == 'TRIMMEDFASTA':
self.output_format = 'trimmedFasta'
elif self.settings.out_format.upper() == 'FASTA':
self.output_format = 'FASTA'
elif self.settings.out_format.upper() == "GRAPH":
self.output_format = 'GRAPH'
else:
raise InvalidOptionException('Invalid output format {0}.'.format(
self.settings.out_format))
def _get_target_sequences(self, databasefile, start=0, end=None):
'''Gets the target (database) sequences from the database file. '''
databasefile = normalize_file_path(databasefile)
if not os.path.exists(databasefile):
raise InvalidOptionException(
'File {0} does not exist.'.format(databasefile))
else:
reader = BioPythonReader(self.logger, databasefile,
self.settings.filetype1,
self.settings.limit_length)
reader.read_records(start, end)
if self.score.score_type == 'DNA_RNA':
reader.complement_records()
elif self.score.score_type == 'PALINDROME':
reader.complement_records_only()
elif self.score.score_type == "IRYS":
reader.reverse_records()
reader.sort_records()
return reader.get_records()
def _set_parameters(self, len_x, len_y, number_sequences, number_targets):
'''
_set_parameters is used to initialize all values before memory initialization and running a Smith-Waterman analysis.
@param len_x: length of each sequence on the len_x axes. len_x % shared_x should be zero
@param len_y: length of each target on the len_y axes. len_y % SHARED_Y should be zero
@param number_sequences: the number of sequences on the len_x axes
@param number_targets: the number of targets on the len_y axes
'''
#self.logger.debug('Setting parameters len_x: {0}, len_y: {1}, '
# 'number_sequences: {2}, '
# 'number_targets: {3} '.format(len_x, len_y, number_sequences, number_targets))
if len_x % self.shared_x > 0 or len_y % self.shared_y > 0:
raise InvalidOptionException("Error in _set_parameters: len_x % " +
str(self.shared_x) + " or len_y % " +
str(self.shared_y) + " != 0 !")
self.length_of_x_sequences = len_x
self.length_of_y_sequences = len_y
self.number_of_sequences = number_sequences
self.number_targets = number_targets
self.x_div_shared_x = int(math.floor((len_x / self.shared_x)))
self.y_div_shared_y = int(math.floor(len_y / self.shared_y))
def _set_path(self, path):
'''Sets the path to the file which contains the records.'''
if self._is_a_readable_file(path):
self.path = path
else:
raise InvalidOptionException('{0} is not a file or the program is not allowed to access it.'.format(path))
def parse_cli(config_file):
'''
parseCLI()
This function parses the command line using the optparse module from the python standard library.
Though deprecated since python 2.7, optparse still is used in stead of argparse because the python
version available at the development systems was 2.6.
The options and arguments are stored in the global variables settings and arguments, respectively.
'''
# Read defaults
config = ConfigParser.ConfigParser()
try:
config.read(config_file)
except ConfigParser.ParsingError:
raise ConfigParser.ParsingError(
"Unable to parse the defaults file ({})".format(config_file))
parser = optparse.OptionParser()
parser.description = (
'This program performs a Smith-Waterman alignment of all sequences in FILE_1'
' against all sequences in FILE_2.\nBoth files should be in the fasta format.'
)
usage = '%prog [options] FILE_1 FILE_2'
parser.usage = usage
# general options
# TODO: Get final naming (convention) for all parameters!!
general_options = optparse.OptionGroup(
parser, 'Options that affect the general operation of the program')
general_options.add_option('-L',
'--logfile',
help='log events to FILE',
metavar="FILE",
dest='logfile')
general_options.add_option(
'--loglevel',
help='log level. Valid options are DEBUG, INFO, WARNING, ERROR'
' and CRITICAL',
dest='loglevel',
default=config.get('General', 'loglevel'))
general_options.add_option(
'-o',
'--out',
help='The file in which the program stores the generated output.'
'\nDefaults to ./output',
dest='out_file',
default=config.get('General', 'out_file'))
general_options.add_option(
'--outputformat',
help='The format of the file in which the program stores the '
'generated output.\nAvailable options are TXT and SAM.\nDefaults to txt',
dest='out_format',
default=config.get('General', 'out_format'))
general_options.add_option(
'-p',
'--program',
help='The program to be executed. Valid options are "aligner"'
' and "trimmer"',
dest='program',
default=config.get('General', 'program'))
general_options.add_option(
'-1',
'--filetype1',
help='File type of the first file. See bioPython IO for'
' available options',
dest='filetype1',
default=config.get('General', 'filetype1'))
general_options.add_option(
'-2',
'--filetype2',
help='File type of the second file. See bioPython IO for'
' available options',
dest='filetype2',
default=config.get('General', 'filetype2'))
general_options.add_option(
'-O',
'--override_output',
help='When output file exists, override it (T/F)',
dest='override_output',
default=config.get('General', 'override_output'))
general_options.add_option('-c',
'--configfile',
help='Give settings using configuration file',
dest='config_file',
default=False)
parser.add_option_group(general_options)
input_options = optparse.OptionGroup(
parser,
'start & stop indices for processing files. Handy for cluster processing. Leave all to zero to process all.'
)
input_options.add_option('--start_query',
help='start index in the query file (1)',
dest='start_query',
default=config.get("Input", "start_query"))
input_options.add_option('--end_query',
help='end index in the query file (1)',
dest='end_query',
default=config.get("Input", "end_query"))
input_options.add_option('--start_target',
help='start index in the target file (2)',
dest='start_target',
default=config.get("Input", "start_target"))
input_options.add_option('--end_target',
help='end index in the target file (2)',
dest='end_target',
default=config.get("Input", "end_target"))
input_options.add_option(
'--sequence_step',
help=
'Number of sequences read from file 2 before processing. Handy when processing NGS files.',
dest='sequence_step',
default=config.get('Input', 'sequence_step'))
input_options.add_option(
'--query_step',
help=
'Number of sequences read from file 1 before processing. Handy when processing NGS files.',
dest='query_step',
default=config.get('Input', 'query_step'))
parser.add_option_group(input_options)
aligner_options = optparse.OptionGroup(
parser, 'Options that affect the alignment.\nAligners include aligner'
' and mapper')
aligner_options.add_option('--customMatrix',
help='the custom matrix that should be used',
dest='custom_matrix')
aligner_options.add_option('-G',
'--gap_score',
help='Float. Penalty for a gap',
dest='gap_score',
default=config.get('Aligner', 'gap_score'))
aligner_options.add_option(
'-g',
'--gap_extension',
help=
'Float. Penalty for a gap extension. Set to zero to ignore this (faster)',
dest='gap_extension',
default=config.get('Aligner', 'gap_extension'))
aligner_options.add_option(
'-M',
'--matrixname',
help='The scoring to be used. Valid options are '
'"DNA-RNA", "BASIC", "Blosum62", "Blosum80" and "CUSTOM"',
dest='matrix_name',
default=config.get('Aligner', 'matrix_name'))
aligner_options.add_option('-q',
'--mismatch_score',
help='Float. Penalty for mismatch',
dest='mismatch_score',
default=config.get('Aligner', 'mismatch_score'))
aligner_options.add_option('-r',
'--match_score',
help='Float. Reward for match',
dest='match_score',
default=config.get('Aligner', 'match_score'))
aligner_options.add_option(
'--any',
help='Float. Score for a character which is neither in the nucleotide'
' list ("ACGTU"), nor equal to the anyNucleotide character ("N").\nOnly relevant'
' for use with the DNA-RNA scoring type.',
dest='any_score',
default=config.get('Aligner', 'any_score'))
aligner_options.add_option(
'--other',
help='Float. Score if the anyNucleotide character ("N") is present in'
' either query or subject.\nOnly relevant for use with the DNA-RNA scoring type.',
dest='other_score',
default=config.get('Aligner', 'other_score'))
aligner_options.add_option(
'--minimum',
help='Float. Sets the minimal score that initiates a back trace.'
' Do not set this very low: output may be flooded by hits.',
dest='minimum_score',
default=config.get('Aligner', 'minimum_score'))
aligner_options.add_option(
'--llimit',
help='Float. Sets the lower limit for the maximum score '
'which will be used to report a hit. pyPaSWAS will then also report hits with '
'a score lowerLimitScore * highest hit score. Set to <= 1.0. ',
dest='lower_limit_score',
default=config.get('Aligner', 'lower_limit_score'))
parser.add_option_group(aligner_options)
filter_options = optparse.OptionGroup(parser,
'Options for filtering the output')
filter_options.add_option(
'--filter_factor',
help='The filter factor to be used. Reports only hits within'
' filterFactor * highest possible score * length shortest sequence (or: defines'
' lowest value of the reported relative score). Set to <= 1.0',
dest='filter_factor',
default=config.get('Filter', 'filter_factor'))
filter_options.add_option('--query_coverage',
help='Minimum query coverage. Set to <= 1.0',
dest='query_coverage',
default=config.get('Filter', 'query_coverage'))
filter_options.add_option('--query_identity',
help='Minimum query identity. Set to <= 1.0',
dest='query_identity',
default=config.get('Filter', 'query_identity'))
filter_options.add_option(
'--relative_score',
help='Minimum relative score, defined by the alignment score'
' divided by the length of the shortest of the two sequences. Set to <= highest possible score'
', for example 5.0 in case of DNA',
dest='relative_score',
default=config.get('Filter', 'relative_score'))
filter_options.add_option(
'--base_score',
help='Minimum base score, defined by the alignment score'
' divided by the length of the alignment (including gaps). Set to <= highest possible score'
', for example 5.0 in case of DNA',
dest='base_score',
default=config.get('Filter', 'base_score'))
parser.add_option_group(filter_options)
graph_options = optparse.OptionGroup(
parser,
'Options to connect to a neo4j graph database and store mappings in a graph'
)
graph_options.add_option('--hostname',
help='Neo4j database host',
default=config.get("GraphDatabase", "hostname"),
dest="hostname")
graph_options.add_option('--username',
help='Neo4j user name',
default=config.get("GraphDatabase", "username"),
dest="username")
graph_options.add_option('--password',
help='Neo4j password',
default=config.get("GraphDatabase", "password"),
dest="password")
graph_options.add_option('--target_node',
help='Target node name',
default=config.get("GraphDatabase",
"target_node"),
dest="target_node")
graph_options.add_option('--sequence_node',
help='Sequence node name',
default=config.get("GraphDatabase",
"sequence_node"),
dest="sequence_node")
parser.add_option_group(graph_options)
device_options = optparse.OptionGroup(
parser, 'Options that affect the usage and settings of the '
'parallel devices')
device_options.add_option(
'--device',
help='the device on which the computations will be performed. '
'This should be an integer.',
dest='device_number',
default=config.get('Device', 'device_number'))
device_options.add_option(
'--number_of_compute_units',
help=
'Number of compute units to use (openCL only). Will not work on every device, recommended for CPU only. Set this 1 to use a single core on the device for example.'
'This should be an integer, using 0 for full device.',
dest='number_of_compute_units',
default=config.get('Device', 'number_of_compute_units'))
device_options.add_option(
'--sub_device',
help=
'the sub device on which the computations will be performed. Only used when number_of_compute_units > 0. '
'This should be an integer.',
dest='sub_device',
default=config.get('Device', 'sub_device'))
device_options.add_option(
'--limit_length',
help='Length of the longest sequence in characters to be read'
' from file. Lower this when memory of GPU is low.',
dest='limit_length',
default=config.get('Device', 'limit_length'))
device_options.add_option(
'--maximum_memory_usage',
help=
'Fraction (<= 1.0) of available device memory to use. Useful when several pyPaSWAS applications are running.',
dest="maximum_memory_usage",
default=config.get('Device', 'maximum_memory_usage'))
device_options.add_option(
'--njobs',
help='Sets the number of jobs run simultaneously on the grid. Will read'
' only part of the sequence file. (not implemented yet)',
dest='number_of_jobs')
device_options.add_option(
'--process_id',
help='Sets the processID of this job in the grid. ',
dest='process_id')
device_options.add_option('--max_genome_length',
help='Deprecated.\nDefaults to 200000',
dest='max_genome_length',
default=config.get('Device',
'max_genome_length'))
device_options.add_option(
'--recompile',
help=
'Recompile CUDA code? Set to F(alse) when sequences are of similar length: much faster.',
dest='recompile',
default=config.get('Device', 'recompile'))
device_options.add_option(
'--short_sequences',
help=
'Set to T(true) when aligning short sequences (trimming?) to maximize memory usage.',
dest='short_sequences',
default=config.get('Device', 'short_sequences'))
parser.add_option_group(device_options)
framework_options = optparse.OptionGroup(
parser,
'Determines which parallel computing framework to use for this program '
)
framework_options.add_option(
'--framework',
help=
'Choose which parallel computing framework to use, can be either CUDA or OpenCL ',
dest='framework',
default=config.get('Framework', 'language'))
parser.add_option_group(framework_options)
ocl_options = optparse.OptionGroup(
parser, 'Options for the usage of the OpenCL framework ')
ocl_options.add_option(
'--device_type',
help=
'Type of device to perform computations on (either CPU, GPU or ACCELARATOR)',
dest='device_type',
default=config.get('OpenCL', 'device_type'))
ocl_options.add_option(
'--platform_name',
help='Platform to run computations on (either Intel, NVIDIA or AMD)',
dest='platform_name',
default=config.get('OpenCL', 'platform_name'))
parser.add_option_group(ocl_options)
(settings, arguments) = parser.parse_args()
# If an extra configuration file is given, override settings as given by this file
if settings.config_file:
(settings, arguments) = _override_settings(settings.config_file,
settings, arguments)
if len(arguments) < 2:
raise InvalidOptionException('Missing input files')
return (settings, arguments)
def __init__(self, logger, score, settings):
"""
Constructor for a Smith-Waterman object.
"""
self.logger = logger
self.logger.debug('Initializing SmithWaterman.')
self.score = score
self.settings = settings
self.number_of_targets = 1
# Added startingpoint instance
self.starting_point = StartingPoint(self.logger)
self.target_block_length = 0
# d_sequences holds the sequences (X) in GPU RAM
self.d_sequences = None
# d_targets holds the target sequences (Y) in GPU RAM
self.d_targets = None
# d_matrix is used to store the SM scoring
self.d_matrix = None
# d_matrix_i is used to store the SM scoring with affine gap
self.d_matrix_i = None
# d_matrix_j is used to store the SM scoring with affine gap
self.d_matrix_j = None
# d_global_maxima holds the maximum values found during the SM calculations
self.d_global_maxima = None
# d_index_increment is the 'auto-increment index' used to store the starting points. Hence, after the
# traceback this attribute holds the number of alignments
self.d_index_increment = None
self.index = 0
# h_global_direction_zero_copy is the direction matrix on the CPU and holds the new values
# after the traceback call
self.h_global_direction_zero_copy = None
self.d_global_direction_zero_copy = None
# h_starting_points_zero_copy is used to store the starting point struct values in a long list.
# Use the StartingPoint python class to get the python object representation
self.h_starting_points_zero_copy = None
self.d_starting_points_zero_copy = None
# h_max_possible_score_zero_copy holds the maximum possible score a sequence (X) can have (for each sequence).
self.h_max_possible_score_zero_copy = None
self.d_max_possible_score_zero_copy = None
self.min_score_np = None
self.max_length = None
self.has_been_compiled = False
self.max_sequences = 0
# Length of the sequences on the X axis. Necessary for the CUDA code.
self.length_of_x_sequences = 0
# Length of the targets on the Y axis. Necessary for the CUDA code.
self.length_of_y_sequences = 0
# Number of sequences. Necessary for the CUDA code.
self.number_of_sequences = 0
# Number of target sequences. Necessary for the CUDA code.
self.number_targets = 0
# Necessary for the CUDA code.
self.x_div_shared_x = 0
# Necessary for the CUDA code.
self.y_div_shared_y = 0
# Necessary for the CUDA code.
self.maximum_number_starting_points = 100
# Changed from ... = StartingPoint.size
self.size_of_startingpoint = self.starting_point.size
# check for gap extension penalty:
self.gap_extension = score.gap_extension != None
if self.gap_extension:
self.logger.info(
"Gap extension penalty detected: using affine gap scoring algorithm"
)
else:
self.logger.info(
"No gap extension penalty detected: using original PaSWAS scoring algorithm"
)
# Defines the number of elements processed in the X direction.
# Don't edit unless you really know what you are doing!
self.shared_x = 8
# Defines the number of elements processed in the Y direction.
# Don't edit unless you really know what you are doing!
self.shared_y = 8
# Filled in later
self.target_array = numpy.array([], dtype=numpy.character)
self.device = 0
self._set_device(self.settings.device_number)
self.filter_factor = 0.7
self.internal_limit = 64000
self.max_genome_length = 10000
self._set_filter_factor(self.settings.filter_factor)
self._set_max_genome_length(self.settings.max_genome_length)
self._set_internal_limit(self.internal_limit)
# set memory usage
try:
self.mem_fill_factor = float(settings.maximum_memory_usage)
except ValueError:
raise InvalidOptionException(
'maximux_memory_usage is not a float'.format(
settings.maximum_memory_usage))
if self.mem_fill_factor > 1.0 or self.mem_fill_factor <= 0.0:
raise InvalidOptionException(
'maximux_memory_usage is not a float between 0.0 and 1.0'.
format(settings.maximum_memory_usage))
def _set_limit_length(self, limit_length):
'''sets the limit of the number of sequences that are to be compared at one time)'''
try:
self.limitlength = int(limit_length)
except ValueError:
raise InvalidOptionException('Limitlength should be an int but is {0}'.format(limit_length))