def sniff(self, filename): """ http://www.mothur.org/wiki/Oligos_File Determines whether the file is a otu (operational taxonomic unit) format >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.oligos' ) >>> Oligos().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.oligos' ) >>> Oligos().sniff( fname ) False """ headers = iter_headers(filename, sep='\t') count = 0 for line in headers: if not line[0].startswith('@') and not line[0].startswith('#'): if len(line) == 2 and line[0] in ['forward', 'reverse']: count += 1 continue elif len(line) == 3 and line[0] == 'barcode': count += 1 continue else: return False if count > 0: return True return False
def sniff(self, filename): """ Determines whether the file is otu (operational taxonomic unit) format >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.otu' ) >>> Otu().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.otu' ) >>> Otu().sniff( fname ) False """ headers = iter_headers(filename, sep='\t') count = 0 for line in headers: if not line[0].startswith('@'): if len(line) < 2: return False if count >= 1: try: check = int(line[1]) if check + 2 != len(line): return False except ValueError: return False count += 1 if count > 2: return True return False
def sniff_prefix(self, file_prefix): """ Determines whether the file is otu (operational taxonomic unit) format label<TAB>count[<TAB>value(1..n)] >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.sabund' ) >>> Sabund().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.sabund' ) >>> Sabund().sniff( fname ) False """ headers = iter_headers(file_prefix, sep='\t') count = 0 for line in headers: if not line[0].startswith('@'): if len(line) < 2: return False try: check = int(line[1]) if check + 2 != len(line): return False for i in range(2, len(line)): int(line[i]) except ValueError: return False count += 1 if count > 0: return True return False
def sniff_prefix(self, file_prefix): """ Try to guess if the file is a PDBQT file. >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname('NuBBE_1_obabel_3D.pdbqt') >>> PDBQT().sniff(fname) True >>> fname = get_test_fname('drugbank_drugs.cml') >>> PDBQT().sniff(fname) False """ headers = iter_headers(file_prefix, sep=' ', count=300) h = t = c = s = k = False for line in headers: section_name = line[0].strip() if section_name == 'REMARK': h = True elif section_name == 'ROOT': t = True elif section_name == 'ENDROOT': c = True elif section_name == 'BRANCH': s = True elif section_name == 'TORSDOF': k = True if h * t * c * s * k: return True else: return False
def sniff(self, filename): """ Try to guess if the file is a PDBQT file. >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname('NuBBE_1_obabel_3D.pdbqt') >>> PDBQT().sniff(fname) True >>> fname = get_test_fname('drugbank_drugs.cml') >>> PDBQT().sniff(fname) False """ headers = iter_headers(filename, sep=' ', count=300) h = t = c = s = k = False for line in headers: section_name = line[0].strip() if section_name == 'REMARK': h = True elif section_name == 'ROOT': t = True elif section_name == 'ENDROOT': c = True elif section_name == 'BRANCH': s = True elif section_name == 'TORSDOF': k = True if h * t * c * s * k: return True else: return False
def set_meta(self, dataset, overwrite=True, **kwd): super(Otu, self).set_meta(dataset, overwrite=overwrite, **kwd) if dataset.has_data(): label_names = set() otulabel_names = set() ncols = 0 data_lines = 0 comment_lines = 0 headers = iter_headers(dataset.file_name, sep='\t', count=-1) first_line = get_headers(dataset.file_name, sep='\t', count=1) # set otulabels if len(first_line) > 2: otulabel_names = first_line[2:] # set label names and number of lines for line in headers: if len(line) >= 2 and not line[0].startswith('@'): data_lines += 1 ncols = max(ncols, len(line)) label_names.add(line[0]) else: comment_lines += 1 # Set the discovered metadata values for the dataset dataset.metadata.data_lines = data_lines dataset.metadata.columns = ncols dataset.metadata.labels = list(label_names) dataset.metadata.labels.sort() dataset.metadata.otulabels = list(otulabel_names) dataset.metadata.otulabels.sort()
def set_meta(self, dataset, overwrite=True, skip=1, **kwd): super(GroupAbund, self).set_meta(dataset, overwrite=overwrite, **kwd) # See if file starts with header line if dataset.has_data(): label_names = set() group_names = set() data_lines = 0 comment_lines = 0 ncols = 0 headers = iter_headers(dataset.file_name, sep='\t', count=-1) for line in headers: if line[0] == 'label' and line[1] == 'Group': skip = 1 comment_lines += 1 else: skip = 0 data_lines += 1 ncols = max(ncols, len(line)) label_names.add(line[0]) group_names.add(line[1]) # Set the discovered metadata values for the dataset dataset.metadata.data_lines = data_lines dataset.metadata.columns = ncols dataset.metadata.labels = list(label_names) dataset.metadata.labels.sort() dataset.metadata.groups = list(group_names) dataset.metadata.groups.sort() dataset.metadata.skip = skip
def sniff(self, filename): """ Determines whether the file is a secondary structure map format A single column with an integer value which indicates the row that this row maps to. Check to make sure if structMap[10] = 380 then structMap[380] = 10 and vice versa. >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.map' ) >>> SecondaryStructureMap().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.map' ) >>> SecondaryStructureMap().sniff( fname ) False """ headers = iter_headers(filename, sep='\t') line_num = 0 rowidxmap = {} for line in headers: line_num += 1 if len(line) > 1: return False try: pointer = int(line[0]) if pointer > line_num: rowidxmap[pointer] = line_num elif pointer > 0 or line_num in rowidxmap: if rowidxmap[line_num] != pointer: return False except (ValueError, KeyError): return False if line_num < 3: return False return True
def sniff(self, filename): """ Determines whether the file is a pairwise distance matrix (Column-formatted distance matrix) format The first and second columns have the sequence names and the third column is the distance between those sequences. >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.pair.dist' ) >>> PairwiseDistanceMatrix().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.pair.dist' ) >>> PairwiseDistanceMatrix().sniff( fname ) False """ headers = iter_headers(filename, sep='\t') count = 0 for line in headers: if not line[0].startswith('@'): if len(line) != 3: return False try: float(line[2]) try: # See if it's also an integer int(line[2]) except ValueError: # At least one value is not an integer all_ints = False except ValueError: return False count += 1 if count > 2: return not all_ints return False
def sniff_prefix(self, file_prefix): """ Determines whether the file is a secondary structure map format A single column with an integer value which indicates the row that this row maps to. Check to make sure if structMap[10] = 380 then structMap[380] = 10 and vice versa. >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.map' ) >>> SecondaryStructureMap().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.map' ) >>> SecondaryStructureMap().sniff( fname ) False """ headers = iter_headers(file_prefix, sep='\t') line_num = 0 rowidxmap = {} for line in headers: line_num += 1 if len(line) > 1: return False try: pointer = int(line[0]) if pointer > line_num: rowidxmap[pointer] = line_num elif pointer > 0 or line_num in rowidxmap: if rowidxmap[line_num] != pointer: return False except (ValueError, KeyError): return False if line_num < 3: return False return True
def sniff(self, filename): """ Determines whether the file is otu (operational taxonomic unit) format label<TAB>count[<TAB>value(1..n)] >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.sabund' ) >>> Sabund().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.sabund' ) >>> Sabund().sniff( fname ) False """ headers = iter_headers(filename, sep='\t') count = 0 for line in headers: if not line[0].startswith('@'): if len(line) < 2: return False try: check = int(line[1]) if check + 2 != len(line): return False for i in range(2, len(line)): int(line[i]) except ValueError: return False count += 1 if count > 0: return True return False
def sniff_prefix(self, file_prefix): """ http://www.mothur.org/wiki/Oligos_File Determines whether the file is a otu (operational taxonomic unit) format >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.oligos' ) >>> Oligos().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.oligos' ) >>> Oligos().sniff( fname ) False """ headers = iter_headers(file_prefix, sep='\t') count = 0 for line in headers: if not line[0].startswith('@') and not line[0].startswith('#'): if len(line) == 2 and line[0] in ['forward', 'reverse']: count += 1 continue elif len(line) == 3 and line[0] == 'barcode': count += 1 continue else: return False if count > 0: return True return False
def sniff_prefix(self, file_prefix): """ >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname('sample.gfa2') >>> Gfa2().sniff(fname) True >>> Gfa1().sniff(fname) False """ found_valid_lines = False for line in iter_headers(file_prefix, "\t"): if line[0].startswith('#'): continue if line[0] == 'H': return len(line) >= 2 and line[1] == 'VN:Z:2.0' elif line[0] == 'S': if len(line) < 3: return False elif line[0] == 'F': if len(line) < 8: return False elif line[0] == 'E': if len(line) < 9: return False elif line[0] == 'G': if len(line) < 6: return False elif line[0] == 'O' or line[0] == 'U': if len(line) < 3: return False else: return False found_valid_lines = True return found_valid_lines
def set_meta(self, dataset, overwrite=True, **kwd): super(Otu, self).set_meta(dataset, overwrite=overwrite, **kwd) if dataset.has_data(): label_names = set() otulabel_names = set() ncols = 0 data_lines = 0 comment_lines = 0 headers = iter_headers(dataset.file_name, sep='\t', count=-1) first_line = get_headers(dataset.file_name, sep='\t', count=1) # set otulabels if len(first_line) > 2: otulabel_names = first_line[2:] # set label names and number of lines for line in headers: if len(line) >= 2 and not line[0].startswith('@'): data_lines += 1 ncols = max(ncols, len(line)) label_names.add(line[0]) else: comment_lines += 1 # Set the discovered metadata values for the dataset dataset.metadata.data_lines = data_lines dataset.metadata.columns = ncols dataset.metadata.labels = list(label_names) dataset.metadata.labels.sort() dataset.metadata.otulabels = list(otulabel_names) dataset.metadata.otulabels.sort()
def sniff_prefix(self, file_prefix): """ Try to guess if the file is a PDB file. >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname('5e5z.pdb') >>> PDB().sniff(fname) True >>> fname = get_test_fname('drugbank_drugs.cml') >>> PDB().sniff(fname) False """ headers = iter_headers(file_prefix, sep=' ', count=300) h = t = c = s = k = e = False for line in headers: section_name = line[0].strip() if section_name == 'HEADER': h = True elif section_name == 'TITLE': t = True elif section_name == 'COMPND': c = True elif section_name == 'SOURCE': s = True elif section_name == 'KEYWDS': k = True elif section_name == 'EXPDTA': e = True if h * t * c * s * k * e: return True else: return False
def set_meta(self, dataset, overwrite=True, skip=1, **kwd): super(GroupAbund, self).set_meta(dataset, overwrite=overwrite, **kwd) # See if file starts with header line if dataset.has_data(): label_names = set() group_names = set() data_lines = 0 comment_lines = 0 ncols = 0 headers = iter_headers(dataset.file_name, sep='\t', count=-1) for line in headers: if line[0] == 'label' and line[1] == 'Group': skip = 1 comment_lines += 1 else: skip = 0 data_lines += 1 ncols = max(ncols, len(line)) label_names.add(line[0]) group_names.add(line[1]) # Set the discovered metadata values for the dataset dataset.metadata.data_lines = data_lines dataset.metadata.columns = ncols dataset.metadata.labels = list(label_names) dataset.metadata.labels.sort() dataset.metadata.groups = list(group_names) dataset.metadata.groups.sort() dataset.metadata.skip = skip
def sniff_prefix(self, file_prefix): """ Try to guess if the file is a PQR file. >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname('5e5z.pqr') >>> PQR().sniff(fname) True >>> fname = get_test_fname('drugbank_drugs.cml') >>> PQR().sniff(fname) False """ prog = self.get_matcher() headers = iter_headers(file_prefix, sep=None, comment_designator='REMARK 5', count=3000) h = a = False for line in headers: section_name = line[0].strip() if section_name == 'REMARK': h = True elif section_name == 'ATOM' or section_name == 'HETATM': if prog.match(' '.join(line)): a = True break if h * a: return True else: return False
def sniff_prefix(self, file_prefix): """ Try to guess if the file is a PQR file. >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname('5e5z.pqr') >>> PQR().sniff(fname) True >>> fname = get_test_fname('drugbank_drugs.cml') >>> PQR().sniff(fname) False """ prog = self.get_matcher() headers = iter_headers(file_prefix, sep=None, comment_designator='REMARK 5', count=3000) h = a = False for line in headers: section_name = line[0].strip() if section_name == 'REMARK': h = True elif section_name == 'ATOM' or section_name == 'HETATM': if prog.match(' '.join(line)): a = True break if h * a: return True else: return False
def sniff_prefix(self, file_prefix): """ Determines whether the file is otu (operational taxonomic unit) format >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.otu' ) >>> Otu().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.otu' ) >>> Otu().sniff( fname ) False """ headers = iter_headers(file_prefix, sep='\t') count = 0 for line in headers: if not line[0].startswith('@'): if len(line) < 2: return False if count >= 1: try: check = int(line[1]) if check + 2 != len(line): return False except ValueError: return False count += 1 if count > 2: return True return False
def sniff_prefix(self, file_prefix): """ Determines whether the file is a pairwise distance matrix (Column-formatted distance matrix) format The first and second columns have the sequence names and the third column is the distance between those sequences. >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.pair.dist' ) >>> PairwiseDistanceMatrix().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.pair.dist' ) >>> PairwiseDistanceMatrix().sniff( fname ) False """ headers = iter_headers(file_prefix, sep='\t') count = 0 for line in headers: if not line[0].startswith('@'): if len(line) != 3: return False try: float(line[2]) try: # See if it's also an integer int(line[2]) except ValueError: # At least one value is not an integer all_ints = False except ValueError: return False count += 1 if count > 2: return not all_ints return False
def sniff(self, filename): """ Try to guess if the file is a PDB file. >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname('5e5z.pdb') >>> PDB().sniff(fname) True >>> fname = get_test_fname('drugbank_drugs.cml') >>> PDB().sniff(fname) False """ headers = iter_headers(filename, sep=' ', count=300) h = t = c = s = k = e = False for line in headers: section_name = line[0].strip() if section_name == 'HEADER': h = True elif section_name == 'TITLE': t = True elif section_name == 'COMPND': c = True elif section_name == 'SOURCE': s = True elif section_name == 'KEYWDS': k = True elif section_name == 'EXPDTA': e = True if h * t * c * s * k * e: return True else: return False
def sniff_prefix(self, file_prefix): """ Determines whether the file is an axes format The first line may have column headings. The following lines have the name in the first column plus float columns for each axis. .. code-block:: group axis1 axis2 forest 0.000000 0.145743 pasture 0.145743 0.000000 .. code-block:: axis1 axis2 U68589 0.262608 -0.077498 U68590 0.027118 0.195197 U68591 0.329854 0.014395 >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.axes' ) >>> Axes().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.axes' ) >>> Axes().sniff( fname ) False """ headers = iter_headers(file_prefix, sep='\t') count = 0 col_cnt = None all_integers = True for line in headers: if count != 0: if col_cnt is None: col_cnt = len(line) if col_cnt < 2: return False else: if len(line) != col_cnt: return False try: for i in range(1, col_cnt): check = float(line[i]) # Check abs value is <= 1.0 if abs(check) > 1.0: return False # Also test for whether value is an integer try: check = int(line[i]) except ValueError: all_integers = False except ValueError: return False count += 1 if count > 0: return not all_integers return False
def set_meta(self, dataset, overwrite=True, skip=None, max_data_lines=None, **kwd): super(Group, self).set_meta(dataset, overwrite, skip, max_data_lines) group_names = set() headers = iter_headers(dataset.file_name, sep='\t', count=-1) for line in headers: if len(line) > 1: group_names.add(line[1]) dataset.metadata.groups = list(group_names)
def set_meta(self, dataset, overwrite=True, skip=None, max_data_lines=None, **kwd): super(Group, self).set_meta(dataset, overwrite, skip, max_data_lines) group_names = set() headers = iter_headers(dataset.file_name, sep='\t', count=-1) for line in headers: if len(line) > 1: group_names.add(line[1]) dataset.metadata.groups = list(group_names)
def sniff_prefix(self, file_prefix): """ Determines whether the file is an axes format The first line may have column headings. The following lines have the name in the first column plus float columns for each axis. ==> 98_sq_phylip_amazon.fn.unique.pca.axes <== group axis1 axis2 forest 0.000000 0.145743 pasture 0.145743 0.000000 ==> 98_sq_phylip_amazon.nmds.axes <== axis1 axis2 U68589 0.262608 -0.077498 U68590 0.027118 0.195197 U68591 0.329854 0.014395 >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.axes' ) >>> Axes().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.axes' ) >>> Axes().sniff( fname ) False """ headers = iter_headers(file_prefix, sep='\t') count = 0 col_cnt = None all_integers = True for line in headers: if count != 0: if col_cnt is None: col_cnt = len(line) if col_cnt < 2: return False else: if len(line) != col_cnt: return False try: for i in range(1, col_cnt): check = float(line[i]) # Check abs value is <= 1.0 if abs(check) > 1.0: return False # Also test for whether value is an integer try: check = int(line[i]) except ValueError: all_integers = False except ValueError: return False count += 1 if count > 0: return not all_integers return False
def set_meta(self, dataset, overwrite=True, skip=0, **kwd): super(DistanceMatrix, self).set_meta(dataset, overwrite=overwrite, skip=skip, **kwd) headers = iter_headers(dataset.file_name, sep='\t') for line in headers: if not line[0].startswith('@'): try: dataset.metadata.sequence_count = int(''.join(line)) # seq count sometimes preceded by tab break except Exception as e: if not isinstance(self, PairwiseDistanceMatrix): log.warning("DistanceMatrix set_meta %s" % e)
def sniff_prefix(self, file_prefix): """ Checks for and does cursory validation on data that looks like AGP >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname('eg1.agp') >>> GoldenPath().sniff(fname) True >>> fname = get_test_fname('eg2.agp') >>> GoldenPath().sniff(fname) True >>> fname = get_test_fname('1.bed') >>> GoldenPath().sniff(fname) False >>> fname = get_test_fname('2.tabular') >>> GoldenPath().sniff(fname) False """ found_non_comment_lines = False try: for line in iter_headers(file_prefix, '\t', comment_designator='#'): if line: if len(line) != 9: return False assert line[4] in [ 'A', 'D', 'F', 'G', 'O', 'P', 'W', 'N', 'U' ] ostensible_numbers = line[1:3] if line[4] in ['U', 'N']: ostensible_numbers.append(line[5]) assert line[6] in [ 'scaffold', 'contig', 'centromere', 'short_arm', 'heterochromatin', 'telomere', 'repeat' ] assert line[7] in ['yes', 'no'] assert line[8] in [ 'na', 'paired-ends', 'align_genus', 'align_xgenus', 'align_trnscript', 'within_clone', 'clone_contig', 'map', 'strobe', 'unspecified' ] else: ostensible_numbers.extend([line[6], line[7]]) assert line[8] in ['+', '-', '?', '0', 'na'] if line[4] == 'U': assert int(line[5]) == 100 assert all( map(lambda x: str(x).isnumeric() and int(x) > 0, ostensible_numbers)) found_non_comment_lines = True except Exception: return False return found_non_comment_lines
def set_meta(self, dataset, overwrite=True, skip=0, **kwd): super(DistanceMatrix, self).set_meta(dataset, overwrite=overwrite, skip=skip, **kwd) headers = iter_headers(dataset.file_name, sep='\t') for line in headers: if not line[0].startswith('@'): try: dataset.metadata.sequence_count = int(''.join(line)) # seq count sometimes preceded by tab break except Exception as e: if not isinstance(self, PairwiseDistanceMatrix): log.warning("DistanceMatrix set_meta %s" % e)
def sniff_prefix(self, file_prefix): """ Determines whether the file is a lower-triangle distance matrix (phylip) format The first line has the number of sequences in the matrix. The remaining lines have the sequence name followed by a list of distances from all preceeding sequences 5 # possibly but not always preceded by a tab :/ U68589 U68590 0.3371 U68591 0.3609 0.3782 U68592 0.4155 0.3197 0.4148 U68593 0.2872 0.1690 0.3361 0.2842 >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.lower.dist' ) >>> LowerTriangleDistanceMatrix().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.lower.dist' ) >>> LowerTriangleDistanceMatrix().sniff( fname ) False """ numlines = 300 headers = iter_headers(file_prefix, sep='\t', count=numlines) line_num = 0 for line in headers: if not line[0].startswith('@'): # first line should contain the number of sequences in the file if line_num == 0: if len(line) > 2: return False else: try: sequence_count = int(''.join(line)) assert sequence_count > 0 except ValueError: return False else: # number of fields should equal the line number if len(line) != (line_num): return False try: # Distances should be floats for column in line[2:]: float(column) except ValueError: return False line_num += 1 # check if the number of lines in the file was as expected if line_num == sequence_count + 1 or line_num == numlines + 1: return True return False
def sniff_prefix(self, file_prefix): """ Determines whether the file is a lower-triangle distance matrix (phylip) format The first line has the number of sequences in the matrix. The remaining lines have the sequence name followed by a list of distances from all preceeding sequences 5 # possibly but not always preceded by a tab :/ U68589 U68590 0.3371 U68591 0.3609 0.3782 U68592 0.4155 0.3197 0.4148 U68593 0.2872 0.1690 0.3361 0.2842 >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.lower.dist' ) >>> LowerTriangleDistanceMatrix().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.lower.dist' ) >>> LowerTriangleDistanceMatrix().sniff( fname ) False """ numlines = 300 headers = iter_headers(file_prefix, sep='\t', count=numlines) line_num = 0 for line in headers: if not line[0].startswith('@'): # first line should contain the number of sequences in the file if line_num == 0: if len(line) > 2: return False else: try: sequence_count = int(''.join(line)) assert sequence_count > 0 except ValueError: return False else: # number of fields should equal the line number if len(line) != (line_num): return False try: # Distances should be floats for column in line[2:]: float(column) except ValueError: return False line_num += 1 # check if the number of lines in the file was as expected if line_num == sequence_count + 1 or line_num == numlines + 1: return True return False
def sniff(self, filename): """ Determines whether the file is a Reference Taxonomy http://www.mothur.org/wiki/Taxonomy_outline A table with 2 or 3 columns: - SequenceName - Taxonomy (semicolon-separated taxonomy in descending order) - integer ? Example: 2-column (http://www.mothur.org/wiki/Taxonomy_outline) X56533.1 Eukaryota;Alveolata;Ciliophora;Intramacronucleata;Oligohymenophorea;Hymenostomatida;Tetrahymenina;Glaucomidae;Glaucoma; X97975.1 Eukaryota;Parabasalidea;Trichomonada;Trichomonadida;unclassified_Trichomonadida; AF052717.1 Eukaryota;Parabasalidea; Example: 3-column (http://vamps.mbl.edu/resources/databases.php) v3_AA008 Bacteria;Firmicutes;Bacilli;Lactobacillales;Streptococcaceae;Streptococcus 5 v3_AA016 Bacteria 120 v3_AA019 Archaea;Crenarchaeota;Marine_Group_I 1 >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.ref.taxonomy' ) >>> RefTaxonomy().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.ref.taxonomy' ) >>> RefTaxonomy().sniff( fname ) False """ headers = iter_headers(filename, sep='\t', count=300) count = 0 pat_prog = re.compile( '^([^ \t\n\r\x0c\x0b;]+([(]\\d+[)])?(;[^ \t\n\r\x0c\x0b;]+([(]\\d+[)])?)*(;)?)$' ) found_semicolons = False for line in headers: if not line[0].startswith('@') and not line[0].startswith('#'): if not (2 <= len(line) <= 3): return False if not pat_prog.match(line[1]): return False if not found_semicolons and line[1].find(';') > -1: found_semicolons = True if len(line) == 3: try: int(line[2]) except Exception: return False count += 1 if count > 0: # Require that at least one entry has semicolons in the 2nd column return found_semicolons return False
def sniff_prefix(self, file_prefix): """ Determines whether the file is a frequency tabular format for chimera analysis .. code-block:: #1.14.0 0 0.000 1 0.000 ... 155 0.975 >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.freq' ) >>> Frequency().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.freq' ) >>> Frequency().sniff( fname ) False >>> # Expression count matrix (EdgeR wrapper) >>> fname = get_test_fname( 'mothur_datatypetest_false_2.mothur.freq' ) >>> Frequency().sniff( fname ) False """ headers = iter_headers(file_prefix, sep='\t') count = 0 for line in headers: if not line[0].startswith('@'): # first line should be #<version string> if count == 0: if not line[0].startswith('#') or len(line) != 1: return False else: # all other lines should be <int> <float> if len(line) != 2: return False try: int(line[0]) float(line[1]) if line[1].find('.') == -1: return False except Exception: return False count += 1 if count > 1: return True return False
def sniff_prefix(self, file_prefix): """ Determines whether the file is a square distance matrix (Column-formatted distance matrix) format The first line has the number of sequences in the matrix. The following lines have the sequence name in the first column plus a column for the distance to each sequence in the row order in which they appear in the matrix. 3 U68589 0.0000 0.3371 0.3610 U68590 0.3371 0.0000 0.3783 U68590 0.3371 0.0000 0.3783 >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.square.dist' ) >>> SquareDistanceMatrix().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.square.dist' ) >>> SquareDistanceMatrix().sniff( fname ) False """ numlines = 300 headers = iter_headers(file_prefix, sep='\t', count=numlines) line_num = 0 for line in headers: if not line[0].startswith('@'): if line_num == 0: if len(line) > 2: return False else: try: sequence_count = int(''.join(line)) assert sequence_count > 0 except ValueError: return False else: # number of fields should equal the number of sequences if len(line) != sequence_count + 1: return False try: # Distances should be floats for column in line[2:]: float(column) except ValueError: return False line_num += 1 # check if the number of lines in the file was as expected if line_num == sequence_count + 1 or line_num == numlines + 1: return True return False
def set_meta(self, dataset, overwrite=True, **kwd): """ Set metadata for Otu files. >>> from galaxy.datatypes.sniff import get_test_fname >>> from galaxy.util.bunch import Bunch >>> dataset = Bunch() >>> dataset.metadata = Bunch >>> otu = Otu() >>> dataset.file_name = get_test_fname( 'mothur_datatypetest_true.mothur.otu' ) >>> dataset.has_data = lambda: True >>> otu.set_meta(dataset) >>> dataset.metadata.columns 100 >>> len(dataset.metadata.labels) == 37 True >>> len(dataset.metadata.otulabels) == 98 True """ super(Otu, self).set_meta(dataset, overwrite=overwrite, **kwd) if dataset.has_data(): label_names = set() otulabel_names = set() ncols = 0 data_lines = 0 comment_lines = 0 headers = iter_headers(dataset.file_name, sep='\t', count=-1) first_line = get_headers(dataset.file_name, sep='\t', count=1) if first_line: first_line = first_line[0] # set otulabels if len(first_line) > 2: otulabel_names = first_line[2:] # set label names and number of lines for line in headers: if len(line) >= 2 and not line[0].startswith('@'): data_lines += 1 ncols = max(ncols, len(line)) label_names.add(line[0]) else: comment_lines += 1 # Set the discovered metadata values for the dataset dataset.metadata.data_lines = data_lines dataset.metadata.columns = ncols dataset.metadata.labels = list(label_names) dataset.metadata.labels.sort() dataset.metadata.otulabels = list(otulabel_names) dataset.metadata.otulabels.sort()
def sniff_prefix(self, file_prefix): """ Determines whether the file is a Reference Taxonomy http://www.mothur.org/wiki/Taxonomy_outline A table with 2 or 3 columns: - SequenceName - Taxonomy (semicolon-separated taxonomy in descending order) - integer ? Example: 2-column (http://www.mothur.org/wiki/Taxonomy_outline) X56533.1 Eukaryota;Alveolata;Ciliophora;Intramacronucleata;Oligohymenophorea;Hymenostomatida;Tetrahymenina;Glaucomidae;Glaucoma; X97975.1 Eukaryota;Parabasalidea;Trichomonada;Trichomonadida;unclassified_Trichomonadida; AF052717.1 Eukaryota;Parabasalidea; Example: 3-column (http://vamps.mbl.edu/resources/databases.php) v3_AA008 Bacteria;Firmicutes;Bacilli;Lactobacillales;Streptococcaceae;Streptococcus 5 v3_AA016 Bacteria 120 v3_AA019 Archaea;Crenarchaeota;Marine_Group_I 1 >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.ref.taxonomy' ) >>> RefTaxonomy().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.ref.taxonomy' ) >>> RefTaxonomy().sniff( fname ) False """ headers = iter_headers(file_prefix, sep='\t', count=300) count = 0 pat_prog = re.compile('^([^ \t\n\r\x0c\x0b;]+([(]\\d+[)])?(;[^ \t\n\r\x0c\x0b;]+([(]\\d+[)])?)*(;)?)$') found_semicolons = False for line in headers: if not line[0].startswith('@') and not line[0].startswith('#'): if not (2 <= len(line) <= 3): return False if not pat_prog.match(line[1]): return False if not found_semicolons and line[1].find(';') > -1: found_semicolons = True if len(line) == 3: try: int(line[2]) except Exception: return False count += 1 if count > 0: # Require that at least one entry has semicolons in the 2nd column return found_semicolons return False
def set_meta(self, dataset, overwrite=True, **kwd): """ Set metadata for Otu files. >>> from galaxy.datatypes.sniff import get_test_fname >>> from galaxy.util.bunch import Bunch >>> dataset = Bunch() >>> dataset.metadata = Bunch >>> otu = Otu() >>> dataset.file_name = get_test_fname( 'mothur_datatypetest_true.mothur.otu' ) >>> dataset.has_data = lambda: True >>> otu.set_meta(dataset) >>> dataset.metadata.columns 100 >>> len(dataset.metadata.labels) == 37 True >>> len(dataset.metadata.otulabels) == 98 True """ super(Otu, self).set_meta(dataset, overwrite=overwrite, **kwd) if dataset.has_data(): label_names = set() otulabel_names = set() ncols = 0 data_lines = 0 comment_lines = 0 headers = iter_headers(dataset.file_name, sep='\t', count=-1) first_line = get_headers(dataset.file_name, sep='\t', count=1) if first_line: first_line = first_line[0] # set otulabels if len(first_line) > 2: otulabel_names = first_line[2:] # set label names and number of lines for line in headers: if len(line) >= 2 and not line[0].startswith('@'): data_lines += 1 ncols = max(ncols, len(line)) label_names.add(line[0]) else: comment_lines += 1 # Set the discovered metadata values for the dataset dataset.metadata.data_lines = data_lines dataset.metadata.columns = ncols dataset.metadata.labels = list(label_names) dataset.metadata.labels.sort() dataset.metadata.otulabels = list(otulabel_names) dataset.metadata.otulabels.sort()
def sniff_prefix(self, file_prefix): """ Determines whether the file is a square distance matrix (Column-formatted distance matrix) format The first line has the number of sequences in the matrix. The following lines have the sequence name in the first column plus a column for the distance to each sequence in the row order in which they appear in the matrix. 3 U68589 0.0000 0.3371 0.3610 U68590 0.3371 0.0000 0.3783 U68590 0.3371 0.0000 0.3783 >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.square.dist' ) >>> SquareDistanceMatrix().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.square.dist' ) >>> SquareDistanceMatrix().sniff( fname ) False """ numlines = 300 headers = iter_headers(file_prefix, sep='\t', count=numlines) line_num = 0 for line in headers: if not line[0].startswith('@'): if line_num == 0: if len(line) > 2: return False else: try: sequence_count = int(''.join(line)) assert sequence_count > 0 except ValueError: return False else: # number of fields should equal the number of sequences if len(line) != sequence_count + 1: return False try: # Distances should be floats for column in line[2:]: float(column) except ValueError: return False line_num += 1 # check if the number of lines in the file was as expected if line_num == sequence_count + 1 or line_num == numlines + 1: return True return False
def sniff(self, filename): """ Determines whether the file is a frequency tabular format for chimera analysis #1.14.0 0 0.000 1 0.000 ... 155 0.975 >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.freq' ) >>> Frequency().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.freq' ) >>> Frequency().sniff( fname ) False # Expression count matrix (EdgeR wrapper) >>> fname = get_test_fname( 'mothur_datatypetest_false_2.mothur.freq' ) >>> Frequency().sniff( fname ) False """ headers = iter_headers(filename, sep='\t') count = 0 for line in headers: if not line[0].startswith('@'): # first line should be #<version string> if count == 0: if not line[0].startswith('#') or len(line) != 1: return False else: # all other lines should be <int> <float> if len(line) != 2: return False try: int(line[0]) float(line[1]) if line[1].find('.') == -1: return False except Exception: return False count += 1 if count > 1: return True return False
def sniff_prefix(self, file_prefix): """ Determines whether the file is a pairwise distance matrix (Column-formatted distance matrix) format The first and second columns have the sequence names and the third column is the distance between those sequences. >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.pair.dist' ) >>> PairwiseDistanceMatrix().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.pair.dist' ) >>> PairwiseDistanceMatrix().sniff( fname ) False """ headers = iter_headers(file_prefix, sep='\t') count = 0 names = [False, False] for line in headers: if line[0].startswith('@'): continue if len(line) != 3: return False # check if col3 contains distances (floats) try: float(line[2]) try: # See if it's also an integer int(line[2]) except ValueError: # At least one value is not an integer all_ints = False except ValueError: return False count += 1 # check if col1 and col2 likely contain names for c in [0, 1]: try: float(line[c]) except ValueError: names[c] = True if not names[0] or not names[1]: return False if count > 2: return not all_ints return False
def sniff(self, filename): """ Determines whether the file is in html format >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'complete.bed' ) >>> Html().sniff( fname ) False >>> fname = get_test_fname( 'file.html' ) >>> Html().sniff( fname ) True """ headers = iter_headers(filename, None) for i, hdr in enumerate(headers): if hdr and hdr[0].lower().find('<html>') >= 0: return True return False
def sniff_prefix(self, file_prefix): """ Determines whether the file is in html format >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'complete.bed' ) >>> Html().sniff( fname ) False >>> fname = get_test_fname( 'file.html' ) >>> Html().sniff( fname ) True """ headers = iter_headers(file_prefix, None) for i, hdr in enumerate(headers): if hdr and hdr[0].lower().find('<html>') >= 0: return True return False
def sniff(self, filename): """ Try to guess if the file is a InChI file. >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname('drugbank_drugs.inchi') >>> InChI().sniff(fname) True >>> fname = get_test_fname('drugbank_drugs.cml') >>> InChI().sniff(fname) False """ inchi_lines = iter_headers(filename, sep=' ', count=10) for inchi in inchi_lines: if not inchi[0].startswith('InChI='): return False return True
def sniff(self, filename): """ Try to guess if the file is a InChI file. >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname('drugbank_drugs.inchi') >>> InChI().sniff(fname) True >>> fname = get_test_fname('drugbank_drugs.cml') >>> InChI().sniff(fname) False """ inchi_lines = iter_headers(filename, sep=' ', count=10) for inchi in inchi_lines: if not inchi[0].startswith('InChI='): return False return True
def sniff_prefix(self, file_prefix): """ Determines whether the file is a quantiles tabular format for chimera analysis .. code-block:: 1 0 0 0 0 0 0 2 0.309198 0.309198 0.37161 0.37161 0.37161 0.37161 3 0.510982 0.563213 0.693529 0.858939 1.07442 1.20608 ... >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.quan' ) >>> Quantile().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.quan' ) >>> Quantile().sniff( fname ) False """ headers = iter_headers(file_prefix, sep='\t') count = 0 for line in headers: if not line[0].startswith('@') and not line[0].startswith('#'): if len(line) != 7: return False try: int(line[0]) float(line[1]) float(line[2]) float(line[3]) float(line[4]) float(line[5]) float(line[6]) except Exception: return False count += 1 if count > 0: return True return False
def sniff(self, filename): """ Determines whether the file is a frequency tabular format for chimera analysis #1.14.0 0 0.000 1 0.000 ... 155 0.975 >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.freq' ) >>> Frequency().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.freq' ) >>> Frequency().sniff( fname ) False """ headers = iter_headers(filename, sep='\t') count = 0 for line in headers: if not line[0].startswith('@'): if count == 0: # first line should be #<version string> if not line[0].startswith('#') and len(line) == 1: return False else: # all other lines should be <int> <float> if len(line) != 2: return False try: int(line[0]) float(line[1]) except Exception: return False count += 1 if count > 1: return True return False
def sniff_prefix(self, file_prefix): """ >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname('big.gfa1') >>> Gfa1().sniff(fname) True >>> Gfa2().sniff(fname) False """ found_valid_lines = False for line in iter_headers(file_prefix, "\t"): if line[0].startswith('#'): continue if line[0] == 'H': return len(line) == 2 and line[1] == 'VN:Z:1.0' elif line[0] == 'S': if len(line) < 3: return False elif line[0] == 'L': if len(line) < 6: return False for i in (2, 4): if line[i] not in ('+', '-'): return False elif line[0] == 'C': if len(line) < 7: return False for i in (2, 4): if line[i] not in ('+', '-'): return False int(line[5]) elif line[0] == 'P': if len(line) < 4: return False else: return False found_valid_lines = True return found_valid_lines
def sniff_prefix(self, file_prefix): """ >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname('A-3105.paf') >>> Paf().sniff(fname) True """ found_valid_lines = False for line in iter_headers(file_prefix, "\t"): if len(line) < 12: return False for i in (1, 2, 3, 6, 7, 8, 9, 10, 11): int(line[i]) if line[4] not in ('+', '-'): return False if not (0 <= int(line[11]) <= 255): return False # Check that the optional columns after the 12th contain SAM-like typed key-value pairs for i in range(12, len(line)): if len(line[i].split(':')) != 3: return False found_valid_lines = True return found_valid_lines
def sniff(self, filename, vals_are_int=False): """ Determines whether the file is a otu (operational taxonomic unit) Shared format label<TAB>group<TAB>count[<TAB>value(1..n)] The first line is column headings as of Mothur v 1.2 >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.shared' ) >>> GroupAbund().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.shared' ) >>> GroupAbund().sniff( fname ) False """ headers = iter_headers(filename, sep='\t') count = 0 for line in headers: if not line[0].startswith('@'): if len(line) < 3: return False if count > 0 or line[0] != 'label': try: check = int(line[2]) if check + 3 != len(line): return False for i in range(3, len(line)): if vals_are_int: int(line[i]) else: float(line[i]) except ValueError: return False count += 1 if count > 1: return True return False
def sniff_prefix(self, file_prefix, vals_are_int=False): """ Determines whether the file is a otu (operational taxonomic unit) Shared format label<TAB>group<TAB>count[<TAB>value(1..n)] The first line is column headings as of Mothur v 1.2 >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.shared' ) >>> GroupAbund().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.shared' ) >>> GroupAbund().sniff( fname ) False """ headers = iter_headers(file_prefix, sep='\t') count = 0 for line in headers: if not line[0].startswith('@'): if len(line) < 3: return False if count > 0 or line[0] != 'label': try: check = int(line[2]) if check + 3 != len(line): return False for i in range(3, len(line)): if vals_are_int: int(line[i]) else: float(line[i]) except ValueError: return False count += 1 if count > 1: return True return False
def sniff_prefix(self, file_prefix): """ Determines whether the file is a quantiles tabular format for chimera analysis 1 0 0 0 0 0 0 2 0.309198 0.309198 0.37161 0.37161 0.37161 0.37161 3 0.510982 0.563213 0.693529 0.858939 1.07442 1.20608 ... >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.quan' ) >>> Quantile().sniff( fname ) True >>> fname = get_test_fname( 'mothur_datatypetest_false.mothur.quan' ) >>> Quantile().sniff( fname ) False """ headers = iter_headers(file_prefix, sep='\t') count = 0 for line in headers: if not line[0].startswith('@') and not line[0].startswith('#'): if len(line) != 7: return False try: int(line[0]) float(line[1]) float(line[2]) float(line[3]) float(line[4]) float(line[5]) float(line[6]) except Exception: return False count += 1 if count > 0: return True return False