def test_teetsv_unicode(): t1 = ((u"name", u"id"), (u"Արամ Խաչատրյան", 1), (u"Johann Strauß", 2), (u"Вагиф Сәмәдоғлу", 3), (u"章子怡", 4)) f1 = NamedTemporaryFile(delete=False) f2 = NamedTemporaryFile(delete=False) (etl.wrap(t1).teetsv(f1.name, encoding="utf-8").selectgt("id", 1).totsv(f2.name, encoding="utf-8")) ieq(t1, etl.fromtsv(f1.name, encoding="utf-8").convertnumbers()) ieq(etl.wrap(t1).selectgt("id", 1), etl.fromtsv(f2.name, encoding="utf-8").convertnumbers())
def test_teetsv(): t1 = (("foo", "bar"), ("a", 2), ("b", 1), ("c", 3)) f1 = NamedTemporaryFile(delete=False) f2 = NamedTemporaryFile(delete=False) (etl.wrap(t1).teetsv(f1.name, encoding="ascii").selectgt("bar", 1).totsv(f2.name, encoding="ascii")) ieq(t1, etl.fromtsv(f1.name, encoding="ascii").convertnumbers()) ieq(etl.wrap(t1).selectgt("bar", 1), etl.fromtsv(f2.name, encoding="ascii").convertnumbers())
def test_teetsv(): t1 = (('foo', 'bar'), ('a', 2), ('b', 1), ('c', 3)) f1 = NamedTemporaryFile(delete=False) f2 = NamedTemporaryFile(delete=False) (etl.wrap(t1).teetsv(f1.name, encoding='ascii').selectgt('bar', 1).totsv(f2.name, encoding='ascii')) ieq(t1, etl.fromtsv(f1.name, encoding='ascii').convertnumbers()) ieq( etl.wrap(t1).selectgt('bar', 1), etl.fromtsv(f2.name, encoding='ascii').convertnumbers())
def init(release_dir): """Initialise data resources. Parameters ---------- release_dir : string Local filesystem path where data from the release are stored. """ # variation ########### global callset, callset_pass variation_dir = os.path.join(release_dir, 'variation') # main callset callset_zarr_fn = os.path.join(variation_dir, 'main', 'zarr2', 'ag1000g.phase1.ar3') if os.path.exists(callset_zarr_fn): callset = zarr.open_group(callset_zarr_fn, mode='r') # main callset, PASS variants only callset_pass_zarr_fn = os.path.join(variation_dir, 'main', 'zarr2', 'ag1000g.phase1.ar3.pass') if os.path.exists(callset_pass_zarr_fn): callset_pass = zarr.open_group(callset_pass_zarr_fn, mode='r') # haplotypes ############ global callset_phased, tbl_haplotypes, lkp_haplotypes, df_haplotypes haplotypes_dir = os.path.join(release_dir, 'haplotypes') # try HDF5 first callset_phased_h5_fn = os.path.join(haplotypes_dir, 'main', 'hdf5', 'ag1000g.phase1.ar3.1.haplotypes.h5') if os.path.exists(callset_phased_h5_fn): callset_phased = h5py.File(callset_phased_h5_fn, mode='r') # prefer Zarr if available # N.B., the Zarr data is not consistent with HDF5 or shapeit outputs, # it is based on a previous phasing run. # #callset_phased_zarr_fn = os.path.join(haplotypes_dir, 'main', 'zarr2', # 'ag1000g.phase1.ar3.1.haplotypes') #if os.path.exists(callset_phased_zarr_fn): # callset_phased = zarr.open_group(callset_phased_zarr_fn, mode='r') # haplotypes metadata haplotypes_fn = os.path.join(haplotypes_dir, 'haplotypes.meta.txt') if os.path.exists(haplotypes_fn): tbl_haplotypes = (etl.fromtsv(haplotypes_fn).convert( ('index', 'kt_2la', 'kt_2rb'), int)) lkp_haplotypes = tbl_haplotypes.recordlookupone('label') df_haplotypes = pandas.read_csv(haplotypes_fn, sep='\t', index_col='index')
def get_table(source=None, nrows=None, skip=None, fields=None, exclude=None, rownumbers=True, **petlargs): """ :param source: full path filename of the delimited file :param nrows: number of rows to include in the table :param skip: number of rows to skip from the file :param fields: selected fields to extract from the file :param exclude: selected fields to be excluded from the file :param rownumbers: Add a rowID column. This is True by default This is similar to pandas.RangeIndex see petl.transform.basics.addrownumbers() Notice: skip and nrows parameters require that addrownumbers() is applied to petl table If `fields` is specified and `rowID` is not included in the list the column will not be included in petl table :param petlargs: see petl.io.csv.fromcsv and petl.io.csv.fromtsv :return: petl table container Notice: petl makes extensive use of lazy evaluation and iterators the file is not loaded in memory instead a container/iterator is returned Examples: etl.get_table('movies.csv', 20, 100, ['rowID', 'movie_title', 'title_year']).lookall() etl.get_table('movies.csv', 20, 100, exclude=['language', 'actor_1_name']).lookall() """ # Get the extension of the filename table = None if source: ext = os.path.splitext(source)[1] # Read all rows from the file and create a pandas dataframe in memory if ext == '.csv': table = petl.fromcsv(source, **petlargs) elif ext == '.tsv': table = petl.fromtsv(source, **petlargs) if rownumbers: table = table.addrownumbers(start=1, step=1, field='rowID') if skip and rownumbers: if nrows: table = table.select(lambda num: num.rowID > skip and num. rowID <= nrows + skip) else: table = table.select(lambda num: num.rowID > skip) if not skip and nrows and rownumbers: table = table.select(lambda num: num.rowID <= nrows) if fields: table = table.cut(*fields) if exclude: table = table.cutout(*exclude) return table
def get_file_header(ftype, fname): ext = os.path.splitext(fname)[1][1:] if not ftype.lower() == ext: raise Exception( f'Failed: Filename extension does not match < ftype={ftype} >') if ftype == 'CSV': return petl.fromcsv(fname).head(0).tol()[0] elif ftype == 'TSV': return petl.fromtsv(fname).head(0).tol()[0]
def test_ZipSource(): # setup table = [('foo', 'bar'), ('a', '1'), ('b', '2')] totsv(table, 'tmp/issue_241.tsv') z = zipfile.ZipFile('tmp/issue_241.zip', mode='w') z.write('tmp/issue_241.tsv', 'data.tsv') z.close() # test actual = fromtsv(ZipSource('tmp/issue_241.zip', 'data.tsv')) ieq(table, actual)
def test_ZipSource(): # setup table = [('foo', 'bar'), ('a', '1'), ('b', '2')] totsv(table, 'tmp/issue_241.tsv') z = zipfile.ZipFile('tmp/issue_241.zip', mode='w') z.write('tmp/issue_241.tsv', 'data.tsv') z.close() # test actual = fromtsv(ZipSource('tmp/issue_241.zip', 'data.tsv')) ieq(table, actual)
def test_fromtsv(): f = NamedTemporaryFile(delete=False) writer = csv.writer(f, delimiter="\t") table = (("foo", "bar"), ("a", 1), ("b", 2), ("c", 2)) for row in table: writer.writerow(row) f.close() actual = fromtsv(f.name) expect = (("foo", "bar"), ("a", "1"), ("b", "2"), ("c", "2")) ieq(expect, actual) ieq(expect, actual) # verify can iterate twice
def test_issue_231(): table = [['foo', 'bar'], ['a', '1'], ['b', '2']] t = cut(table, 'foo') totsv(t, 'tmp/issue_231.tsv') u = fromtsv('tmp/issue_231.tsv') ieq(t, u) tocsv(t, 'tmp/issue_231.csv') u = fromcsv('tmp/issue_231.csv') ieq(t, u) topickle(t, 'tmp/issue_231.pickle') u = frompickle('tmp/issue_231.pickle') ieq(t, u)
def test_issue_231(): table = [['foo', 'bar'], ['a', '1'], ['b', '2']] t = cut(table, 'foo') totsv(t, 'tmp/issue_231.tsv') u = fromtsv('tmp/issue_231.tsv') ieq(t, u) tocsv(t, 'tmp/issue_231.csv') u = fromcsv('tmp/issue_231.csv') ieq(t, u) topickle(t, 'tmp/issue_231.pickle') u = frompickle('tmp/issue_231.pickle') ieq(t, u)
def test_fromtsv(): f = NamedTemporaryFile(delete=False) writer = csv.writer(f, delimiter='\t') table = (('foo', 'bar'), ('a', 1), ('b', 2), ('c', 2)) for row in table: writer.writerow(row) f.close() actual = fromtsv(f.name) expect = (('foo', 'bar'), ('a', '1'), ('b', '2'), ('c', '2')) ieq(expect, actual) ieq(expect, actual) # verify can iterate twice
def test_teetsv_unicode(): t1 = ( (u'name', u'id'), (u'Արամ Խաչատրյան', 1), (u'Johann Strauß', 2), (u'Вагиф Сәмәдоғлу', 3), (u'章子怡', 4), ) f1 = NamedTemporaryFile(delete=False) f2 = NamedTemporaryFile(delete=False) (etl.wrap(t1).teetsv(f1.name, encoding='utf-8').selectgt('id', 1).totsv(f2.name, encoding='utf-8')) ieq(t1, etl.fromtsv(f1.name, encoding='utf-8').convertnumbers()) ieq( etl.wrap(t1).selectgt('id', 1), etl.fromtsv(f2.name, encoding='utf-8').convertnumbers())
def test_zipsource(): # setup tbl = [('foo', 'bar'), ('a', '1'), ('b', '2')] fn_tsv = NamedTemporaryFile().name etl.totsv(tbl, fn_tsv) fn_zip = NamedTemporaryFile().name z = zipfile.ZipFile(fn_zip, mode='w') z.write(fn_tsv, 'data.tsv') z.close() # test actual = etl.fromtsv(ZipSource(fn_zip, 'data.tsv')) ieq(tbl, actual)
def test_zipsource(): # setup tbl = [('foo', 'bar'), ('a', '1'), ('b', '2')] fn_tsv = NamedTemporaryFile().name etl.totsv(tbl, fn_tsv) fn_zip = NamedTemporaryFile().name z = zipfile.ZipFile(fn_zip, mode='w') z.write(fn_tsv, 'data.tsv') z.close() # test actual = etl.fromtsv(ZipSource(fn_zip, 'data.tsv')) ieq(tbl, actual)
def test_fromtsv(): f = NamedTemporaryFile(delete=False) writer = csv.writer(f, delimiter='\t') table = (('foo', 'bar'), ('a', 1), ('b', 2), ('c', 2)) for row in table: writer.writerow(row) f.close() actual = fromtsv(f.name) expect = (('foo', 'bar'), ('a', '1'), ('b', '2'), ('c', '2')) ieq(expect, actual) ieq(expect, actual) # verify can iterate twice
def xls_tidy(xls,qvalue): d=etl.fromtsv(xls) sd=etl.select(d,lambda x: float(x.PepQValue) <=float(qvalue)) psmsummary=sd ssd=etl.cut(sd, 'Peptide', 'Protein', 'PepQValue') #remove the mod info in peptide. ssd=etl.transform.regex.sub(ssd,'Peptide', r'^[\w-]\.(.+)\.[\w-]$', r'\1') ssd=etl.transform.regex.sub(ssd,'Peptide', r'[\d\.\+]+', r'') aggregation = OrderedDict() aggregation['SpecCount'] = len cssd=etl.aggregate(ssd, 'Peptide', aggregation) fssd=etl.groupselectfirst(ssd, key=('Peptide','Protein',"PepQValue")) aggregation = OrderedDict() aggregation['Protein'] = 'Protein', etl.strjoin(';') aggregation['PepQValue'] = 'PepQValue', etl.strjoin(';') assd=etl.aggregate(fssd, 'Peptide', aggregation) pepsummary=etl.join(assd, cssd, key='Peptide') return (psmsummary, pepsummary)
def condense(path=".", use_tsv=True): """ Couldn't be more pleased with this thing. utility method for moi - all I want it to do is generate a bunch of PETL objects that will allow me to extract columns. Should take some of the output in such a way that I can populate a database with patterns for rapid matching later. I do want to add my scoring mechanism in to the row/column generator when I can. """ objects = [] # I'll use os.listdir(path) to try and seek everything try: x = os.listdir(path) # get the state at the start. log_output(x) # make a note of what was found. for a in x: pair = [] try: if use_tsv: pair.append(petl.fromtsv(a)) else: pair.append(petl.fromcsv(a)) pair.append(os.path.basename(a)) log_output("Added petl object for %s" % (a)) objects.append(tuple(pair)) except Exception as ECHO: log_output(ECHO, "./error_log.log") log_output("Exception has occurred: %s" % (ECHO)) except Exception as eddy: log_output(ECHO, "./error_log.log") log_output("Exception has occurred: %s" % (eddy)) return objects
def init(release_dir, load_geneset=False): """Initialise data resources. Parameters ---------- release_dir : string Local filesystem path where data from the release are stored. load_geneset : string If True, load geneset into memory. """ # reference sequence #################### global genome_fn, genome genome_dir = os.path.join(release_dir, 'genome') genome_fn = os.path.join(genome_dir, 'Anopheles-gambiae-PEST_CHROMOSOMES_AgamP3.fa') if os.path.exists(genome_fn): genome = pyfasta.Fasta(genome_fn) # genome annotations #################### global geneset_agamp42_fn, geneset_agamp42 geneset_dir = os.path.join(release_dir, 'geneset') geneset_agamp42_fn = os.path.join( geneset_dir, 'Anopheles-gambiae-PEST_BASEFEATURES_AgamP4.2.sorted.gff3.gz') if os.path.exists(geneset_agamp42_fn) and load_geneset: geneset_agamp42 = allel.FeatureTable.from_gff3(geneset_agamp42_fn) # variant callsets ################## global callset, callset_pass variation_dir = os.path.join(release_dir, 'variation') # main callset callset_h5_fn = os.path.join(variation_dir, 'main', 'hdf5', 'ag1000g.phase1.ar3.h5') if os.path.exists(callset_h5_fn): callset = h5py.File(callset_h5_fn, mode='r') # main callset, PASS variants only callset_pass_h5_fn = os.path.join(variation_dir, 'main', 'hdf5', 'ag1000g.phase1.ar3.pass.h5') if os.path.exists(callset_pass_h5_fn): callset_pass = h5py.File(callset_pass_h5_fn, mode='r') # accessibility ############### global accessibility accessibility_dir = os.path.join(release_dir, 'accessibility') accessibility_fn = os.path.join(accessibility_dir, 'accessibility.h5') if os.path.exists(accessibility_fn): accessibility = h5py.File(accessibility_fn, mode='r') # sample metadata ################# global samples_fn, tbl_samples, lkp_samples, sample_ids, df_samples samples_dir = os.path.join(release_dir, 'samples') samples_fn = os.path.join(samples_dir, 'samples.all.txt') if os.path.exists(samples_fn): tbl_samples = (etl.fromtsv(samples_fn).convert( ('index', 'year', 'n_sequences', 'kt_2la', 'kt_2rb'), int).convert( ('mean_coverage', 'latitude', 'longitude') + tuple(range(20, 36)), float)) lkp_samples = tbl_samples.recordlookupone('ox_code') sample_ids = tbl_samples.values('ox_code').list() df_samples = pandas.read_csv(samples_fn, sep='\t', index_col='index') # extras ######## global allele_counts, allele_counts_gq10, outgroup_alleles, outgroup_allele_counts, \ outgroup_species extras_dir = os.path.join(release_dir, 'extras') # allele counts allele_counts_fn = os.path.join(extras_dir, 'allele_counts.h5') if os.path.exists(allele_counts_fn): allele_counts = h5py.File(allele_counts_fn, mode='r') allele_counts_gq10_fn = os.path.join(extras_dir, 'allele_counts.gq10.h5') if os.path.exists(allele_counts_gq10_fn): allele_counts_gq10 = h5py.File(allele_counts_gq10_fn, mode='r') # outgroup data outgroup_species = 'arab', 'meru', 'mela', 'quad', 'epir', 'chri' outgroup_alleles_fn = os.path.join(extras_dir, 'outgroup_alleles.h5') if os.path.exists(outgroup_alleles_fn): outgroup_alleles = h5py.File(outgroup_alleles_fn, mode='r') outgroup_allele_counts_fn = os.path.join(extras_dir, 'outgroup_allele_counts.h5') if os.path.exists(outgroup_allele_counts_fn): outgroup_allele_counts = h5py.File(outgroup_allele_counts_fn, mode='r')
def fromgff3(filename, region=None): """ Extract feature rows from a GFF3 file, e.g.:: >>> import petl as etl >>> # activate bio extensions ... import petlx.bio >>> table1 = etl.fromgff3('fixture/sample.gff') >>> table1.look(truncate=30) +--------------+---------+---------------+-------+---------+-------+--------+-------+--------------------------------+ | seqid | source | type | start | end | score | strand | phase | attributes | +==============+=========+===============+=======+=========+=======+========+=======+================================+ | 'apidb|MAL1' | 'ApiDB' | 'supercontig' | 1 | 643292 | '.' | '+' | '.' | {'localization': 'nuclear', 'o | +--------------+---------+---------------+-------+---------+-------+--------+-------+--------------------------------+ | 'apidb|MAL2' | 'ApiDB' | 'supercontig' | 1 | 947102 | '.' | '+' | '.' | {'localization': 'nuclear', 'o | +--------------+---------+---------------+-------+---------+-------+--------+-------+--------------------------------+ | 'apidb|MAL3' | 'ApiDB' | 'supercontig' | 1 | 1060087 | '.' | '+' | '.' | {'localization': 'nuclear', 'o | +--------------+---------+---------------+-------+---------+-------+--------+-------+--------------------------------+ | 'apidb|MAL4' | 'ApiDB' | 'supercontig' | 1 | 1204112 | '.' | '+' | '.' | {'localization': 'nuclear', 'o | +--------------+---------+---------------+-------+---------+-------+--------+-------+--------------------------------+ | 'apidb|MAL5' | 'ApiDB' | 'supercontig' | 1 | 1343552 | '.' | '+' | '.' | {'localization': 'nuclear', 'o | +--------------+---------+---------------+-------+---------+-------+--------+-------+--------------------------------+ ... A region query string of the form '[seqid]' or '[seqid]:[start]-[end]' may be given for the `region` argument. If given, requires the GFF3 file to be position sorted, bgzipped and tabix indexed. Requires pysam to be installed. E.g.:: >>> # extract from a specific genome region via tabix ... table2 = etl.fromgff3('fixture/sample.sorted.gff.gz', ... region='apidb|MAL5:1289593-1289595') >>> table2.look(truncate=30) +--------------+---------+---------------+---------+---------+-------+--------+-------+--------------------------------+ | seqid | source | type | start | end | score | strand | phase | attributes | +==============+=========+===============+=========+=========+=======+========+=======+================================+ | 'apidb|MAL5' | 'ApiDB' | 'supercontig' | 1 | 1343552 | '.' | '+' | '.' | {'localization': 'nuclear', 'o | +--------------+---------+---------------+---------+---------+-------+--------+-------+--------------------------------+ | 'apidb|MAL5' | 'ApiDB' | 'exon' | 1289594 | 1291685 | '.' | '+' | '.' | {'size': '2092', 'Parent': 'ap | +--------------+---------+---------------+---------+---------+-------+--------+-------+--------------------------------+ | 'apidb|MAL5' | 'ApiDB' | 'gene' | 1289594 | 1291685 | '.' | '+' | '.' | {'ID': 'apidb|MAL5_18S', 'web_ | +--------------+---------+---------------+---------+---------+-------+--------+-------+--------------------------------+ | 'apidb|MAL5' | 'ApiDB' | 'rRNA' | 1289594 | 1291685 | '.' | '+' | '.' | {'ID': 'apidb|rna_MAL5_18S-1', | +--------------+---------+---------------+---------+---------+-------+--------+-------+--------------------------------+ """ if region is None: # parse file as tab-delimited table = etl.fromtsv(filename) else: # extract via tabix table = etl.fromtabix(filename, region=region) return ( table .pushheader(GFF3_HEADER) .skipcomments('#') # ignore any row not 9 values long (e.g., trailing fasta) .rowlenselect(9) # parse attributes into a dict .convert('attributes', gff3_parse_attributes) # parse coordinates .convert(('start', 'end'), int) )
import petl as etl readFile = etl.fromtsv("donedeal_data_sample.tsv") tmpTable = etl.addfield(readFile, 'InKms', lambda rec: rec['mileage']) tmpTable2File = etl.convert(tmpTable, 'InKms', lambda v: int(float(v) * 1.6), where=lambda r: r.mileageType == 'miles') etl.totsv(tmpTable2File, 'donedeal_inKms.tsv')
import petl as etl table1 = etl.fromtsv("D:\JOB\BI_Developer_Challenge\donedeal_data_sample.tsv") table2 = etl.convert(table1, 'mileage', float) table3 = etl.convert(table2, 'mileage', lambda v: v * 1.60934, where=lambda r: r.mileageType == 'miles') table4 = etl.convert(table3, 'mileageType', lambda v: 'km', where=lambda r: r.mileageType in ('miles', 'kilometres')) table4 = etl.convert(table3, 'mileageType', lambda v: 'NA', where=lambda r: r.mileageType not in ('km')) etl.totsv(table4, "D:\JOB\BI_Developer_Challenge\donedeal_data_etl.tsv")
def init(release_dir, load_geneset=False, geneset_attributes=None): """Initialise data resources. Parameters ---------- release_dir : string Local filesystem path where data from the release are stored. load_geneset : string If True, load geneset into memory. geneset_attributes : dict-like Attributes to load. """ # reference sequence #################### global genome_agamp3, genome_agamp4, genome_dir genome_dir = os.path.join(release_dir, 'genome') genome_agamp3_dir = os.path.join(genome_dir, 'agamP3') genome_agamp3_fn = os.path.join( genome_agamp3_dir, 'Anopheles-gambiae-PEST_CHROMOSOMES_AgamP3.fa') if os.path.exists(genome_agamp3_fn): genome_agamp3 = pyfasta.Fasta(genome_agamp3_fn, key_fn=lambda v: v.split()[0]) genome_agamp4_dir = os.path.join(genome_dir, 'agamP4') genome_agamp4_fn = os.path.join( genome_agamp4_dir, 'Anopheles-gambiae-PEST_CHROMOSOMES_AgamP4.fa') if os.path.exists(genome_agamp4_fn): genome_agamp4 = pyfasta.Fasta(genome_agamp4_fn, key_fn=lambda v: v.split()[0]) # genome annotations #################### global geneset_agamp44_fn, geneset_agamp44, geneset_dir geneset_dir = os.path.join(release_dir, 'geneset') geneset_agamp44_fn = os.path.join( geneset_dir, 'Anopheles-gambiae-PEST_BASEFEATURES_AgamP4.4.sorted.gff3.gz') if load_geneset: geneset_agamp44 = allel.FeatureTable.from_gff3( geneset_agamp44_fn, attributes=geneset_attributes) # variant callsets ################## global callset, callset_pass, callset_pass_biallelic, variation_dir, \ callset_snpeff_agamp42 variation_dir = os.path.join(release_dir, 'variation') # main callset callset_h5_fn = os.path.join(variation_dir, 'main', 'hdf5', 'all', 'ag1000g.phase2.ar1.h5') callset_lite_h5_fn = os.path.join(variation_dir, 'main', 'hdf5', 'lite', 'ag1000g.phase2.ar1.lite.h5') callset_zarr_fn = os.path.join(variation_dir, 'main', 'zarr', 'all', 'ag1000g.phase2.ar1') # preference: zarr > hdf5 > hdf5 (lite) if os.path.exists(callset_zarr_fn): callset = zarr.open_group(callset_zarr_fn, mode='r') elif os.path.exists(callset_h5_fn): callset = h5py.File(callset_h5_fn, mode='r') elif os.path.exists(callset_lite_h5_fn): callset = h5py.File(callset_lite_h5_fn, mode='r') # main callset, PASS variants only callset_pass_h5_fn = os.path.join(variation_dir, 'main', 'hdf5', 'pass', 'ag1000g.phase2.ar1.pass.h5') callset_pass_lite_h5_fn = os.path.join(variation_dir, 'main', 'hdf5', 'lite', 'ag1000g.phase2.ar1.pass.lite.h5') callset_pass_zarr_fn = os.path.join(variation_dir, 'main', 'zarr', 'pass', 'ag1000g.phase2.ar1.pass') # preference: zarr > hdf5 > hdf5 (lite) if os.path.exists(callset_pass_zarr_fn): callset_pass = zarr.open_group(callset_pass_zarr_fn, mode='r') elif os.path.exists(callset_pass_h5_fn): callset_pass = h5py.File(callset_pass_h5_fn, mode='r') elif os.path.exists(callset_pass_lite_h5_fn): callset_pass = h5py.File(callset_pass_lite_h5_fn, mode='r') # main callset, PASS biallelic variants only callset_pass_biallelic_h5_fn = os.path.join( variation_dir, 'main', 'hdf5', 'biallelic', 'ag1000g.phase2.ar1.pass.biallelic.h5') callset_pass_biallelic_lite_h5_fn = os.path.join( variation_dir, 'main', 'hdf5', 'lite', 'ag1000g.phase2.ar1.pass.biallelic.lite.h5') callset_pass_biallelic_zarr_fn = os.path.join( variation_dir, 'main', 'zarr', 'biallelic', 'ag1000g.phase2.ar1.pass.biallelic') # preference: zarr > hdf5 > hdf5 (lite) if os.path.exists(callset_pass_biallelic_zarr_fn): callset_pass_biallelic = zarr.open_group( callset_pass_biallelic_zarr_fn, mode='r') elif os.path.exists(callset_pass_biallelic_h5_fn): callset_pass_biallelic = h5py.File(callset_pass_biallelic_h5_fn, mode='r') elif os.path.exists(callset_pass_biallelic_lite_h5_fn): callset_pass_biallelic = h5py.File(callset_pass_biallelic_lite_h5_fn, mode='r') # SNPEFF annotations callset_snpeff_agamp42_h5_fn_template = os.path.join( variation_dir, 'main', 'hdf5', 'all_snpeff', 'ag1000g.phase2.ar1.snpeff.AgamP4.2.{chrom}.h5') # work around broken link file callset_snpeff_agamp42 = dict() for chrom in '2L', '2R', '3L', '3R', 'X': fn = callset_snpeff_agamp42_h5_fn_template.format(chrom=chrom) if os.path.exists(fn): callset_snpeff_agamp42[chrom] = h5py.File(fn, mode='r')[chrom] # accessibility ############### global accessibility, accessibility_dir accessibility_dir = os.path.join(release_dir, 'accessibility') accessibility_fn = os.path.join(accessibility_dir, 'accessibility.h5') if os.path.exists(accessibility_fn): accessibility = h5py.File(accessibility_fn, mode='r') # sample metadata ################# global tbl_samples, lkp_samples, sample_ids, df_samples, samples_dir samples_dir = os.path.join(release_dir, 'samples') samples_fn = os.path.join(samples_dir, 'samples.meta.txt') if os.path.exists(samples_fn): tbl_samples = (etl.fromtsv(samples_fn).convert( ('year', 'n_sequences'), int).convert(('mean_coverage', ), float)) lkp_samples = tbl_samples.recordlookupone('ox_code') sample_ids = tbl_samples.values('ox_code').list() df_samples = pandas.read_csv(samples_fn, sep='\t', index_col='ox_code') # extras ######## global allele_counts extras_dir = os.path.join(release_dir, 'extras') # allele counts allele_counts_fn = os.path.join(extras_dir, 'allele_counts.h5') if os.path.exists(allele_counts_fn): allele_counts = h5py.File(allele_counts_fn, mode='r') # haplotypes ############ global haplotypes_dir, callset_phased, tbl_haplotypes, df_haplotypes, lkp_haplotypes haplotypes_dir = os.path.join(release_dir, 'haplotypes') # no HDF5 link file, load up as dict for now callset_phased_hdf5_fn_template = os.path.join( haplotypes_dir, 'main', 'hdf5', 'ag1000g.phase2.ar1.haplotypes.{chrom}.h5') callset_phased = dict() for chrom in '2L', '2R', '3L', '3R', 'X': fn = callset_phased_hdf5_fn_template.format(chrom=chrom) if os.path.exists(fn): callset_phased[chrom] = h5py.File(fn, mode='r')[chrom] # no haplotypes file, create here for now # TODO source this from file Nick has created if '3R' in callset_phased: phased_samples = callset_phased['3R']['samples'][:].astype('U') haplotype_labels = list( itertools.chain(*[[s + 'a', s + 'b'] for s in phased_samples])) tbl_haplotypes = (etl.empty().addcolumn( 'label', haplotype_labels).addrownumbers(start=0).rename( 'row', 'index' ).addfield('ox_code', lambda row: row.label[:-1]).hashleftjoin( tbl_samples, key='ox_code').addfield( 'label_aug', lambda row: '%s [%s, %s, %s, %s]' % (row.label, row.country, row.location, row.m_s, row.sex))) lkp_haplotypes = tbl_haplotypes.recordlookupone('label') df_haplotypes = tbl_haplotypes.todataframe(index='index')
def fromgff3(filename, region=None): """ Extract feature rows from a GFF3 file, e.g.:: >>> import petl as etl >>> # activate bio extensions ... import petlx.bio >>> table1 = etl.fromgff3('fixture/sample.gff') >>> table1.look(truncate=30) +--------------+---------+---------------+-------+---------+-------+--------+-------+--------------------------------+ | seqid | source | type | start | end | score | strand | phase | attributes | +==============+=========+===============+=======+=========+=======+========+=======+================================+ | 'apidb|MAL1' | 'ApiDB' | 'supercontig' | 1 | 643292 | '.' | '+' | '.' | {'localization': 'nuclear', 'o | +--------------+---------+---------------+-------+---------+-------+--------+-------+--------------------------------+ | 'apidb|MAL2' | 'ApiDB' | 'supercontig' | 1 | 947102 | '.' | '+' | '.' | {'localization': 'nuclear', 'o | +--------------+---------+---------------+-------+---------+-------+--------+-------+--------------------------------+ | 'apidb|MAL3' | 'ApiDB' | 'supercontig' | 1 | 1060087 | '.' | '+' | '.' | {'localization': 'nuclear', 'o | +--------------+---------+---------------+-------+---------+-------+--------+-------+--------------------------------+ | 'apidb|MAL4' | 'ApiDB' | 'supercontig' | 1 | 1204112 | '.' | '+' | '.' | {'localization': 'nuclear', 'o | +--------------+---------+---------------+-------+---------+-------+--------+-------+--------------------------------+ | 'apidb|MAL5' | 'ApiDB' | 'supercontig' | 1 | 1343552 | '.' | '+' | '.' | {'localization': 'nuclear', 'o | +--------------+---------+---------------+-------+---------+-------+--------+-------+--------------------------------+ ... A region query string of the form '[seqid]' or '[seqid]:[start]-[end]' may be given for the `region` argument. If given, requires the GFF3 file to be position sorted, bgzipped and tabix indexed. Requires pysam to be installed. E.g.:: >>> # extract from a specific genome region via tabix ... table2 = etl.fromgff3('fixture/sample.sorted.gff.gz', ... region='apidb|MAL5:1289593-1289595') >>> table2.look(truncate=30) +--------------+---------+---------------+---------+---------+-------+--------+-------+--------------------------------+ | seqid | source | type | start | end | score | strand | phase | attributes | +==============+=========+===============+=========+=========+=======+========+=======+================================+ | 'apidb|MAL5' | 'ApiDB' | 'supercontig' | 1 | 1343552 | '.' | '+' | '.' | {'localization': 'nuclear', 'o | +--------------+---------+---------------+---------+---------+-------+--------+-------+--------------------------------+ | 'apidb|MAL5' | 'ApiDB' | 'exon' | 1289594 | 1291685 | '.' | '+' | '.' | {'size': '2092', 'Parent': 'ap | +--------------+---------+---------------+---------+---------+-------+--------+-------+--------------------------------+ | 'apidb|MAL5' | 'ApiDB' | 'gene' | 1289594 | 1291685 | '.' | '+' | '.' | {'ID': 'apidb|MAL5_18S', 'web_ | +--------------+---------+---------------+---------+---------+-------+--------+-------+--------------------------------+ | 'apidb|MAL5' | 'ApiDB' | 'rRNA' | 1289594 | 1291685 | '.' | '+' | '.' | {'ID': 'apidb|rna_MAL5_18S-1', | +--------------+---------+---------------+---------+---------+-------+--------+-------+--------------------------------+ """ if region is None: # parse file as tab-delimited table = etl.fromtsv(filename) else: # extract via tabix table = etl.fromtabix(filename, region=region) return (table.pushheader(GFF3_HEADER).skipcomments('#') # ignore any row not 9 values long (e.g., trailing fasta) .rowlenselect(9) # parse attributes into a dict .convert('attributes', gff3_parse_attributes) # parse coordinates .convert(('start', 'end'), int))
import petl as etl import IPython import pandas as pd _DEBUG = True _TIME_TEST = True # Test case on run time complement vs antijoin. # normally these would be toggles but for testing we set both to true _COMPLEMENT = True _ANTI_JOIN = True # csv = comma delimited, tsv = tab delimited pre_etl_time = time.time() a = etl.fromtsv('snpdata.csv') post_etl_time = time.time() b = etl.fromtsv('popdata.csv') pre_df_time = time.time() df_a = pd.read_csv('snpdata.csv', sep='\t', header=0) post_df_time = time.time() print("ETL time to load A file: {} Pandas time to load A file: {}".format( post_etl_time - pre_etl_time, post_df_time - pre_df_time)) df_b = pd.read_csv('popdata.csv', sep='\t', header=0) header_a = etl.header(a) header_b = etl.header(b) if _DEBUG:
def convert_folder(base_source_dir, base_target_dir, tmp_dir, tika=False, ocr=False, merge=False, tsv_source_path=None, tsv_target_path=None, make_unique=True, sample=False, zip=False): # WAIT: Legg inn i gui at kan velge om skal ocr-behandles txt_target_path = base_target_dir + '_result.txt' json_tmp_dir = base_target_dir + '_tmp' converted_now = False errors = False originals = False if merge is False: # TODO: Trengs begge argumentene? make_unique = False if tsv_source_path is None: tsv_source_path = base_target_dir + '.tsv' else: txt_target_path = os.path.splitext( tsv_source_path)[1][1:] + '_result.txt' if tsv_target_path is None: tsv_target_path = base_target_dir + '_processed.tsv' if os.path.exists(tsv_target_path): os.remove(tsv_target_path) Path(base_target_dir).mkdir(parents=True, exist_ok=True) # TODO: Viser mime direkte om er pdf/a eller må en sjekke mot ekstra felt i de to under? Forsjekk om Tika og siegfried? # TODO: Trengs denne sjekk om tsv her. Gjøres sjekk før kaller denne funskjonen og slik at unødvendig? if not os.path.isfile(tsv_source_path): if tika: run_tika(tsv_source_path, base_source_dir, json_tmp_dir, zip) else: run_siegfried(base_source_dir, tmp_dir, tsv_source_path, zip) # TODO: Legg inn test på at tsv-fil ikke er tom replace_text_in_file(tsv_source_path, '\0', '') table = etl.fromtsv(tsv_source_path) table = etl.rename(table, { 'filename': 'source_file_path', 'tika_batch_fs_relative_path': 'source_file_path', 'filesize': 'file_size', 'mime': 'mime_type', 'Content_Type': 'mime_type', 'Version': 'version' }, strict=False) thumbs_table = etl.select( table, lambda rec: Path(rec.source_file_path).name == 'Thumbs.db') if etl.nrows(thumbs_table) > 0: thumbs_paths = etl.values(thumbs_table, 'source_file_path') for path in thumbs_paths: if '/' not in path: path = os.path.join(base_source_dir, path) if os.path.isfile(path): os.remove(path) table = etl.select( table, lambda rec: Path(rec.source_file_path).name != 'Thumbs.db') table = etl.select(table, lambda rec: rec.source_file_path != '') table = etl.select(table, lambda rec: '#' not in rec.source_file_path) # WAIT: Ikke fullgod sjekk på embedded dokument i linje over da # faktisk kan forekomme i filnavn row_count = etl.nrows(table) file_count = sum([len(files) for r, d, files in os.walk(base_source_dir)]) if row_count == 0: print('No files to convert. Exiting.') return 'Error', file_count elif file_count != row_count: print('Row count: ' + str(row_count)) print('File count: ' + str(file_count)) print("Files listed in '" + tsv_source_path + "' doesn't match files on disk. Exiting.") return 'Error', file_count elif not zip: print('Converting files..') # WAIT: Legg inn sjekk på filstørrelse før og etter konvertering append_fields = ('version', 'norm_file_path', 'result', 'original_file_copy', 'id') table = add_fields(append_fields, table) cut_fields = ('0', '1', 'X_TIKA_EXCEPTION_runtime', 'X_TIKA_EXCEPTION_warn') table = remove_fields(cut_fields, table) header = etl.header(table) append_tsv_row(tsv_target_path, header) # Treat csv (detected from extension only) as plain text: table = etl.convert(table, 'mime_type', lambda v, row: 'text/plain' if row.id == 'x-fmt/18' else v, pass_row=True) # Update for missing mime types where id is known: table = etl.convert(table, 'mime_type', lambda v, row: 'application/xml' if row.id == 'fmt/979' else v, pass_row=True) if os.path.isfile(txt_target_path): os.remove(txt_target_path) data = etl.dicts(table) count = 0 for row in data: count += 1 count_str = ('(' + str(count) + '/' + str(file_count) + '): ') source_file_path = row['source_file_path'] if '/' not in source_file_path: source_file_path = os.path.join(base_source_dir, source_file_path) mime_type = row['mime_type'] # TODO: Virker ikke når Tika brukt -> finn hvorfor if ';' in mime_type: mime_type = mime_type.split(';')[0] version = row['version'] result = None old_result = row['result'] if not mime_type: if os.path.islink(source_file_path): mime_type = 'n/a' # kind = filetype.guess(source_file_path) extension = os.path.splitext(source_file_path)[1][1:].lower() if extension == 'xml': mime_type = 'application/xml' if not zip: print_path = os.path.relpath(source_file_path, Path(base_source_dir).parents[1]) print(count_str + '.../' + print_path + ' (' + mime_type + ')') if mime_type not in mime_to_norm.keys(): # print("|" + mime_type + "|") errors = True converted_now = True result = 'Conversion not supported' append_txt_file( txt_target_path, result + ': ' + source_file_path + ' (' + mime_type + ')') row['norm_file_path'] = '' row['original_file_copy'] = '' else: keep_original = mime_to_norm[mime_type][0] if keep_original: originals = True if zip: keep_original = False function = mime_to_norm[mime_type][1] # Ensure unique file names in dir hierarchy: norm_ext = mime_to_norm[mime_type][2] if not norm_ext: norm_ext = 'none' if make_unique: norm_ext = (base64.b32encode( bytes( str(count), encoding='ascii'))).decode('utf8').replace( '=', '').lower() + '.' + norm_ext target_dir = os.path.dirname( source_file_path.replace(base_source_dir, base_target_dir)) normalized = file_convert(source_file_path, mime_type, function, target_dir, tmp_dir, None, norm_ext, version, ocr, keep_original, zip=zip) if normalized['result'] == 0: errors = True result = 'Conversion failed' append_txt_file( txt_target_path, result + ': ' + source_file_path + ' (' + mime_type + ')') elif normalized['result'] == 1: result = 'Converted successfully' converted_now = True elif normalized['result'] == 2: errors = True result = 'Conversion not supported' append_txt_file( txt_target_path, result + ': ' + source_file_path + ' (' + mime_type + ')') elif normalized['result'] == 3: if old_result not in ('Converted successfully', 'Manually converted'): result = 'Manually converted' converted_now = True else: result = old_result elif normalized['result'] == 4: converted_now = True errors = True result = normalized['error'] append_txt_file( txt_target_path, result + ': ' + source_file_path + ' (' + mime_type + ')') elif normalized['result'] == 5: result = 'Not a document' if normalized['norm_file_path']: row['norm_file_path'] = relpath(normalized['norm_file_path'], base_target_dir) file_copy_path = normalized['original_file_copy'] if file_copy_path: file_copy_path = relpath(file_copy_path, base_target_dir) row['original_file_copy'] = file_copy_path row['result'] = result row_values = list(row.values()) # TODO: Fikset med å legge inn escapechar='\\' i append_tsv_row -> vil det skal problemer senere? # row_values = [r.replace('\n', ' ') for r in row_values if r is not None] append_tsv_row(tsv_target_path, row_values) if sample and count > 9: break if not sample: shutil.move(tsv_target_path, tsv_source_path) # TODO: Legg inn valg om at hvis merge = true kopieres alle filer til mappe på øverste nivå og så slettes tomme undermapper msg = None if sample: msg = 'Sample files converted.' if errors: msg = "Not all sample files were converted. See '" + txt_target_path + "' for details." else: if converted_now: msg = 'All files converted succcessfully.' if errors: msg = "Not all files were converted. See '" + txt_target_path + "' for details." else: msg = 'All files converted previously.' return msg, file_count, errors, originals # TODO: Fiks så bruker denne heller for oppsummering til slutt når flere mapper konvertert