Esempio n. 1
0
def test_teehtml():

    t1 = (('foo', 'bar'), ('a', 2), ('b', 1), ('c', 3))

    f1 = NamedTemporaryFile(delete=False)
    f2 = NamedTemporaryFile(delete=False)
    etl.wrap(t1).teehtml(f1.name).selectgt('bar', 1).topickle(f2.name)

    ieq(t1, etl.fromxml(f1.name, './/tr', ('th', 'td')).convertnumbers())
    ieq(etl.wrap(t1).selectgt('bar', 1), etl.frompickle(f2.name))
Esempio n. 2
0
def test_teepickle():

    t1 = (('foo', 'bar'), ('a', 2), ('b', 1), ('c', 3))

    f1 = NamedTemporaryFile(delete=False)
    f2 = NamedTemporaryFile(delete=False)
    etl.wrap(t1).teepickle(f1.name).selectgt('bar', 1).topickle(f2.name)

    ieq(t1, etl.frompickle(f1.name))
    ieq(etl.wrap(t1).selectgt('bar', 1), etl.frompickle(f2.name))
Esempio n. 3
0
def test_container():
    table = (('foo', 'bar'), ('a', 1), ('b', 2), ('c', 2))
    actual = etl.wrap(table)[0]
    expect = ('foo', 'bar')
    eq_(expect, actual)
    actual = etl.wrap(table)['bar']
    expect = (1, 2, 2)
    ieq(expect, actual)
    actual = len(etl.wrap(table))
    expect = 4
    eq_(expect, actual)
Esempio n. 4
0
def test_teetsv():

    t1 = (('foo', 'bar'), ('a', 2), ('b', 1), ('c', 3))

    f1 = NamedTemporaryFile(delete=False)
    f2 = NamedTemporaryFile(delete=False)

    etl.wrap(t1).teetsv(f1.name).selectgt('bar', 1).totsv(f2.name)

    ieq(t1, etl.fromtsv(f1.name).convertnumbers())
    ieq(etl.wrap(t1).selectgt('bar', 1), etl.fromtsv(f2.name).convertnumbers())
Esempio n. 5
0
def test_teeuhtml():

    t1 = ((u'foo', u'bar'), (u'Արամ Խաչատրյան', 2), (u'Johann Strauß', 1),
          (u'Вагиф Сәмәдоғлу', 3))

    f1 = NamedTemporaryFile(delete=False)
    f2 = NamedTemporaryFile(delete=False)
    etl.wrap(t1).teeuhtml(f1.name).selectgt('bar', 1).topickle(f2.name)

    ieq(t1, etl.fromxml(f1.name, './/tr', ('th', 'td')).convertnumbers())
    ieq(etl.wrap(t1).selectgt('bar', 1), etl.frompickle(f2.name))
Esempio n. 6
0
def test_teepickle():

    t1 = (('foo', 'bar'),
          ('a', 2),
          ('b', 1),
          ('c', 3))

    f1 = NamedTemporaryFile(delete=False)
    f2 = NamedTemporaryFile(delete=False)
    etl.wrap(t1).teepickle(f1.name).selectgt('bar', 1).topickle(f2.name)

    ieq(t1, etl.frompickle(f1.name))
    ieq(etl.wrap(t1).selectgt('bar', 1), etl.frompickle(f2.name))
Esempio n. 7
0
def test_teeuhtml():

    t1 = ((u'foo', u'bar'),
          (u'Արամ Խաչատրյան', 2),
          (u'Johann Strauß', 1),
          (u'Вагиф Сәмәдоғлу', 3))

    f1 = NamedTemporaryFile(delete=False)
    f2 = NamedTemporaryFile(delete=False)
    etl.wrap(t1).teeuhtml(f1.name).selectgt('bar', 1).topickle(f2.name)

    ieq(t1, etl.fromxml(f1.name, './/tr', ('th', 'td')).convertnumbers())
    ieq(etl.wrap(t1).selectgt('bar', 1), etl.frompickle(f2.name))
Esempio n. 8
0
def test_teehtml():

    t1 = (('foo', 'bar'),
          ('a', 2),
          ('b', 1),
          ('c', 3))

    f1 = NamedTemporaryFile(delete=False)
    f2 = NamedTemporaryFile(delete=False)
    etl.wrap(t1).teehtml(f1.name).selectgt('bar', 1).topickle(f2.name)

    ieq(t1, etl.fromxml(f1.name, './/tr', ('th', 'td')).convertnumbers())
    ieq(etl.wrap(t1).selectgt('bar', 1), etl.frompickle(f2.name))
Esempio n. 9
0
def test_container():
    table = (('foo', 'bar'),
             ('a', 1),
             ('b', 2),
             ('c', 2))
    actual = etl.wrap(table)[0]
    expect = ('foo', 'bar')
    eq_(expect, actual)
    actual = etl.wrap(table)['bar']
    expect = (1, 2, 2)
    ieq(expect, actual)
    actual = len(etl.wrap(table))
    expect = 4
    eq_(expect, actual)
Esempio n. 10
0
def test_teetext():

    t1 = (('foo', 'bar'),
          ('a', 2),
          ('b', 1),
          ('c', 3))

    f1 = NamedTemporaryFile(delete=False)
    f2 = NamedTemporaryFile(delete=False)

    prologue = 'foo,bar\n'
    template = '{foo},{bar}\n'
    epilogue = 'd,4'
    (etl
     .wrap(t1)
     .teetext(f1.name,
              template=template,
              prologue=prologue,
              epilogue=epilogue)
     .selectgt('bar', 1)
     .topickle(f2.name))

    ieq(t1 + (('d', 4),),
        etl.fromcsv(f1.name).convertnumbers())
    ieq(etl.wrap(t1).selectgt('bar', 1),
        etl.frompickle(f2.name))
Esempio n. 11
0
def test_values_container_convenience_methods():
    table = etl.wrap((('foo', 'bar'),
                 ('a', 1),
                 ('b', 2),
                 ('c', 2)))
    
    actual = table.values('foo').set()
    expect = set(['a', 'b', 'c'])
    eq_(expect, actual)
    
    actual = table.values('foo').list()
    expect = ['a', 'b', 'c']
    eq_(expect, actual)
    
    actual = table.values('foo').tuple()
    expect = ('a', 'b', 'c')
    eq_(expect, actual)
    
    actual = table.values('bar').sum()
    expect = 5
    eq_(expect, actual)
    
    actual = table.data().dict()
    expect = {'a': 1, 'b': 2, 'c': 2}
    eq_(expect, actual)
Esempio n. 12
0
def test_teeutext():

    t1 = ((u'foo', u'bar'),
          (u'Արամ Խաչատրյան', 2),
          (u'Johann Strauß', 1),
          (u'Вагиф Сәмәдоғлу', 3))

    f1 = NamedTemporaryFile(delete=False)
    f2 = NamedTemporaryFile(delete=False)

    prologue = u'foo,bar\n'
    template = u'{foo},{bar}\n'
    epilogue = u'章子怡,4'
    (etl
     .wrap(t1)
     .teeutext(f1.name,
               template=template,
               prologue=prologue,
               epilogue=epilogue)
     .selectgt('bar', 1)
     .topickle(f2.name))

    ieq(t1 + ((u'章子怡', 4),),
        etl.fromucsv(f1.name).convertnumbers())
    ieq(etl.wrap(t1).selectgt('bar', 1),
        etl.frompickle(f2.name))
Esempio n. 13
0
def test_values_container_convenience_methods():
    table = etl.wrap((('foo', 'bar'),
                 ('a', 1),
                 ('b', 2),
                 ('c', 2)))
    
    actual = table.values('foo').set()
    expect = set(['a', 'b', 'c'])
    eq_(expect, actual)
    
    actual = table.values('foo').list()
    expect = ['a', 'b', 'c']
    eq_(expect, actual)
    
    actual = table.values('foo').tuple()
    expect = ('a', 'b', 'c')
    eq_(expect, actual)
    
    actual = table.values('bar').sum()
    expect = 5
    eq_(expect, actual)
    
    actual = table.data().dict()
    expect = {'a': 1, 'b': 2, 'c': 2}
    eq_(expect, actual)
Esempio n. 14
0
def test_teetsv():

    t1 = (('foo', 'bar'),
          ('a', 2),
          ('b', 1),
          ('c', 3))

    f1 = NamedTemporaryFile(delete=False)
    f2 = NamedTemporaryFile(delete=False)

    etl.wrap(t1).teetsv(f1.name).selectgt('bar', 1).totsv(f2.name)

    ieq(t1,
        etl.fromtsv(f1.name).convertnumbers())
    ieq(etl.wrap(t1).selectgt('bar', 1),
        etl.fromtsv(f2.name).convertnumbers())
Esempio n. 15
0
def test_teeutsv():

    t1 = (
        (u'name', u'id'),
        (u'Արամ Խաչատրյան', 1),
        (u'Johann Strauß', 2),
        (u'Вагиф Сәмәдоғлу', 3),
        (u'章子怡', 4),
    )

    f1 = NamedTemporaryFile(delete=False)
    f2 = NamedTemporaryFile(delete=False)

    etl.wrap(t1).teeutsv(f1.name).selectgt('id', 1).toutsv(f2.name)

    ieq(t1, etl.fromutsv(f1.name).convertnumbers())
    ieq(etl.wrap(t1).selectgt('id', 1), etl.fromutsv(f2.name).convertnumbers())
Esempio n. 16
0
def test_valuesarray_explicit_dtype():

    t = [("foo", "bar", "baz"), ("apples", 1, 2.5), ("oranges", 3, 4.4), ("pears", 7, 0.1)]

    expect = np.array([1, 3, 7], dtype="i2")
    actual = etl.wrap(t).values("bar").array(dtype="i2")
    eq_(expect.dtype, actual.dtype)
    assert np.all(expect == actual)
Esempio n. 17
0
def test_teeutsv():

    t1 = ((u'name', u'id'),
          (u'Արամ Խաչատրյան', 1),
          (u'Johann Strauß', 2),
          (u'Вагиф Сәмәдоғлу', 3),
          (u'章子怡', 4),
          )

    f1 = NamedTemporaryFile(delete=False)
    f2 = NamedTemporaryFile(delete=False)

    etl.wrap(t1).teeutsv(f1.name).selectgt('id', 1).toutsv(f2.name)

    ieq(t1,
        etl.fromutsv(f1.name).convertnumbers())
    ieq(etl.wrap(t1).selectgt('id', 1),
        etl.fromutsv(f2.name).convertnumbers())
Esempio n. 18
0
def test_teetext():

    t1 = (('foo', 'bar'), ('a', 2), ('b', 1), ('c', 3))

    f1 = NamedTemporaryFile(delete=False)
    f2 = NamedTemporaryFile(delete=False)

    prologue = 'foo,bar\n'
    template = '{foo},{bar}\n'
    epilogue = 'd,4'
    (etl.wrap(t1).teetext(f1.name,
                          template=template,
                          prologue=prologue,
                          epilogue=epilogue).selectgt('bar',
                                                      1).topickle(f2.name))

    ieq(t1 + (('d', 4), ), etl.fromcsv(f1.name).convertnumbers())
    ieq(etl.wrap(t1).selectgt('bar', 1), etl.frompickle(f2.name))
Esempio n. 19
0
def test_valuesarray_explicit_dtype():

    t = [('foo', 'bar', 'baz'), ('apples', 1, 2.5), ('oranges', 3, 4.4),
         ('pears', 7, .1)]

    expect = np.array([1, 3, 7], dtype='i2')
    actual = etl.wrap(t).values('bar').array(dtype='i2')
    eq_(expect.dtype, actual.dtype)
    assert np.all(expect == actual)
Esempio n. 20
0
def test_teeutext():

    t1 = ((u'foo', u'bar'), (u'Արամ Խաչատրյան', 2), (u'Johann Strauß', 1),
          (u'Вагиф Сәмәдоғлу', 3))

    f1 = NamedTemporaryFile(delete=False)
    f2 = NamedTemporaryFile(delete=False)

    prologue = u'foo,bar\n'
    template = u'{foo},{bar}\n'
    epilogue = u'章子怡,4'
    (etl.wrap(t1).teeutext(f1.name,
                           template=template,
                           prologue=prologue,
                           epilogue=epilogue).selectgt('bar',
                                                       1).topickle(f2.name))

    ieq(t1 + ((u'章子怡', 4), ), etl.fromucsv(f1.name).convertnumbers())
    ieq(etl.wrap(t1).selectgt('bar', 1), etl.frompickle(f2.name))
Esempio n. 21
0
def test_integration():
    #apidb|MAL1    ApiDB    gene    56913    57116    .    -    .    ID=apidb|PFA0035c;Name=PFA0035c;description=hypothetical+protein%2C+conserved+in+P.+falciparum;size=204;web_id=PFA0035c;locus_tag=PFA0035c;size=204;Alias=MAL1P4.06b
    snps = etl.wrap((('chr', 'pos'),
                     ('apidb|MAL1', 56911),
                     ('apidb|MAL1', 56915)))
    features = etl.fromgff3(plasmodb_gff3_file.name)
    genes = features.selecteq('type', 'gene')
    actual = snps.gff3join(genes, seqid='chr', start='pos', end='pos')
    expect = (('chr', 'pos', 'seqid', 'source', 'type', 'start', 'end', 'score', 'strand', 'phase', 'attributes'),
              ('apidb|MAL1', 56915, 'apidb|MAL1', 'ApiDB', 'gene', 56913, 57116, '.', '-', '.', gff3_parse_attributes("ID=apidb|PFA0035c;Name=PFA0035c;description=hypothetical+protein%2C+conserved+in+P.+falciparum;size=204;web_id=PFA0035c;locus_tag=PFA0035c;size=204;Alias=MAL1P4.06b")))
    ieq(expect, actual)
    ieq(expect, actual)
Esempio n. 22
0
def test_integration():
    
    left = etl.wrap((('begin', 'end', 'quux'),
                     (1, 2, 'a'),
                     (2, 4, 'b'),
                     (2, 5, 'c'),
                     (9, 14, 'd'),
                     (9, 140, 'e'),
                     (1, 1, 'f'),
                     (2, 2, 'g'),
                     (4, 4, 'h'),
                     (5, 5, 'i'),
                     (1, 8, 'j')))

    right = etl.wrap((('start', 'stop', 'value'),
                      (1, 4, 'foo'),
                      (3, 7, 'bar'),
                      (4, 9, 'baz')))

    actual = left.intervaljoin(right, 
                               lstart='begin', lstop='end',
                               rstart='start', rstop='stop')
    expect = (('begin', 'end', 'quux', 'start', 'stop', 'value'),
              (1, 2, 'a', 1, 4, 'foo'),
              (2, 4, 'b', 1, 4, 'foo'),
              (2, 4, 'b', 3, 7, 'bar'),
              (2, 5, 'c', 1, 4, 'foo'),
              (2, 5, 'c', 3, 7, 'bar'),
              (2, 5, 'c', 4, 9, 'baz'),
              (2, 2, 'g', 1, 4, 'foo'),
              (4, 4, 'h', 3, 7, 'bar'),
              (5, 5, 'i', 3, 7, 'bar'),
              (5, 5, 'i', 4, 9, 'baz'),
              (1, 8, 'j', 1, 4, 'foo'),
              (1, 8, 'j', 3, 7, 'bar'),
              (1, 8, 'j', 4, 9, 'baz'))
    ieq(expect, actual)
    ieq(expect, actual)
Esempio n. 23
0
def test_integration():
    
    left = etl.wrap((('begin', 'end', 'quux'),
                     (1, 2, 'a'),
                     (2, 4, 'b'),
                     (2, 5, 'c'),
                     (9, 14, 'd'),
                     (9, 140, 'e'),
                     (1, 1, 'f'),
                     (2, 2, 'g'),
                     (4, 4, 'h'),
                     (5, 5, 'i'),
                     (1, 8, 'j')))

    right = etl.wrap((('start', 'stop', 'value'),
                      (1, 4, 'foo'),
                      (3, 7, 'bar'),
                      (4, 9, 'baz')))

    actual = left.intervaljoin(right, 
                               lstart='begin', lstop='end',
                               rstart='start', rstop='stop')
    expect = (('begin', 'end', 'quux', 'start', 'stop', 'value'),
              (1, 2, 'a', 1, 4, 'foo'),
              (2, 4, 'b', 1, 4, 'foo'),
              (2, 4, 'b', 3, 7, 'bar'),
              (2, 5, 'c', 1, 4, 'foo'),
              (2, 5, 'c', 3, 7, 'bar'),
              (2, 5, 'c', 4, 9, 'baz'),
              (2, 2, 'g', 1, 4, 'foo'),
              (4, 4, 'h', 3, 7, 'bar'),
              (5, 5, 'i', 3, 7, 'bar'),
              (5, 5, 'i', 4, 9, 'baz'),
              (1, 8, 'j', 1, 4, 'foo'),
              (1, 8, 'j', 3, 7, 'bar'),
              (1, 8, 'j', 4, 9, 'baz'))
    ieq(expect, actual)
    ieq(expect, actual)
Esempio n. 24
0
def test_basics():

    t1 = (('foo', 'bar'), ('A', 1), ('B', 2))
    w1 = etl.wrap(t1)

    eq_(('foo', 'bar'), w1.header())
    eq_(petl.header(w1), w1.header())
    ieq((('A', 1), ('B', 2)), w1.data())
    ieq(petl.data(w1), w1.data())

    w2 = w1.cut('bar', 'foo')
    expect2 = (('bar', 'foo'), (1, 'A'), (2, 'B'))
    ieq(expect2, w2)
    ieq(petl.cut(w1, 'bar', 'foo'), w2)

    w3 = w1.cut('bar', 'foo').cut('foo', 'bar')
    ieq(t1, w3)
Esempio n. 25
0
def test_basics():
    
    t1 = (('foo', 'bar'),
         ('A', 1),
         ('B', 2))
    w1 = etl.wrap(t1)
    
    eq_(('foo', 'bar'), w1.header())
    eq_(petl.header(w1), w1.header())
    ieq((('A', 1), ('B', 2)), w1.data())
    ieq(petl.data(w1), w1.data())
    
    w2 = w1.cut('bar', 'foo')
    expect2 = (('bar', 'foo'),
               (1, 'A'),
               (2, 'B'))
    ieq(expect2, w2)
    ieq(petl.cut(w1, 'bar', 'foo'), w2)
    
    w3 = w1.cut('bar', 'foo').cut('foo', 'bar')
    ieq(t1, w3)
Esempio n. 26
0
def test_integration():
    
    # set up a new hdf5 table to work with
    h5file = openFile("test4.h5", mode="w", title="Test file")
    h5file.createGroup('/', 'testgroup', 'Test Group')
    class FooBar(IsDescription):
        foo = Int32Col(pos=0)
        bar = StringCol(6, pos=2)
    h5file.createTable('/testgroup', 'testtable', FooBar, 'Test Table')
    h5file.flush()
    h5file.close()
    
    # load some initial data via tohdf5()
    table1 = etl.wrap((('foo', 'bar'),
                       (1, 'asdfgh'),
                       (2, 'qwerty'),
                       (3, 'zxcvbn')))
    table1.tohdf5('test4.h5', '/testgroup', 'testtable')
    ieq(table1, etl.fromhdf5('test4.h5', '/testgroup', 'testtable'))

    # append some more data
    table1.appendhdf5('test4.h5', '/testgroup', 'testtable')
    ieq(chain(table1, table1[1:]), etl.fromhdf5('test4.h5', '/testgroup', 'testtable'))
Esempio n. 27
0
def test_integration():
    t = etl.wrap([("foo", "bar", "baz"), ("apples", 1, 2.5), ("oranges", 3, 4.4), ("pears", 7, 0.1)])
    a = t.toarray()
    u = etl.fromarray(a).convert("bar", int)
    ieq(t, u)
Esempio n. 28
0
                m = self.regex.match(line)
                if m:            
                    yield  m.groups()
                else:
                    raise ValueError("Failed to match regex on line: "+line)
                    
    def cachetag(self):
        try:
            return hash((self.source.checksum(), self.args, tuple(self.kwargs.items())))
        except Exception as e:
            raise Exception(e)



  
#
# Fluentize
#

import sys
from petl.fluent import FluentWrapper, wrap

#
# Add all of the functions in this module into the FluentWrapper as
# methods
#

for n, c in sys.modules[__name__].__dict__.items():
    if callable(c):
        setattr(FluentWrapper, n, wrap(c)) 
        
Esempio n. 29
0
            for line in f:
                m = self.regex.match(line)
                if m:
                    yield m.groups()
                else:
                    raise ValueError("Failed to match regex on line: " + line)

    def cachetag(self):
        try:
            return hash((self.source.checksum(), self.args,
                         tuple(self.kwargs.items())))
        except Exception as e:
            raise Exception(e)


#
# Fluentize
#

import sys
from petl.fluent import FluentWrapper, wrap

#
# Add all of the functions in this module into the FluentWrapper as
# methods
#

for n, c in sys.modules[__name__].__dict__.items():
    if callable(c):
        setattr(FluentWrapper, n, wrap(c))
Esempio n. 30
0
def test_integration():
    t = etl.wrap([('foo', 'bar', 'baz'), ('apples', 1, 2.5),
                  ('oranges', 3, 4.4), ('pears', 7, .1)])
    a = t.toarray()
    u = etl.fromarray(a).convert('bar', int)
    ieq(t, u)