Esempio n. 1
0
def test_tocsv_appendcsv_gz():

    # exercise function
    table = (('foo', 'bar'), ('a', 1), ('b', 2), ('c', 2))
    f = NamedTemporaryFile(delete=False)
    fn = f.name + '.gz'
    f.close()
    tocsv(table, fn, delimiter='\t')

    # check what it did
    o = gzip.open(fn, 'rb')
    try:
        actual = csv.reader(o, delimiter='\t')
        expect = [['foo', 'bar'], ['a', '1'], ['b', '2'], ['c', '2']]
        ieq(expect, actual)
    finally:
        o.close()

    # check appending
    table2 = (('foo', 'bar'), ('d', 7), ('e', 9), ('f', 1))
    appendcsv(table2, fn, delimiter='\t')

    # check what it did
    o = gzip.open(fn, 'rb')
    try:
        actual = csv.reader(o, delimiter='\t')
        expect = [['foo', 'bar'], ['a', '1'], ['b', '2'], ['c', '2'],
                  ['d', '7'], ['e', '9'], ['f', '1']]
        ieq(expect, actual)
    finally:
        o.close()
Esempio n. 2
0
def _write_read_file_into_url(base_url, filename, compression=None, pkg=None):
    if not _is_installed(pkg, filename):
        return
    source_url = _build_source_url_from(base_url, filename, compression)
    if source_url is None:
        return
    actual = None
    if ".avro" in filename:
        toavro(_table, source_url)
        actual = fromavro(source_url)
    elif ".xlsx" in filename:
        toxlsx(_table, source_url, 'test1', mode='overwrite')
        toxlsx(_table2, source_url, 'test2', mode='add')
        actual = fromxlsx(source_url, 'test1')
    elif ".xls" in filename:
        toxls(_table, source_url, 'test')
        actual = fromxls(source_url, 'test')
    elif ".json" in filename:
        tojson(_table, source_url)
        actual = fromjson(source_url)
    elif ".csv" in filename:
        tocsv(_table, source_url, encoding="ascii", lineterminator="\n")
        actual = fromcsv(source_url, encoding="ascii")

    if actual is not None:
        _show__rows_from("Expected:", _table)
        _show__rows_from("Actual:", actual)
        ieq(_table, actual)
        ieq(_table, actual)  # verify can iterate twice
    else:
        print("\n    - %s SKIPPED " % filename, file=sys.stderr, end="")
Esempio n. 3
0
def _write_read_file_into_url(base_url, filename, compression=None):
    if ".avro" in filename and not _has_avro:
        return
    is_local = base_url.startswith("./")
    if compression is not None:
        if is_local:
            return
        filename = filename + "." + compression
        codec = fsspec.utils.infer_compression(filename)
        if codec is None:
            print("\n    - %s SKIPPED " % filename, file=sys.stderr, end="")
            return
    print("\n    - %s " % filename, file=sys.stderr, end="")

    if is_local:
        source_url = base_url + filename
    else:
        source_url = os.path.join(base_url, filename)

    _show__rows_from("Expected:", _table)

    if ".avro" in filename:
        toavro(_table, source_url)
        actual = fromavro(source_url)
    else:
        tocsv(_table, source_url, encoding="ascii", lineterminator="\n")
        actual = fromcsv(source_url, encoding="ascii")

    _show__rows_from("Actual:", actual)
    ieq(_table, actual)
    ieq(_table, actual)  # verify can iterate twice
Esempio n. 4
0
def test_totsv_appendtsv():

    # exercise function
    table = (('foo', 'bar'), ('a', 1), ('b', 2), ('c', 2))
    f = NamedTemporaryFile(delete=False)
    totsv(table, f.name)

    # check what it did
    with open(f.name, 'rb') as o:
        actual = csv.reader(o, delimiter='\t')
        expect = [['foo', 'bar'], ['a', '1'], ['b', '2'], ['c', '2']]
        ieq(expect, actual)

    # check appending
    table2 = (('foo', 'bar'), ('d', 7), ('e', 9), ('f', 1))
    appendtsv(table2, f.name)

    # check what it did
    with open(f.name, 'rb') as o:
        actual = csv.reader(o, delimiter='\t')
        expect = [['foo', 'bar'], ['a', '1'], ['b', '2'], ['c', '2'],
                  ['d', '7'], ['e', '9'], ['f', '1']]
        ieq(expect, actual)

    # check explicit no header
    table = (('foo', 'bar'), ('a', 1), ('b', 2), ('c', 2))
    f = NamedTemporaryFile(delete=False)
    tocsv(table, f.name, delimiter='\t', write_header=False)

    # check what it did
    with open(f.name, 'rb') as o:
        actual = csv.reader(o, delimiter='\t')
        expect = [['a', '1'], ['b', '2'], ['c', '2']]
        ieq(expect, actual)
Esempio n. 5
0
def test_tocsv():

    tbl = ((u'name', u'id'), (u'Արամ Խաչատրյան', 1), (u'Johann Strauß', 2),
           (u'Вагиф Сәмәдоғлу', 3), (u'章子怡', 4))
    fn = NamedTemporaryFile().name
    tocsv(tbl, fn, encoding='utf-8', lineterminator='\n')

    expect = (u"name,id\n"
              u"Արամ Խաչատրյան,1\n"
              u"Johann Strauß,2\n"
              u"Вагиф Сәмәдоғлу,3\n"
              u"章子怡,4\n")
    uf = io.open(fn, encoding='utf-8', mode='rt', newline='')
    actual = uf.read()
    eq_(expect, actual)

    # Test with write_header=False
    tbl = ((u'name', u'id'), (u'Արամ Խաչատրյան', 1), (u'Johann Strauß', 2),
           (u'Вагиф Сәмәдоғлу', 3), (u'章子怡', 4))
    tocsv(tbl, fn, encoding='utf-8', lineterminator='\n', write_header=False)

    expect = (u"Արամ Խաչատրյան,1\n"
              u"Johann Strauß,2\n"
              u"Вагиф Сәмәдоғлу,3\n"
              u"章子怡,4\n")
    uf = io.open(fn, encoding='utf-8', mode='rt', newline='')
    actual = uf.read()
    eq_(expect, actual)
def test_tocsv_appendcsv():

    # exercise function
    table = (('foo', 'bar'), ('a', 1), ('b', 2), ('c', 2))
    f = NamedTemporaryFile(delete=False)
    f.close()
    tocsv(table, f.name, encoding='ascii', lineterminator='\n')

    # check what it did
    with open(f.name, 'rb') as o:
        data = [b'foo,bar', b'a,1', b'b,2', b'c,2']
        # don't forget final terminator
        expect = b'\n'.join(data) + b'\n'
        actual = o.read()
        eq_(expect, actual)

    # check appending
    table2 = (('foo', 'bar'), ('d', 7), ('e', 9), ('f', 1))
    appendcsv(table2, f.name, encoding='ascii', lineterminator='\n')

    # check what it did
    with open(f.name, 'rb') as o:
        data = [b'foo,bar', b'a,1', b'b,2', b'c,2', b'd,7', b'e,9', b'f,1']
        # don't forget final terminator
        expect = b'\n'.join(data) + b'\n'
        actual = o.read()
        eq_(expect, actual)
Esempio n. 7
0
def fetch(request):
    people = fetch_people_table()
    uid = uuid.uuid4()
    fname = '{0}.csv'.format(uid)
    full_fname = os.path.join(settings.CSV_DIR, fname)
    tocsv(people, full_fname)
    CSVDownload.objects.create(uuid=uid)
    return redirect(to=reverse('csv_list'))
Esempio n. 8
0
def test_tocsv_none():

    tbl = ((u'col1', u'colNone'), (u'a', 1), (u'b', None), (u'c', None), (u'd',
                                                                          4))
    fn = NamedTemporaryFile().name
    tocsv(tbl, fn, encoding='utf-8', lineterminator='\n')

    expect = (u'col1,colNone\n' u'a,1\n' u'b,\n' u'c,\n' u'd,4\n')

    uf = io.open(fn, encoding='utf-8', mode='rt', newline='')
    actual = uf.read()
    eq_(expect, actual)
Esempio n. 9
0
def test_tocsv_appendcsv():

    # exercise function
    table = (('foo', 'bar'),
             ('a', 1),
             ('b', 2),
             ('c', 2))
    f = NamedTemporaryFile(delete=False)
    tocsv(table, f.name, delimiter='\t')

    # check what it did
    with open(f.name, 'rb') as o:
        actual = csv.reader(o, delimiter='\t')
        expect = [['foo', 'bar'],
                  ['a', '1'],
                  ['b', '2'],
                  ['c', '2']]
        ieq(expect, actual)

    # check appending
    table2 = (('foo', 'bar'),
              ('d', 7),
              ('e', 9),
              ('f', 1))
    appendcsv(table2, f.name, delimiter='\t')

    # check what it did
    with open(f.name, 'rb') as o:
        actual = csv.reader(o, delimiter='\t')
        expect = [['foo', 'bar'],
                  ['a', '1'],
                  ['b', '2'],
                  ['c', '2'],
                  ['d', '7'],
                  ['e', '9'],
                  ['f', '1']]
        ieq(expect, actual)

    # check explicit no header
    table = (('foo', 'bar'),
             ('a', 1),
             ('b', 2),
             ('c', 2))
    f = NamedTemporaryFile(delete=False)
    tocsv(table, f.name, delimiter='\t', write_header=False)

    # check what it did
    with open(f.name, 'rb') as o:
        actual = csv.reader(o, delimiter='\t')
        expect = [['a', '1'],
                  ['b', '2'],
                  ['c', '2']]
        ieq(expect, actual)
Esempio n. 10
0
def test_tocsv_appendcsv_gz():

    # exercise function
    table = (('foo', 'bar'),
             ('a', 1),
             ('b', 2),
             ('c', 2))
    f = NamedTemporaryFile(delete=False)
    fn = f.name + '.gz'
    f.close()
    tocsv(table, fn, encoding='ascii', lineterminator='\n')

    # check what it did
    o = gzip.open(fn, 'rb')
    try:
        data = [b'foo,bar',
                b'a,1',
                b'b,2',
                b'c,2']
        # don't forget final terminator
        expect = b'\n'.join(data) + b'\n'
        actual = o.read()
        eq_(expect, actual)
    finally:
        o.close()

    # check appending
    table2 = (('foo', 'bar'),
              ('d', 7),
              ('e', 9),
              ('f', 1))
    appendcsv(table2, fn, encoding='ascii', lineterminator='\n')

    # check what it did
    o = gzip.open(fn, 'rb')
    try:
        data = [b'foo,bar',
                b'a,1',
                b'b,2',
                b'c,2',
                b'd,7',
                b'e,9',
                b'f,1']
        # don't forget final terminator
        expect = b'\n'.join(data) + b'\n'
        actual = o.read()
        eq_(expect, actual)
    finally:
        o.close()
Esempio n. 11
0
def _write_read_file_into_url(base_url, filename, compression=None, pkg=None):
    if pkg is not None:
        if not _is_installed(pkg):
            print("\n    - %s SKIPPED " % filename, file=sys.stderr, end="")
            return
    is_local = base_url.startswith("./")
    if compression is not None:
        if is_local:
            return
        filename = filename + "." + compression
        codec = fsspec.utils.infer_compression(filename)
        if codec is None:
            print("\n    - %s SKIPPED " % filename, file=sys.stderr, end="")
            return
    print("\n    - %s " % filename, file=sys.stderr, end="")

    if is_local:
        source_url = base_url + filename
    else:
        source_url = os.path.join(base_url, filename)

    actual = None
    if ".avro" in filename:
        toavro(_table, source_url)
        actual = fromavro(source_url)
    elif ".xlsx" in filename:
        toxlsx(_table, source_url, 'test1', mode='overwrite')
        toxlsx(_table2, source_url, 'test2', mode='add')
        actual = fromxlsx(source_url, 'test1')
    elif ".xls" in filename:
        toxls(_table, source_url, 'test')
        actual = fromxls(source_url, 'test')
    elif ".json" in filename:
        tojson(_table, source_url)
        actual = fromjson(source_url)
    elif ".csv" in filename:
        tocsv(_table, source_url, encoding="ascii", lineterminator="\n")
        actual = fromcsv(source_url, encoding="ascii")

    if actual is not None:
        _show__rows_from("Expected:", _table)
        _show__rows_from("Actual:", actual)
        ieq(_table, actual)
        ieq(_table, actual)  # verify can iterate twice
    else:
        print("\n    - %s SKIPPED " % filename, file=sys.stderr, end="")
Esempio n. 12
0
def test_tocsv_appendcsv_gz():

    # exercise function
    table = (('foo', 'bar'),
             ('a', 1),
             ('b', 2),
             ('c', 2))
    f = NamedTemporaryFile(delete=False)
    fn = f.name + '.gz'
    f.close()
    tocsv(table, fn, delimiter='\t')

    # check what it did
    o = gzip.open(fn, 'rb')
    try:
        actual = csv.reader(o, delimiter='\t')
        expect = [['foo', 'bar'],
                  ['a', '1'],
                  ['b', '2'],
                  ['c', '2']]
        ieq(expect, actual)
    finally:
        o.close()

    # check appending
    table2 = (('foo', 'bar'),
              ('d', 7),
              ('e', 9),
              ('f', 1))
    appendcsv(table2, fn, delimiter='\t')

    # check what it did
    o = gzip.open(fn, 'rb')
    try:
        actual = csv.reader(o, delimiter='\t')
        expect = [['foo', 'bar'],
                  ['a', '1'],
                  ['b', '2'],
                  ['c', '2'],
                  ['d', '7'],
                  ['e', '9'],
                  ['f', '1']]
        ieq(expect, actual)
    finally:
        o.close()
def test_tocsv_noheader():

    # check explicit no header
    table = (('foo', 'bar'), ('a', 1), ('b', 2), ('c', 2))
    f = NamedTemporaryFile(delete=False)
    tocsv(table,
          f.name,
          encoding='ascii',
          lineterminator='\n',
          write_header=False)

    # check what it did
    with open(f.name, 'rb') as o:
        data = [b'a,1', b'b,2', b'c,2']
        # don't forget final terminator
        expect = b'\n'.join(data) + b'\n'
        actual = o.read()
        eq_(expect, actual)
Esempio n. 14
0
def test_tocsv_noheader():

    # check explicit no header
    table = (('foo', 'bar'),
             ('a', 1),
             ('b', 2),
             ('c', 2))
    f = NamedTemporaryFile(delete=False)
    tocsv(table, f.name, encoding='ascii', lineterminator='\n',
          write_header=False)

    # check what it did
    with open(f.name, 'rb') as o:
        data = [b'a,1',
                b'b,2',
                b'c,2']
        # don't forget final terminator
        expect = b'\n'.join(data) + b'\n'
        actual = o.read()
        eq_(expect, actual)
Esempio n. 15
0
def test_tocsv_none():

    tbl = ((u'col1', u'colNone'),
           (u'a', 1),
           (u'b', None),
           (u'c', None),
           (u'd', 4))
    fn = NamedTemporaryFile().name
    tocsv(tbl, fn, encoding='utf-8', lineterminator='\n')

    expect = (
        u'col1,colNone\n'
        u'a,1\n'
        u'b,\n'
        u'c,\n'
        u'd,4\n'
    )

    uf = io.open(fn, encoding='utf-8', mode='rt', newline='')
    actual = uf.read()
    eq_(expect, actual)
Esempio n. 16
0
def test_stringsource():

    table1 = (('foo', 'bar'), ('a', '1'), ('b', '2'), ('c', '2'))

    # test writing to a string buffer
    ss = StringSource()
    tocsv(table1, ss)
    expect = "foo,bar\r\na,1\r\nb,2\r\nc,2\r\n"
    actual = ss.getvalue()
    eq_(expect, actual)

    # test reading from a string buffer
    table2 = fromcsv(StringSource(actual))
    ieq(table1, table2)
    ieq(table1, table2)

    # test appending
    appendcsv(table1, ss)
    actual = ss.getvalue()
    expect = "foo,bar\r\na,1\r\nb,2\r\nc,2\r\na,1\r\nb,2\r\nc,2\r\n"
    eq_(expect, actual)
Esempio n. 17
0
def test_stringsource():

    table1 = (("foo", "bar"), ("a", "1"), ("b", "2"), ("c", "2"))

    # test writing to a string buffer
    ss = StringSource()
    tocsv(table1, ss)
    expect = "foo,bar\r\na,1\r\nb,2\r\nc,2\r\n"
    actual = ss.getvalue()
    eq_(expect, actual)

    # test reading from a string buffer
    table2 = fromcsv(StringSource(actual))
    ieq(table1, table2)
    ieq(table1, table2)

    # test appending
    appendcsv(table1, ss)
    actual = ss.getvalue()
    expect = "foo,bar\r\na,1\r\nb,2\r\nc,2\r\na,1\r\nb,2\r\nc,2\r\n"
    eq_(expect, actual)
Esempio n. 18
0
def test_tocsv():

    tbl = ((u'name', u'id'),
           (u'Արամ Խաչատրյան', 1),
           (u'Johann Strauß', 2),
           (u'Вагиф Сәмәдоғлу', 3),
           (u'章子怡', 4))
    fn = NamedTemporaryFile().name
    tocsv(tbl, fn, encoding='utf-8', lineterminator='\n')

    expect = (
        u"name,id\n"
        u"Արամ Խաչատրյան,1\n"
        u"Johann Strauß,2\n"
        u"Вагиф Сәмәдоғлу,3\n"
        u"章子怡,4\n"
    )
    uf = io.open(fn, encoding='utf-8', mode='rt', newline='')
    actual = uf.read()
    eq_(expect, actual)

    # Test with write_header=False
    tbl = ((u'name', u'id'),
           (u'Արամ Խաչատրյան', 1),
           (u'Johann Strauß', 2),
           (u'Вагиф Сәмәдоғлу', 3),
           (u'章子怡', 4))
    tocsv(tbl, fn, encoding='utf-8', lineterminator='\n', write_header=False)

    expect = (
        u"Արամ Խաչատրյան,1\n"
        u"Johann Strauß,2\n"
        u"Вагиф Сәмәдоғлу,3\n"
        u"章子怡,4\n"
    )
    uf = io.open(fn, encoding='utf-8', mode='rt', newline='')
    actual = uf.read()
    eq_(expect, actual)