예제 #1
0
def test_memory_from_table(numpy):
    table = asciitable.get_reader(numpy=numpy, Reader=asciitable.Daophot)
    data = table.read('t/daophot.dat')

    mem_table = asciitable.get_reader(Reader=asciitable.Memory, numpy=numpy)
    mem_data = mem_table.read(data)
    assert(data.dtype.names == mem_data.dtype.names)
    _test_values_equal(data, mem_data, numpy)

    mem_data = mem_table.read(mem_table)
    assert(data.dtype.names == mem_data.dtype.names)
    _test_values_equal(data, mem_data, numpy)
예제 #2
0
def test_write_table():
    table = asciitable.get_reader(Reader=asciitable.Daophot)
    data = table.read('t/daophot.dat')

    for test_def in test_defs:
        yield check_write_table, test_def, table
        yield check_write_table, test_def, data
예제 #3
0
def test_write_table_no_numpy():
    table = asciitable.get_reader(Reader=asciitable.Daophot, numpy=False)
    data = table.read("t/daophot.dat")

    for test_def in test_defs:
        yield check_write_table, test_def, table
        yield check_write_table, test_def, data
예제 #4
0
def test_write_table_no_numpy():
    table = asciitable.get_reader(Reader=asciitable.Daophot, numpy=False)
    data = table.read('t/daophot.dat')

    for test_def in test_defs:
        check_write_table(test_def, table)
        check_write_table(test_def, data)
예제 #5
0
def test_custom_process_lines(numpy):
    def process_lines(lines):
        bars_at_ends = re.compile(r'^\| | \|$', re.VERBOSE)
        striplines = (x.strip() for x in lines)
        return [bars_at_ends.sub('', x) for x in striplines if len(x) > 0]
    reader = asciitable.get_reader(delimiter='|', numpy=numpy)
    reader.inputter.process_lines = process_lines
    data = reader.read('t/bars_at_ends.txt')
    assert_equal(data.dtype.names, ('obsid', 'redshift', 'X', 'Y', 'object', 'rad'))
    assert_equal(len(data), 3)
예제 #6
0
def test_custom_process_line(numpy):
    def process_line(line):
        line_out = re.sub(r'^\|\s*', '', line.strip())
        return line_out
    reader = asciitable.get_reader(data_start=2, delimiter='|', numpy=numpy)
    reader.header.splitter.process_line = process_line
    reader.data.splitter.process_line = process_line
    data = reader.read('t/nls1_stackinfo.dbout')
    cols = get_testfiles('t/nls1_stackinfo.dbout')['cols']
    assert_equal(data.dtype.names, cols[1:])
예제 #7
0
    def __init__(self):
        self.autoRerun = ""
        self.parseKnownIssues = ""
        self.errataName = ""
        self.errataLname = ""
        self.rerunedRSId = []
        self.force = False
        self.__parseArgs()
        self.errataInfo = ErrataInfo(self.errataName, self.errataLname, False)

        self.resultPath = "./result"
        self.jobStatePath = "%s/%s.%s" % (self.resultPath, self.errataInfo.errataId, "jobstate")
        genLogger.info("jobStatePath      : %s", self.jobStatePath)
        self.jobState = ConfigObj(self.jobStatePath, encoding="utf8")

        if self.parseKnownIssues == "y":
            self.knownIssuesPath = []
            self.knownIssues = []
            self.knownIssuesRPath = "./known_issues"
            self.knownIssuesPath.append(
                "%s/%s.%s" % (self.knownIssuesRPath, self.errataInfo.rhel_version, "known_issues")
            )
            self.knownIssuesPath.append(
                "%s/RHEL-%s.%s" % (self.knownIssuesRPath, self.errataInfo.major, "known_issues")
            )
            for i in range(0, len(self.knownIssuesPath)):
                str = "%d  : %s" % (i, self.knownIssuesPath[i])
                genLogger.info("knownIssuesPath%s" % str)
                self.knownIssues.append(ConfigObj(self.knownIssuesPath[i], encoding="utf8"))

            self.knownIssuesResult = "%s/%s.%s" % (self.resultPath, self.errataInfo.errataId, "knownIssues")
            self.unknownIssuesResult = "%s/%s.%s" % (self.resultPath, self.errataInfo.errataId, "unknownIssues")
            self.tableTemple = {
                "Path": ["---"],
                "TaskName": ["---"],
                "TaskResult": ["---"],
                "TaskStatus": ["---"],
                "ResultPath": ["---"],
                "PathResult": ["---"],
                "Checked": ["---"],
            }
            self.columns = ["Path", "TaskName", "TaskResult", "TaskStatus", "ResultPath", "PathResult", "Checked"]
            if not os.path.exists(self.knownIssuesResult):
                asciitable.write(
                    self.tableTemple, self.knownIssuesResult, names=self.columns, Writer=asciitable.FixedWidth
                )
            if not os.path.exists(self.unknownIssuesResult):
                asciitable.write(
                    self.tableTemple, self.unknownIssuesResult, names=self.columns, Writer=asciitable.FixedWidth
                )
            reader = asciitable.get_reader(Reader=asciitable.FixedWidth)
            self.knownIssuesTable = reader.read(self.knownIssuesResult)
            self.unknownIssuesTable = reader.read(self.unknownIssuesResult)
def run_ascii():
    """
    Load records with asciitable.

    * PyPy: OK. Development stopped (was moved into the Astropy project
    as astropy.io.ascii).
    * Source: https://github.com/taldcroft/asciitable
    * Docs: Decent
    * Independent: NumPy not required but recommended.
    * Small: no
    * Can specify column data types: yes. If trying to guess, will parse zips
    as numbers.
    * Can read in chunks: no
    * Can skip columns: yes
    * Can stream: no
    * Return type: wrapper around file or iterable, each row is a tuple
    * Memory usage: ~ 60 Mb
    * Timing: around 0.7 sec
    """
    import asciitable
    import numpy

    reader = asciitable.get_reader(
        Reader=asciitable.FixedWidth,
        header_start=None,
        data_start=2,
        col_starts=(0, 5, 7, 35, 36, 41, 48, 56, 59, 65, 66, 67, 71, 75, 78),
        col_ends=(4, 6, 34, 35, 40, 47, 55, 58, 64, 65, 66, 70, 74, 77, 80),
        names=(
            'zip_code', 'state_code', 'city_name', 'zip_type',
            'county_code', 'latitude', 'longitude', 'area_code',
            'finance_code', 'city_official', 'facility', 'msa_code',
            'pmsa_code', 'filler'
        ),
        converters={
            'zip_code': [asciitable.convert_numpy(numpy.str)]
        },
        include_names=(
            'zip_code', 'state_code', 'city_name', 'county_code', 'latitude',
            'longitude', 'area_code', 'msa_code', 'pmsa_code'
        ),
    )

    data = reader.read(
        'data/ZIP.DAT'
    )

    records = 0
    for row in data:
        records += 1

    print 'Records:', records
예제 #9
0
def test_read_normal_names(numpy):
    """Nice, typical fixed format table with col names provided"""
    table = """
# comment (with blank line above)
|  Col1  |  Col2   |
|  1.2   | "hello" |
|  2.4   |'s worlds|
"""
    reader = asciitable.get_reader(Reader=asciitable.FixedWidth,
                                   names=('name1', 'name2'))
    dat = reader.read(table)
    assert_equal(reader.header.colnames, ('name1', 'name2'))
    assert_almost_equal(dat[1][0], 2.4)
예제 #10
0
def test_read_normal_exclude(numpy):
    """Nice, typical fixed format table with col name excluded"""
    table = """
# comment (with blank line above)
|  Col1  |  Col2   |
|  1.2   | "hello" |
|  2.4   |'s worlds|
"""
    reader = asciitable.get_reader(Reader=asciitable.FixedWidth,
                                   exclude_names=('Col1',))
    dat = reader.read(table)
    assert_equal(reader.header.colnames, ('Col2',))
    assert_equal(dat[1][0], "'s worlds")
예제 #11
0
def test_read_weird(numpy):
    """Weird input table with data values chopped by col extent """
    table = """
  Col1  |  Col2 |
  1.2       "hello" 
  2.4   sdf's worlds
"""
    reader = asciitable.get_reader(Reader=asciitable.FixedWidth)
    dat = reader.read(table)
    assert_equal(reader.header.colnames, ('Col1', 'Col2'))
    assert_almost_equal(dat[1][0], 2.4)
    assert_equal(dat[0][1], '"hel')
    assert_equal(dat[1][1], "df's wo")
예제 #12
0
def test_custom_splitters(numpy):
    reader = asciitable.get_reader(numpy=numpy)
    reader.header.splitter = asciitable.BaseSplitter()
    reader.data.splitter = asciitable.BaseSplitter()
    f = 't/test4.dat'
    data = reader.read(f)
    testfile = get_testfiles(f)
    assert_equal(data.dtype.names, testfile['cols'])
    assert_equal(len(data), testfile['nrows'])
    assert_almost_equal(data.field('zabs1.nh')[2], 0.0839710433091)
    assert_almost_equal(data.field('p1.gamma')[2], 1.25997502704)
    assert_almost_equal(data.field('p1.ampl')[2], 0.000696444029148)
    assert_equal(data.field('statname')[2], 'chi2modvar')
    assert_almost_equal(data.field('statval')[2], 497.56468441)
예제 #13
0
def test_read_normal(numpy):
    """Nice, typical fixed format table"""
    table = """
# comment (with blank line above)
|  Col1  |  Col2   |
|  1.2   | "hello" |
|  2.4   |'s worlds|
"""
    reader = asciitable.get_reader(Reader=asciitable.FixedWidth)
    dat = reader.read(table)
    assert_equal(reader.header.colnames, ('Col1', 'Col2'))
    assert_almost_equal(dat[1][0], 2.4)
    assert_equal(dat[0][1], '"hello"')
    assert_equal(dat[1][1], "'s worlds")
예제 #14
0
def test_daophot_header_keywords(numpy):
    reader = asciitable.get_reader(Reader=asciitable.DaophotReader, numpy=numpy)
    table = reader.read('t/daophot.dat')
    expected_keywords = (('NSTARFILE', 'test.nst.1', 'filename', '%-23s'),
                         ('REJFILE', 'hello world', 'filename', '%-23s'),
                         ('SCALE', '1.',  'units/pix', '%-23.7g'),)

    for name, value, units, format_ in expected_keywords:
        for keyword in reader.keywords:
            if keyword.name == name:
                assert_equal(keyword.value, value)
                assert_equal(keyword.units, units)
                assert_equal(keyword.format, format_)
                break
        else:
            raise ValueError('Keyword not found')
예제 #15
0
def test_memory_from_LOL2(numpy):
    data = [[1, 2, 3], [4, 5.2, 6.1], [8, 9, 'hello']]
    mem_table = asciitable.get_reader(Reader=asciitable.Memory, numpy=numpy, names=('c1','c2','c3'))
    mem_data = mem_table.read(data)
    print(mem_data.dtype.names)
    assert(mem_data.dtype.names == ('c1', 'c2', 'c3'))
    if numpy:
        assert(mem_data[0][0] == 1)
        assert(mem_data[0][1] == 2)
        assert(mem_data[0][2] == '3')
        assert((mem_data['c2'] == np.array([2, 5.2, 9])).all())
        assert((mem_data['c3'] == np.array([3, 6.1, 'hello'])).all())
    else:
        assert(mem_data[0] == [1, 2, 3])
        assert(mem_data['c2'] == [2, 5.2, 9])
        assert(mem_data['c3'] == [3, 6.1, 'hello'])
예제 #16
0
def test_ipac_read_types(numpy):
    table = r"""\
|     ra   |    dec   |   sai   |-----v2---|    sptype        |
|    real  |   float  |   l     |    real  |     char         |
|    unit  |   unit   |   unit  |    unit  |     ergs         |
|    null  |   null   |   null  |    null  |     -999         |
   2.09708   2956        73765    2.06000   B8IVpMnHg
"""
    reader = asciitable.get_reader(Reader=asciitable.Ipac, numpy=numpy)
    dat = reader.read(table)
    types = [asciitable.FloatType,
             asciitable.FloatType,
             asciitable.IntType,
             asciitable.FloatType,
             asciitable.StrType]
    for (col, expected_type) in zip(reader.cols, types):
        assert_equal(col.type, expected_type)
예제 #17
0
def run_ascii():
    """
    Load records with asciitable.

    * PyPy: OK. Development stopped (was moved into the Astropy project
    as astropy.io.ascii).
    * Source: https://github.com/taldcroft/asciitable
    * Docs: Decent
    * Independent: NumPy not required but recommended.
    * Small: no
    * Can specify column data types: yes. If trying to guess, will parse zips
    as numbers.
    * Can read in chunks: no
    * Can skip columns: yes
    * Can stream: no
    * Return type: wrapper around file or iterable, each row is a tuple
    * Memory usage: ~ 60 Mb
    * Timing: around 0.7 sec
    """
    import asciitable
    import numpy

    reader = asciitable.get_reader(
        Reader=asciitable.FixedWidth,
        header_start=None,
        data_start=2,
        col_starts=(0, 5, 7, 35, 36, 41, 48, 56, 59, 65, 66, 67, 71, 75, 78),
        col_ends=(4, 6, 34, 35, 40, 47, 55, 58, 64, 65, 66, 70, 74, 77, 80),
        names=('zip_code', 'state_code', 'city_name', 'zip_type',
               'county_code', 'latitude', 'longitude', 'area_code',
               'finance_code', 'city_official', 'facility', 'msa_code',
               'pmsa_code', 'filler'),
        converters={'zip_code': [asciitable.convert_numpy(numpy.str)]},
        include_names=('zip_code', 'state_code', 'city_name', 'county_code',
                       'latitude', 'longitude', 'area_code', 'msa_code',
                       'pmsa_code'),
    )

    data = reader.read('data/ZIP.DAT')

    records = 0
    for row in data:
        records += 1

    print 'Records:', records
예제 #18
0
def test_memory_from_DOL(numpy):
    data = {'c1': [1, 2, 3],
            'c2': [4, 5.2, 6.1],
            'c3': [8, 9, 'hello']}
    mem_table = asciitable.get_reader(Reader=asciitable.Memory, numpy=numpy,
                                      names=sorted(data.keys()))
    mem_data = mem_table.read(data)
    assert(mem_data.dtype.names == ('c1', 'c2', 'c3'))
    if numpy:
        assert(mem_data[0][0] == 1)
        assert(mem_data[0][1] == 4)
        assert(mem_data[0][2] == '8')
        assert((mem_data['c2'] == np.array([4, 5.2, 6.1])).all())
        assert((mem_data['c3'] == np.array([8, 9, 'hello'])).all())
    else:
        assert(mem_data[0] == [1, 4, 8])
        assert(mem_data['c2'] == [4, 5.2, 6.1])
        assert(mem_data['c3'] == [8, 9, 'hello'])
예제 #19
0
def test_types_from_dat(numpy):
    if numpy:
        converters = {'a': [asciitable.convert_numpy(np.float)],
                      'e': [asciitable.convert_numpy(np.str)]}
    else:
        converters = {'a': [asciitable.convert_list(float)],
                      'e': [asciitable.convert_list(str)]}

    dat = asciitable.read(['a b c d e', '1 1 cat 2.1 4.2'], Reader=asciitable.Basic,
                          converters=converters, numpy=numpy)

    reader = asciitable.get_reader(Reader=asciitable.Memory, numpy=numpy)
    reader.read(dat)

    print('numpy=%s' % numpy)
    print('dat=%s' % repr(dat))
    print('reader.table=%s' % repr(reader.table))
    print('types=%s' % repr([x.type for x in reader.cols]))

    assert_true(issubclass(reader.cols[0].type, asciitable.FloatType))
    assert_true(issubclass(reader.cols[1].type, asciitable.IntType))
    assert_true(issubclass(reader.cols[2].type, asciitable.StrType))
    assert_true(issubclass(reader.cols[3].type, asciitable.FloatType))
    assert_true(issubclass(reader.cols[4].type, asciitable.StrType))
def read_table2(readme, data):
    reader = asciitable.get_reader(Reader=asciitable.Cds, readme=readme)
    if asciitable.has_numpy:
        reader.outputter = asciitable.NumpyOutputter()
    return reader.read(data)
예제 #21
0
def test_comment_lines(numpy):
    table = asciitable.get_reader(Reader=asciitable.RdbReader, numpy=numpy)
    data = table.read('t/apostrophe.rdb')
    assert_equal(table.comment_lines, ['# first comment', '  # second comment'])
예제 #22
0
"""
Test case for CDS table writing

Reads in an IPAC table and writes it out as a CDS table
Unfortunately, as of 7/7/11, IPAC tables do not have a 'descr' attribute
"""
import asciitable

# Generate an IPAC reader object
ipac_reader = asciitable.get_reader(Reader=asciitable.Ipac)
# Fill the IPAC reader from a table...
ipac_table = ipac_reader.read('t/ipac2.dat')

# create a CDS object
cds_writer = asciitable.get_writer(Writer=asciitable.Cds)
# write the ipac_reader Table using the CDS writer
# (I think there should be a wrapper for this...)
asciitable.write(ipac_reader,'t/ipac_convert_to_cds.cds',Writer=asciitable.Cds)


cds_reader = asciitable.get_reader(Reader=asciitable.Cds)
cds_table = cds_reader.read('t/cds2.dat')
asciitable.write(cds_reader,'t/cds_convert_to_cds.cds',Writer=asciitable.Cds)


test_read_ipaccds = cds_reader.read('t/ipac_convert_to_cds.cds')
print test_read_ipaccds
test_read_cdscds = cds_reader.read('t/cds_convert_to_cds.cds')
print test_read_cdscds[:5]