示例#1
0
文件: batch.py 项目: zari277/pwtools
def default_repl_keys():
    """Return a dict of default keys for replacement in a
    :class:`ParameterStudy`. Each key will in practice be processed by
    :class:`FileTemplate`, such that e.g. the key 'foo' becomes the placeholder
    'XXXFOO'.

    Each of these placeholders can be used in any parameter study in any file
    template, indenpendent from `params_lst` to :class:`ParameterStudy`.

    Notes
    -----
    If this function is called, dummy files are written to a temp dir, the
    datsbase is read, and the file are deleted.
    """
    # HACK!!! Due to the way ParameterStudy.write_input() is coded, we really
    # need to set up a dummy ParameterStudy and read out the database to get
    # the replacement keys :-)
    import tempfile
    calc_root = tempfile.mkdtemp()
    jobfn_templ = pj(calc_root, 'foo.job')
    common.file_write(jobfn_templ, 'dummy job template, go away!')
    m = Machine(hostname='foo',
                template=FileTemplate(basename='foo.job', templ_dir=calc_root))
    print("writing test files to: %s, will be deleted" % calc_root)
    params_lst = [[SQLEntry(key='dummy', sqlval=1)]]
    study = ParameterStudy(machines=m,
                           params_lst=params_lst,
                           calc_root=calc_root)
    study.write_input()
    db = SQLiteDB(study.dbfn, table='calc')
    db_keys = [str(x[0]) for x in db.get_header()]
    for kk in ['dummy', 'hostname']:
        db_keys.pop(db_keys.index(kk))
    db.finish()
    ret = {
        'ParameterStudy': db_keys,
        'Machine': list(m.get_sql_record().keys())
    }
    shutil.rmtree(calc_root)
    return ret
示例#2
0
文件: batch.py 项目: elcorto/pwtools
def default_repl_keys():
    """Return a dict of default keys for replacement in a
    :class:`ParameterStudy`. Each key will in practice be processed by
    :class:`FileTemplate`, such that e.g. the key 'foo' becomes the placeholder
    'XXXFOO'.

    Each of these placeholders can be used in any parameter study in any file
    template, indenpendent from `params_lst` to :class:`ParameterStudy`.

    Notes
    -----
    If this function is called, dummy files are written to a temp dir, the
    datsbase is read, and the file are deleted.
    """
    # HACK!!! Due to the way ParameterStudy.write_input() is coded, we really
    # need to set up a dummy ParameterStudy and read out the database to get
    # the replacement keys :-)
    import tempfile
    calc_root = tempfile.mkdtemp()
    jobfn_templ = pj(calc_root, 'foo.job')
    common.file_write(jobfn_templ, 'dummy job template, go away!')
    m = Machine(hostname='foo', 
                template=FileTemplate(basename='foo.job',
                                      templ_dir=calc_root))
    print ("writing test files to: %s, will be deleted" %calc_root)
    params_lst = [[SQLEntry(key='dummy', sqlval=1)]]
    study = ParameterStudy(machines=m, params_lst=params_lst,
                           calc_root=calc_root)
    study.write_input()
    db = SQLiteDB(study.dbfn, table='calc')
    db_keys = [str(x[0]) for x in db.get_header()]
    for kk in ['dummy', 'hostname']:
        db_keys.pop(db_keys.index(kk))
    db.finish()
    ret = {'ParameterStudy' : db_keys,
           'Machine' : m.get_sql_record().keys()}
    shutil.rmtree(calc_root)
    return ret
示例#3
0
def test_sql():
    # Check for sqlite3 command line tool. In Python 3.3, we can use
    # shutil.which().
    have_sqlite3 = False
    for pp in sys.path:
        exe = pj(pp, 'sqlite3')
        if os.path.isfile(exe):
            print("found:", exe)
            have_sqlite3 = True
            break

    # --- SQLiteDB ----------------------------------------------------
    dbfn = pj(testdir, 'test.db')
    if os.path.exists(dbfn):
        os.remove(dbfn)

    header = [('idx', 'INTEGER'), ('foo', 'REAL'), ('bar', 'TEXT')]
    db = SQLiteDB(dbfn, table='calc')
    db.execute("CREATE TABLE calc (%s)" %','.join("%s %s" %(x[0], x[1]) \
                                                  for x in header))

    vals = [[0, 1.1, 'a'], [1, 2.2, 'b'], [2, 3.3, 'c']]
    for lst in vals:
        db.execute("INSERT INTO calc (idx, foo, bar) VALUES (?,?,?)",
                   tuple(lst))
    db.commit()

    # get_max_rowid
    assert db.get_max_rowid() == 3

    # has_table
    assert db.has_table('calc')
    assert not db.has_table('foo')

    # has_column
    assert db.has_column('idx')
    assert not db.has_column('grrr')

    # get_single
    assert float(db.get_single("select foo from calc where idx==0")) == 1.1

    assert header == db.get_header()

    if have_sqlite3:
        # call sqlite3, the cmd line interface
        assert common.backtick("sqlite3 %s 'select * from calc'" %dbfn) \
            == '0|1.1|a\n1|2.2|b\n2|3.3|c\n'

    # ret =
    # [(0, 1.1000000000000001, u'a'),
    # (1, 2.2000000000000002, u'b'),
    # (2, 3.2999999999999998, u'c')]
    ret = db.execute("select * from calc").fetchall()
    for idx, lst in enumerate(vals):
        assert list(ret[idx]) == lst

    # generator object, yields
    # tup = (0, 1.1000000000000001, u'a')
    # tup = (1, 2.2000000000000002, u'b')
    # tup = (2, 3.2999999999999998, u'c')
    itr = db.execute("select * from calc")
    for idx, tup in enumerate(itr):
        assert list(tup) == vals[idx]

    # [(0, 1.1000000000000001, u'a')]
    assert db.execute("select * from calc where idx==0").fetchall() == \
        [tuple(vals[0])]
    # (0, 1.1000000000000001, u'a')
    assert db.execute("select * from calc where idx==0").fetchone() == \
        tuple(vals[0])

    assert db.execute("select bar from calc where idx==0").fetchone()[0] == \
        'a'

    # get_list1d(), get_array1d(), get_array()
    assert db.get_list1d("select idx from calc") == [0, 1, 2]
    np.testing.assert_array_equal(db.get_array1d("select idx from calc"),
                                  np.array([0, 1, 2]))
    np.testing.assert_array_equal(db.get_array("select idx from calc"),
                                  np.array([0, 1, 2])[:, None])
    np.testing.assert_array_equal(
        db.get_array("select idx,foo from calc"),
        np.array(vals, dtype='S3')[:, :2].astype(float))

    # add_column(), fill with values
    db.add_column('baz', 'TEXT')
    add_header = [('baz', 'TEXT')]
    header += add_header
    assert db.get_header() == header
    db.execute("UPDATE %s SET baz='xx' where idx==0" % db.table)
    db.execute("UPDATE %s SET baz='yy' where idx==1" % db.table)
    db.execute("UPDATE %s SET baz=? where idx==2" % db.table, ('zz', ))
    db.commit()
    if have_sqlite3:
        print(common.backtick("sqlite3 %s 'select * from calc'" % dbfn))
    print(db.execute("select baz from calc").fetchall())
    assert db.execute("select baz from calc").fetchall() == \
        [('xx',), ('yy',), ('zz',)]

    # add even more cols with add_columns()
    add_header = [('bob', 'TEXT'), ('alice', 'BLOB')]
    header += add_header
    db.add_columns(add_header)
    assert db.get_header() == header

    # create_table()
    dbfn2 = pj(testdir, 'test2.db')
    header2 = [('a', 'REAL'), ('b', 'TEXT')]
    db2 = SQLiteDB(dbfn2, table='foo')
    db2.create_table(header2)
    assert db2.get_header() == header2

    # get_dict()
    dct = db.get_dict("select foo,bar from calc")
    cols = [x[0] for x in db.get_header()]
    for key in ['foo', 'bar']:
        assert key in cols
    foo = db.get_list1d("select foo from calc")
    bar = db.get_list1d("select bar from calc")
    assert foo == dct['foo']
    assert bar == dct['bar']
示例#4
0
def test_sql():
    # Check for sqlite3 command line tool. In Python 3.3, we can use
    # shutil.which().
    have_sqlite3 = False
    for pp in sys.path:
        exe = pj(pp, 'sqlite3')
        if os.path.isfile(exe):
            print "found:", exe
            have_sqlite3 = True
            break
        
    # --- SQLiteDB ----------------------------------------------------
    dbfn = pj(testdir, 'test.db')
    if os.path.exists(dbfn):
        os.remove(dbfn)

    header = [('idx', 'INTEGER'), ('foo', 'REAL'), ('bar', 'TEXT')]
    db = SQLiteDB(dbfn, table='calc')
    db.execute("CREATE TABLE calc (%s)" %','.join("%s %s" %(x[0], x[1]) \
                                                  for x in header)) 

    vals = [[0, 1.1, 'a'],
            [1, 2.2, 'b'],
            [2, 3.3, 'c']]
    for lst in vals:
        db.execute("INSERT INTO calc (idx, foo, bar) VALUES (?,?,?)", tuple(lst))
    db.commit()
    
    # get_max_rowid
    assert db.get_max_rowid() == 3

    # has_table
    assert db.has_table('calc')
    assert not db.has_table('foo')
    
    # has_column
    assert db.has_column('idx')
    assert not db.has_column('grrr')

    # get_single
    assert float(db.get_single("select foo from calc where idx==0")) == 1.1

    assert header == db.get_header()
    
    if have_sqlite3:
        # call sqlite3, the cmd line interface
        assert common.backtick("sqlite3 %s 'select * from calc'" %dbfn) \
            == '0|1.1|a\n1|2.2|b\n2|3.3|c\n'

    # ret = 
    # [(0, 1.1000000000000001, u'a'),
    # (1, 2.2000000000000002, u'b'),
    # (2, 3.2999999999999998, u'c')]
    ret = db.execute("select * from calc").fetchall()
    for idx, lst in enumerate(vals):
        assert list(ret[idx]) == lst

    # generator object, yields
    # tup = (0, 1.1000000000000001, u'a')
    # tup = (1, 2.2000000000000002, u'b')
    # tup = (2, 3.2999999999999998, u'c')
    itr = db.execute("select * from calc")
    for idx, tup in enumerate(itr):
        assert list(tup) == vals[idx]

    # [(0, 1.1000000000000001, u'a')]
    assert db.execute("select * from calc where idx==0").fetchall() == \
        [tuple(vals[0])]
    # (0, 1.1000000000000001, u'a')
    assert db.execute("select * from calc where idx==0").fetchone() == \
        tuple(vals[0])

    assert db.execute("select bar from calc where idx==0").fetchone()[0] == \
        'a'
    
    # get_list1d(), get_array1d(), get_array()
    assert db.get_list1d("select idx from calc") == [0,1,2]
    np.testing.assert_array_equal(db.get_array1d("select idx from calc"),
                                  np.array([0,1,2]))
    np.testing.assert_array_equal(db.get_array("select idx from calc"), 
                                  np.array([0,1,2])[:,None])
    np.testing.assert_array_equal(db.get_array("select idx,foo from calc"), 
                                  np.array(vals, dtype='S3')[:,:2].astype(float))

    # add_column(), fill with values
    db.add_column('baz', 'TEXT')
    add_header = [('baz', 'TEXT')]
    header += add_header
    assert db.get_header() == header
    db.execute("UPDATE %s SET baz='xx' where idx==0" %db.table)
    db.execute("UPDATE %s SET baz='yy' where idx==1" %db.table)
    db.execute("UPDATE %s SET baz=? where idx==2" %db.table, ('zz',))
    db.commit()
    if have_sqlite3:
        print common.backtick("sqlite3 %s 'select * from calc'" %dbfn)
    print db.execute("select baz from calc").fetchall()
    assert db.execute("select baz from calc").fetchall() == \
        [(u'xx',), (u'yy',), (u'zz',)]
    
    # add even more cols with add_columns()
    add_header = [('bob', 'TEXT'), ('alice', 'BLOB')]
    header += add_header
    db.add_columns(add_header)
    assert db.get_header() == header
    
    # create_table()
    dbfn2 = pj(testdir, 'test2.db')
    header2 = [('a', 'REAL'), ('b', 'TEXT')]
    db2 = SQLiteDB(dbfn2, table='foo')
    db2.create_table(header2)
    assert db2.get_header() == header2
    
    # get_dict()
    dct = db.get_dict("select foo,bar from calc")
    cols = [x[0] for x in db.get_header()]
    for key in ['foo', 'bar']:
        assert key in cols
    foo = db.get_list1d("select foo from calc")
    bar = db.get_list1d("select bar from calc")
    assert foo == dct['foo']
    assert bar == dct['bar']