def test_reflect_tables_into_session(blank_postgis_session):
engine = blank_postgis_session.get_bind()
create_test_database(engine)
res = blank_postgis_session.execute(
"SELECT table_name FROM information_schema.tables WHERE table_schema = 'public';"
)
res = [x[0] for x in res.fetchall()]
assert set(res).issuperset(
set(
[
"meta_sensor",
"meta_contact",
"climo_obs_count_mv",
"obs_count_per_month_history_mv",
"meta_network_geoserver",
"vars_per_history_mv",
"crmp_network_geoserver",
"meta_history",
"meta_vars",
"meta_network",
"meta_station",
"obs_with_flags",
"obs_raw",
"meta_native_flag",
"obs_raw_native_flags",
]
)
)
def main(opts, args):
log_conf = yaml.load(opts.log_conf)
if opts.log:
log_conf["handlers"]["file"]["filename"] = opts.log
else:
opts.log = log_conf["handlers"]["file"]["filename"]
if opts.error_email:
log_conf["handlers"]["mail"]["toaddrs"] = opts.error_email
logging.config.dictConfig(log_conf)
engine = create_engine(opts.connection_string)
if opts.new_db:
create_test_database(engine)
if opts.source_dsn:
copy_stations_to_sqlite(opts.source_dsn, opts.connection_string)
sesh = sessionmaker(bind=engine)()
files = glob(os.sep.join([opts.cache_dir, "*"]))
for file_ in files:
print(file_)
et = parse(file_)
try:
rv = process(sesh, et)
logging.info("Processed file",
extra={
"file": file_,
"results": rv
})
except Exception as e:
logging.error(e)
def test_can_create_crmp_subset_db(blank_postgis_session):
engine = blank_postgis_session.get_bind()
create_test_database(engine)
insert_crmp_data(blank_postgis_session)
q = blank_postgis_session.query(History)
assert q.count() > 0
def test_can_create_test_db(blank_postgis_session):
engine = blank_postgis_session.get_bind()
create_test_database(engine)
create_test_data(engine)
# Get some data
q = blank_postgis_session.query(Contact)
assert len(q.all()) == 2
def main(opts, args):
log_conf = yaml.load(opts.log_conf)
if opts.log:
log_conf['handlers']['file']['filename'] = opts.log
else:
opts.log = log_conf['handlers']['file']['filename']
if opts.error_email:
log_conf['handlers']['mail']['toaddrs'] = opts.error_email
logging.config.dictConfig(log_conf)
engine = create_engine(opts.connection_string)
if opts.new_db:
create_test_database(engine)
if opts.source_dsn:
copy_stations_to_sqlite(opts.source_dsn, opts.connection_string)
sesh = sessionmaker(bind=engine)()
files = glob(os.sep.join([opts.cache_dir, '*']))
for file_ in files:
print(file_)
et = parse(file_)
try:
rv = process(sesh, et)
logging.info("Processed file",
extra={'file': file_, 'results': rv})
except Exception as e:
logging.error(e)
def test_reflect_tables_into_session(blank_postgis_session):
engine = blank_postgis_session.get_bind()
create_test_database(engine)
res = blank_postgis_session.execute("SELECT table_name FROM information_schema.tables WHERE table_schema = 'crmp';")
res = [x[0] for x in res.fetchall()]
assert set(res).issuperset(set(['meta_sensor', 'meta_contact', 'climo_obs_count_mv',
'obs_count_per_month_history_mv', 'meta_network_geoserver',
'vars_per_history_mv', 'crmp_network_geoserver', 'meta_history',
'meta_vars', 'meta_network', 'meta_station', 'obs_with_flags',
'obs_raw', 'meta_native_flag', 'obs_raw_native_flags']))
import sys
import logging
from argparse import ArgumentParser
from pkg_resources import resource_filename
from sqlalchemy import create_engine
from pycds.util import create_test_database, create_test_data
if __name__ == '__main__':
parser = ArgumentParser(description="Script to create a test CRMP database and write it to a test database. DSN strings are of form:\n\tdialect+driver://username:password@host:port/database\nExamples:\n\tpostgresql://scott:tiger@localhost/mydatabase\n\tpostgresql+psycopg2://scott:tiger@localhost/mydatabase\n\tpostgresql+pg8000://scott:tiger@localhost/mydatabase")
parser.add_argument("-d", "--dsn", help="Source database DSN from which to read ")
parser.add_argument("-t", "--testdsn", help="Destination DSN to write to")
args = parser.parse_args()
logger = logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
read_engine = create_engine(args.dsn)
write_engine = create_engine(args.testdsn)
create_test_database(write_engine)
create_test_data(write_engine)
