def test_reflect_tables_into_session(blank_postgis_session): engine = blank_postgis_session.get_bind() create_test_database(engine) res = blank_postgis_session.execute( "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public';" ) res = [x[0] for x in res.fetchall()] assert set(res).issuperset( set( [ "meta_sensor", "meta_contact", "climo_obs_count_mv", "obs_count_per_month_history_mv", "meta_network_geoserver", "vars_per_history_mv", "crmp_network_geoserver", "meta_history", "meta_vars", "meta_network", "meta_station", "obs_with_flags", "obs_raw", "meta_native_flag", "obs_raw_native_flags", ] ) )
def main(opts, args): log_conf = yaml.load(opts.log_conf) if opts.log: log_conf["handlers"]["file"]["filename"] = opts.log else: opts.log = log_conf["handlers"]["file"]["filename"] if opts.error_email: log_conf["handlers"]["mail"]["toaddrs"] = opts.error_email logging.config.dictConfig(log_conf) engine = create_engine(opts.connection_string) if opts.new_db: create_test_database(engine) if opts.source_dsn: copy_stations_to_sqlite(opts.source_dsn, opts.connection_string) sesh = sessionmaker(bind=engine)() files = glob(os.sep.join([opts.cache_dir, "*"])) for file_ in files: print(file_) et = parse(file_) try: rv = process(sesh, et) logging.info("Processed file", extra={ "file": file_, "results": rv }) except Exception as e: logging.error(e)
def test_can_create_crmp_subset_db(blank_postgis_session): engine = blank_postgis_session.get_bind() create_test_database(engine) insert_crmp_data(blank_postgis_session) q = blank_postgis_session.query(History) assert q.count() > 0
def test_can_create_test_db(blank_postgis_session): engine = blank_postgis_session.get_bind() create_test_database(engine) create_test_data(engine) # Get some data q = blank_postgis_session.query(Contact) assert len(q.all()) == 2
def main(opts, args): log_conf = yaml.load(opts.log_conf) if opts.log: log_conf['handlers']['file']['filename'] = opts.log else: opts.log = log_conf['handlers']['file']['filename'] if opts.error_email: log_conf['handlers']['mail']['toaddrs'] = opts.error_email logging.config.dictConfig(log_conf) engine = create_engine(opts.connection_string) if opts.new_db: create_test_database(engine) if opts.source_dsn: copy_stations_to_sqlite(opts.source_dsn, opts.connection_string) sesh = sessionmaker(bind=engine)() files = glob(os.sep.join([opts.cache_dir, '*'])) for file_ in files: print(file_) et = parse(file_) try: rv = process(sesh, et) logging.info("Processed file", extra={'file': file_, 'results': rv}) except Exception as e: logging.error(e)
def test_reflect_tables_into_session(blank_postgis_session): engine = blank_postgis_session.get_bind() create_test_database(engine) res = blank_postgis_session.execute("SELECT table_name FROM information_schema.tables WHERE table_schema = 'crmp';") res = [x[0] for x in res.fetchall()] assert set(res).issuperset(set(['meta_sensor', 'meta_contact', 'climo_obs_count_mv', 'obs_count_per_month_history_mv', 'meta_network_geoserver', 'vars_per_history_mv', 'crmp_network_geoserver', 'meta_history', 'meta_vars', 'meta_network', 'meta_station', 'obs_with_flags', 'obs_raw', 'meta_native_flag', 'obs_raw_native_flags']))
import sys import logging from argparse import ArgumentParser from pkg_resources import resource_filename from sqlalchemy import create_engine from pycds.util import create_test_database, create_test_data if __name__ == '__main__': parser = ArgumentParser(description="Script to create a test CRMP database and write it to a test database. DSN strings are of form:\n\tdialect+driver://username:password@host:port/database\nExamples:\n\tpostgresql://scott:tiger@localhost/mydatabase\n\tpostgresql+psycopg2://scott:tiger@localhost/mydatabase\n\tpostgresql+pg8000://scott:tiger@localhost/mydatabase") parser.add_argument("-d", "--dsn", help="Source database DSN from which to read ") parser.add_argument("-t", "--testdsn", help="Destination DSN to write to") args = parser.parse_args() logger = logging.getLogger(__name__) logging.basicConfig(stream=sys.stdout, level=logging.INFO) read_engine = create_engine(args.dsn) write_engine = create_engine(args.testdsn) create_test_database(write_engine) create_test_data(write_engine)