async def main(workers=WORKERS, make_engine=True):
# when testing we do not want an engine.
if make_engine:
engine = db_helper.make_engine(section="docker",
environment=KILO_ENVIRONMENT_OVERRIDES)
db_helper.set_session(engine)
await run_workers(workers=workers)
def main(make_engine=True):
desc = "Clean data and import into db."
inputparser = argparse.ArgumentParser(desc)
inputparser.add_argument("--debug",
action="store_true",
default=False,
help="Enable debugging")
inputparser.add_argument("--link_areas",
action="store_true",
default=False,
help="Link with neighbourhood areas")
args = inputparser.parse_args()
if args.debug:
log.setLevel(logging.DEBUG)
logging.getLogger("urllib3").setLevel(logging.DEBUG)
start = time.time()
if make_engine:
engine = db_helper.make_engine()
db_helper.set_session(engine)
if args.link_areas:
session = db_helper.session
link_areas(session, 'importer_trafficorder')
else:
importer = TrafficOrderImporter()
importer.start_import()
log.info("Total time: %s", time.time() - start)
def main(make_engine=True):
desc = "Clean data and import into db."
inputparser = argparse.ArgumentParser(desc)
inputparser.add_argument("--link_areas",
action="store_true",
default=False,
help="Link stations with neighbourhood areas")
args = inputparser.parse_args()
start = time.time()
if make_engine:
engine = db_helper.make_engine()
db_helper.set_session(engine)
session = db_helper.session
if args.link_areas:
link_areas(session, OvFiets.__tablename__)
else:
start_import()
log.info("Took: %s", time.time() - start)
session.close()
def start_import(make_engine, endpoint):
if make_engine:
engine = db_helper.make_engine(section='docker')
db_helper.set_session(engine)
data = fetch_json(endpoint)
store(data, endpoint)
async def main(args):
"""Main."""
engine = db_helper.make_engine(section="docker")
db_helper.set_session(engine)
drop_views(args)
drop_tables(args)
LOG.warning("CREATING DEFINED TABLES")
# recreate tables
Base.metadata.create_all(engine)
create_views(args)
def main(make_engine=True):
"""Start ETL.
When testing we can turn of endgine creation.
"""
if make_engine:
engine = db_helper.make_engine(
section="docker",
environment=KILO_ENVIRONMENT_OVERRIDES)
db_helper.set_session(engine)
extract_measurements()
# using wfs services which are loaded in the maintenance
# jenkins job. we merge enevo fill levels with datapunt afval sites.
# based on distance.
update_site_ids()
def main(make_engine=True):
desc = "Clean data and import into db."
inputparser = argparse.ArgumentParser(desc)
inputparser.add_argument(
"endpoint",
type=str,
default="traveltime",
choices=ENDPOINTS,
help="Provide Endpoint to scrape",
nargs=1,
)
inputparser.add_argument(
"--debug",
action="store_true",
default=False,
help="Enable debugging"
)
inputparser.add_argument(
"--exclude_areas",
action="store_true",
default=False,
help="Link areas to model"
)
args = inputparser.parse_args()
if args.debug:
log.setLevel(logging.DEBUG)
start = time.time()
if make_engine:
engine = db_helper.make_engine()
db_helper.set_session(engine)
importer = ENDPOINT_IMPORTER[args.endpoint[0]]()
if args.exclude_areas:
importer.link_areas = False
importer.start_import()
log.info("Total time: %s", time.time() - start)
def setup_module():
global transaction, connection, engine, session
settings.TESTING["running"] = True
db_helper.create_db()
engine = db_helper.make_engine(section="test")
connection = engine.connect()
transaction = connection.begin()
session = db_helper.set_session(engine)
session.execute("CREATE EXTENSION IF NOT EXISTS postgis;")
session.commit()
models.Base.metadata.drop_all(bind=engine)
models.Base.metadata.create_all(bind=engine)
def main(make_engine=True):
desc = "Clean data and import into db."
inputparser = argparse.ArgumentParser(desc)
inputparser.add_argument(
"endpoint",
type=str,
default="parking_location",
choices=ENDPOINTS,
help="Provide Endpoint to scrape",
nargs=1,
)
inputparser.add_argument("--link_areas",
action="store_true",
default=False,
help="Link stations with neighbourhood areas")
args = inputparser.parse_args()
start = time.time()
if make_engine:
engine = db_helper.make_engine()
db_helper.set_session(engine)
session = db_helper.session
endpoint = args.endpoint[0]
raw_model = ENDPOINT_RAW_MODEL[endpoint]
clean_model = ENDPOINT_MODEL[endpoint]
store_func = ENDPOINTS_STORE_FUNC[endpoint]
if args.link_areas:
link_areas(session, clean_model.__tablename__)
else:
start_import(store_func, raw_model, clean_model)
log.info("Took: %s", time.time() - start)
session.close()
def main(make_engine=True):
desc = "Clean data and import into db."
inputparser = argparse.ArgumentParser(desc)
inputparser.add_argument("--debug",
action="store_true",
default=False,
help="Enable debugging")
args = inputparser.parse_args()
if args.debug:
log.setLevel(logging.DEBUG)
start = time.time()
if make_engine:
engine = db_helper.make_engine()
db_helper.set_session(engine)
WifiInfoImporter().start_import()
log.info("Total time: %s", time.time() - start)
async def main(args):
"""Main."""
engine = db_helper.make_engine(section="docker")
session = db_helper.set_session(engine)
if args.drop:
# resets everything
LOG.warning("DROPPING ALL DEFINED TABLES")
for table in OVFIETS_TABLES:
session.execute(f"DROP table if exists {table};")
session.commit()
LOG.warning("CREATING DEFINED TABLES")
# recreate tables
Base.metadata.create_all(engine)
async def create_tables(args):
"""Main."""
engine = db_helper.make_engine(environment=KILO_ENVIRONMENT_OVERRIDES)
session = db_helper.set_session(engine)
if args.drop:
# resets everything
LOG.warning("DROPPING ALL DEFINED TABLES")
for table in SIDCON_TABLES:
session.execute(f"DROP table if exists {table};")
session.commit()
LOG.warning("CREATING DEFINED TABLES")
# recreate tables
Base.metadata.create_all(engine)
db_helper.alembic_migrate(engine)
async def create_tables(args):
"""Main."""
engine = db_helper.make_engine(environment=KILO_ENVIRONMENT_OVERRIDES)
session = db_helper.set_session(engine)
if args.drop:
# resets the target table
LOG.warning("DROPPING TARGET TABLE")
#for table in KILOGRAM_TABLES:
session.execute(f"DROP table if exists kilogram_weigh_measurement;")
session.commit()
# Base.metadata.drop_all(engine)
LOG.warning("CREATING DEFINED TABLES")
# recreate tables
Base.metadata.create_all(engine)
db_helper.alembic_migrate(engine)
async def main(args):
"""Table management."""
engine = db_helper.make_engine(environment=KILO_ENVIRONMENT_OVERRIDES, )
session = db_helper.set_session(engine)
if args.dropall:
LOG.warning("DROPPING ALL DEFINED TABLES")
# resets everything
Base.metadata.drop_all(engine)
if args.drop:
# resets non raw tables
LOG.warning("DROPPING ALL DROP TABLES")
for table in DROP_TABLES:
table = table.__table__.name
session.execute(f"DROP table if exists {table};")
for seq in DROP_SEQUENCE:
session.execute(f"DROP SEQUENCE IF EXISTS {seq};")
session.commit()
if args.live:
LOG.warning("CREATING LIVE RELATED TABLES")
meta = MetaData()
meta.bind = engine
meta.reflect()
for table in LIVE_TABLES:
table_name = table.__table__.name
if table_name not in meta.tables:
table.__table__.create(engine)
# stop creating more tables
return
LOG.warning("CREATING ALL DEFINED TABLES")
# recreate tables
Base.metadata.create_all(engine)
def get_session():
engine = db_helper.make_engine(environment=KILO_ENVIRONMENT_OVERRIDES)
session = db_helper.set_session(engine)
return engine, session
def setup_db(self, make_engine):
if make_engine:
self.engine = db_helper.make_engine(section='docker')
db_helper.set_session(self.engine)
inputparser = argparse.ArgumentParser(desc)
inputparser.add_argument(
"endpoint",
type=str,
default="",
choices=list(OPTIONS.keys()),
help="Provide Endpoint to copy",
nargs=1,
)
inputparser.add_argument(
"--link_container_slots",
action="store_true",
default=False,
help="Link containerslots with sites and sitecontenttypes")
inputparser.add_argument("--validate_containers",
action="store_true",
default=False,
help="Validate the customer_keys are in bammens")
args = inputparser.parse_args()
engine = db_helper.make_engine()
session = db_helper.set_session(engine)
main()
session.close()
