def dummy_2019(config):
def _daterange(start_date, end_date):
for n in range(int((end_date - start_date).days)):
yield start_date + timedelta(n)
start_date = date(config["year"], 1, 1)
end_date = date(config["year"], 12, 31)
dates = []
for single_date in _daterange(start_date, end_date):
dates.append({
"year": single_date.year,
"month": single_date.month,
"day": single_date.day,
"dow": single_date.weekday() + 1,
"sum_length": 0,
})
df = pd.DataFrame(dates)
sql = Path(__file__).cwd() / config["path"]
query = generate_query(sql, config)
query_athena(query, config)
_save_local(df, config, df.columns)
def _add_table(config):
sql = Path(__file__).cwd() / config["path"]
query = generate_query(sql, config)
query_athena(query, config)
def start(config):
sql_path = Path(__file__).cwd() / config["path"]
# create table
if should_create_table(config):
query_athena(generate_query(sql_path / "create_table.sql", config),
config)
config["force"] = False
# fill table
partition_query(sql_path / "fill_table.sql", config)
# partition
sql = generate_query(sql_path / "partition.sql", config)
query_athena(sql, config)
def start(config):
if config['name'] in globals().keys():
config = globals()[config['name']](config)
sql = (Path(__file__).cwd() / config['path'])
query = utils.generate_query(sql, config)
utils.query_athena(query, config)
def start(config, insert_groups=None):
sql_path = Path(__file__).cwd() / config["path"]
# create table
if should_create_table(config):
query_athena(generate_query(sql_path / "create_table.sql", config),
config)
# fill table
insert_into(sql_path / "insert_into.sql", config, insert_groups)
def partition_query(query_path, config):
"""Entrypoint function.
In order to run paralel queries to populate the same table, the trick
is to create an empty table that points to a specific S3 path. Then,
a bunch of temporary tables are through an Athena process that points
to the same path. This populates the initial table that now can be
queried normally by Athena.
This also allows us to do that asynchronously. This function implements
that by using `multiprocessing.Pool` from python standard library.
First, it creates a list of dictionaries, `queries`, with two objects:
- `make`: a query that creates the table
- `drop`: a query that drops the same table
Then, this list is called by a Pool that has a number of jobs set by
the user in the config.yaml file, {{ number_of_athena_jobs }}.
Parameters
----------
query_path : str
path to the query that creates the table
config : dict
variables from config.yaml
"""
queries = []
partitions = globals()[config["name"]](config)
if partitions is None:
return
for partition in partitions:
config.update(deepcopy(partition))
queries.append(
dict(
make=generate_query(query_path, config),
drop=
f"drop table {config['athena_database']}.{config['slug']}_{config['raw_table']}_{config['name']}_{config['p_name']}",
config=config,
p_path=deepcopy(config["p_path"]),
partition=config["partition"],
))
with Pool(config["number_of_athena_jobs"]) as p:
p.map(partial(perform_query), queries)
def insert_into(query_path, config, insert_groups):
config["force"] = False
if not isinstance(insert_groups, list):
insert_groups = globals()[config["name"]](config)
queries = []
for insert_group in insert_groups:
config.update(deepcopy(insert_group))
queries.append(
dict(make=generate_query(query_path, config),
config=deepcopy(config)))
with Pool(config["number_of_athena_jobs"]) as p:
p.map(partial(perform_query), queries)