def dummy_2019(config):
def _daterange(start_date, end_date):
for n in range(int((end_date - start_date).days)):
yield start_date + timedelta(n)
start_date = date(config["year"], 1, 1)
end_date = date(config["year"], 12, 31)
dates = []
for single_date in _daterange(start_date, end_date):
dates.append({
"year": single_date.year,
"month": single_date.month,
"day": single_date.day,
"dow": single_date.weekday() + 1,
"sum_length": 0,
})
df = pd.DataFrame(dates)
sql = Path(__file__).cwd() / config["path"]
query = generate_query(sql, config)
query_athena(query, config)
_save_local(df, config, df.columns)
def _add_table(config):
sql = Path(__file__).cwd() / config["path"]
query = generate_query(sql, config)
query_athena(query, config)
def start(config):
if config['name'] in globals().keys():
config = globals()[config['name']](config)
sql = (Path(__file__).cwd() / config['path'])
query = utils.generate_query(sql, config)
utils.query_athena(query, config)
def start(config, insert_groups=None):
sql_path = Path(__file__).cwd() / config["path"]
# create table
if should_create_table(config):
query_athena(generate_query(sql_path / "create_table.sql", config),
config)
# fill table
insert_into(sql_path / "insert_into.sql", config, insert_groups)
def start(config):
sql_path = Path(__file__).cwd() / config["path"]
# create table
if should_create_table(config):
query_athena(generate_query(sql_path / "create_table.sql", config),
config)
config["force"] = False
# fill table
partition_query(sql_path / "fill_table.sql", config)
# partition
sql = generate_query(sql_path / "partition.sql", config)
query_athena(sql, config)
def perform_query(query):
"""Simply calls Athena and logs the exceptions.
Parameters
----------
query : dict
dict with two objects, `make` and `drop`. The first to create
a table and the second to drop the same table.
"""
for i in range(query["config"]["n_tries"]):
try:
delete_s3_path(query["p_path"], query["config"])
query_athena(query["drop"], query["config"])
query_athena(query["make"], query["config"])
query_athena(query["drop"], query["config"])
break
except Exception as e:
if query["config"]["verbose"]:
print(e)
if i == query["config"]["n_tries"] - 1:
raise e
continue