def dummy_2019(config): def _daterange(start_date, end_date): for n in range(int((end_date - start_date).days)): yield start_date + timedelta(n) start_date = date(config["year"], 1, 1) end_date = date(config["year"], 12, 31) dates = [] for single_date in _daterange(start_date, end_date): dates.append({ "year": single_date.year, "month": single_date.month, "day": single_date.day, "dow": single_date.weekday() + 1, "sum_length": 0, }) df = pd.DataFrame(dates) sql = Path(__file__).cwd() / config["path"] query = generate_query(sql, config) query_athena(query, config) _save_local(df, config, df.columns)
def _add_table(config): sql = Path(__file__).cwd() / config["path"] query = generate_query(sql, config) query_athena(query, config)
def start(config): sql_path = Path(__file__).cwd() / config["path"] # create table if should_create_table(config): query_athena(generate_query(sql_path / "create_table.sql", config), config) config["force"] = False # fill table partition_query(sql_path / "fill_table.sql", config) # partition sql = generate_query(sql_path / "partition.sql", config) query_athena(sql, config)
def start(config): if config['name'] in globals().keys(): config = globals()[config['name']](config) sql = (Path(__file__).cwd() / config['path']) query = utils.generate_query(sql, config) utils.query_athena(query, config)
def start(config, insert_groups=None): sql_path = Path(__file__).cwd() / config["path"] # create table if should_create_table(config): query_athena(generate_query(sql_path / "create_table.sql", config), config) # fill table insert_into(sql_path / "insert_into.sql", config, insert_groups)
def partition_query(query_path, config): """Entrypoint function. In order to run paralel queries to populate the same table, the trick is to create an empty table that points to a specific S3 path. Then, a bunch of temporary tables are through an Athena process that points to the same path. This populates the initial table that now can be queried normally by Athena. This also allows us to do that asynchronously. This function implements that by using `multiprocessing.Pool` from python standard library. First, it creates a list of dictionaries, `queries`, with two objects: - `make`: a query that creates the table - `drop`: a query that drops the same table Then, this list is called by a Pool that has a number of jobs set by the user in the config.yaml file, {{ number_of_athena_jobs }}. Parameters ---------- query_path : str path to the query that creates the table config : dict variables from config.yaml """ queries = [] partitions = globals()[config["name"]](config) if partitions is None: return for partition in partitions: config.update(deepcopy(partition)) queries.append( dict( make=generate_query(query_path, config), drop= f"drop table {config['athena_database']}.{config['slug']}_{config['raw_table']}_{config['name']}_{config['p_name']}", config=config, p_path=deepcopy(config["p_path"]), partition=config["partition"], )) with Pool(config["number_of_athena_jobs"]) as p: p.map(partial(perform_query), queries)
def insert_into(query_path, config, insert_groups): config["force"] = False if not isinstance(insert_groups, list): insert_groups = globals()[config["name"]](config) queries = [] for insert_group in insert_groups: config.update(deepcopy(insert_group)) queries.append( dict(make=generate_query(query_path, config), config=deepcopy(config))) with Pool(config["number_of_athena_jobs"]) as p: p.map(partial(perform_query), queries)