def queue_calc_imeis_jobs(self, executor, app_config, run_id, curr_date):
"""
Method to queue jobs to calculate the IMEIs that are met by this condition.
Arguments:
executor: instance of the python executor class, to submit back the results
app_config: dirbs app current configuration, to extract various configs required for the job
run_id: run id of the current classification job
curr_date: current date of the system
"""
with create_db_connection(app_config.db_config) as conn, conn.cursor() as cursor:
cursor.execute(sql.SQL("""CREATE UNLOGGED TABLE {intermediate_tbl} (
imei_norm TEXT NOT NULL,
virt_imei_shard SMALLINT NOT NULL
)
PARTITION BY RANGE (virt_imei_shard)""")
.format(intermediate_tbl=self.intermediate_tbl_id(run_id)))
partition_utils.create_imei_shard_partitions(conn, tbl_name=self.intermediate_tbl_name(run_id),
unlogged=True)
parallel_shards = partition_utils.num_physical_imei_shards(conn)
# Done with connection -- temp tables should now be committed
virt_imei_shard_ranges = partition_utils.virt_imei_shard_bounds(parallel_shards)
for virt_imei_range_start, virt_imei_range_end in virt_imei_shard_ranges:
yield executor.submit(self._calc_imeis_job,
app_config,
run_id,
curr_date,
virt_imei_range_start,
virt_imei_range_end)
def queue_update_classification_state_jobs(self, executor, app_config,
run_id, curr_date):
"""Method to queue jobs to update the classification_state table after the IMEIs have been calculated."""
with create_db_connection(app_config.db_config) as conn:
parallel_shards = partition_utils.num_physical_imei_shards(conn)
virt_imei_shard_ranges = partition_utils.virt_imei_shard_bounds(
parallel_shards)
for virt_imei_range_start, virt_imei_range_end in virt_imei_shard_ranges:
yield executor.submit(self._update_classification_state_job,
app_config, run_id, curr_date,
virt_imei_range_start,
virt_imei_range_end)
def queue_update_classification_state_jobs(self, executor, app_config, run_id, curr_date):
"""
Method to queue jobs to update the classification_state table after the IMEIs have been calculated.
Arguments:
executor: job executor instance to submit back the results to the queue
app_config: current dirbs app config object to use configuration from
run_id: run_id of the current running classification job
curr_date: current date of the system
"""
with create_db_connection(app_config.db_config) as conn:
parallel_shards = partition_utils.num_physical_imei_shards(conn)
virt_imei_shard_ranges = partition_utils.virt_imei_shard_bounds(parallel_shards)
for virt_imei_range_start, virt_imei_range_end in virt_imei_shard_ranges:
yield executor.submit(self._update_classification_state_job,
app_config,
run_id,
curr_date,
virt_imei_range_start,
virt_imei_range_end)
def queue_calc_imeis_jobs(self, executor, app_config, run_id, curr_date):
"""Method to queue jobs to calculate the IMEIs that are met by this condition."""
with create_db_connection(
app_config.db_config) as conn, conn.cursor() as cursor:
cursor.execute(
sql.SQL("""CREATE UNLOGGED TABLE {intermediate_tbl} (
imei_norm TEXT NOT NULL,
virt_imei_shard SMALLINT NOT NULL
)
PARTITION BY RANGE (virt_imei_shard)""").
format(intermediate_tbl=self.intermediate_tbl_id(run_id)))
partition_utils.create_imei_shard_partitions(
conn,
tbl_name=self.intermediate_tbl_name(run_id),
unlogged=True)
parallel_shards = partition_utils.num_physical_imei_shards(conn)
# Done with connection -- temp tables should now be committed
virt_imei_shard_ranges = partition_utils.virt_imei_shard_bounds(
parallel_shards)
for virt_imei_range_start, virt_imei_range_end in virt_imei_shard_ranges:
yield executor.submit(self._calc_imeis_job, app_config, run_id,
curr_date, virt_imei_range_start,
virt_imei_range_end)