def submit_records(self, record_type: Union[Tuple[str, str], str],
prepared_records: List[Tuple[Any]]):
"""Submit/save prepared records to db or csv"""
if self.store_csv:
if self.dry_run:
return
self.csv_output.store_append_batch_to_csv(
self.batched_lbsn_records[record_type], self.count_round,
record_type)
if self.db_cursor:
# get sql escaped values list
sql_escaped_values_list = [
self.prepare_sqlescaped_values(record)
for record in prepared_records
]
# concat to single sql str
values_str = HF.concat_values_str(sql_escaped_values_list)
insert_sql = self.insert_sql_selector(values_str, record_type)
# clear line
sys.stdout.write("\033[K")
print(f'Submitting {len(prepared_records)}..', end='\r')
self.submit_batch(insert_sql)
def get_prepared_hll_records(self, batch_item: Dict[str, Any]):
"""Turns propietary hll classes into prepared sql value tuples
This includes calculation of shards from individual items
using the hll_worker
"""
hll_items = [] # (base_key, metric_key, item)
hll_base_records = [] # (base_key, attr1, attr2)
# the following iteration will
# loop keys in case of dict
# and values in case of list
for index, record_item in enumerate(batch_item.values()):
# get base record and value
base = record_item.get_prepared_record()
if base.record:
hll_base_records.append(base.record)
base_metric_item_tuples = HLF.concat_base_metric_item(
index, base.metrics)
# format tuple-values as sql-escaped strings
value_str = [
self.prepare_sqlescaped_values(record)
for record in base_metric_item_tuples
]
# add to global list of items to be upserted
hll_items.extend(value_str)
# format sql for shard generation
# get sql escaped values list
values_str = HF.concat_values_str(hll_items)
# clear line
sys.stdout.write("\033[K")
print(f'Calculating hll shards for {len(values_str)} values..',
end='\r')
# calculate hll shards from raw values
hll_shards = HLF.calculate_item_shards(self.hllworker_cursor,
values_str)
prepared_records = HLF.concat_base_shards(hll_base_records, hll_shards)
return prepared_records