def process_extract(extract_id, limit=None, date_range=None):
def update_status(total, current):
meta = {'current': current, 'total': total}
current_task.update_state(state='PROGRESS', meta=meta)
mapping = SqlExtractMapping.get(extract_id)
get_extractor(mapping.backend).extract(mapping, limit=limit, date_range=date_range, status_callback=update_status)
def clear_data(request, mapping_id, domain=None):
if mapping_id:
try:
mapping = SqlExtractMapping.get(mapping_id)
get_extractor(mapping.backend).clear_all_data(mapping)
return json_response({'status': 'success', 'message': _('Data successfully cleared.')})
except ResourceNotFound:
return json_response({'status': 'error', 'message': _('Mapping not found.')})
return json_response({'status': 'error', 'message': _('Mapping not found.')})
def handle(self, *args, **options):
drop_all_tables = options.get('all_tables', False)
delete_all_mappings = options.get('all_mappings', False)
dry_run = options.get('dry_run', False)
if dry_run:
print("\n-------- DRY RUN --------\n")
all_tables = get_db_tables(settings.SQL_REPORTING_DATABASE_URL)
extractor = get_extractor('SQL')
domains = get_call_center_domains()
all_mappings = SqlExtractMapping.all()
for mapping_name in mapping_names:
for domain_lite in domains:
domain = domain_lite.name
print("Processing domain", domain)
mapping = get_mapping(domain, mapping_name)
if mapping.table_name in all_tables:
print("\tDropping SQL table", mapping.table_name)
if not dry_run:
extractor.clear_all_data(mapping)
if not mapping.new_document:
print("\tDeleting ctable mapping", mapping.name)
if not dry_run:
mapping.delete()
missed_tables = [t for t in all_tables if t.endswith(mapping_name)]
if missed_tables:
print('\nSome tables are still hanging around:')
with extractor.backend as backend:
for table in missed_tables:
if not drop_all_tables:
print('\t*', table)
else:
print("\tDeleting table", table)
backend.op.drop_table(table)
if not drop_all_tables:
print("\n To delete these tables run with '--all-tables'")
missed_mappings = [m for m in all_mappings if m.name == mapping_name]
if missed_mappings:
print('\nSome mappings are still hanging around:')
for mapping in missed_mappings:
if not delete_all_mappings:
print('\t*', mapping.name, 'for domains', ', '.join(mapping.domains))
else:
print('\tDeleting mapping', mapping.name, 'for domains', ', '.join(mapping.domains))
mapping.delete()
if not delete_all_mappings:
print("\n To delete these mappings run with '--all-mappings'")
def process_fluff_diff(sender, diff=None, **kwargs):
from ctable.util import get_extractor
from .util import get_backend_name_for_fluff_pillow
backend_name = get_backend_name_for_fluff_pillow(diff['doc_type'])
if diff and backend_name:
get_extractor(backend_name).process_fluff_diff(diff, backend_name)