def get_mapping(domain, mapping_name):
mapping = SqlExtractMapping.by_name(domain, mapping_name)
if not mapping:
mapping = SqlExtractMapping()
mapping.domains = [domain]
mapping.name = mapping_name
return mapping
def get_mapping(domain, mapping_name):
mapping = SqlExtractMapping.by_name(domain, mapping_name)
if not mapping:
mapping = SqlExtractMapping()
mapping.domains = [domain]
mapping.name = mapping_name
return mapping
def view(request, domain=None, template='ctable/list_mappings.html'):
if domain:
mappings = SqlExtractMapping.by_domain(domain)
else:
mappings = SqlExtractMapping.all()
return render(request, template, {
'domain': domain,
'mappings': mappings
})
def test(request, mapping_id, domain=None, template='ctable/test_mapping.html'):
if mapping_id:
try:
limit = request.GET.get('limit', 100)
if not limit or limit == 'undefined':
limit = None
elif limit:
limit = int(limit)
limit = min(limit, 1000)
mapping = SqlExtractMapping.get(mapping_id)
test_extractor = get_test_extractor()
backend = get_backend(mapping.backend)
with backend:
checks = backend.check_mapping(mapping)
rows_processed, rows_with_value = test_extractor.extract(mapping, date_range=-1, limit=limit)
checks.update({
'domain': domain,
'mapping': mapping,
'rows_processed': rows_processed,
'rows_with_value': rows_with_value,
'data': test_extractor.backend.data,
})
return render(request, template, checks)
except ResourceNotFound:
raise Http404()
kwargs = {'domain': domain} if domain else {}
return redirect('sql_mappings_list', **kwargs)
def clear_data(request, mapping_id, domain=None):
if mapping_id:
try:
mapping = SqlExtractMapping.get(mapping_id)
get_extractor(mapping.backend).clear_all_data(mapping)
return json_response({'status': 'success', 'message': _('Data successfully cleared.')})
except ResourceNotFound:
return json_response({'status': 'error', 'message': _('Mapping not found.')})
return json_response({'status': 'error', 'message': _('Mapping not found.')})
def handle(self, *args, **options):
drop_all_tables = options.get('all_tables', False)
delete_all_mappings = options.get('all_mappings', False)
dry_run = options.get('dry_run', False)
if dry_run:
print("\n-------- DRY RUN --------\n")
all_tables = get_db_tables(settings.SQL_REPORTING_DATABASE_URL)
extractor = get_extractor('SQL')
domains = get_call_center_domains()
all_mappings = SqlExtractMapping.all()
for mapping_name in mapping_names:
for domain_lite in domains:
domain = domain_lite.name
print("Processing domain", domain)
mapping = get_mapping(domain, mapping_name)
if mapping.table_name in all_tables:
print("\tDropping SQL table", mapping.table_name)
if not dry_run:
extractor.clear_all_data(mapping)
if not mapping.new_document:
print("\tDeleting ctable mapping", mapping.name)
if not dry_run:
mapping.delete()
missed_tables = [t for t in all_tables if t.endswith(mapping_name)]
if missed_tables:
print('\nSome tables are still hanging around:')
with extractor.backend as backend:
for table in missed_tables:
if not drop_all_tables:
print('\t*', table)
else:
print("\tDeleting table", table)
backend.op.drop_table(table)
if not drop_all_tables:
print("\n To delete these tables run with '--all-tables'")
missed_mappings = [m for m in all_mappings if m.name == mapping_name]
if missed_mappings:
print('\nSome mappings are still hanging around:')
for mapping in missed_mappings:
if not delete_all_mappings:
print('\t*', mapping.name, 'for domains', ', '.join(mapping.domains))
else:
print('\tDeleting mapping', mapping.name, 'for domains', ', '.join(mapping.domains))
mapping.delete()
if not delete_all_mappings:
print("\n To delete these mappings run with '--all-mappings'")
def create(self):
if not self.name or not self.domains:
raise Exception('Missing name or domains property')
mapping = SqlExtractMapping()
for domain in self.domains:
existing = SqlExtractMapping.by_name(domain, self.name)
if existing:
mapping = existing
break
mapping.active = self.schedule_active
mapping.auto_generated = True
mapping.name = self.name
mapping.domains = self.domains
mapping.couch_view = self.couch_view
mapping.columns = self.columns
self.customize(mapping)
mapping.save()
def get_ctable_tables(self):
prefix = 0
if hasattr(settings, 'CTABLE_PREFIX'):
prefix = len(settings.CTABLE_PREFIX) + 1
else:
cont = raw_input("No CTABLE_PREFIX setting. Do you want to continue (yes/no): ")
if cont != 'yes':
sys.exit(1)
return {(m.table_name[prefix:], m.table_name)
for m in SqlExtractMapping.view('ctable/by_name', include_docs=True)}
def get_or_create_mapping(domain, mapping_name, date_range=2):
mapping = SqlExtractMapping.by_name(domain.name, mapping_name)
if not mapping:
mapping = SqlExtractMapping()
mapping.auto_generated = True
mapping.domains = [domain.name]
mapping.name = mapping_name
mapping.active = True
mapping.couch_date_range = date_range
mapping.couch_view_params = {}
return mapping
def toggle(request, mapping_id, domain=None):
if mapping_id:
try:
mapping = SqlExtractMapping.get(mapping_id)
# don't activate 'fluff' mappings
if not mapping.auto_generated:
mapping.active = not mapping.active
mapping.save()
except ResourceNotFound:
raise Http404()
kwargs = {'domain': domain} if domain else {}
return redirect('sql_mappings_list', **kwargs)
def delete(request, mapping_id, domain=None):
if mapping_id:
try:
mapping = SqlExtractMapping.get(mapping_id)
if domain:
assert domain in mapping.domains
assert mapping.doc_type == SqlExtractMapping._doc_type
mapping.delete()
except ResourceNotFound:
raise Http404()
kwargs = {'domain': domain} if domain else {}
return redirect('sql_mappings_list', **kwargs)
def get_ctable_tables(self):
prefix = 0
if hasattr(settings, 'CTABLE_PREFIX'):
prefix = len(settings.CTABLE_PREFIX) + 1
else:
cont = raw_input(
"No CTABLE_PREFIX setting. Do you want to continue (yes/no): ")
if cont != 'yes':
sys.exit(1)
return {(m.table_name[prefix:], m.table_name)
for m in SqlExtractMapping.view('ctable/by_name',
include_docs=True)}
def edit(request, mapping_id, domain=None, template='ctable/edit_mapping.html'):
if request.method == 'POST':
d = _to_kwargs(request)
if domain and domain not in d['domains']:
d['domains'].append(domain)
mapping = SqlExtractMapping.from_json(d)
if mapping.couch_key_prefix and mapping.couch_key_prefix[0] == '':
mapping.couch_key_prefix = None
# check for unique name
for dom in mapping.domains:
existing = SqlExtractMapping.by_name(dom, mapping.name)
if existing and existing._id != mapping._id:
args = {'domain': dom}
return json_response({'error': _("Mapping with the same name exists "
"in the '%(domain)s' domain.") % args})
mapping.save()
kwargs = {'domain': domain} if domain else {}
return json_response({'redirect': reverse('sql_mappings_list', kwargs=kwargs)})
if mapping_id:
try:
mapping = SqlExtractMapping.get(mapping_id)
except ResourceNotFound:
raise Http404()
else:
mapping = SqlExtractMapping()
return render(request, template, {
'domain': domain,
'mapping': mapping,
'backends': backends()
})
def get_test_extractor():
return CtableExtractor(SqlExtractMapping.get_db(), InMemoryBackend())
def get_or_create_mapping(domain, mapping_name, date_range=2):
mapping = SqlExtractMapping.by_name(domain, mapping_name)
if not mapping:
mapping = SqlExtractMapping()
mapping.auto_generated = True
mapping.domains = [domain]
mapping.name = mapping_name
mapping.active = True
mapping.couch_date_range = date_range
mapping.couch_view_params = {}
mapping.schedule_type = 'hourly'
mapping.schedule_hour = -1
mapping.schedule_day = -1
return mapping
def get_extractor(backend_name):
return CtableExtractor(SqlExtractMapping.get_db(), get_backend(backend_name))
def get_or_create_mapping(domain, mapping_name, date_range=2):
mapping = SqlExtractMapping.by_name(domain, mapping_name)
if not mapping:
mapping = SqlExtractMapping()
mapping.auto_generated = True
mapping.domains = [domain]
mapping.name = mapping_name
mapping.active = True
mapping.couch_date_range = date_range
mapping.couch_view_params = {}
mapping.schedule_type = 'hourly'
mapping.schedule_hour = -1
mapping.schedule_day = -1
return mapping