def get_mapping(domain, mapping_name): mapping = SqlExtractMapping.by_name(domain, mapping_name) if not mapping: mapping = SqlExtractMapping() mapping.domains = [domain] mapping.name = mapping_name return mapping
def view(request, domain=None, template='ctable/list_mappings.html'): if domain: mappings = SqlExtractMapping.by_domain(domain) else: mappings = SqlExtractMapping.all() return render(request, template, { 'domain': domain, 'mappings': mappings })
def test(request, mapping_id, domain=None, template='ctable/test_mapping.html'): if mapping_id: try: limit = request.GET.get('limit', 100) if not limit or limit == 'undefined': limit = None elif limit: limit = int(limit) limit = min(limit, 1000) mapping = SqlExtractMapping.get(mapping_id) test_extractor = get_test_extractor() backend = get_backend(mapping.backend) with backend: checks = backend.check_mapping(mapping) rows_processed, rows_with_value = test_extractor.extract(mapping, date_range=-1, limit=limit) checks.update({ 'domain': domain, 'mapping': mapping, 'rows_processed': rows_processed, 'rows_with_value': rows_with_value, 'data': test_extractor.backend.data, }) return render(request, template, checks) except ResourceNotFound: raise Http404() kwargs = {'domain': domain} if domain else {} return redirect('sql_mappings_list', **kwargs)
def clear_data(request, mapping_id, domain=None): if mapping_id: try: mapping = SqlExtractMapping.get(mapping_id) get_extractor(mapping.backend).clear_all_data(mapping) return json_response({'status': 'success', 'message': _('Data successfully cleared.')}) except ResourceNotFound: return json_response({'status': 'error', 'message': _('Mapping not found.')}) return json_response({'status': 'error', 'message': _('Mapping not found.')})
def handle(self, *args, **options): drop_all_tables = options.get('all_tables', False) delete_all_mappings = options.get('all_mappings', False) dry_run = options.get('dry_run', False) if dry_run: print("\n-------- DRY RUN --------\n") all_tables = get_db_tables(settings.SQL_REPORTING_DATABASE_URL) extractor = get_extractor('SQL') domains = get_call_center_domains() all_mappings = SqlExtractMapping.all() for mapping_name in mapping_names: for domain_lite in domains: domain = domain_lite.name print("Processing domain", domain) mapping = get_mapping(domain, mapping_name) if mapping.table_name in all_tables: print("\tDropping SQL table", mapping.table_name) if not dry_run: extractor.clear_all_data(mapping) if not mapping.new_document: print("\tDeleting ctable mapping", mapping.name) if not dry_run: mapping.delete() missed_tables = [t for t in all_tables if t.endswith(mapping_name)] if missed_tables: print('\nSome tables are still hanging around:') with extractor.backend as backend: for table in missed_tables: if not drop_all_tables: print('\t*', table) else: print("\tDeleting table", table) backend.op.drop_table(table) if not drop_all_tables: print("\n To delete these tables run with '--all-tables'") missed_mappings = [m for m in all_mappings if m.name == mapping_name] if missed_mappings: print('\nSome mappings are still hanging around:') for mapping in missed_mappings: if not delete_all_mappings: print('\t*', mapping.name, 'for domains', ', '.join(mapping.domains)) else: print('\tDeleting mapping', mapping.name, 'for domains', ', '.join(mapping.domains)) mapping.delete() if not delete_all_mappings: print("\n To delete these mappings run with '--all-mappings'")
def create(self): if not self.name or not self.domains: raise Exception('Missing name or domains property') mapping = SqlExtractMapping() for domain in self.domains: existing = SqlExtractMapping.by_name(domain, self.name) if existing: mapping = existing break mapping.active = self.schedule_active mapping.auto_generated = True mapping.name = self.name mapping.domains = self.domains mapping.couch_view = self.couch_view mapping.columns = self.columns self.customize(mapping) mapping.save()
def get_ctable_tables(self): prefix = 0 if hasattr(settings, 'CTABLE_PREFIX'): prefix = len(settings.CTABLE_PREFIX) + 1 else: cont = raw_input("No CTABLE_PREFIX setting. Do you want to continue (yes/no): ") if cont != 'yes': sys.exit(1) return {(m.table_name[prefix:], m.table_name) for m in SqlExtractMapping.view('ctable/by_name', include_docs=True)}
def get_or_create_mapping(domain, mapping_name, date_range=2): mapping = SqlExtractMapping.by_name(domain.name, mapping_name) if not mapping: mapping = SqlExtractMapping() mapping.auto_generated = True mapping.domains = [domain.name] mapping.name = mapping_name mapping.active = True mapping.couch_date_range = date_range mapping.couch_view_params = {} return mapping
def toggle(request, mapping_id, domain=None): if mapping_id: try: mapping = SqlExtractMapping.get(mapping_id) # don't activate 'fluff' mappings if not mapping.auto_generated: mapping.active = not mapping.active mapping.save() except ResourceNotFound: raise Http404() kwargs = {'domain': domain} if domain else {} return redirect('sql_mappings_list', **kwargs)
def delete(request, mapping_id, domain=None): if mapping_id: try: mapping = SqlExtractMapping.get(mapping_id) if domain: assert domain in mapping.domains assert mapping.doc_type == SqlExtractMapping._doc_type mapping.delete() except ResourceNotFound: raise Http404() kwargs = {'domain': domain} if domain else {} return redirect('sql_mappings_list', **kwargs)
def get_ctable_tables(self): prefix = 0 if hasattr(settings, 'CTABLE_PREFIX'): prefix = len(settings.CTABLE_PREFIX) + 1 else: cont = raw_input( "No CTABLE_PREFIX setting. Do you want to continue (yes/no): ") if cont != 'yes': sys.exit(1) return {(m.table_name[prefix:], m.table_name) for m in SqlExtractMapping.view('ctable/by_name', include_docs=True)}
def edit(request, mapping_id, domain=None, template='ctable/edit_mapping.html'): if request.method == 'POST': d = _to_kwargs(request) if domain and domain not in d['domains']: d['domains'].append(domain) mapping = SqlExtractMapping.from_json(d) if mapping.couch_key_prefix and mapping.couch_key_prefix[0] == '': mapping.couch_key_prefix = None # check for unique name for dom in mapping.domains: existing = SqlExtractMapping.by_name(dom, mapping.name) if existing and existing._id != mapping._id: args = {'domain': dom} return json_response({'error': _("Mapping with the same name exists " "in the '%(domain)s' domain.") % args}) mapping.save() kwargs = {'domain': domain} if domain else {} return json_response({'redirect': reverse('sql_mappings_list', kwargs=kwargs)}) if mapping_id: try: mapping = SqlExtractMapping.get(mapping_id) except ResourceNotFound: raise Http404() else: mapping = SqlExtractMapping() return render(request, template, { 'domain': domain, 'mapping': mapping, 'backends': backends() })
def get_test_extractor(): return CtableExtractor(SqlExtractMapping.get_db(), InMemoryBackend())
def get_or_create_mapping(domain, mapping_name, date_range=2): mapping = SqlExtractMapping.by_name(domain, mapping_name) if not mapping: mapping = SqlExtractMapping() mapping.auto_generated = True mapping.domains = [domain] mapping.name = mapping_name mapping.active = True mapping.couch_date_range = date_range mapping.couch_view_params = {} mapping.schedule_type = 'hourly' mapping.schedule_hour = -1 mapping.schedule_day = -1 return mapping
def get_extractor(backend_name): return CtableExtractor(SqlExtractMapping.get_db(), get_backend(backend_name))