def sitemap(request): # TODO: REFACTOR ME? urls = [ reverse("wagtail_serve", args=[""]), reverse("wagtail_serve", args=["about/"]), reverse("wagtail_serve", args=["api/"]), reverse("wagtail_serve", args=["news/"]), reverse("regions_home"), reverse("business_intelligence"), ] for news in NewsPage.objects.live(): urls.append(news.url) search = Declaration.search().params(search_type="count") search.aggs.bucket( 'per_region', 'terms', field='general.post.region', size=0) for r in search.execute().aggregations.per_region.buckets: if r.key == "": continue urls.append(reverse("region", kwargs={"region_name": r.key})) subsearch = Declaration.search()\ .filter( Term(general__post__region=r.key) & Not(Term(general__post__office='')))\ .params(search_type="count") subsearch.aggs.bucket( 'per_office', 'terms', field='general.post.office', size=0) for subr in subsearch.execute().aggregations.per_office.buckets: urls.append(reverse( "region_office", kwargs={"region_name": r.key, "office_name": subr.key})) search = Declaration.search().params(search_type="count") search.aggs.bucket( 'per_office', 'terms', field='general.post.office', size=0) for r in search.execute().aggregations.per_office.buckets: if r.key == "": continue urls.append(reverse("office", kwargs={"office_name": r.key})) search = Declaration.search().extra(fields=[], size=100000) for r in search.execute(): urls.append(reverse("details", kwargs={"declaration_id": r._id})) return render(request, "sitemap.jinja", {"urls": urls}, content_type="application/xml")
def sitemap(request): # TODO: REFACTOR ME? urls = [ reverse("wagtail_serve", args=[""]), reverse("wagtail_serve", args=["about/"]), reverse("wagtail_serve", args=["api/"]), reverse("regions_home"), ] search = Declaration.search().params(search_type="count") search.aggs.bucket('per_region', 'terms', field='general.post.region', size=0) for r in search.execute().aggregations.per_region.buckets: urls.append(reverse("region", kwargs={"region_name": r.key})) subsearch = Declaration.search()\ .filter( Term(general__post__region=r.key) & Not(Term(general__post__office='')))\ .params(search_type="count") subsearch.aggs.bucket('per_office', 'terms', field='general.post.office', size=0) for subr in subsearch.execute().aggregations.per_office.buckets: urls.append( reverse("region_office", kwargs={ "region_name": r.key, "office_name": subr.key })) search = Declaration.search().params(search_type="count") search.aggs.bucket('per_office', 'terms', field='general.post.office', size=0) for r in search.execute().aggregations.per_office.buckets: urls.append(reverse("office", kwargs={"office_name": r.key})) search = Declaration.search().extra(fields=[], size=100000) for r in search.execute(): urls.append(reverse("details", kwargs={"declaration_id": r._id})) return render(request, "sitemap.jinja", {"urls": urls}, content_type="application/xml")
def handle(self, *args, **options): all_decls = Declaration.search().query('match_all') if options["to"] is not None: all_decls = all_decls[options["from"]:options["to"]].execute() elif options["from"]: all_decls = all_decls[options["from"]:].execute() else: all_decls = all_decls.scan() w = DictWriter(options["outfile"], fieldnames=["_id"] + options["field"]) w.writeheader() for decl in all_decls: decl_dict = decl.to_dict() row = { field: self.fetch_field(decl_dict, field) for field in options["field"] } row["_id"] = decl.meta.id w.writerow(row)
def handle(self, *args, **options): dump_to_file = len(args) > 0 all_decls = Declaration.search().query('match_all').scan() table = self._generate_table(all_decls) report = self._run_knitr(table, fragment_only=not dump_to_file) if dump_to_file: file_path = args[0] with open(file_path, 'w', encoding='utf-8') as f: f.write(report) else: root_page = Site.objects.get(is_default_site=True).root_page try: analytics_page = root_page.get_children().get( slug=settings.ANALYTICS_SLUG).specific() except Page.DoesNotExist: page_instance = RawHTMLPage(owner=None, title=settings.ANALYTICS_TITLE, slug=settings.ANALYTICS_SLUG) analytics_page = root_page.add_child(instance=page_instance) analytics_page.body = '<div class="analytics-wrapper">' + report + '</div>' revision = analytics_page.save_revision(user=None) revision.publish() self.stdout.write('Analytics page "{}" has been published.'.format( analytics_page.url))
def details(request, declaration_id): try: declaration = Declaration.get(id=declaration_id) except (ValueError, NotFoundError): raise Http404("Таких не знаємо!") return {"declaration": declaration}
def handle(self, *args, **options): try: file_path = args[0] except IndexError: raise CommandError('First argument must be a source file') with open(file_path, 'r', newline='', encoding='utf-8') as source: reader = csv.DictReader(source, delimiter=";") counter = 0 Declaration.init() # Apparently this is required to init mappings for row in reader: item = Declaration(**self.map_fields(row)) item.save() counter += 1 self.stdout.write( 'Loaded {} items to persistence storage'.format(counter))
def load_declarations(new_ids, limit=LOAD_DECLS_LIMIT): decl_list = list() fields = ['meta.id', 'general.*', 'intro.declaration_year'] if len(new_ids) > limit: logger.error("load new_ids %d limit %d exceed", len(new_ids), limit) new_ids = new_ids[:limit] decl_list = NACPDeclaration.mget(new_ids, raise_on_error=False, missing='skip', _source=fields) if not decl_list: decl_list = [] if len(decl_list) < len(new_ids): add_list = Declaration.mget(new_ids, raise_on_error=False, missing='skip', _source=fields) if add_list: decl_list.extend(add_list) if len(decl_list) < len(new_ids): logger.error("load new_ids %d docs not found", len(new_ids) - len(decl_list)) return decl_list
def handle(self, *args, **options): self.apply_migrations() all_decls = Declaration.search().query('match_all').scan() for decl in all_decls: sys.stdout.write('Processing decl for {}\n'.format( decl.general.full_name)) sys.stdout.flush() decl.general.full_name = replace_apostrophes( decl.general.full_name) decl.general.name = replace_apostrophes(decl.general.name) decl.general.last_name = replace_apostrophes( decl.general.last_name) decl.general.patronymic = replace_apostrophes( decl.general.patronymic) decl.general.full_name_suggest = { 'input': [ decl.general.full_name, ' '.join([ decl.general.name, decl.general.patronymic, decl.general.last_name ]), ' '.join([decl.general.name, decl.general.last_name]) ] } decl.ft_src = "\n".join(filter_only_interesting(decl.to_dict())) decl.general.full_name_for_sorting = keyword_for_sorting( decl.general.full_name) decl.index_card = concat_fields(decl.to_dict(), Declaration.INDEX_CARD_FIELDS) decl.save()
def office(request, office_name): search = Declaration.search()\ .filter('term', general__post__office=office_name) return { 'query': office_name, 'results': paginated_search(request, search) }
def search(request): query = request.GET.get("q", "") search = Declaration.search() if query: search = search.query("match", _all=query) else: search = search.query('match_all') return {"query": query, "results": paginated_search(request, search)}
def region_office(request, region_name, office_name): search = Declaration.search()\ .filter('term', general__post__region=region_name)\ .filter('term', general__post__office=office_name) return { 'query': office_name, 'results': paginated_search(request, search), }
def details(request, declaration_id): try: declaration = Declaration.get(id=int(declaration_id)) except (ValueError, NotFoundError): raise Http404("Таких не знаємо!") return { "declaration": declaration }
def stats_processor(request): s = Declaration.search() # res = s.params(search_type="count").aggs.metric( # "distinct_names", "cardinality", field="full_name").execute() return { 'total_declarations': s.count(), 'total_persons': s.count() # res.aggregations.distinct_names.value }
def regions_home(request): search = Declaration.search().params(search_type="count") search.aggs.bucket( 'per_region', 'terms', field='general.post.region', size=0) res = search.execute() return { 'facets': res.aggregations.per_region.buckets }
def handle(self, *args, **options): all_decls = Declaration.search().query('match_all').scan() for decl in all_decls: print('Processing decl for {}'.format(decl.general.full_name)) for parent, field_key in self._decl_list_fields(decl): field = getattr(parent, field_key, None) if field: filtered_field = list(filter(self._values_filter, field)) setattr(parent, field_key, filtered_field) decl.save()
def regions_home(request): search = Declaration.search().params(search_type="count") search.aggs.bucket('per_region', 'terms', field='general.post.region', size=0) res = search.execute() return {'facets': res.aggregations.per_region.buckets}
def handle(self, *args, **options): translator = Translator() translator.fetch_full_dict_from_db() self.apply_migrations() all_decls = Declaration.search().query('match_all').scan() for decl in tqdm.tqdm(all_decls): decl_dct = decl.to_dict() decl.general.full_name = replace_apostrophes(decl.general.full_name) decl.general.name = replace_apostrophes(decl.general.name) decl.general.last_name = replace_apostrophes(decl.general.last_name) decl.general.patronymic = replace_apostrophes(decl.general.patronymic) decl.general.full_name_suggest = { 'input': [ decl.general.full_name, ' '.join([decl.general.name, decl.general.patronymic, decl.general.last_name]), ' '.join([decl.general.name, decl.general.last_name]) ] } decl_dct["ft_src"] = "" terms = filter_only_interesting(decl_dct) terms += [translator.translate(x)["translation"] for x in terms] decl.ft_src = "\n".join(terms) decl.general.full_name_for_sorting = keyword_for_sorting(decl.general.full_name) decl.index_card = concat_fields(decl_dct, Declaration.INDEX_CARD_FIELDS) extracted_names = [(decl.general.last_name, decl.general.name, decl.general.patronymic, None)] persons = set() names_autocomplete = set() for person in decl.general.family: l, f, p, _ = parse_fullname(person.family_name) extracted_names.append((l, f, p, person.relations)) for name in extracted_names: persons |= generate_all_names( *name ) names_autocomplete |= autocomplete_suggestions( concat_name(*name[:-1]) ) decl.persons = list(filter(None, persons)) decl.names_autocomplete = list(filter(None, names_autocomplete)) decl.save()
def sitemap(request): # TODO: REFACTOR ME? urls = [ reverse("home"), reverse("about"), reverse("regions_home"), ] search = Declaration.search().params(search_type="count") search.aggs.bucket( 'per_region', 'terms', field='general.post.region', size=0) for r in search.execute().aggregations.per_region.buckets: urls.append(reverse("region", kwargs={"region_name": r.key})) subsearch = Declaration.search()\ .filter( Term(general__post__region=r.key) & Not(Term(general__post__office='')))\ .params(search_type="count") subsearch.aggs.bucket( 'per_office', 'terms', field='general.post.office', size=0) for subr in subsearch.execute().aggregations.per_office.buckets: urls.append(reverse( "region_office", kwargs={"region_name": r.key, "office_name": subr.key})) search = Declaration.search().params(search_type="count") search.aggs.bucket( 'per_office', 'terms', field='general.post.office', size=0) for r in search.execute().aggregations.per_office.buckets: urls.append(reverse("office", kwargs={"office_name": r.key})) search = Declaration.search().extra(fields=[], size=100000) for r in search.execute(): urls.append(reverse("details", kwargs={"declaration_id": r._id})) return render(request, "sitemap.jinja", {"urls": urls}, content_type="application/xml")
def search(request): query = request.GET.get("q", "") search = Declaration.search() if query: search = search.query("match", _all=query) else: search = search.query('match_all') return { "query": query, "results": paginated_search(request, search) }
def search(request): query = request.GET.get("q", "") search = Declaration.search() if query: search = search.query("match", _all={ "query": query, "operator": "and" }) if not search.count(): search = Declaration.search().query("match", _all={ "query": query, "operator": "or", "minimum_should_match": "2" }) else: search = search.query('match_all') return {"query": query, "results": paginated_search(request, search)}
def handle(self, *args, **options): try: file_path = args[0] except IndexError: raise CommandError('First argument must be a source file') with open(file_path, 'r', newline='', encoding='utf-8') as source: decls = json.load(source) counter = 0 Declaration.init() # Apparently this is required to init mappings for row in decls: mapped = self.map_fields(row) res = Declaration.search().filter( Term(general__last_name=mapped[ 'general']['last_name'].lower().split('-')) & Term(general__name=mapped[ 'general']['name'].lower().split('-')) & Term(intro__declaration_year=mapped[ 'intro']['declaration_year']) ) if mapped['general']['patronymic']: res = res.filter(Term(general__patronymic=mapped[ 'general']['patronymic'].lower())) res = res.execute() if not res.hits: item = Declaration(**mapped) item.save() counter += 1 self.stdout.write( 'Loaded {} items to persistence storage'.format(counter))
def handle(self, *args, **options): self.apply_migrations() all_decls = Declaration.search().query('match_all').scan() for decl in all_decls: decl_dct = decl.to_dict() sys.stdout.write('Processing decl for {}\n'.format( decl.general.full_name)) sys.stdout.flush() decl.general.full_name = replace_apostrophes( decl.general.full_name) decl.general.name = replace_apostrophes(decl.general.name) decl.general.last_name = replace_apostrophes( decl.general.last_name) decl.general.patronymic = replace_apostrophes( decl.general.patronymic) decl.general.full_name_suggest = { 'input': [ decl.general.full_name, ' '.join([ decl.general.name, decl.general.patronymic, decl.general.last_name ]), ' '.join([decl.general.name, decl.general.last_name]) ] } decl_dct["ft_src"] = "" decl.ft_src = "\n".join(filter_only_interesting(decl_dct)) decl.general.full_name_for_sorting = keyword_for_sorting( decl.general.full_name) decl.index_card = concat_fields(decl_dct, Declaration.INDEX_CARD_FIELDS) extracted_names = [(decl.general.last_name, decl.general.name, decl.general.patronymic, None)] persons = set() names_autocomplete = set() for person in decl.general.family: l, f, p, _ = parse_fullname(person.family_name) extracted_names.append((l, f, p, person.relations)) for name in extracted_names: persons |= generate_all_names(*name) names_autocomplete |= autocomplete_suggestions( concat_name(*name[:-1])) decl.persons = list(filter(None, persons)) decl.names_autocomplete = list(filter(None, names_autocomplete)) decl.save()
def handle(self, *args, **options): try: file_path = args[0] id_prefix = args[1] except IndexError: raise CommandError( 'First argument must be a source file and second is a id prefix' ) groups = defaultdict(list) with open(file_path, 'r', newline='', encoding='utf-8') as source: reader = csv.DictReader(source, delimiter=',') counter = 0 for row in reader: status_col = 'Status' if 'Status' in row else 'Статус' if row[status_col] == '' or row[status_col] == 'Ок': groups[row[self._group_column(row)]].append(row) counter += 1 self.stdout.write( 'Read {} valid rows from the input file'.format(counter)) Declaration.init() # Apparently this is required to init mappings declarations = map(self.merge_group, groups.values()) counter = 0 for declaration in declarations: mapped = self.map_fields(declaration, id_prefix) res = Declaration.search().filter( Term(general__last_name=mapped['general'] ['last_name'].lower().split('-')) & Term( general__name=mapped['general']['name'].lower().split('-')) & Term(intro__declaration_year=mapped['intro'] ['declaration_year'])) if mapped['general']['patronymic']: res = res.filter( Term(general__patronymic=mapped['general'] ['patronymic'].lower())) res = res.execute() if res.hits: self.stdout.write("%s (%s) already exists" % (mapped['general']['full_name'], mapped['intro']['declaration_year'])) mapped['_id'] = res.hits[0]._id item = Declaration(**mapped) item.save() counter += 1 self.stdout.write( 'Loaded {} items to persistence storage'.format(counter))
def search(request): query = request.GET.get("q", "") search = Declaration.search() if query: search = search.query( "match", _all={"query": query, "operator": "and"}) if not search.count(): search = Declaration.search().query( "match", _all={ "query": query, "operator": "or", "minimum_should_match": "2" } ) else: search = search.query('match_all') return { "query": query, "results": paginated_search(request, search) }
def region(request, region_name): search = Declaration.search()\ .filter( Term(general__post__region=region_name) & Not(Term(general__post__office='')))\ .params(search_type="count") search.aggs.bucket( 'per_office', 'terms', field='general.post.office', size=0) res = search.execute() return { 'facets': res.aggregations.per_office.buckets, 'region_name': region_name }
def handle(self, *args, **options): all_decls = Declaration.search().query('match_all').scan() urls_in_list = [] on_unshred_it = 0 on_static = 0 on_chesno = 0 no_url_no_id = 0 not_found = 0 duplicates = 0 for decl in all_decls: try: url = decl["declaration"]["url"] except KeyError: url = None if not url: no_url_no_id += 1 print(decl.meta.id) continue if url in urls_in_list: duplicates += 1 print("Duplicates", url, decl.meta.id) continue if "unshred.it" in url.lower(): on_unshred_it += 1 if not self.check_url(url, options["destination"]): not_found += 1 print("Not found", url, decl.meta.id) else: decl["declaration"]["url"] = url.replace( "http://unshred.it/static/", "http://static.declarations.com.ua/") # decl.save() elif "static.declarations.com.ua" in url.lower(): on_static += 1 if not self.check_url(url, options["destination"]): print(url) not_found += 1 else: on_chesno += 1 print(on_unshred_it, on_static, on_chesno, no_url_no_id) print(not_found, duplicates)
def handle(self, *args, **options): try: file_path = args[0] except IndexError: raise CommandError('First argument must be a source file') with open(file_path, 'r', newline='', encoding='utf-8') as source: decls = json.load(source) counter = 0 Declaration.init() # Apparently this is required to init mappings for row in decls: if "fields" not in row["details"]: continue mapped = self.map_fields(row) res = Declaration.search().query( Q('terms', general__last_name=mapped['general'] ['last_name'].lower().split("-")) & Q('terms', general__name=mapped['general']['name'].lower().split( "-")) & Q('term', intro__declaration_year=mapped['intro'] ['declaration_year'])) if mapped['general']['patronymic']: res = res.query('term', general__patronymic=mapped['general'] ['patronymic'].lower()) self.stdout.write("Checking %s (%s)" % (mapped['general']['full_name'], mapped['intro']['declaration_year'])) res = res.execute() if not res.hits: item = Declaration(**mapped) item.save() counter += 1 else: self.stdout.write("%s (%s) already exists" % (mapped['general']['full_name'], mapped['intro']['declaration_year'])) self.stdout.write( 'Loaded {} items to persistence storage'.format(counter))
def region(request, region_name): search = Declaration.search()\ .filter( Term(general__post__region=region_name) & Not(Term(general__post__office='')))\ .params(search_type="count") search.aggs.bucket('per_office', 'terms', field='general.post.office', size=0) res = search.execute() return { 'facets': res.aggregations.per_office.buckets, 'region_name': region_name }
def handle(self, *args, **options): try: file_path = args[0] id_prefix = args[1] except IndexError: raise CommandError( 'First argument must be a source file and second is a id prefix') groups = defaultdict(list) with open(file_path, 'r', newline='', encoding='utf-8') as source: reader = csv.DictReader(source, delimiter=',') counter = 0 for row in reader: status_col = 'Status' if 'Status' in row else 'Статус' if row[status_col] == '' or row[status_col] == 'Ок': groups[row[self._group_column(row)]].append(row) counter += 1 self.stdout.write('Read {} valid rows from the input file'.format(counter)) Declaration.init() # Apparently this is required to init mappings declarations = map(self.merge_group, groups.values()) counter = 0 for declaration in declarations: mapped = self.map_fields(declaration, id_prefix) res = Declaration.search().filter( Term(general__last_name=mapped[ 'general']['last_name'].lower().split('-')) & Term(general__name=mapped[ 'general']['name'].lower().split('-')) & Term(intro__declaration_year=mapped[ 'intro']['declaration_year']) ) if mapped['general']['patronymic']: res = res.filter(Term(general__patronymic=mapped[ 'general']['patronymic'].lower())) res = res.execute() if res.hits: self.stdout.write( "%s (%s) already exists" % ( mapped['general']['full_name'], mapped['intro']['declaration_year'])) mapped['_id'] = res.hits[0]._id item = Declaration(**mapped) item.save() counter += 1 self.stdout.write('Loaded {} items to persistence storage'.format(counter))
def handle(self, *args, **options): try: file_path = args[0] id_prefix = args[1] except IndexError: raise CommandError("First argument must be a source file and second is a id prefix") groups = defaultdict(list) with open(file_path, "r", newline="", encoding="utf-8") as source: reader = csv.DictReader(source, delimiter=",") counter = 0 for row in reader: status_col = "Status" if "Status" in row else "Статус" if row[status_col] == "" or row[status_col] == "Ок": groups[row[self._group_column(row)]].append(row) counter += 1 self.stdout.write("Read {} valid rows from the input file".format(counter)) Declaration.init() # Apparently this is required to init mappings declarations = map(self.merge_group, groups.values()) counter = 0 for declaration in declarations: mapped = self.map_fields(declaration, id_prefix) res = Declaration.search().filter( Terms(general__last_name=mapped["general"]["last_name"].lower().split("-")) & Terms(general__name=mapped["general"]["name"].lower().split("-")) & Term(intro__declaration_year=mapped["intro"]["declaration_year"]) ) if mapped["general"]["patronymic"]: res = res.filter(Term(general__patronymic=mapped["general"]["patronymic"].lower())) res = res.execute() if res.hits: self.stdout.write( "%s (%s) already exists" % (mapped["general"]["full_name"], mapped["intro"]["declaration_year"]) ) mapped["_id"] = res.hits[0]._id else: self.stdout.write( "%s (%s) created" % (mapped["general"]["full_name"], mapped["intro"]["declaration_year"]) ) item = Declaration(**mapped) item.save() counter += 1 self.stdout.write("Loaded {} items to persistence storage".format(counter))
def handle(self, *args, **options): dump_to_file = len(args) > 0 all_decls = Declaration.search().query('match_all').scan() table = self._generate_table(all_decls) report = self._run_knitr(table, fragment_only=not dump_to_file) if dump_to_file: file_path = args[0] with open(file_path, 'w', encoding='utf-8') as f: f.write(report) else: root_page = Site.objects.get(is_default_site=True).root_page try: analytics_page = root_page.get_children().get(slug=settings.ANALYTICS_SLUG).specific except Page.DoesNotExist: page_instance = RawHTMLPage(owner=None, title=settings.ANALYTICS_TITLE, slug=settings.ANALYTICS_SLUG) analytics_page = root_page.add_child(instance=page_instance) analytics_page.body = '<div class="analytics-wrapper">' + report + '</div>' revision = analytics_page.save_revision(user=None) revision.publish() self.stdout.write('Analytics page "{}" has been published.'.format(analytics_page.url))
def assume(q, fuzziness): search = Declaration.search()\ .suggest( 'name', q, completion={ 'field': 'general.full_name_suggest', 'size': 10, 'fuzzy': { 'fuzziness': fuzziness, 'unicode_aware': 1 } } ) res = search.execute() if res.success(): return [val['text'] for val in res.suggest['name'][0]['options']] else: []
def region(request, region_name): search = Declaration.search()\ .filter( Term(general__post__region=region_name) & Not(Term(general__post__office='')))\ .params(search_type="count") meta_data = MetaData.objects.filter(region_id=region_name, office_id=None).first() search.aggs.bucket('per_office', 'terms', field='general.post.office', size=0) res = search.execute() return { 'facets': res.aggregations.per_office.buckets, 'region_name': region_name, 'title': meta_data.title if meta_data else "", 'meta_desc': meta_data.description if meta_data else "", }
def region(request, region_name): search = Declaration.search()\ .filter( Term(general__post__region=region_name) & Not(Term(general__post__office='')))\ .params(search_type="count") meta_data = MetaData.objects.filter( region_id=region_name, office_id=None ).first() search.aggs.bucket( 'per_office', 'terms', field='general.post.office', size=0) res = search.execute() return { 'facets': res.aggregations.per_office.buckets, 'region_name': region_name, 'title': meta_data.title if meta_data else "", 'meta_desc': meta_data.description if meta_data else "", }
def suggest(request): search = Declaration.search()\ .suggest( 'name', request.GET.get('q', ''), completion={ 'field': 'general.full_name_suggest', 'size': 10, 'fuzzy': { 'fuzziness': 3, 'unicode_aware': 1 } } ) res = search.execute() if res.success(): return JsonResponse( [val['text'] for val in res.suggest['name'][0]['options']], safe=False) else: return JsonResponse([], safe=False)
def handle(self, *args, **options): all_decls = Declaration.search().query('match_all').scan() for decl in all_decls: print('Processing decl for {}'.format(decl.general.full_name)) decl.general.full_name = replace_apostrophes(decl.general.full_name) decl.general.name = replace_apostrophes(decl.general.name) decl.general.last_name = replace_apostrophes(decl.general.last_name) decl.general.patronymic = replace_apostrophes(decl.general.patronymic) decl.general.full_name_suggest = { 'input': [ decl.general.full_name, ' '.join([decl.general.name, decl.general.patronymic, decl.general.last_name]), ' '.join([decl.general.name, decl.general.last_name]) ] } decl.ft_src = "\n".join(filter_only_interesting(decl.to_dict())) decl.save()
def suggest(request): search = Declaration.search()\ .suggest( 'name', request.GET.get('q', ''), completion={ 'field': 'general.full_name_suggest', 'size': 10, 'fuzzy': { 'fuzziness': 3, 'unicode_aware': 1 } } ) res = search.execute() if res.success(): return JsonResponse( [val['text'] for val in res.suggest['name'][0]['options']], safe=False ) else: return JsonResponse([], safe=False)
def handle(self, *args, **options): try: file_path = args[0] except IndexError: raise CommandError('First argument must be a source file') with open(file_path, 'r', newline='', encoding='utf-8') as source: reader = csv.DictReader(source, delimiter=";") counter = 0 Declaration.init() # Apparently this is required to init mappings for row in reader: mapped = self.map_fields(row) res = Declaration.search().query( Q('terms', general__last_name=mapped['general'] ['last_name'].lower().split("-")) & Q('terms', general__name=mapped['general']['name'].lower().split( "-")) & Q('term', intro__declaration_year=mapped['intro'] ['declaration_year'])) if mapped['general']['patronymic']: res = res.query('term', general__patronymic=mapped['general'] ['patronymic'].lower()) res = res.execute() if res.hits: mapped["_id"] = res.hits[0]._id item = Declaration(**mapped) item.save() counter += 1 self.stdout.write( 'Loaded {} items to persistence storage'.format(counter))
def handle(self, *args, **options): try: file_path = args[0] except IndexError: raise CommandError('First argument must be a source file') with open(file_path, 'r', newline='', encoding='utf-8') as source: reader = csv.DictReader(source, delimiter=";") counter = 0 Declaration.init() # Apparently this is required to init mappings for row in reader: mapped = self.map_fields(row) res = Declaration.search().filter( Term(general__last_name=mapped[ "general"]["last_name"].lower().split("-")) & Term(general__name=mapped[ "general"]["name"].lower().split("-")) & Term(intro__declaration_year=mapped[ "intro"]["declaration_year"]) ) if mapped["general"]["patronymic"]: res = res.filter(Term(general__patronymic=mapped[ "general"]["patronymic"].lower())) res = res.execute() if res.hits: mapped["id"] = res.hits[0]._id item = Declaration(**mapped) item.save() counter += 1 self.stdout.write( 'Loaded {} items to persistence storage'.format(counter))
def handle(self, *args, **options): try: file_path = options["file_path"] id_prefix = options["prefix"] except IndexError: raise CommandError( 'First argument must be a source file and second is a id prefix' ) if hasattr(sys.stdin, 'isatty') and not sys.stdin.isatty(): self.stdout.write( "To import something you need to run this command in TTY.") return groups = defaultdict(list) with open(file_path, 'r', newline='', encoding='utf-8') as source: reader = csv.DictReader(source, delimiter='\t') counter = 0 for row in reader: status_col = 'Status' if 'Status' in row else 'Статус' if row[status_col] == '' or row[status_col] == 'Ок': groups[row[self._group_column(row)]].append(row) counter += 1 self.stdout.write( 'Read {} valid rows from the input file'.format(counter)) Declaration.init() # Apparently this is required to init mappings declarations = map(self.merge_group, groups.values()) counter = 0 for declaration in declarations: mapped = self.map_fields(declaration, id_prefix) res = Declaration.search().query( Q('terms', general__last_name=mapped['general'] ['last_name'].lower().split("-")) & Q('terms', general__name=mapped['general']['name'].lower().split("-")) & Q('term', intro__declaration_year=mapped['intro'] ['declaration_year'])) if mapped['general']['patronymic']: res = res.query('term', general__patronymic=mapped['general'] ['patronymic'].lower()) res = res.execute() if res.hits: self.stdout.write( "Person\n%s (%s, %s, %s, %s)\n%s\nalready exists" % (mapped['general']['full_name'], mapped['general']['post']['post'], mapped['general']['post']['office'], mapped['general']['post']['region'], mapped['intro']['declaration_year'], mapped['declaration']['url'])) for i, hit in enumerate(res.hits): self.stdout.write("%s: %s (%s, %s, %s, %s), %s\n%s" % (i + 1, hit['general']['full_name'], hit['general']['post']['post'], hit['general']['post']['office'], hit['general']['post']['region'], hit['intro']['declaration_year'], hit._id, hit['declaration']['url'])) msg = ( "Select one of persons above to replace, or press [s] " + "to skip current record or [c] to create new (default): ") r = input(msg).lower() or "c" if r == "s": self.stdout.write("Ok, skipping") continue if r.isdigit() and int(r) <= len(res.hits): r = int(r) - 1 mapped['_id'] = res.hits[r]._id self.stdout.write("Ok, replacing %s" % res.hits[r]._id) else: self.stdout.write("Ok, adding new record") else: self.stdout.write("%s (%s) created" % (mapped['general']['full_name'], mapped['intro']['declaration_year'])) item = Declaration(**mapped) item.save() counter += 1 self.stdout.write( 'Loaded {} items to persistence storage'.format(counter))