def sitemap(request):
# TODO: REFACTOR ME?
urls = [
reverse("wagtail_serve", args=[""]),
reverse("wagtail_serve", args=["about/"]),
reverse("wagtail_serve", args=["api/"]),
reverse("wagtail_serve", args=["news/"]),
reverse("regions_home"),
reverse("business_intelligence"),
]
for news in NewsPage.objects.live():
urls.append(news.url)
search = Declaration.search().params(search_type="count")
search.aggs.bucket(
'per_region', 'terms', field='general.post.region', size=0)
for r in search.execute().aggregations.per_region.buckets:
if r.key == "":
continue
urls.append(reverse("region", kwargs={"region_name": r.key}))
subsearch = Declaration.search()\
.filter(
Term(general__post__region=r.key) &
Not(Term(general__post__office='')))\
.params(search_type="count")
subsearch.aggs.bucket(
'per_office', 'terms', field='general.post.office', size=0)
for subr in subsearch.execute().aggregations.per_office.buckets:
urls.append(reverse(
"region_office",
kwargs={"region_name": r.key, "office_name": subr.key}))
search = Declaration.search().params(search_type="count")
search.aggs.bucket(
'per_office', 'terms', field='general.post.office', size=0)
for r in search.execute().aggregations.per_office.buckets:
if r.key == "":
continue
urls.append(reverse("office", kwargs={"office_name": r.key}))
search = Declaration.search().extra(fields=[], size=100000)
for r in search.execute():
urls.append(reverse("details", kwargs={"declaration_id": r._id}))
return render(request, "sitemap.jinja",
{"urls": urls}, content_type="application/xml")
def sitemap(request):
# TODO: REFACTOR ME?
urls = [
reverse("wagtail_serve", args=[""]),
reverse("wagtail_serve", args=["about/"]),
reverse("wagtail_serve", args=["api/"]),
reverse("regions_home"),
]
search = Declaration.search().params(search_type="count")
search.aggs.bucket('per_region',
'terms',
field='general.post.region',
size=0)
for r in search.execute().aggregations.per_region.buckets:
urls.append(reverse("region", kwargs={"region_name": r.key}))
subsearch = Declaration.search()\
.filter(
Term(general__post__region=r.key) &
Not(Term(general__post__office='')))\
.params(search_type="count")
subsearch.aggs.bucket('per_office',
'terms',
field='general.post.office',
size=0)
for subr in subsearch.execute().aggregations.per_office.buckets:
urls.append(
reverse("region_office",
kwargs={
"region_name": r.key,
"office_name": subr.key
}))
search = Declaration.search().params(search_type="count")
search.aggs.bucket('per_office',
'terms',
field='general.post.office',
size=0)
for r in search.execute().aggregations.per_office.buckets:
urls.append(reverse("office", kwargs={"office_name": r.key}))
search = Declaration.search().extra(fields=[], size=100000)
for r in search.execute():
urls.append(reverse("details", kwargs={"declaration_id": r._id}))
return render(request,
"sitemap.jinja", {"urls": urls},
content_type="application/xml")
def handle(self, *args, **options):
all_decls = Declaration.search().query('match_all')
if options["to"] is not None:
all_decls = all_decls[options["from"]:options["to"]].execute()
elif options["from"]:
all_decls = all_decls[options["from"]:].execute()
else:
all_decls = all_decls.scan()
w = DictWriter(options["outfile"],
fieldnames=["_id"] + options["field"])
w.writeheader()
for decl in all_decls:
decl_dict = decl.to_dict()
row = {
field: self.fetch_field(decl_dict, field)
for field in options["field"]
}
row["_id"] = decl.meta.id
w.writerow(row)
def handle(self, *args, **options):
dump_to_file = len(args) > 0
all_decls = Declaration.search().query('match_all').scan()
table = self._generate_table(all_decls)
report = self._run_knitr(table, fragment_only=not dump_to_file)
if dump_to_file:
file_path = args[0]
with open(file_path, 'w', encoding='utf-8') as f:
f.write(report)
else:
root_page = Site.objects.get(is_default_site=True).root_page
try:
analytics_page = root_page.get_children().get(
slug=settings.ANALYTICS_SLUG).specific()
except Page.DoesNotExist:
page_instance = RawHTMLPage(owner=None,
title=settings.ANALYTICS_TITLE,
slug=settings.ANALYTICS_SLUG)
analytics_page = root_page.add_child(instance=page_instance)
analytics_page.body = '<div class="analytics-wrapper">' + report + '</div>'
revision = analytics_page.save_revision(user=None)
revision.publish()
self.stdout.write('Analytics page "{}" has been published.'.format(
analytics_page.url))
def details(request, declaration_id):
try:
declaration = Declaration.get(id=declaration_id)
except (ValueError, NotFoundError):
raise Http404("Таких не знаємо!")
return {"declaration": declaration}
def handle(self, *args, **options):
try:
file_path = args[0]
except IndexError:
raise CommandError('First argument must be a source file')
with open(file_path, 'r', newline='', encoding='utf-8') as source:
reader = csv.DictReader(source, delimiter=";")
counter = 0
Declaration.init() # Apparently this is required to init mappings
for row in reader:
item = Declaration(**self.map_fields(row))
item.save()
counter += 1
self.stdout.write(
'Loaded {} items to persistence storage'.format(counter))
def load_declarations(new_ids, limit=LOAD_DECLS_LIMIT):
decl_list = list()
fields = ['meta.id', 'general.*', 'intro.declaration_year']
if len(new_ids) > limit:
logger.error("load new_ids %d limit %d exceed", len(new_ids), limit)
new_ids = new_ids[:limit]
decl_list = NACPDeclaration.mget(new_ids,
raise_on_error=False,
missing='skip',
_source=fields)
if not decl_list:
decl_list = []
if len(decl_list) < len(new_ids):
add_list = Declaration.mget(new_ids,
raise_on_error=False,
missing='skip',
_source=fields)
if add_list:
decl_list.extend(add_list)
if len(decl_list) < len(new_ids):
logger.error("load new_ids %d docs not found",
len(new_ids) - len(decl_list))
return decl_list
def handle(self, *args, **options):
self.apply_migrations()
all_decls = Declaration.search().query('match_all').scan()
for decl in all_decls:
sys.stdout.write('Processing decl for {}\n'.format(
decl.general.full_name))
sys.stdout.flush()
decl.general.full_name = replace_apostrophes(
decl.general.full_name)
decl.general.name = replace_apostrophes(decl.general.name)
decl.general.last_name = replace_apostrophes(
decl.general.last_name)
decl.general.patronymic = replace_apostrophes(
decl.general.patronymic)
decl.general.full_name_suggest = {
'input': [
decl.general.full_name, ' '.join([
decl.general.name, decl.general.patronymic,
decl.general.last_name
]), ' '.join([decl.general.name, decl.general.last_name])
]
}
decl.ft_src = "\n".join(filter_only_interesting(decl.to_dict()))
decl.general.full_name_for_sorting = keyword_for_sorting(
decl.general.full_name)
decl.index_card = concat_fields(decl.to_dict(),
Declaration.INDEX_CARD_FIELDS)
decl.save()
def office(request, office_name):
search = Declaration.search()\
.filter('term', general__post__office=office_name)
return {
'query': office_name,
'results': paginated_search(request, search)
}
def search(request):
query = request.GET.get("q", "")
search = Declaration.search()
if query:
search = search.query("match", _all=query)
else:
search = search.query('match_all')
return {"query": query, "results": paginated_search(request, search)}
def region_office(request, region_name, office_name):
search = Declaration.search()\
.filter('term', general__post__region=region_name)\
.filter('term', general__post__office=office_name)
return {
'query': office_name,
'results': paginated_search(request, search),
}
def details(request, declaration_id):
try:
declaration = Declaration.get(id=int(declaration_id))
except (ValueError, NotFoundError):
raise Http404("Таких не знаємо!")
return {
"declaration": declaration
}
def stats_processor(request):
s = Declaration.search()
# res = s.params(search_type="count").aggs.metric(
# "distinct_names", "cardinality", field="full_name").execute()
return {
'total_declarations': s.count(),
'total_persons': s.count() # res.aggregations.distinct_names.value
}
def regions_home(request):
search = Declaration.search().params(search_type="count")
search.aggs.bucket(
'per_region', 'terms', field='general.post.region', size=0)
res = search.execute()
return {
'facets': res.aggregations.per_region.buckets
}
def handle(self, *args, **options):
all_decls = Declaration.search().query('match_all').scan()
for decl in all_decls:
print('Processing decl for {}'.format(decl.general.full_name))
for parent, field_key in self._decl_list_fields(decl):
field = getattr(parent, field_key, None)
if field:
filtered_field = list(filter(self._values_filter, field))
setattr(parent, field_key, filtered_field)
decl.save()
def regions_home(request):
search = Declaration.search().params(search_type="count")
search.aggs.bucket('per_region',
'terms',
field='general.post.region',
size=0)
res = search.execute()
return {'facets': res.aggregations.per_region.buckets}
def handle(self, *args, **options):
translator = Translator()
translator.fetch_full_dict_from_db()
self.apply_migrations()
all_decls = Declaration.search().query('match_all').scan()
for decl in tqdm.tqdm(all_decls):
decl_dct = decl.to_dict()
decl.general.full_name = replace_apostrophes(decl.general.full_name)
decl.general.name = replace_apostrophes(decl.general.name)
decl.general.last_name = replace_apostrophes(decl.general.last_name)
decl.general.patronymic = replace_apostrophes(decl.general.patronymic)
decl.general.full_name_suggest = {
'input': [
decl.general.full_name,
' '.join([decl.general.name,
decl.general.patronymic,
decl.general.last_name]),
' '.join([decl.general.name,
decl.general.last_name])
]
}
decl_dct["ft_src"] = ""
terms = filter_only_interesting(decl_dct)
terms += [translator.translate(x)["translation"] for x in terms]
decl.ft_src = "\n".join(terms)
decl.general.full_name_for_sorting = keyword_for_sorting(decl.general.full_name)
decl.index_card = concat_fields(decl_dct,
Declaration.INDEX_CARD_FIELDS)
extracted_names = [(decl.general.last_name, decl.general.name, decl.general.patronymic, None)]
persons = set()
names_autocomplete = set()
for person in decl.general.family:
l, f, p, _ = parse_fullname(person.family_name)
extracted_names.append((l, f, p, person.relations))
for name in extracted_names:
persons |= generate_all_names(
*name
)
names_autocomplete |= autocomplete_suggestions(
concat_name(*name[:-1])
)
decl.persons = list(filter(None, persons))
decl.names_autocomplete = list(filter(None, names_autocomplete))
decl.save()
def sitemap(request):
# TODO: REFACTOR ME?
urls = [
reverse("home"),
reverse("about"),
reverse("regions_home"),
]
search = Declaration.search().params(search_type="count")
search.aggs.bucket(
'per_region', 'terms', field='general.post.region', size=0)
for r in search.execute().aggregations.per_region.buckets:
urls.append(reverse("region", kwargs={"region_name": r.key}))
subsearch = Declaration.search()\
.filter(
Term(general__post__region=r.key) &
Not(Term(general__post__office='')))\
.params(search_type="count")
subsearch.aggs.bucket(
'per_office', 'terms', field='general.post.office', size=0)
for subr in subsearch.execute().aggregations.per_office.buckets:
urls.append(reverse(
"region_office",
kwargs={"region_name": r.key, "office_name": subr.key}))
search = Declaration.search().params(search_type="count")
search.aggs.bucket(
'per_office', 'terms', field='general.post.office', size=0)
for r in search.execute().aggregations.per_office.buckets:
urls.append(reverse("office", kwargs={"office_name": r.key}))
search = Declaration.search().extra(fields=[], size=100000)
for r in search.execute():
urls.append(reverse("details", kwargs={"declaration_id": r._id}))
return render(request, "sitemap.jinja",
{"urls": urls}, content_type="application/xml")
def search(request):
query = request.GET.get("q", "")
search = Declaration.search()
if query:
search = search.query("match", _all=query)
else:
search = search.query('match_all')
return {
"query": query,
"results": paginated_search(request, search)
}
def search(request):
query = request.GET.get("q", "")
search = Declaration.search()
if query:
search = search.query("match",
_all={
"query": query,
"operator": "and"
})
if not search.count():
search = Declaration.search().query("match",
_all={
"query": query,
"operator": "or",
"minimum_should_match": "2"
})
else:
search = search.query('match_all')
return {"query": query, "results": paginated_search(request, search)}
def handle(self, *args, **options):
try:
file_path = args[0]
except IndexError:
raise CommandError('First argument must be a source file')
with open(file_path, 'r', newline='', encoding='utf-8') as source:
decls = json.load(source)
counter = 0
Declaration.init() # Apparently this is required to init mappings
for row in decls:
mapped = self.map_fields(row)
res = Declaration.search().filter(
Term(general__last_name=mapped[
'general']['last_name'].lower().split('-')) &
Term(general__name=mapped[
'general']['name'].lower().split('-')) &
Term(intro__declaration_year=mapped[
'intro']['declaration_year'])
)
if mapped['general']['patronymic']:
res = res.filter(Term(general__patronymic=mapped[
'general']['patronymic'].lower()))
res = res.execute()
if not res.hits:
item = Declaration(**mapped)
item.save()
counter += 1
self.stdout.write(
'Loaded {} items to persistence storage'.format(counter))
def handle(self, *args, **options):
self.apply_migrations()
all_decls = Declaration.search().query('match_all').scan()
for decl in all_decls:
decl_dct = decl.to_dict()
sys.stdout.write('Processing decl for {}\n'.format(
decl.general.full_name))
sys.stdout.flush()
decl.general.full_name = replace_apostrophes(
decl.general.full_name)
decl.general.name = replace_apostrophes(decl.general.name)
decl.general.last_name = replace_apostrophes(
decl.general.last_name)
decl.general.patronymic = replace_apostrophes(
decl.general.patronymic)
decl.general.full_name_suggest = {
'input': [
decl.general.full_name, ' '.join([
decl.general.name, decl.general.patronymic,
decl.general.last_name
]), ' '.join([decl.general.name, decl.general.last_name])
]
}
decl_dct["ft_src"] = ""
decl.ft_src = "\n".join(filter_only_interesting(decl_dct))
decl.general.full_name_for_sorting = keyword_for_sorting(
decl.general.full_name)
decl.index_card = concat_fields(decl_dct,
Declaration.INDEX_CARD_FIELDS)
extracted_names = [(decl.general.last_name, decl.general.name,
decl.general.patronymic, None)]
persons = set()
names_autocomplete = set()
for person in decl.general.family:
l, f, p, _ = parse_fullname(person.family_name)
extracted_names.append((l, f, p, person.relations))
for name in extracted_names:
persons |= generate_all_names(*name)
names_autocomplete |= autocomplete_suggestions(
concat_name(*name[:-1]))
decl.persons = list(filter(None, persons))
decl.names_autocomplete = list(filter(None, names_autocomplete))
decl.save()
def handle(self, *args, **options):
try:
file_path = args[0]
id_prefix = args[1]
except IndexError:
raise CommandError(
'First argument must be a source file and second is a id prefix'
)
groups = defaultdict(list)
with open(file_path, 'r', newline='', encoding='utf-8') as source:
reader = csv.DictReader(source, delimiter=',')
counter = 0
for row in reader:
status_col = 'Status' if 'Status' in row else 'Статус'
if row[status_col] == '' or row[status_col] == 'Ок':
groups[row[self._group_column(row)]].append(row)
counter += 1
self.stdout.write(
'Read {} valid rows from the input file'.format(counter))
Declaration.init() # Apparently this is required to init mappings
declarations = map(self.merge_group, groups.values())
counter = 0
for declaration in declarations:
mapped = self.map_fields(declaration, id_prefix)
res = Declaration.search().filter(
Term(general__last_name=mapped['general']
['last_name'].lower().split('-'))
& Term(
general__name=mapped['general']['name'].lower().split('-'))
& Term(intro__declaration_year=mapped['intro']
['declaration_year']))
if mapped['general']['patronymic']:
res = res.filter(
Term(general__patronymic=mapped['general']
['patronymic'].lower()))
res = res.execute()
if res.hits:
self.stdout.write("%s (%s) already exists" %
(mapped['general']['full_name'],
mapped['intro']['declaration_year']))
mapped['_id'] = res.hits[0]._id
item = Declaration(**mapped)
item.save()
counter += 1
self.stdout.write(
'Loaded {} items to persistence storage'.format(counter))
def search(request):
query = request.GET.get("q", "")
search = Declaration.search()
if query:
search = search.query(
"match", _all={"query": query, "operator": "and"})
if not search.count():
search = Declaration.search().query(
"match",
_all={
"query": query,
"operator": "or",
"minimum_should_match": "2"
}
)
else:
search = search.query('match_all')
return {
"query": query,
"results": paginated_search(request, search)
}
def region(request, region_name):
search = Declaration.search()\
.filter(
Term(general__post__region=region_name) &
Not(Term(general__post__office='')))\
.params(search_type="count")
search.aggs.bucket(
'per_office', 'terms', field='general.post.office', size=0)
res = search.execute()
return {
'facets': res.aggregations.per_office.buckets,
'region_name': region_name
}
def handle(self, *args, **options):
all_decls = Declaration.search().query('match_all').scan()
urls_in_list = []
on_unshred_it = 0
on_static = 0
on_chesno = 0
no_url_no_id = 0
not_found = 0
duplicates = 0
for decl in all_decls:
try:
url = decl["declaration"]["url"]
except KeyError:
url = None
if not url:
no_url_no_id += 1
print(decl.meta.id)
continue
if url in urls_in_list:
duplicates += 1
print("Duplicates", url, decl.meta.id)
continue
if "unshred.it" in url.lower():
on_unshred_it += 1
if not self.check_url(url, options["destination"]):
not_found += 1
print("Not found", url, decl.meta.id)
else:
decl["declaration"]["url"] = url.replace(
"http://unshred.it/static/",
"http://static.declarations.com.ua/")
# decl.save()
elif "static.declarations.com.ua" in url.lower():
on_static += 1
if not self.check_url(url, options["destination"]):
print(url)
not_found += 1
else:
on_chesno += 1
print(on_unshred_it, on_static, on_chesno, no_url_no_id)
print(not_found, duplicates)
def handle(self, *args, **options):
try:
file_path = args[0]
except IndexError:
raise CommandError('First argument must be a source file')
with open(file_path, 'r', newline='', encoding='utf-8') as source:
decls = json.load(source)
counter = 0
Declaration.init() # Apparently this is required to init mappings
for row in decls:
if "fields" not in row["details"]:
continue
mapped = self.map_fields(row)
res = Declaration.search().query(
Q('terms',
general__last_name=mapped['general']
['last_name'].lower().split("-"))
& Q('terms',
general__name=mapped['general']['name'].lower().split(
"-")) & Q('term',
intro__declaration_year=mapped['intro']
['declaration_year']))
if mapped['general']['patronymic']:
res = res.query('term',
general__patronymic=mapped['general']
['patronymic'].lower())
self.stdout.write("Checking %s (%s)" %
(mapped['general']['full_name'],
mapped['intro']['declaration_year']))
res = res.execute()
if not res.hits:
item = Declaration(**mapped)
item.save()
counter += 1
else:
self.stdout.write("%s (%s) already exists" %
(mapped['general']['full_name'],
mapped['intro']['declaration_year']))
self.stdout.write(
'Loaded {} items to persistence storage'.format(counter))
def region(request, region_name):
search = Declaration.search()\
.filter(
Term(general__post__region=region_name) &
Not(Term(general__post__office='')))\
.params(search_type="count")
search.aggs.bucket('per_office',
'terms',
field='general.post.office',
size=0)
res = search.execute()
return {
'facets': res.aggregations.per_office.buckets,
'region_name': region_name
}
def handle(self, *args, **options):
try:
file_path = args[0]
id_prefix = args[1]
except IndexError:
raise CommandError(
'First argument must be a source file and second is a id prefix')
groups = defaultdict(list)
with open(file_path, 'r', newline='', encoding='utf-8') as source:
reader = csv.DictReader(source, delimiter=',')
counter = 0
for row in reader:
status_col = 'Status' if 'Status' in row else 'Статус'
if row[status_col] == '' or row[status_col] == 'Ок':
groups[row[self._group_column(row)]].append(row)
counter += 1
self.stdout.write('Read {} valid rows from the input file'.format(counter))
Declaration.init() # Apparently this is required to init mappings
declarations = map(self.merge_group, groups.values())
counter = 0
for declaration in declarations:
mapped = self.map_fields(declaration, id_prefix)
res = Declaration.search().filter(
Term(general__last_name=mapped[
'general']['last_name'].lower().split('-')) &
Term(general__name=mapped[
'general']['name'].lower().split('-')) &
Term(intro__declaration_year=mapped[
'intro']['declaration_year'])
)
if mapped['general']['patronymic']:
res = res.filter(Term(general__patronymic=mapped[
'general']['patronymic'].lower()))
res = res.execute()
if res.hits:
self.stdout.write(
"%s (%s) already exists" % (
mapped['general']['full_name'],
mapped['intro']['declaration_year']))
mapped['_id'] = res.hits[0]._id
item = Declaration(**mapped)
item.save()
counter += 1
self.stdout.write('Loaded {} items to persistence storage'.format(counter))
def handle(self, *args, **options):
try:
file_path = args[0]
id_prefix = args[1]
except IndexError:
raise CommandError("First argument must be a source file and second is a id prefix")
groups = defaultdict(list)
with open(file_path, "r", newline="", encoding="utf-8") as source:
reader = csv.DictReader(source, delimiter=",")
counter = 0
for row in reader:
status_col = "Status" if "Status" in row else "Статус"
if row[status_col] == "" or row[status_col] == "Ок":
groups[row[self._group_column(row)]].append(row)
counter += 1
self.stdout.write("Read {} valid rows from the input file".format(counter))
Declaration.init() # Apparently this is required to init mappings
declarations = map(self.merge_group, groups.values())
counter = 0
for declaration in declarations:
mapped = self.map_fields(declaration, id_prefix)
res = Declaration.search().filter(
Terms(general__last_name=mapped["general"]["last_name"].lower().split("-"))
& Terms(general__name=mapped["general"]["name"].lower().split("-"))
& Term(intro__declaration_year=mapped["intro"]["declaration_year"])
)
if mapped["general"]["patronymic"]:
res = res.filter(Term(general__patronymic=mapped["general"]["patronymic"].lower()))
res = res.execute()
if res.hits:
self.stdout.write(
"%s (%s) already exists" % (mapped["general"]["full_name"], mapped["intro"]["declaration_year"])
)
mapped["_id"] = res.hits[0]._id
else:
self.stdout.write(
"%s (%s) created" % (mapped["general"]["full_name"], mapped["intro"]["declaration_year"])
)
item = Declaration(**mapped)
item.save()
counter += 1
self.stdout.write("Loaded {} items to persistence storage".format(counter))
def handle(self, *args, **options):
dump_to_file = len(args) > 0
all_decls = Declaration.search().query('match_all').scan()
table = self._generate_table(all_decls)
report = self._run_knitr(table, fragment_only=not dump_to_file)
if dump_to_file:
file_path = args[0]
with open(file_path, 'w', encoding='utf-8') as f:
f.write(report)
else:
root_page = Site.objects.get(is_default_site=True).root_page
try:
analytics_page = root_page.get_children().get(slug=settings.ANALYTICS_SLUG).specific
except Page.DoesNotExist:
page_instance = RawHTMLPage(owner=None, title=settings.ANALYTICS_TITLE, slug=settings.ANALYTICS_SLUG)
analytics_page = root_page.add_child(instance=page_instance)
analytics_page.body = '<div class="analytics-wrapper">' + report + '</div>'
revision = analytics_page.save_revision(user=None)
revision.publish()
self.stdout.write('Analytics page "{}" has been published.'.format(analytics_page.url))
def assume(q, fuzziness):
search = Declaration.search()\
.suggest(
'name',
q,
completion={
'field': 'general.full_name_suggest',
'size': 10,
'fuzzy': {
'fuzziness': fuzziness,
'unicode_aware': 1
}
}
)
res = search.execute()
if res.success():
return [val['text'] for val in res.suggest['name'][0]['options']]
else:
[]
def assume(q, fuzziness):
search = Declaration.search()\
.suggest(
'name',
q,
completion={
'field': 'general.full_name_suggest',
'size': 10,
'fuzzy': {
'fuzziness': fuzziness,
'unicode_aware': 1
}
}
)
res = search.execute()
if res.success():
return [val['text'] for val in res.suggest['name'][0]['options']]
else:
[]
def handle(self, *args, **options):
try:
file_path = args[0]
except IndexError:
raise CommandError('First argument must be a source file')
with open(file_path, 'r', newline='', encoding='utf-8') as source:
reader = csv.DictReader(source, delimiter=";")
counter = 0
Declaration.init() # Apparently this is required to init mappings
for row in reader:
item = Declaration(**self.map_fields(row))
item.save()
counter += 1
self.stdout.write(
'Loaded {} items to persistence storage'.format(counter))
def region(request, region_name):
search = Declaration.search()\
.filter(
Term(general__post__region=region_name) &
Not(Term(general__post__office='')))\
.params(search_type="count")
meta_data = MetaData.objects.filter(region_id=region_name,
office_id=None).first()
search.aggs.bucket('per_office',
'terms',
field='general.post.office',
size=0)
res = search.execute()
return {
'facets': res.aggregations.per_office.buckets,
'region_name': region_name,
'title': meta_data.title if meta_data else "",
'meta_desc': meta_data.description if meta_data else "",
}
def region(request, region_name):
search = Declaration.search()\
.filter(
Term(general__post__region=region_name) &
Not(Term(general__post__office='')))\
.params(search_type="count")
meta_data = MetaData.objects.filter(
region_id=region_name,
office_id=None
).first()
search.aggs.bucket(
'per_office', 'terms', field='general.post.office', size=0)
res = search.execute()
return {
'facets': res.aggregations.per_office.buckets,
'region_name': region_name,
'title': meta_data.title if meta_data else "",
'meta_desc': meta_data.description if meta_data else "",
}
def suggest(request):
search = Declaration.search()\
.suggest(
'name',
request.GET.get('q', ''),
completion={
'field': 'general.full_name_suggest',
'size': 10,
'fuzzy': {
'fuzziness': 3,
'unicode_aware': 1
}
}
)
res = search.execute()
if res.success():
return JsonResponse(
[val['text'] for val in res.suggest['name'][0]['options']],
safe=False)
else:
return JsonResponse([], safe=False)
def handle(self, *args, **options):
all_decls = Declaration.search().query('match_all').scan()
for decl in all_decls:
print('Processing decl for {}'.format(decl.general.full_name))
decl.general.full_name = replace_apostrophes(decl.general.full_name)
decl.general.name = replace_apostrophes(decl.general.name)
decl.general.last_name = replace_apostrophes(decl.general.last_name)
decl.general.patronymic = replace_apostrophes(decl.general.patronymic)
decl.general.full_name_suggest = {
'input': [
decl.general.full_name,
' '.join([decl.general.name,
decl.general.patronymic,
decl.general.last_name]),
' '.join([decl.general.name,
decl.general.last_name])
]
}
decl.ft_src = "\n".join(filter_only_interesting(decl.to_dict()))
decl.save()
def suggest(request):
search = Declaration.search()\
.suggest(
'name',
request.GET.get('q', ''),
completion={
'field': 'general.full_name_suggest',
'size': 10,
'fuzzy': {
'fuzziness': 3,
'unicode_aware': 1
}
}
)
res = search.execute()
if res.success():
return JsonResponse(
[val['text'] for val in res.suggest['name'][0]['options']],
safe=False
)
else:
return JsonResponse([], safe=False)
def handle(self, *args, **options):
try:
file_path = args[0]
except IndexError:
raise CommandError('First argument must be a source file')
with open(file_path, 'r', newline='', encoding='utf-8') as source:
reader = csv.DictReader(source, delimiter=";")
counter = 0
Declaration.init() # Apparently this is required to init mappings
for row in reader:
mapped = self.map_fields(row)
res = Declaration.search().query(
Q('terms',
general__last_name=mapped['general']
['last_name'].lower().split("-"))
& Q('terms',
general__name=mapped['general']['name'].lower().split(
"-")) & Q('term',
intro__declaration_year=mapped['intro']
['declaration_year']))
if mapped['general']['patronymic']:
res = res.query('term',
general__patronymic=mapped['general']
['patronymic'].lower())
res = res.execute()
if res.hits:
mapped["_id"] = res.hits[0]._id
item = Declaration(**mapped)
item.save()
counter += 1
self.stdout.write(
'Loaded {} items to persistence storage'.format(counter))
def handle(self, *args, **options):
try:
file_path = args[0]
except IndexError:
raise CommandError('First argument must be a source file')
with open(file_path, 'r', newline='', encoding='utf-8') as source:
reader = csv.DictReader(source, delimiter=";")
counter = 0
Declaration.init() # Apparently this is required to init mappings
for row in reader:
mapped = self.map_fields(row)
res = Declaration.search().filter(
Term(general__last_name=mapped[
"general"]["last_name"].lower().split("-")) &
Term(general__name=mapped[
"general"]["name"].lower().split("-")) &
Term(intro__declaration_year=mapped[
"intro"]["declaration_year"])
)
if mapped["general"]["patronymic"]:
res = res.filter(Term(general__patronymic=mapped[
"general"]["patronymic"].lower()))
res = res.execute()
if res.hits:
mapped["id"] = res.hits[0]._id
item = Declaration(**mapped)
item.save()
counter += 1
self.stdout.write(
'Loaded {} items to persistence storage'.format(counter))
def handle(self, *args, **options):
try:
file_path = options["file_path"]
id_prefix = options["prefix"]
except IndexError:
raise CommandError(
'First argument must be a source file and second is a id prefix'
)
if hasattr(sys.stdin, 'isatty') and not sys.stdin.isatty():
self.stdout.write(
"To import something you need to run this command in TTY.")
return
groups = defaultdict(list)
with open(file_path, 'r', newline='', encoding='utf-8') as source:
reader = csv.DictReader(source, delimiter='\t')
counter = 0
for row in reader:
status_col = 'Status' if 'Status' in row else 'Статус'
if row[status_col] == '' or row[status_col] == 'Ок':
groups[row[self._group_column(row)]].append(row)
counter += 1
self.stdout.write(
'Read {} valid rows from the input file'.format(counter))
Declaration.init() # Apparently this is required to init mappings
declarations = map(self.merge_group, groups.values())
counter = 0
for declaration in declarations:
mapped = self.map_fields(declaration, id_prefix)
res = Declaration.search().query(
Q('terms',
general__last_name=mapped['general']
['last_name'].lower().split("-"))
& Q('terms',
general__name=mapped['general']['name'].lower().split("-"))
& Q('term',
intro__declaration_year=mapped['intro']
['declaration_year']))
if mapped['general']['patronymic']:
res = res.query('term',
general__patronymic=mapped['general']
['patronymic'].lower())
res = res.execute()
if res.hits:
self.stdout.write(
"Person\n%s (%s, %s, %s, %s)\n%s\nalready exists" %
(mapped['general']['full_name'],
mapped['general']['post']['post'],
mapped['general']['post']['office'],
mapped['general']['post']['region'],
mapped['intro']['declaration_year'],
mapped['declaration']['url']))
for i, hit in enumerate(res.hits):
self.stdout.write("%s: %s (%s, %s, %s, %s), %s\n%s" %
(i + 1, hit['general']['full_name'],
hit['general']['post']['post'],
hit['general']['post']['office'],
hit['general']['post']['region'],
hit['intro']['declaration_year'],
hit._id, hit['declaration']['url']))
msg = (
"Select one of persons above to replace, or press [s] " +
"to skip current record or [c] to create new (default): ")
r = input(msg).lower() or "c"
if r == "s":
self.stdout.write("Ok, skipping")
continue
if r.isdigit() and int(r) <= len(res.hits):
r = int(r) - 1
mapped['_id'] = res.hits[r]._id
self.stdout.write("Ok, replacing %s" % res.hits[r]._id)
else:
self.stdout.write("Ok, adding new record")
else:
self.stdout.write("%s (%s) created" %
(mapped['general']['full_name'],
mapped['intro']['declaration_year']))
item = Declaration(**mapped)
item.save()
counter += 1
self.stdout.write(
'Loaded {} items to persistence storage'.format(counter))
