def matching_france(region=None):
mode = request.args.get('mode', 'none')
if region is None:
config = Config('./config/config.yml')
factory = DocFactory(config.get('mongodb'))
internal = factory.internal_collection()
objects = internal.aggregate([{
'$match': {
'name': {
'$exists': True,
'$not': {
'$size': 0
}
},
'$and': [{
'admin_hierarchy.ADMIN_LEVEL_1.name': 'France'
}]
}
}, {
'$group': {
'_id': '$admin_hierarchy.ADMIN_LEVEL_2.name',
'count': {
'$sum': 1
}
}
}])
return render_template('admin/matching-france/region-list.html',
data=objects,
mode=mode)
else:
return render_template('admin/matching-france/list.html',
region=region,
mode=mode)
gmaps.create_index([('_id', 1)])
gmaps.create_index([('code', 1)])
gmaps.create_index([('name', 1)])
gmaps.create_index([('admin_hierarchy', 1)])
insee = factory.insee_collection()
insee.drop_indexes()
insee.create_index([('_id', 1)])
insee.create_index([('code', 1)])
insee.create_index([('name', 1)])
istat = factory.istat_collection()
istat.drop_indexes()
insee.create_index([('_id', 1)])
insee.create_index([('code', 1)])
insee.create_index([('name', 1)])
internal = factory.internal_collection()
internal.drop_indexes()
internal.create_index([('_id', 1)])
internal.create_index([('code', 1)])
internal.create_index([('name', 1)])
internal.create_index([('admin_hierarchy', 1)])
]
config = Config('./config/config.yml')
doc_factory = DocFactory(config.get('mongodb'))
language = 'fr'
spider = Spider(loader_factory=LoaderFactory,
gmap_parser=MapFactory.france,
wiki_parser=ParserFranceWiki,
doc_factory=doc_factory,
language=language,
config=config,
use_cache=True)
internal_collection = doc_factory.internal_collection()
i = 0
hash_lib = hash()
for csv_file in files:
with open(csv_file, encoding='utf-8') as admin_div_CSV:
for line in csv.DictReader(admin_div_CSV, delimiter='\t'):
try:
i = i + 1
print(i)
dct = {}
wiki_parsed = {}
wiki = parser_wiki(line)
gmap = parser_gmap(line)
wiki_obj = hash_lib.make(wiki.get('url'))
def matching_france_js(region):
region = unquote_plus(region)
#mode = request.args.get('mode', 'none')
config = Config('./config/config.yml')
factory = DocFactory(config.get('mongodb'))
internal = factory.internal_collection()
wiki = factory.wiki_collection()
gmap = factory.gmaps_collection()
insee = factory.insee_collection()
objects = internal.find({
'name': {
'$exists': True,
'$not': {
'$size': 0
}
},
'$and': [{
'admin_hierarchy.ADMIN_LEVEL_1.name': 'France'
}, {
'admin_hierarchy.ADMIN_LEVEL_2.name': region
}],
})
result = []
for item in objects:
dic = {'internal': item}
wiki_res = {}
if item.get('source', {}).get('wiki'):
wiki_res = wiki.find_one(
{'code': item.get('source', {}).get('wiki')})
dic.update(wiki=wiki_res)
gmap_res = {}
if item.get('source', {}).get('gmap'):
gmap_res = gmap.find_one(
{'code': item.get('source', {}).get('gmap')})
dic.update(gmap=gmap_res)
insee_res = {}
if item.get('source', {}).get('insee'):
insee_res = insee.find_one(
{'code': item.get('source', {}).get('insee')})
dic.update(insee=insee_res)
compare_res = {}
compare_res.update({
'insee_code!=wiki_code':
1 if not (insee_res.get('InseeXls_CodeCommune')
== wiki_res.get('commune_codes')) else 0
})
compare_res.update({
'insee_name!=wiki_name':
1 if
not (insee_res.get('InseeXls_NameCommune') == wiki_res.get('name'))
else 0
})
compare_res.update({
'wiki_name!=gmaps_name':
1
if not (wiki_res.get('true_name', wiki_res.get('name'))
== gmap_res.get('true_name', gmap_res.get('name'))) else 0
})
compare_res.update({
'wiki_post!=gmaps_post':
1 if not (str(wiki_res.get('postal_codes')) == str(
gmap_res.get('postal_code'))) else 0
})
compare_res.update({
'wiki_admin!=gmaps_admin':
1 if not (str(wiki_res.get('admin_hierarchy')) == str(
gmap_res.get('admin_hierarchy'))) else 0
})
try:
max_meters_in_distance = 5000
compare_res.update({
'wiki_posinion>gmaps_position':
1 if Comparison.by_distance(wiki_res.get('center'),
gmap_res.get('center')) >
max_meters_in_distance else 0
})
except:
compare_res.update({'wiki_posinion>gmaps_position': 1})
dic.update(compare=compare_res)
# if mode != 'none':
# if mode == 'wiki_adapte':
# if dic.get('wiki', {}).get('name', '').lower() != dic.get('insee', {}).get('name', '').lower():
# result.append(dic)
# elif mode == 'gmap_adapte':
# if dic.get('gmap', {}).get('name', '').lower() != dic.get('insee', {}).get('name', '').lower():
# result.append(dic)
# else:
# result.append(dic)
result.append(dic)
return render_template('admin/matching-france/list.js',
e=escape,
items=result)