def matching_france(region=None):
mode = request.args.get('mode', 'none')
if region is None:
config = Config('./config/config.yml')
factory = DocFactory(config.get('mongodb'))
internal = factory.internal_collection()
objects = internal.aggregate([{
'$match': {
'name': {
'$exists': True,
'$not': {
'$size': 0
}
},
'$and': [{
'admin_hierarchy.ADMIN_LEVEL_1.name': 'France'
}]
}
}, {
'$group': {
'_id': '$admin_hierarchy.ADMIN_LEVEL_2.name',
'count': {
'$sum': 1
}
}
}])
return render_template('admin/matching-france/region-list.html',
data=objects,
mode=mode)
else:
return render_template('admin/matching-france/list.html',
region=region,
mode=mode)
def by_place_id(list_places, city_id):
config = Config('./config/config.yml')
Key = Keygen()
keyAPI = Key.get_key_geocode()
if not keyAPI:
sys.exit()
cnf = {'googlemaps': {'geocoding': {'key': keyAPI}}}
config.set(cnf)
language = 'uk'
doc_factory = DocFactory(config.get('mongodb'))
spider = Spider(loader_factory=LoaderFactory,
gmap_parser=MapFactory.spain,
doc_factory=doc_factory,
language=language,
config=config,
use_cache=True)
for loc in list_places:
objects = spider.get_gmap_place_id(loc)
gmap = {}
try:
gmap = objects[0].get_document()
gmap["city_id"] = city_id
exixts = db.ukraine_city_sublocal.find({
"city_id": city_id,
"code": gmap['code']
}).count()
if exixts < 1:
gmap.pop('_id', None)
db.ukraine_city_sublocal.save(gmap)
print(gmap)
except Exception as e:
print(str(e))
class Keygen:
config = Config('./config/config.yml')
mongo_config = config.get('mongodb')
connection = MongoClient(mongo_config['host'], mongo_config['port'])
def __init__(self):
self.db = self.connection.local
def get_key_geocode(self):
key = self.db.keygen.find_one_and_update({'geocode': {'$lt': 2500}},
{'$inc': {'geocode': 1}},
projection={'key': True, '_id': False})
try:
return key['key']
except Exception as e:
return False
def get_key_place(self):
key = self.db.keygen.find_one_and_update({'place': {'$lt': 1000}},
{'$inc': {'place': 1}},
projection={'key': True, '_id': False})
try:
return key['key']
except Exception as e:
return False
def get_key_distance(self):
key = self.db.keygen.find_one_and_update({'distance': {'$lt': 2500}},
{'$inc': {'distance': 1}},
projection={'key': True, '_id': False})
try:
return key['key']
except Exception as e:
return False
def createGmapsKey(self, key):
data = {
"key":key,
"geocode":0,
"place":0,
"distance":0
}
post = self.db.keygen.find_one({'key': key})
if post is None:
result = self.db.keygen.insert_one(data)
print("Created",key)
else:
print("KEY EXIST",key)
def changeLimites(self):
self.db.keygen.update({}, {'$set': {'geocode': 0, "place":0, "distance":0}}, False, True)
def main():
config = Config('./config/config.yml')
mongo_config = config.get('mongodb')
connection = MongoClient(mongo_config['host'], mongo_config['port'])
db = connection.local
user = input("Enter your phone (format: 38063000000): ")
password = input("Enter your password: "******"_id": user, "password": pass_hash})
print("User created.")
except DuplicateKeyError:
print("User already present in DB.")
def _get_common_data(self, list_name, force=False):
config = Config('./config/config.yml')
loader = LoaderFactory.loader_gmaps_with_cache(
gmaps_config=config.get('googlemaps'),
storage_config=config.get('mongodb')
)
document_factory = DocFactory(config.get('mongodb'))
options = {
'loader': loader,
'doc_factory': document_factory,
'parser': getattr(MapFactory, self._country),
'force_update': force
}
storage = MongoDB('{}_{}'.format(list_name, self._country), config.get('mongodb'))
return options, storage
from lib.job.TaskListMongoDB import TaskListMongoDB
from lib.job.map.google.PositionTask import PositionTask
from lib.job.ExecutorWithLimit import ExecutorWithLimit
from lib.job.storage.MongoDB import MongoDB as Storage
from lib.config.Yaml import Yaml as Config
from lib.factory.Loader import Loader as LoaderFactory
from lib.factory.StorageLocation import StorageLocation as DocFactory
from lib.parser.map.google.Italy import Italy
from lib.logger.MongoDB import MongoDB as Log
from lib.Counter.CounterMongoDB import CounterMongoDB
from pymongo import MongoClient
force = True
config = Config('./config/config.yml')
country = 'Italy'
options = {}
gmaps_config = config.get('googlemaps')
gmaps_config.update(language='it')
loader = LoaderFactory.loader_gmaps_with_cache(
gmaps_config=gmaps_config,
storage_config=config.get('mongodb')
)
options.update(loader=loader)
doc_factory = DocFactory(config.get('mongodb'))
from pymongo import MongoClient
from lib.config.Yaml import Yaml as Config
from lib.factory.StorageLocation import StorageLocation as DocFactory
config = Config('./config/config.yml').get('mongodb')
connection = MongoClient(config['host'], config['port'])
factory = DocFactory(config)
wiki = factory.wiki_collection()
wiki.drop_indexes()
wiki.create_index([('_id', 1)])
wiki.create_index([('code', 1)])
wiki.create_index([('name', 1)])
wiki.create_index([('admin_hierarchy', 1)])
gmaps = factory.gmaps_collection()
gmaps.drop_indexes()
gmaps.create_index([('_id', 1)])
gmaps.create_index([('code', 1)])
gmaps.create_index([('name', 1)])
gmaps.create_index([('admin_hierarchy', 1)])
def insee_code_unit(id):
config = Config('./config/config.yml')
factory = DocFactory(config.get('mongodb'))
collection = factory.insee_collection()
obj = collection.find_one({'code': id})
return render_template('admin/other/unit.html', data=obj)
def matching_france_js(region):
region = unquote_plus(region)
#mode = request.args.get('mode', 'none')
config = Config('./config/config.yml')
factory = DocFactory(config.get('mongodb'))
internal = factory.internal_collection()
wiki = factory.wiki_collection()
gmap = factory.gmaps_collection()
insee = factory.insee_collection()
objects = internal.find({
'name': {
'$exists': True,
'$not': {
'$size': 0
}
},
'$and': [{
'admin_hierarchy.ADMIN_LEVEL_1.name': 'France'
}, {
'admin_hierarchy.ADMIN_LEVEL_2.name': region
}],
})
result = []
for item in objects:
dic = {'internal': item}
wiki_res = {}
if item.get('source', {}).get('wiki'):
wiki_res = wiki.find_one(
{'code': item.get('source', {}).get('wiki')})
dic.update(wiki=wiki_res)
gmap_res = {}
if item.get('source', {}).get('gmap'):
gmap_res = gmap.find_one(
{'code': item.get('source', {}).get('gmap')})
dic.update(gmap=gmap_res)
insee_res = {}
if item.get('source', {}).get('insee'):
insee_res = insee.find_one(
{'code': item.get('source', {}).get('insee')})
dic.update(insee=insee_res)
compare_res = {}
compare_res.update({
'insee_code!=wiki_code':
1 if not (insee_res.get('InseeXls_CodeCommune')
== wiki_res.get('commune_codes')) else 0
})
compare_res.update({
'insee_name!=wiki_name':
1 if
not (insee_res.get('InseeXls_NameCommune') == wiki_res.get('name'))
else 0
})
compare_res.update({
'wiki_name!=gmaps_name':
1
if not (wiki_res.get('true_name', wiki_res.get('name'))
== gmap_res.get('true_name', gmap_res.get('name'))) else 0
})
compare_res.update({
'wiki_post!=gmaps_post':
1 if not (str(wiki_res.get('postal_codes')) == str(
gmap_res.get('postal_code'))) else 0
})
compare_res.update({
'wiki_admin!=gmaps_admin':
1 if not (str(wiki_res.get('admin_hierarchy')) == str(
gmap_res.get('admin_hierarchy'))) else 0
})
try:
max_meters_in_distance = 5000
compare_res.update({
'wiki_posinion>gmaps_position':
1 if Comparison.by_distance(wiki_res.get('center'),
gmap_res.get('center')) >
max_meters_in_distance else 0
})
except:
compare_res.update({'wiki_posinion>gmaps_position': 1})
dic.update(compare=compare_res)
# if mode != 'none':
# if mode == 'wiki_adapte':
# if dic.get('wiki', {}).get('name', '').lower() != dic.get('insee', {}).get('name', '').lower():
# result.append(dic)
# elif mode == 'gmap_adapte':
# if dic.get('gmap', {}).get('name', '').lower() != dic.get('insee', {}).get('name', '').lower():
# result.append(dic)
# else:
# result.append(dic)
result.append(dic)
return render_template('admin/matching-france/list.js',
e=escape,
items=result)
