def matching_france(region=None): mode = request.args.get('mode', 'none') if region is None: config = Config('./config/config.yml') factory = DocFactory(config.get('mongodb')) internal = factory.internal_collection() objects = internal.aggregate([{ '$match': { 'name': { '$exists': True, '$not': { '$size': 0 } }, '$and': [{ 'admin_hierarchy.ADMIN_LEVEL_1.name': 'France' }] } }, { '$group': { '_id': '$admin_hierarchy.ADMIN_LEVEL_2.name', 'count': { '$sum': 1 } } }]) return render_template('admin/matching-france/region-list.html', data=objects, mode=mode) else: return render_template('admin/matching-france/list.html', region=region, mode=mode)
def by_place_id(list_places, city_id): config = Config('./config/config.yml') Key = Keygen() keyAPI = Key.get_key_geocode() if not keyAPI: sys.exit() cnf = {'googlemaps': {'geocoding': {'key': keyAPI}}} config.set(cnf) language = 'uk' doc_factory = DocFactory(config.get('mongodb')) spider = Spider(loader_factory=LoaderFactory, gmap_parser=MapFactory.spain, doc_factory=doc_factory, language=language, config=config, use_cache=True) for loc in list_places: objects = spider.get_gmap_place_id(loc) gmap = {} try: gmap = objects[0].get_document() gmap["city_id"] = city_id exixts = db.ukraine_city_sublocal.find({ "city_id": city_id, "code": gmap['code'] }).count() if exixts < 1: gmap.pop('_id', None) db.ukraine_city_sublocal.save(gmap) print(gmap) except Exception as e: print(str(e))
class Keygen: config = Config('./config/config.yml') mongo_config = config.get('mongodb') connection = MongoClient(mongo_config['host'], mongo_config['port']) def __init__(self): self.db = self.connection.local def get_key_geocode(self): key = self.db.keygen.find_one_and_update({'geocode': {'$lt': 2500}}, {'$inc': {'geocode': 1}}, projection={'key': True, '_id': False}) try: return key['key'] except Exception as e: return False def get_key_place(self): key = self.db.keygen.find_one_and_update({'place': {'$lt': 1000}}, {'$inc': {'place': 1}}, projection={'key': True, '_id': False}) try: return key['key'] except Exception as e: return False def get_key_distance(self): key = self.db.keygen.find_one_and_update({'distance': {'$lt': 2500}}, {'$inc': {'distance': 1}}, projection={'key': True, '_id': False}) try: return key['key'] except Exception as e: return False def createGmapsKey(self, key): data = { "key":key, "geocode":0, "place":0, "distance":0 } post = self.db.keygen.find_one({'key': key}) if post is None: result = self.db.keygen.insert_one(data) print("Created",key) else: print("KEY EXIST",key) def changeLimites(self): self.db.keygen.update({}, {'$set': {'geocode': 0, "place":0, "distance":0}}, False, True)
def main(): config = Config('./config/config.yml') mongo_config = config.get('mongodb') connection = MongoClient(mongo_config['host'], mongo_config['port']) db = connection.local user = input("Enter your phone (format: 38063000000): ") password = input("Enter your password: "******"_id": user, "password": pass_hash}) print("User created.") except DuplicateKeyError: print("User already present in DB.")
def _get_common_data(self, list_name, force=False): config = Config('./config/config.yml') loader = LoaderFactory.loader_gmaps_with_cache( gmaps_config=config.get('googlemaps'), storage_config=config.get('mongodb') ) document_factory = DocFactory(config.get('mongodb')) options = { 'loader': loader, 'doc_factory': document_factory, 'parser': getattr(MapFactory, self._country), 'force_update': force } storage = MongoDB('{}_{}'.format(list_name, self._country), config.get('mongodb')) return options, storage
from lib.job.TaskListMongoDB import TaskListMongoDB from lib.job.map.google.PositionTask import PositionTask from lib.job.ExecutorWithLimit import ExecutorWithLimit from lib.job.storage.MongoDB import MongoDB as Storage from lib.config.Yaml import Yaml as Config from lib.factory.Loader import Loader as LoaderFactory from lib.factory.StorageLocation import StorageLocation as DocFactory from lib.parser.map.google.Italy import Italy from lib.logger.MongoDB import MongoDB as Log from lib.Counter.CounterMongoDB import CounterMongoDB from pymongo import MongoClient force = True config = Config('./config/config.yml') country = 'Italy' options = {} gmaps_config = config.get('googlemaps') gmaps_config.update(language='it') loader = LoaderFactory.loader_gmaps_with_cache( gmaps_config=gmaps_config, storage_config=config.get('mongodb') ) options.update(loader=loader) doc_factory = DocFactory(config.get('mongodb'))
from pymongo import MongoClient from lib.config.Yaml import Yaml as Config from lib.factory.StorageLocation import StorageLocation as DocFactory config = Config('./config/config.yml').get('mongodb') connection = MongoClient(config['host'], config['port']) factory = DocFactory(config) wiki = factory.wiki_collection() wiki.drop_indexes() wiki.create_index([('_id', 1)]) wiki.create_index([('code', 1)]) wiki.create_index([('name', 1)]) wiki.create_index([('admin_hierarchy', 1)]) gmaps = factory.gmaps_collection() gmaps.drop_indexes() gmaps.create_index([('_id', 1)]) gmaps.create_index([('code', 1)]) gmaps.create_index([('name', 1)]) gmaps.create_index([('admin_hierarchy', 1)])
def insee_code_unit(id): config = Config('./config/config.yml') factory = DocFactory(config.get('mongodb')) collection = factory.insee_collection() obj = collection.find_one({'code': id}) return render_template('admin/other/unit.html', data=obj)
def matching_france_js(region): region = unquote_plus(region) #mode = request.args.get('mode', 'none') config = Config('./config/config.yml') factory = DocFactory(config.get('mongodb')) internal = factory.internal_collection() wiki = factory.wiki_collection() gmap = factory.gmaps_collection() insee = factory.insee_collection() objects = internal.find({ 'name': { '$exists': True, '$not': { '$size': 0 } }, '$and': [{ 'admin_hierarchy.ADMIN_LEVEL_1.name': 'France' }, { 'admin_hierarchy.ADMIN_LEVEL_2.name': region }], }) result = [] for item in objects: dic = {'internal': item} wiki_res = {} if item.get('source', {}).get('wiki'): wiki_res = wiki.find_one( {'code': item.get('source', {}).get('wiki')}) dic.update(wiki=wiki_res) gmap_res = {} if item.get('source', {}).get('gmap'): gmap_res = gmap.find_one( {'code': item.get('source', {}).get('gmap')}) dic.update(gmap=gmap_res) insee_res = {} if item.get('source', {}).get('insee'): insee_res = insee.find_one( {'code': item.get('source', {}).get('insee')}) dic.update(insee=insee_res) compare_res = {} compare_res.update({ 'insee_code!=wiki_code': 1 if not (insee_res.get('InseeXls_CodeCommune') == wiki_res.get('commune_codes')) else 0 }) compare_res.update({ 'insee_name!=wiki_name': 1 if not (insee_res.get('InseeXls_NameCommune') == wiki_res.get('name')) else 0 }) compare_res.update({ 'wiki_name!=gmaps_name': 1 if not (wiki_res.get('true_name', wiki_res.get('name')) == gmap_res.get('true_name', gmap_res.get('name'))) else 0 }) compare_res.update({ 'wiki_post!=gmaps_post': 1 if not (str(wiki_res.get('postal_codes')) == str( gmap_res.get('postal_code'))) else 0 }) compare_res.update({ 'wiki_admin!=gmaps_admin': 1 if not (str(wiki_res.get('admin_hierarchy')) == str( gmap_res.get('admin_hierarchy'))) else 0 }) try: max_meters_in_distance = 5000 compare_res.update({ 'wiki_posinion>gmaps_position': 1 if Comparison.by_distance(wiki_res.get('center'), gmap_res.get('center')) > max_meters_in_distance else 0 }) except: compare_res.update({'wiki_posinion>gmaps_position': 1}) dic.update(compare=compare_res) # if mode != 'none': # if mode == 'wiki_adapte': # if dic.get('wiki', {}).get('name', '').lower() != dic.get('insee', {}).get('name', '').lower(): # result.append(dic) # elif mode == 'gmap_adapte': # if dic.get('gmap', {}).get('name', '').lower() != dic.get('insee', {}).get('name', '').lower(): # result.append(dic) # else: # result.append(dic) result.append(dic) return render_template('admin/matching-france/list.js', e=escape, items=result)