def by_place_id(list_places, city_id): config = Config('./config/config.yml') Key = Keygen() keyAPI = Key.get_key_geocode() if not keyAPI: sys.exit() cnf = {'googlemaps': {'geocoding': {'key': keyAPI}}} config.set(cnf) language = 'uk' doc_factory = DocFactory(config.get('mongodb')) spider = Spider(loader_factory=LoaderFactory, gmap_parser=MapFactory.spain, doc_factory=doc_factory, language=language, config=config, use_cache=True) for loc in list_places: objects = spider.get_gmap_place_id(loc) gmap = {} try: gmap = objects[0].get_document() gmap["city_id"] = city_id exixts = db.ukraine_city_sublocal.find({ "city_id": city_id, "code": gmap['code'] }).count() if exixts < 1: gmap.pop('_id', None) db.ukraine_city_sublocal.save(gmap) print(gmap) except Exception as e: print(str(e))
from argparse import ArgumentParser import sys import json import requests import pymongo from bson.json_util import dumps from lib.keygen.gmap_keygen import Keygen # from lib.parser.wiki.Spain import Spain as ParserSpain country = 'Spain' config = Config('./config/config.yml') mongo_config = config.get('mongodb') conn = pymongo.MongoClient(mongo_config['host'], mongo_config['port']) Key = Keygen() cnf = {'geocoding': {'key': Key.get_key_geocode()}} config.set(cnf) db = conn.location coll = db.sinoplik_romania # print(config.get('googlemaps').get('geocoding').get('key')) doc_factory = DocFactory(config.get('mongodb')) # try: # skiprows = sys.argv[1] # except Exception as e: # skiprows = 0 # df = pd.read_csv('./data/spain/Spain_notDublicate.csv', skiprows=int(skiprows), low_memory=False) loader = Loader.loader_with_mongodb(config.get('mongodb')) headers = {'User-Agent': 'Mozilla/5.0'} language = 'ro' # spider = Spider( # loader_factory=LoaderFactory,