class GoogleMap: keys = {} clients = [] nyc = { 'new york county', 'bronx county', 'kings county', 'new york county', 'queens county', 'richmond county' } def __init__(self): self.mongo = MongoDict() self.logger = getLogger("GoogleMap") getLogger('requests.packages.urllib3.connectionpool').setLevel( logging.ERROR) file = os.path.join( os.path.dirname(os.path.abspath( inspect.getsourcefile(DummyClass))), 'api.key') # dynamically load key from api.key with open(file) as f: for line in f: [app_name, app_key] = line.strip().split(" ") self.logger.info("Found api key:" + app_key) self.keys[app_name] = app_key self.clients.append( Client(key=app_key, timeout=None, retry_timeout=40)) self.logger.info("Registered api key:" + app_key) def get_client(self) -> Client: return random.choice(self.clients) def check_if_nyc(self, address) -> str: if address is None: return "None" return str(address.lower() in self.nyc) def geocode(self, address): if [COLLECTION.GEO_CACHE, address] not in self.mongo: value = self.get_client().geocode(address) self.mongo.put(COLLECTION.GEO_CACHE, address, value) return value # self.logger.critical("cached") else: return self.mongo.get(COLLECTION.GEO_CACHE, address)
class WaybackTimePipeline(object): def __init__(self): self.mongo_dict = MongoDict() def process_item(self, item: TimeItem, spider): self.spider = spider self.add_item_to_db(item) return item def add_item_to_db(self, item: TimeItem) -> None: # self.mongo_dict.insert_item(COLLECTION.TEST, item) if [COLLECTION.OT_CATALOG, item['version_datetime']] not in self.mongo_dict: self.spider.logger.debug(item['version_datetime'] + " stored to DB") self.mongo_dict.put(COLLECTION.OT_CATALOG, item['version_datetime'], dict(item)) else: self.spider.logger.debug(item['version_datetime'] + " exist. Skipped") pass
from mongotable.mongo_dict import MongoDict, COLLECTION mh = MongoDict() test = COLLECTION.TEST print([test, "doesnotexist"] in mh) mh.put(test, "a", "value1") mh.put(test, "a", "value2") mh.put(test, "b", {"ad": 123}) mh.put(test, "list", [{"ad": 123}])