class GoogleMap:
keys = {}
clients = []
nyc = {
'new york county', 'bronx county', 'kings county', 'new york county',
'queens county', 'richmond county'
}
def __init__(self):
self.mongo = MongoDict()
self.logger = getLogger("GoogleMap")
getLogger('requests.packages.urllib3.connectionpool').setLevel(
logging.ERROR)
file = os.path.join(
os.path.dirname(os.path.abspath(
inspect.getsourcefile(DummyClass))), 'api.key')
# dynamically load key from api.key
with open(file) as f:
for line in f:
[app_name, app_key] = line.strip().split(" ")
self.logger.info("Found api key:" + app_key)
self.keys[app_name] = app_key
self.clients.append(
Client(key=app_key, timeout=None, retry_timeout=40))
self.logger.info("Registered api key:" + app_key)
def get_client(self) -> Client:
return random.choice(self.clients)
def check_if_nyc(self, address) -> str:
if address is None:
return "None"
return str(address.lower() in self.nyc)
def geocode(self, address):
if [COLLECTION.GEO_CACHE, address] not in self.mongo:
value = self.get_client().geocode(address)
self.mongo.put(COLLECTION.GEO_CACHE, address, value)
return value
# self.logger.critical("cached")
else:
return self.mongo.get(COLLECTION.GEO_CACHE, address)
class WaybackTimePipeline(object):
def __init__(self):
self.mongo_dict = MongoDict()
def process_item(self, item: TimeItem, spider):
self.spider = spider
self.add_item_to_db(item)
return item
def add_item_to_db(self, item: TimeItem) -> None:
# self.mongo_dict.insert_item(COLLECTION.TEST, item)
if [COLLECTION.OT_CATALOG,
item['version_datetime']] not in self.mongo_dict:
self.spider.logger.debug(item['version_datetime'] +
" stored to DB")
self.mongo_dict.put(COLLECTION.OT_CATALOG,
item['version_datetime'], dict(item))
else:
self.spider.logger.debug(item['version_datetime'] +
" exist. Skipped")
pass
from mongotable.mongo_dict import MongoDict, COLLECTION
mh = MongoDict()
test = COLLECTION.TEST
print([test, "doesnotexist"] in mh)
mh.put(test, "a", "value1")
mh.put(test, "a", "value2")
mh.put(test, "b", {"ad": 123})
mh.put(test, "list", [{"ad": 123}])