def __init__(self, cache_db=None, logger=None): if cache_db: # TODO: use ignored_parameters="key" # TODO: read from providers their key name and add dynamically - don't miss a key... requests_cache.install_cache( cache_db, ignored_parameters=['key', 'apiKey', 'user_key'], allowable_methods=('GET', 'POST')) # Set up a new logger or append to existing one self._setup_logger(logger) # Log the python interpreter version self.logger.info( 'Initializing EnrichmentService (python version: %s.%s.%s, cache db: %s)', sys.version_info[0], sys.version_info[1], sys.version_info[2], str(cache_db)) # Set up the store and connect Store.connect() # Set up providers self.registered_providers = {} self.registered_providers["FullContact"] = FullContactEngager() self.registered_providers["Clearbit"] = ClearbitEngager() self.registered_providers["Pipl"] = PiplEngager() self.registered_providers["System"] = SystemEngager() self.registered_providers["Test"] = TestEngager() self.registered_providers["CrunchBase"] = CrunchBaseEngager() self.registered_providers["CrunchBaseBot"] = CrunchBaseBotEngager() self.registered_providers[ "CrunchBaseScraper"] = CrunchBaseScraperEngager() self.registered_providers[ "BloombergScraper"] = BloombergScraperEngager() self.registered_providers[ "AngelListScraper"] = AngelListScraperEngager() self.registered_providers["OpenCnam"] = OpenCnamEngager() # self.registered_providers["ZoomInfo"] = ZoomInfoEngager() self.registered_providers[ "StartupNationCentral"] = StartupNationCentralEngager() self.registered_providers["CircleBack"] = CircleBackEngager() self.registered_providers["Twitter"] = TwitterEngager() self.registered_providers["WhitePages"] = WhitePagesEngager() # Create providers map self._providers_symbols_map = {} for provider_name, provider in self.registered_providers.items(): self._providers_symbols_map[ provider.get_short_symbol()] = provider_name # Setup queues self.fc_companies_queue = QueueWrapper(DBWrapper.db(), "fc_companies", consumer_id="consumer-1", timeout=300, max_attempts=3) self.fc_people_queue = QueueWrapper(DBWrapper.db(), "fc_people", consumer_id="consumer-1", timeout=300, max_attempts=3) #queue.put({"name": "appilog", "requestor": "Israel David"}) #queue.put({"name": "VolcanicData", "requestor": "Uzy Hadad"}) #d1 = queue.next() #d2 = queue.next() pass
def __init__(self): self.logger = logging.getLogger(EnrichmentServiceConfig.LOGGER_NAME) # Make sure store is connected from store.store import Store Store.connect() pass