def __call__(*args, **kwargs):
"""returns a newly created connection, which is not maintained by the _pool"""
connection = psycopg2.connect(
*args,
**parse(DATABASE_CONFIG_PATH,
'postgresql',
unwanted_fields=["minconn", "maxconn"]), **kwargs)
Connection.get_connection_status(connection)
return connection
def __init__(self):
super().__init__()
self.wait_time = 1
# this is not balanced, rather just parsing from config directly
self.api = parse(TWITTER_API_CONFIG_PATH)["twitter-covid-19-API"]
self.data: List = []
self.keywords = []
self.total_crawled_count = 0
def __init__(self,
extractor: ExtractorBase = None,
dumper: DumperBase = None):
super().__init__(extractor, dumper)
self.api = twitter.Api(**parse(TWITTER_API_CONFIG_PATH, 'twitter-API'))
self.data = []
self.keywords = []
self.total_crawled_count = 0
self.crawled_id_set: Set[int] = set()
self.data_from_db_count = 0
class TwitterAPILoadBalancer:
iter_index = 0
apis = [twitter.Api(**config, sleep_on_rate_limit=True) for config in
parse(TWITTER_API_CONFIG_PATH).values() if config.get('access_token_key')]
lock = Lock()
@staticmethod
def get():
TwitterAPILoadBalancer.lock.acquire()
TwitterAPILoadBalancer.iter_index += 1
if TwitterAPILoadBalancer.iter_index == len(TwitterAPILoadBalancer.apis):
TwitterAPILoadBalancer.iter_index = 0
api = TwitterAPILoadBalancer.apis[TwitterAPILoadBalancer.iter_index]
TwitterAPILoadBalancer.lock.release()
return api
def config():
return parse(DATABASE_CONFIG_PATH, 'postgresql')