def __init__(self, config: MutableMapping) -> None:
self.config = config
self.dataset_conf_id, self.target_ds_structure, self.ds_meta = None, None, None
if self.config.experiment.db_functionality_enabled:
self.cnxp = get_cnxp_handle()
if not self.config.data_source.skip_db_refresh:
refresh_db(self.config.data_source.db_conf, self.cnxp)
self.device = utils.core_utils.device_config(self.config)
self.dataset_conf = self.init_dataset_conf()
self.base_falsehood_embed_cache = Path(
f'{self.config.experiment.dirs.model_cache_dir}/'
f'{constants.BASE_FALSEHOOD_EMBED_CACHE_NAME}')
self.base_truth_embed_cache = Path(
f'{self.config.experiment.dirs.model_cache_dir}/'
f'{constants.BASE_TRUTH_EMBED_CACHE_NAME}')
self.falsehood_embed_cache = Path(
f'{self.config.experiment.dirs.model_cache_dir}/'
f'{constants.FALSEHOOD_EMBED_CACHE_NAME}')
self.truth_embed_cache = Path(
f'{self.config.experiment.dirs.model_cache_dir}/'
f'{constants.TRUTH_EMBED_CACHE_NAME}')
self.file_suffix = self.config.data_source.file_suffix
if self.config.data_source.update_ds_db_metadata_only and self.config.experiment.db_functionality_enabled:
self.update_ds_db_metadata()
else:
# build dataset from DB if a cached file version doesn't already exist
self.load_datasets()
def batch_inference(self) -> None:
if not self.config.experiment.infsvc.skip_db_refresh:
refresh_db(self.config.data_source.db_conf,
self.cnxp,
self.infsvc_dbconf,
self.svc_auth['twitter'],
batch_infsvc=True)
self.publish_flow()
def update_non_twit_sources(self):
refresh_db(self.config.data_source.db_conf,
self.cnxp,
self.infsvc_dbconf,
api_handle=self.svc_auth['twitter'],
nontwtr_update=True)
update_nt = False
last_nt_update_cnt = 0
return update_nt, last_nt_update_cnt
def update_non_twit_sources(self):
refresh_db(self.config.data_source.db_conf,
self.cnxp,
self.tweetbot_dbconf,
api_handle=self.api,
nontwtr_update=True)
update_nt = False
last_nt_update_cnt = 0
self.maybe_publish('stmts')
return update_nt, last_nt_update_cnt
def poll_and_analyze(self) -> NoReturn:
last_nt_update_cnt = 0
update_nt = True
while True:
self.verify_date()
self.purge_old_reports()
if self.non_twitter_updatefreq and update_nt:
update_nt, last_nt_update_cnt = self.update_non_twit_sources()
refresh_db(self.config.data_source.db_conf, self.cnxp,
self.tweetbot_dbconf, self.api)
self.maybe_publish('tweets')
time.sleep(self.config.experiment.tweetbot.dcbot_poll_interval)
if self.non_twitter_updatefreq:
last_nt_update_cnt += 1
if last_nt_update_cnt >= self.non_twitter_updatefreq:
update_nt = True
def poll_and_analyze(self) -> NoReturn:
last_nt_update_cnt = 0
update_nt = True
twitter_enabled = self.config.experiment.infsvc.twitter_enabled
while True:
if self.non_twitter_updatefreq and update_nt:
update_nt, last_nt_update_cnt = self.update_non_twit_sources()
if twitter_enabled:
refresh_db(self.config.data_source.db_conf, self.cnxp,
self.infsvc_dbconf, self.svc_auth['twitter'])
self.publish_flow()
time.sleep(self.config.experiment.infsvc.dcbot_poll_interval)
if self.non_twitter_updatefreq:
last_nt_update_cnt += 1
if last_nt_update_cnt >= self.non_twitter_updatefreq:
update_nt = True