def _init_api_connector(self, start_time: datetime):
"""Set up uplink and send initial message.
Args:
start_time: monitoring start timestamp
"""
try:
configuration = load_api_configuration()
except RuntimeError as ex:
msg = 'Could not load API settings: {0}'.format(ex)
logging.error(msg)
self._should_close = True
self._close()
return
camera_pk = configuration.camera_pk
logging.info('Setting up uplink...')
self._api_connector = ApiConnector(
camera_pk, configuration, start_time,
)
def __init__(self, conf, specs):
#TODO: verify logger setup
#set up the logger
self.logger = logging.getLogger("fdp")
self.logger.setLevel(logging.DEBUG)
fh = logging.FileHandler('errors.log')
fh.setLevel(logging.DEBUG)
self.logger.addHandler(fh)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
self.logger.addHandler(ch)
# initialize variables and validate the configuration
self.config = conf
self.specs = specs
self.logger.debug("Validating the FDP configuration.")
# TODO self.validate_configuration(conf, specs)
self.logger.debug("Initializing the input API.")
self.radar_api = ApiConnector(host=conf['api_host'],
user=conf['api_user'],
password=conf['api_password'])
curr_titles = TitleSaver.read_title_file(startfile)
curr_titles -= all_discards
curr_titles -= all_redirects
all_pending = utils.limit_titles(all_titles, curr_titles, tot_num_titles)
if len(curr_titles) == 0:
sys.exit("No new titles to crawl. Quitting...")
# Process a batch, use links from the batch in a future batch, ...
all_content = []
while curr_level <= args['levels'] and len(curr_titles) > 0 and len(all_content) < args['maxpages']:
if not args['seed']: print("Level {} >>>".format(curr_level))
print("Processing batch of {} {} titles...".format(len(curr_titles), context))
areader = ArticleReader(transcludes=cfg['transcludes'], restricted=args['restricted'])
bproc = BatchProcessor(ApiConnector(**cfg['api']).func, 1, areader, seed=args['seed'])
articles = bproc.batch_call_api(curr_titles)
print("Reading {} articles...".format(len(articles)))
contents, next_titles, trans_titles = bproc.read_articles(articles)
# Don't add duplicates
uniq_contents = []
for content in contents:
currid = str(content['pageid'])
if currid not in all_ids:
uniq_contents.append(content)
all_ids.add(currid)
if cfg['transcludes']['add_to_curr_level']:
# Adding to current level is aggressive