def _ensure_size_in_range(feed, filedata, min_lines, max_lines):
num_lines = int(filedata['lines'])
if not (num_lines >= min_lines and num_lines <= max_lines):
post_alert(
"warning: most recent {}'s {} data feed file had {} lines, expected between {} and {}"
.format(feed['interval'], filedata['filetype'], num_lines,
min_lines, max_lines))
def handle_papertrail_alert(alert):
payload = json.loads(alert.values['payload'])
pp_alert = json.dumps(payload, indent=2)
logger.info(u'got this papertrail alert:\n{}'.format(pp_alert))
post_alert(pp_alert)
return alert
def _ensure_max_age(feed, filedata, max_age):
file_date = dateutil.parser.parse(filedata['last_modified'])
file_age = datetime.utcnow() - file_date
if file_age > max_age:
post_alert(
"warning: most recent {}'s {} data feed file was generated {}".
format(feed['interval'], filedata['filetype'], file_date))
def test_changefile_listing_endpoint(feed):
api_key = random.choice(valid_changefile_api_keys())
url = u'https://api.unpaywall.org/{}/changefiles?api_key={}'.format(feed['endpoint'], api_key)
start = time()
r = requests.get(url)
et = elapsed(start)
if et > 25:
post_alert(u'warning: changefile listing at {} took {} seconds'.format(url, et))
if r.status_code != 200:
post_alert(u'warning: HTTP status {} from {}'.format(r.status_code, url))
try:
file_listing = r.json()
logger.info(u'got response from {} in {} seconds: {} files listed'.format(
url,
et,
len(file_listing['list'])
))
except Exception as e:
post_alert(u'warning: changefile listing at {} not valid JSON ({}): {}'.format(url, e.message, r.content))