def check_mirror_url(mirror_url): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) log = MirrorLog(url=mirror_url, check_time=utc_now()) try: start = time.time() result = urllib2.urlopen(url, timeout=10) data = result.read() result.close() end = time.time() # lastsync should be an epoch value created by us parsed_time = None try: parsed_time = datetime.utcfromtimestamp(int(data)) parsed_time = parsed_time.replace(tzinfo=utc) except ValueError: # it is bad news to try logging the lastsync value; # sometimes we get a crazy-encoded web page. pass log.last_sync = parsed_time # if we couldn't parse a time, this is a failure if parsed_time is None: log.error = "Could not parse time from lastsync" log.is_success = False log.duration = end - start logger.debug("success: %s, %.2f", url, log.duration) except urllib2.HTTPError, e: if e.code == 404: # we have a duration, just not a success end = time.time() log.duration = end - start log.is_success = False log.error = str(e) logger.debug("failed: %s, %s", url, log.error)
def check_mirror_url(mirror_url): url = mirror_url.url + 'lastsync' logger.info("checking URL %s" % url) log = MirrorLog(url=mirror_url, check_time=datetime.utcnow()) try: start = time.time() result = urllib2.urlopen(url, timeout=10) data = result.read() result.close() end = time.time() # lastsync should be an epoch value, but some mirrors # are creating their own in RFC-3339 format: # '2010-09-02 11:05:06+02:00' parsed_time = None try: parsed_time = datetime.utcfromtimestamp(int(data)) except ValueError: # it is bad news to try logging the lastsync value; # sometimes we get a crazy-encoded web page. logger.info("attempting to parse generated lastsync file" " from mirror %s" % url) parsed_time = parse_rfc3339_datetime(data) log.last_sync = parsed_time # if we couldn't parse a time, this is a failure if parsed_time == None: log.error = "Could not parse time from lastsync" log.is_success = False log.duration = end - start logger.debug("success: %s, %.2f" % (url, log.duration)) except urllib2.HTTPError, e: if e.code == 404: # we have a duration, just not a success end = time.time() log.duration = end - start log.is_success = False log.error = str(e) logger.debug("failed: %s, %s" % (url, log.error))