def check_rsync_url(mirror_url, location, timeout):
url = mirror_url.url + 'lastsync'
logger.info("checking URL %s", url)
log = MirrorLog(url=mirror_url, check_time=now(), location=location)
tempdir = tempfile.mkdtemp()
ipopt = ''
if location:
if location.family == socket.AF_INET6:
ipopt = '--ipv6'
elif location.family == socket.AF_INET:
ipopt = '--ipv4'
lastsync_path = os.path.join(tempdir, 'lastsync')
rsync_cmd = ["rsync", "--quiet", "--contimeout=%d" % timeout,
"--timeout=%d" % timeout]
if ipopt:
rsync_cmd.append(ipopt)
rsync_cmd.append(url)
rsync_cmd.append(lastsync_path)
try:
with open(os.devnull, 'w') as devnull:
if logger.isEnabledFor(logging.DEBUG):
logger.debug("rsync cmd: %s", ' '.join(rsync_cmd))
start = time.time()
proc = subprocess.Popen(rsync_cmd, stdout=devnull,
stderr=subprocess.PIPE)
_, errdata = proc.communicate()
end = time.time()
log.duration = end - start
if proc.returncode != 0:
logger.debug("error: %s, %s", url, errdata)
log.is_success = False
log.error = errdata.strip().decode('utf-8')
# look at rsync error code- if we had a command error or timed out,
# don't record a duration as it is misleading
if proc.returncode in (1, 30, 35):
log.duration = None
else:
logger.debug("success: %s, %.2f", url, log.duration)
if os.path.exists(lastsync_path):
with open(lastsync_path, 'r') as lastsync:
parse_lastsync(log, lastsync.read())
else:
parse_lastsync(log, None)
finally:
if os.path.exists(lastsync_path):
os.unlink(lastsync_path)
os.rmdir(tempdir)
return log
def check_mirror_url(mirror_url):
url = mirror_url.url + 'lastsync'
logger.info("checking URL %s", url)
log = MirrorLog(url=mirror_url, check_time=utc_now())
try:
start = time.time()
result = urllib2.urlopen(url, timeout=10)
data = result.read()
result.close()
end = time.time()
# lastsync should be an epoch value created by us
parsed_time = None
try:
parsed_time = datetime.utcfromtimestamp(int(data))
parsed_time = parsed_time.replace(tzinfo=utc)
except ValueError:
# it is bad news to try logging the lastsync value;
# sometimes we get a crazy-encoded web page.
pass
log.last_sync = parsed_time
# if we couldn't parse a time, this is a failure
if parsed_time is None:
log.error = "Could not parse time from lastsync"
log.is_success = False
log.duration = end - start
logger.debug("success: %s, %.2f", url, log.duration)
except urllib2.HTTPError, e:
if e.code == 404:
# we have a duration, just not a success
end = time.time()
log.duration = end - start
log.is_success = False
log.error = str(e)
logger.debug("failed: %s, %s", url, log.error)
def check_rsync_url(mirror_url, timeout):
url = mirror_url.url + 'lastsync'
logger.info("checking URL %s", url)
log = MirrorLog(url=mirror_url, check_time=now())
tempdir = tempfile.mkdtemp()
lastsync_path = os.path.join(tempdir, 'lastsync')
rsync_cmd = ["rsync", "--quiet", "--contimeout=%d" % timeout,
"--timeout=%d" % timeout, url, lastsync_path]
try:
with open(os.devnull, 'w') as devnull:
proc = subprocess.Popen(rsync_cmd, stdout=devnull,
stderr=subprocess.PIPE)
start = time.time()
_, errdata = proc.communicate()
end = time.time()
log.duration = end - start
if proc.returncode != 0:
logger.debug("error: %s, %s", url, errdata)
log.is_success = False
log.error = errdata.strip()
# look at rsync error code- if we had a command error or timed out,
# don't record a duration as it is misleading
if proc.returncode in (1, 30, 35):
log.duration = None
else:
logger.debug("success: %s, %.2f", url, log.duration)
with open(lastsync_path, 'r') as lastsync:
parse_lastsync(log, lastsync.read())
finally:
if os.path.exists(lastsync_path):
os.unlink(lastsync_path)
os.rmdir(tempdir)
return log
def check_rsync_url(mirror_url, location, timeout):
url = mirror_url.url + 'lastsync'
logger.info("checking URL %s", url)
log = MirrorLog(url=mirror_url, check_time=now(), location=location)
tempdir = tempfile.mkdtemp()
ipopt = ''
if location:
if location.family == socket.AF_INET6:
ipopt = '--ipv6'
elif location.family == socket.AF_INET:
ipopt = '--ipv4'
lastsync_path = os.path.join(tempdir, 'lastsync')
rsync_cmd = [
"rsync", "--quiet",
"--contimeout=%d" % timeout,
"--timeout=%d" % timeout
]
if ipopt:
rsync_cmd.append(ipopt)
rsync_cmd.append(url)
rsync_cmd.append(lastsync_path)
try:
with open(os.devnull, 'w') as devnull:
if logger.isEnabledFor(logging.DEBUG):
logger.debug("rsync cmd: %s", ' '.join(rsync_cmd))
start = time.time()
proc = subprocess.Popen(rsync_cmd,
stdout=devnull,
stderr=subprocess.PIPE)
_, errdata = proc.communicate()
end = time.time()
log.duration = end - start
if proc.returncode != 0:
logger.debug("error: %s, %s", url, errdata)
log.is_success = False
log.error = errdata.strip()
# look at rsync error code- if we had a command error or timed out,
# don't record a duration as it is misleading
if proc.returncode in (1, 30, 35):
log.duration = None
else:
logger.debug("success: %s, %.2f", url, log.duration)
if os.path.exists(lastsync_path):
with open(lastsync_path, 'r') as lastsync:
parse_lastsync(log, lastsync.read())
else:
parse_lastsync(log, None)
finally:
if os.path.exists(lastsync_path):
os.unlink(lastsync_path)
os.rmdir(tempdir)
return log
def check_mirror_url(mirror_url):
url = mirror_url.url + 'lastsync'
logger.info("checking URL %s" % url)
log = MirrorLog(url=mirror_url, check_time=datetime.utcnow())
try:
start = time.time()
result = urllib2.urlopen(url, timeout=10)
data = result.read()
result.close()
end = time.time()
# lastsync should be an epoch value, but some mirrors
# are creating their own in RFC-3339 format:
# '2010-09-02 11:05:06+02:00'
parsed_time = None
try:
parsed_time = datetime.utcfromtimestamp(int(data))
except ValueError:
# it is bad news to try logging the lastsync value;
# sometimes we get a crazy-encoded web page.
logger.info("attempting to parse generated lastsync file"
" from mirror %s" % url)
parsed_time = parse_rfc3339_datetime(data)
log.last_sync = parsed_time
# if we couldn't parse a time, this is a failure
if parsed_time == None:
log.error = "Could not parse time from lastsync"
log.is_success = False
log.duration = end - start
logger.debug("success: %s, %.2f" % (url, log.duration))
except urllib2.HTTPError, e:
if e.code == 404:
# we have a duration, just not a success
end = time.time()
log.duration = end - start
log.is_success = False
log.error = str(e)
logger.debug("failed: %s, %s" % (url, log.error))
def check_mirror_url(mirror_url, location, timeout):
url = mirror_url.url + 'lastsync'
logger.info("checking URL %s", url)
log = MirrorLog(url=mirror_url, check_time=now(), location=location)
headers = {'User-Agent': 'archweb/1.0'}
req = urllib2.Request(url, None, headers)
start = time.time()
try:
result = urllib2.urlopen(req, timeout=timeout)
data = result.read()
result.close()
end = time.time()
parse_lastsync(log, data)
log.duration = end - start
logger.debug("success: %s, %.2f", url, log.duration)
except urllib2.HTTPError as e:
if e.code == 404:
# we have a duration, just not a success
end = time.time()
log.duration = end - start
log.is_success = False
log.error = str(e)
logger.debug("failed: %s, %s", url, log.error)
except urllib2.URLError as e:
log.is_success = False
log.error = e.reason
if isinstance(e.reason, types.StringTypes) and \
re.search(r'550.*No such file', e.reason):
# similar to 404 case above, still record duration
end = time.time()
log.duration = end - start
if isinstance(e.reason, socket.timeout):
log.error = "Connection timed out."
elif isinstance(e.reason, socket.error):
log.error = e.reason.args[-1]
logger.debug("failed: %s, %s", url, log.error)
except HTTPException:
# e.g., BadStatusLine
log.is_success = False
log.error = "Exception in processing HTTP request."
logger.debug("failed: %s, %s", url, log.error)
except ssl.CertificateError as e:
log.is_success = False
log.error = str(e)
logger.debug("failed: %s, %s", url, log.error)
except socket.timeout:
log.is_success = False
log.error = "Connection timed out."
logger.debug("failed: %s, %s", url, log.error)
except socket.error as e:
log.is_success = False
log.error = str(e)
logger.debug("failed: %s, %s", url, log.error)
return log
def check_mirror_url(mirror_url, location, timeout):
url = mirror_url.url + 'lastsync'
logger.info("checking URL %s", url)
log = MirrorLog(url=mirror_url, check_time=now(), location=location)
headers = {'User-Agent': 'archweb/1.0'}
req = urllib.request.Request(url, None, headers)
start = time.time()
try:
result = urllib.request.urlopen(req, timeout=timeout)
data = result.read()
result.close()
end = time.time()
parse_lastsync(log, data)
log.duration = end - start
logger.debug("success: %s, %.2f", url, log.duration)
except urllib.error.HTTPError as e:
if e.code == 404:
# we have a duration, just not a success
end = time.time()
log.duration = end - start
log.is_success = False
log.error = str(e)
logger.debug("failed: %s, %s", url, log.error)
except urllib.error.URLError as e:
log.is_success = False
log.error = e.reason
if isinstance(e.reason, str) and \
re.search(r'550.*No such file', e.reason):
# similar to 404 case above, still record duration
end = time.time()
log.duration = end - start
if isinstance(e.reason, socket.timeout):
log.error = "Connection timed out."
elif isinstance(e.reason, socket.error):
log.error = e.reason.args[-1]
logger.debug("failed: %s, %s", url, log.error)
except HTTPException:
# e.g., BadStatusLine
log.is_success = False
log.error = "Exception in processing HTTP request."
logger.debug("failed: %s, %s", url, log.error)
except ssl.CertificateError as e:
log.is_success = False
log.error = str(e)
logger.debug("failed: %s, %s", url, log.error)
except socket.timeout:
log.is_success = False
log.error = "Connection timed out."
logger.debug("failed: %s, %s", url, log.error)
except socket.error as e:
log.is_success = False
log.error = str(e)
logger.debug("failed: %s, %s", url, log.error)
return log