def get_available_days(url, session=None, auth=None):
soup = get_page_soup_with_checks(url, session, auth)
if not soup:
log.debug("Failed to open days pages {}".format(url))
return
links = soup.find_all("a")
day_links = []
for link in links:
if link.text.endswith("UTC/"):
day_links.append((link.text, url + link.get("href")))
return day_links
def get_available_locations(url, session=None, auth=None):
soup = get_page_soup_with_checks(url, session, auth)
if not soup:
log.debug("Failed to open equinix-location pages {}".format(url))
return
links = soup.find_all("a")
link_locations = []
for link_location in links:
if "equinix" in link_location.text:
link_locations.append(
(link_location.text, url + "/" + link_location.get("href")))
return link_locations
def get_page_soup_with_checks(url, session=None, auth=None):
current_retry = 0
soup = get_page_soup(url, session, auth)
if is_authorization_required(soup):
log.debug("Unauthorized access to {}".format(url))
return
while is_service_unavailable(soup):
current_retry += 1
time.sleep(current_retry * 0.5)
soup = get_page_soup(url, session, auth)
if current_retry == MAX_RETRY:
log.error("Max retries trying to get {}".format(url))
return
return soup
def get_available_links(url, session=None, auth=None):
soup = get_page_soup_with_checks(url, session, auth)
if not soup:
log.debug("Failed to open traces/timestamps pages {}".format(url))
return
links = soup.find_all("a")
download_links = {"pcaps": [], "timestamps": [], "stats": []}
for link in links:
if link.text.endswith("pcap.gz"):
download_links["pcaps"].append((link.text, url + link.get("href")))
elif link.text.endswith("times.gz"):
download_links["timestamps"].append(
(link.text, url + link.get("href")))
elif link.text.endswith("pcap.stats"):
download_links["stats"].append((link.text, url + link.get("href")))
return download_links
def run_check(cmd, shell=True):
log.debug(cmd)
return subprocess.check_output(cmd, shell=shell)
def run_cmd(cmd, shell=True):
log.debug(cmd)
subprocess.call(cmd, shell=shell)