def test_create_base_link(filename, result, deed_result, rdf_result): args = link_checker.parse_argument([]) baseURL = create_base_link(args, filename) assert baseURL == result baseURL = create_base_link(args, filename, for_deeds=True) assert baseURL == deed_result baseURL = create_base_link(args, filename, for_rdfs=True) assert baseURL == rdf_result
def test_output_write(tmpdir): # output_errors is set and written to output_file = tmpdir.join("errorlog.txt") args = link_checker.parse_argument( ["--output-errors", output_file.strpath]) output_write(args, "Output enabled") args.output_errors.flush() assert output_file.read() == "Output enabled\n"
def test_write_response(tmpdir): # Set config output_file = tmpdir.join("errorlog.txt") args = link_checker.parse_argument( ["--output-errors", output_file.strpath]) # Text to extract valid_anchors text = ("<a href='http://httpbin.org/status/200'>Response 200</a>," " <a href='file://link3'>Invalid Scheme</a>," " <a href='http://httpbin.org/status/400'>Response 400</a>") soup = BeautifulSoup(text, "lxml") valid_anchors = soup.find_all("a") # Setup function params all_links = [ "http://httpbin.org/status/200", "file://link3", "http://httpbin.org/status/400", ] rs = (grequests.get(link) for link in all_links) response = grequests.map(rs, exception_handler=exception_handler) base_url = "https://baseurl/goes/here" license_name = "by-cc-nd_2.0" # Set output to external file caught_errors = write_response( args, all_links, response, base_url, license_name, valid_anchors, license_name, False, ) assert caught_errors == 2 args.output_errors.flush() lines = output_file.readlines() i = 0 assert lines[i] == "\n" i += 1 assert lines[i] == "by-cc-nd_2.0\n" i += 1 assert lines[i] == "URL: https://baseurl/goes/here\n" i += 1 assert lines[i] == f' {"Invalid Schema":<24}file://link3\n' i += 1 assert lines[i] == f'{"":<26}<a href="file://link3">Invalid Scheme</a>\n' i += 1 assert lines[i] == f' {"400":<24}http://httpbin.org/status/400\n' i += 1 assert lines[i] == ( f'{"":<26}<a href="http://httpbin.org/status/400">Response 400</a>\n')
def test_output_summary(reset_global, tmpdir): # output_errors is set and written to output_file = tmpdir.join("errorlog.txt") args = link_checker.parse_argument( ["--output-errors", output_file.strpath]) utils.MAP_BROKEN_LINKS = { "https://link1.demo": [ "https://file1.url/here", "https://file2.url/goes/here", ], "https://link2.demo": ["https://file4.url/here"], } all_links = ["some link"] * 5 output_summary(args, all_links, 3) args.output_errors.flush() lines = output_file.readlines() i = 0 assert lines[i] == "\n" i += 1 assert lines[i] == "\n" i += 1 assert lines[i] == "***************************************\n" i += 1 assert lines[i] == " SUMMARY\n" i += 1 assert lines[i] == "***************************************\n" i += 1 assert lines[i] == "\n" i += 1 assert str(lines[i]).startswith("Timestamp:") i += 1 assert lines[i] == "Total files checked: 5\n" i += 1 assert lines[i] == "Number of error links: 3\n" i += 1 assert lines[i] == "Number of unique broken links: 2\n" i += 1 assert lines[i] == "\n" i += 1 assert lines[i] == "\n" i += 1 assert lines[i] == "Broken link - https://link1.demo found in:\n" i += 1 assert lines[i] == "https://file1.url/here\n" i += 1 assert lines[i] == "https://file2.url/goes/here\n" i += 1 assert lines[i] == "\n" i += 1 assert lines[i] == "Broken link - https://link2.demo found in:\n" i += 1 assert lines[i] == "https://file4.url/here\n"
def test_parse_argument(tmpdir): # Test default options args = link_checker.parse_argument([]) assert args.log_level == 30 assert bool(args.output_errors) is False assert args.local is False assert args.root_url == "https://creativecommons.org" # Test --licenses args = link_checker.parse_argument(["--legalcode"]) assert args.legalcode is True args = link_checker.parse_argument(["legalcode"]) assert args.func.__name__ == "check_legalcode" args = link_checker.parse_argument(["legalcode", "--local"]) assert args.local is True # Test --deeds args = link_checker.parse_argument(["--deeds"]) assert args.deeds is True args = link_checker.parse_argument(["deeds"]) assert args.func.__name__ == "check_deeds" args = link_checker.parse_argument(["deeds", "--local"]) assert args.local is True # Test --rdf args = link_checker.parse_argument(["--rdf"]) assert args.rdf is True args = link_checker.parse_argument(["rdf"]) assert args.func.__name__ == "check_rdfs" args = link_checker.parse_argument(["rdf", "--index"]) assert args.index is True args = link_checker.parse_argument(["rdf", "--local"]) assert args.local is True # Test --index args = link_checker.parse_argument(["--index"]) assert args.index is True # Test --local args = link_checker.parse_argument(["--local"]) assert args.local is True # Test Logging Levels -q/--quiet args = link_checker.parse_argument(["-q"]) assert args.log_level == 40 args = link_checker.parse_argument(["-qq"]) assert args.log_level == 50 args = link_checker.parse_argument(["-qqq"]) assert args.log_level == 50 args = link_checker.parse_argument(["-q", "--quiet"]) assert args.log_level == 50 # Test Logging Levels -v/--verbose args = link_checker.parse_argument(["-v"]) assert args.log_level == 20 args = link_checker.parse_argument(["-vv"]) assert args.log_level == 10 args = link_checker.parse_argument(["-vvv"]) assert args.log_level == 10 args = link_checker.parse_argument(["-v", "--verbose"]) assert args.log_level == 10 # Test Logging Levels with both -v and -q args = link_checker.parse_argument(["-vq"]) assert args.log_level == 30 args = link_checker.parse_argument(["-vvq"]) assert args.log_level == 20 args = link_checker.parse_argument(["-vqq"]) assert args.log_level == 40 # Test default value of --output-errors args = link_checker.parse_argument(["--output-errors"]) assert bool(args.output_errors) is True assert args.output_errors.name == "errorlog.txt" # Test custom value of --output-errors output_file = tmpdir.join("errorlog.txt") args = link_checker.parse_argument( ["--output-errors", output_file.strpath]) assert bool(args.output_errors) is True assert args.output_errors.name == output_file.strpath
def test_get_scrapable_links(): args = link_checker.parse_argument([]) test_file = ("<a name='hello'>without href</a>," " <a href='#hello'>internal link</a>," " <a href='mailto:[email protected]'>mailto protocol</a>," " <a href='https://creativecommons.ca'>Absolute link</a>," " <a href='/index'>Relative Link</a>") soup = BeautifulSoup(test_file, "lxml") test_case = soup.find_all("a") base_url = "https://www.demourl.com/dir1/dir2" valid_anchors, valid_links, _ = get_scrapable_links( args, base_url, test_case, None, False) assert str(valid_anchors) == ( '[<a href="https://creativecommons.ca">Absolute link</a>,' ' <a href="/index">Relative Link</a>]') assert (str(valid_links) == "['https://creativecommons.ca', 'https://www.demourl.com/index']") # Testing RDF args = link_checker.parse_argument(["--local"]) rdf_obj_list = get_index_rdf(args, local_path=constants.TEST_RDF_LOCAL_PATH) rdf_obj = rdf_obj_list[0] base_url = rdf_obj["rdf:about"] links_found = get_links_from_rdf(rdf_obj) valid_anchors, valid_links, _ = get_scrapable_links( args, base_url, links_found, None, False, rdf=True, ) assert str(valid_anchors) == ( "[<cc:permits " 'rdf:resource="http://creativecommons.org/ns#DerivativeWorks"/>, ' "<cc:permits " 'rdf:resource="http://creativecommons.org/ns#Reproduction"/>, ' "<cc:permits " 'rdf:resource="http://creativecommons.org/ns#Distribution"/>, ' "<cc:jurisdiction " 'rdf:resource="http://creativecommons.org/international/ch/"/>, ' "<foaf:logo " 'rdf:resource="https://i.creativecommons.org/' 'l/by-nc-sa/2.5/ch/88x31.png"/>, ' "<foaf:logo " 'rdf:resource="https://i.creativecommons.org/' 'l/by-nc-sa/2.5/ch/80x15.png"/>, ' "<cc:legalcode " 'rdf:resource="http://creativecommons.org/' 'licenses/by-nc-sa/2.5/ch/legalcode.de"/>, ' "<dc:source " 'rdf:resource="http://creativecommons.org/licenses/by-nc-sa/2.5/"/>, ' "<dc:creator " 'rdf:resource="http://creativecommons.org"/>, ' "<cc:prohibits " 'rdf:resource="http://creativecommons.org/ns#CommercialUse"/>, ' "<cc:licenseClass " 'rdf:resource="http://creativecommons.org/license/"/>, ' "<cc:requires " 'rdf:resource="http://creativecommons.org/ns#ShareAlike"/>, ' "<cc:requires " 'rdf:resource="http://creativecommons.org/ns#Attribution"/>, ' "<cc:requires " 'rdf:resource="http://creativecommons.org/ns#Notice"/>]') assert str(valid_links) == ( "['http://creativecommons.org/ns#DerivativeWorks', " "'http://creativecommons.org/ns#Reproduction', " "'http://creativecommons.org/ns#Distribution', " "'http://creativecommons.org/international/ch/', " "'https://i.creativecommons.org/l/by-nc-sa/2.5/ch/88x31.png', " "'https://i.creativecommons.org/l/by-nc-sa/2.5/ch/80x15.png', " "'http://creativecommons.org/licenses/by-nc-sa/2.5/ch/legalcode.de', " "'http://creativecommons.org/licenses/by-nc-sa/2.5/', " "'http://creativecommons.org', " "'http://creativecommons.org/ns#CommercialUse', " "'http://creativecommons.org/license/', " "'http://creativecommons.org/ns#ShareAlike', " "'http://creativecommons.org/ns#Attribution', " "'http://creativecommons.org/ns#Notice']")