def main(): """ Parse arguments and call principal function print_links""" # Set description and version of the program parser = argparse.ArgumentParser( description="Let is craaaawl the Internet", version="%(prog)s 0.1", epilog="The Birds are coming!!! Close your windows.", ) # Parse the target url parser.add_argument("url", nargs="?", help="target URL") # Parse argument to show license parser.add_argument("-l", "--license", help="show license", action="store_true") # Parse argument to get the number of levels for scanning parser.add_argument("-n", "--number-of-levels", type=int, default=1, help="how deep the craaaawl will go") # Parse argument to set different bullet, default * parser.add_argument( "-b", "--bullet", default="*", help="Set different bullet character to print levels of scanning" ) # Parse argument to do verbose of the program parser.add_argument("--verbosity", help="increase output verbosity", action="store_true") args = parser.parse_args() # Print license if args.license: print st.WS_LICENSE sys.exit(0) # set verbosity value if args.verbosity: print "Info: Verbosity turned on" # Change default character for bullet if args.bullet: st.WS_BULLET = args.bullet[0] bullet = args.bullet # Limit of levels that will be explored depth = args.number_of_levels # Url to scan url = args.url if url == None: print "Warning: URL is required" print """usage: webspider.py [-h] [-v] [-l] [-n NUMBER_OF_LEVELS] [-b BULLET] [--verbosity] [url]""" sys.exit(0) if args.verbosity: print "Info: depth=%s." % depth print "Info: url=%s." % url print "Info: Bullet: %s." % st.WS_BULLET # Print all links in recursive mode pywebspider.print_links(url, depth, bullet)
def test_third_level (self): url = "http://herraiz.org" depth = 3 pywebspider.print_links(url, depth) output = len(sys.stdout.getvalue()) self.assertEqual(output, LENGTH_LEVEL3)