def main():
""" Parse arguments and call principal function print_links"""
# Set description and version of the program
parser = argparse.ArgumentParser(
description="Let is craaaawl the Internet",
version="%(prog)s 0.1",
epilog="The Birds are coming!!! Close your windows.",
)
# Parse the target url
parser.add_argument("url", nargs="?", help="target URL")
# Parse argument to show license
parser.add_argument("-l", "--license", help="show license", action="store_true")
# Parse argument to get the number of levels for scanning
parser.add_argument("-n", "--number-of-levels", type=int, default=1, help="how deep the craaaawl will go")
# Parse argument to set different bullet, default *
parser.add_argument(
"-b", "--bullet", default="*", help="Set different bullet character to print levels of scanning"
)
# Parse argument to do verbose of the program
parser.add_argument("--verbosity", help="increase output verbosity", action="store_true")
args = parser.parse_args()
# Print license
if args.license:
print st.WS_LICENSE
sys.exit(0)
# set verbosity value
if args.verbosity:
print "Info: Verbosity turned on"
# Change default character for bullet
if args.bullet:
st.WS_BULLET = args.bullet[0]
bullet = args.bullet
# Limit of levels that will be explored
depth = args.number_of_levels
# Url to scan
url = args.url
if url == None:
print "Warning: URL is required"
print """usage: webspider.py [-h] [-v] [-l] [-n NUMBER_OF_LEVELS] [-b BULLET]
[--verbosity]
[url]"""
sys.exit(0)
if args.verbosity:
print "Info: depth=%s." % depth
print "Info: url=%s." % url
print "Info: Bullet: %s." % st.WS_BULLET
# Print all links in recursive mode
pywebspider.print_links(url, depth, bullet)
def test_third_level (self):
url = "http://herraiz.org"
depth = 3
pywebspider.print_links(url, depth)
output = len(sys.stdout.getvalue())
self.assertEqual(output, LENGTH_LEVEL3)
