def main():
global kill_me
print "starting"
home = parse_args()
backtime = -1
signal.signal(signal.SIGTERM, dead)
image_dir = imaget.create_image_dir("images")
##initialize selenium
browser = init_browser()
##setup mysql db
con, cur = dblib.setup_db()
kill_me = browser
try:
scrape_thread(kill_me, home, con, cur)
except:
# pdb.post_mortem()
dead()
raise
print "done"
"""
def main():
global kill_me
print "starting"
home = parse_args()
backtime = -1
signal.signal(signal.SIGTERM, dead)
image_dir = imaget.create_image_dir("images")
##initialize selenium
browser = init_browser()
##setup mysql db
con, cur = dblib.setup_db()
kill_me = browser
try:
scrape_thread(kill_me, home, con, cur)
except:
#pdb.post_mortem()
dead()
raise
print "done"
"""
vbulletin = False
#archives = False
re_sort = re.compile(r"\d*(?=\.html)")
re_uid = re.compile(r"\d*$")
i = None
timeout = 10
home = ""
hdir = ""
archive_link = ""
P = None
state = [0, 0]
pfile = ""
save_files = False
con, cur = dblib.setup_db()
delay = 0
delay_range = 0
def parse_args(args):
parser = argparse.ArgumentParser(description="Scrape a forum", add_help=False)
parser.add_argument("url")
parser.add_argument("num")
parser.add_argument("--authfile")
parser.add_argument("-d", "--delay", type=int)
parser.add_argument("-r", "--delay_range", type=int)
parser.add_argument("--save_files", action="store_true")
type_scrape = parser.add_mutually_exclusive_group(required=True)
#type_scrape.add_argument("--archives", action="store_true")
type_scrape.add_argument("--vbulletin", action="store_true")
