crawl_policy.START_FILTER[URL] = value
argv.append(value)
if op == "--apply_time_interval":
crawl_policy.APPLY_TIME_INTERVAL = True
argv.append(value)
if op == "--time_st":
try:
datetime.datetime.strptime(value, "%Y-%m-%d")
crawl_policy.TIME_INTERVAL_ST = value
argv.append(value)
except Exception, e:
print e.message
exit(1)
if op == "--time_ed":
try:
datetime.datetime.strptime(value, "%Y-%m-%d")
crawl_policy.TIME_INTERVAL_ED = value
argv.append(value)
except Exception, e:
print e.message
exit(1)
if op == "-h" or op == "--help":
show_help = True
argv.append(value)
if len(argv) == 0 or show_help:
usage()
if len(argv) > 0:
start_crawlers(crawl_policy)
# start_crawlers(crawl_policy)
