def task_verifyIP(log_file=None):
print("task_verifyIP 接收到的值", log_file)
if log_file == None:
log_file = getRandomLogFileName("verifyIP-beat")
os.chdir(PROXY_SPIDER_DIR)
log_file_abs = os.path.join(PROXY_SPIDER_LOG_DIR, log_file)
cmd = 'scrapy crawl verify -s LOG_FILE={}'.format(log_file_abs)
print(cmd)
# os.system(cmd)
subprocess.Popen(cmd)
# cmdline.execute(cmd.split())
return log_file
def runVerifyIP(request):
print("sss")
logger.info("得到请求runVerifyIP")
result = None
try:
lf = getRandomLogFileName("verifyIP")
print("开始调用", lf)
result = task_verifyIP.delay(lf)
except BaseException as e:
print(e)
logger.error(e)
if result == None:
return JsonResponse({"flag": False, "massage": "启动失败"})
return JsonResponse({"flag": True, "massage": "启动成功"})
def runSpider(request):
print(request.POST)
spider_name = request.POST.get("spiderName", None)
if spider_name == None:
return HttpResponse("错误的请求")
db = MongoDBCli()
spider_config = db.getOneSpiderFromSpiderName(spider_name)
if spider_config == None:
return HttpResponse("没有这个爬虫")
result = task_runSpider.delay(
spider_config["config"]["name"],
getRandomLogFileName(spider_config["config"]["name"]),
"-a si={} -a ei={}".format(
spider_config["startIndex"],
spider_config["endIndex"],
))
return HttpResponse(result)
def task_runSpider(spider_name, log_file=None, param=""):
if log_file == None:
log_file = getRandomLogFileName(spider_name + "-beat")
db = MongoDBCli()
spider_config = db.getOneSpiderFromSpiderName(spider_name)
param = "-a si={} -a ei={}".format(
spider_config["startIndex"],
spider_config["endIndex"],
)
os.chdir(PROXY_SPIDER_DIR)
log_file_abs = os.path.join(PROXY_SPIDER_LOG_DIR, log_file)
print(log_file_abs)
cmd = 'scrapy crawl genericSpider -a cn={} -s LOG_FILE={} {}'.format(
spider_name, log_file_abs, param)
print(cmd)
# cmdline.execute(cmd.split())
# subprocess.Popen("notepad")
# os.system(cmd)
subprocess.Popen(cmd)
return log_file
def clearIP(log_file=None):
if log_file == None:
log_file = getRandomLogFileName("clearIP-beat")
proxys = Proxy.objects.filter(https=0, http=0)
log_file_abs = os.path.join(PROXY_SPIDER_LOG_DIR, log_file)
with open(log_file_abs, 'w', encoding="utf-8") as fp:
fp.write("\t %-15s %-6s %-10s %-10s %-34s %-10s %-10s %-10s\n" %
("ip", "port", "protocol", "anonymity", "verify_time", "http",
"https", "source"))
fp.write("\t" + ("-" * 112) + "\n")
for proxy in proxys:
fp.write(
"\t %-15s %-6s %-10s %-10s %-34s %-10s %-10s %-10s\n" %
(proxy.ip, proxy.port, proxy.protocol, proxy.anonymity,
proxy.verify_time, proxy.http, proxy.https, proxy.source))
try:
proxys.delete()
except:
return "删除失败"
return "删除成功"
def post(self, request):
print("开始调用")
result = clearIP.delay(getRandomLogFileName("clearIP"))
print(result)
return JsonResponse({"flag": True, "massage": "启动成功"})