def get(self, request, spider_name):
# print(time.time())
# print(request.path)
db = MongoDBCli()
spider = db.getOneSpiderFromSpiderName(spider_name)
# print("spider", spider)
if spider == None:
return render(request, '404.html')
content = {
"spider": spider,
"spider_config_json": json.dumps(spider["config"]),
}
# print("content", content)
return render(request, 'proxy/editSpiderConfig.html', context=content)
def runSpider(request):
print(request.POST)
spider_name = request.POST.get("spiderName", None)
if spider_name == None:
return HttpResponse("错误的请求")
db = MongoDBCli()
spider_config = db.getOneSpiderFromSpiderName(spider_name)
if spider_config == None:
return HttpResponse("没有这个爬虫")
result = task_runSpider.delay(
spider_config["config"]["name"],
getRandomLogFileName(spider_config["config"]["name"]),
"-a si={} -a ei={}".format(
spider_config["startIndex"],
spider_config["endIndex"],
))
return HttpResponse(result)
def task_runSpider(spider_name, log_file=None, param=""):
if log_file == None:
log_file = getRandomLogFileName(spider_name + "-beat")
db = MongoDBCli()
spider_config = db.getOneSpiderFromSpiderName(spider_name)
param = "-a si={} -a ei={}".format(
spider_config["startIndex"],
spider_config["endIndex"],
)
os.chdir(PROXY_SPIDER_DIR)
log_file_abs = os.path.join(PROXY_SPIDER_LOG_DIR, log_file)
print(log_file_abs)
cmd = 'scrapy crawl genericSpider -a cn={} -s LOG_FILE={} {}'.format(
spider_name, log_file_abs, param)
print(cmd)
# cmdline.execute(cmd.split())
# subprocess.Popen("notepad")
# os.system(cmd)
subprocess.Popen(cmd)
return log_file
def post(self, request, spider_name):
# receive_spider = json.loads()
db = MongoDBCli()
spider = db.getOneSpiderFromSpiderName(spider_name)
if spider == None:
return JsonResponse({
"flag": False,
"massage": "没有这个爬虫",
})
r_tmp = checkJson(request)
# print(r_tmp)
if r_tmp == False:
return JsonResponse({
"flag": False,
"massage": "spider info 参数错误",
})
# print(spider, "\n>>\n", r_tmp)
db_ret = db.setOneSpider(r_tmp)
if db_ret != None:
return JsonResponse({"flag": True, "massage": "OK"})
return JsonResponse({"flag": False, "massage": "编辑错误"})