def schedule_del(request):
id = request.GET.get('id')
if id:
Schedule.objects.get(id=id).delete()
return JSONResponse({'status': 'ok'})
else:
return JSONResponse({'status': 'error'})
def job_delete(request):
id = request.GET.get('job')
if id:
Job.objects.get(id=id).delete()
Log.objects.get(id=id).delete()
return JSONResponse({'status': 'ok'})
else:
return JSONResponse({'status': 'error'})
def job_cancel(request):
project_id = request.GET.get('project')
job_id = request.GET.get('job')
res = _request_post("%s/cancel.json?project=%s&job=%s" %
(settings.SCRAPYD_URL, project_id, job_id))
if res:
result = res.json()
if result.get("status", '') == 'ok':
return JSONResponse({'status': 'ok'})
return JSONResponse({'status': 'error'})
def capabilities(request):
capabilities = {
'custom': settings.CUSTOM,
'username': request.user.username,
'capabilities': settings.CAPABILITIES,
}
return JSONResponse(capabilities)
def schedule_add(request):
project = request.GET.get('project')
spider = request.GET.get('spider')
interval = request.GET.get('interval')
times = request.GET.get('times')
if project and spider and interval:
schedule = Schedule(id=uuid.uuid1().hex,
project=project,
spider=spider,
start_time=int(time.time() * 1000),
interval=interval,
times=times,
date_update=int(time.time() * 1000))
schedule.save()
return JSONResponse({'status': 'ok'})
else:
return JSONResponse({'status': 'error'})
def article_del(request):
spider_id = request.GET.get('spider')
job_id = request.GET.get('job')
job_item_id = request.GET.get('job_item')
if spider_id:
jobItems = JobItem.objects.filter(spider=spider_id)
for item in jobItems:
item.delete()
return JSONResponse({'status': 'ok'})
elif job_id:
jobItems = JobItem.objects.filter(job=job_id)
for item in jobItems:
item.delete()
return JSONResponse({'status': 'ok'})
elif job_item_id:
JobItem.objects.get(id=job_item_id).delete()
return JSONResponse({'status': 'ok'})
else:
return JSONResponse({'status': 'error'})
def article_detail(request):
result = {}
job_item_id = request.GET.get('job_item')
job_items = JobItem.objects(id=job_item_id)
if job_items[0]:
for name, value in job_items[0].__dict__.iteritems():
if not name.startswith('_') and not inspect.ismethod(value):
#value = getattr(item, name )
result[name] = value
return JSONResponse(result)
def article_list(request):
result = []
job = request.GET.get('job')
items = JobItem.objects(job=job)
for item in items:
res = {
'id': str(item.id),
'item-display-name': 'item',
'job': item.job,
'spider': item.spider,
'url': item.url,
'time': item.time.strftime("%Y-%m-%d %H:%M:%S")
}
result.append(res)
return JSONResponse(result)
def job_log(request):
result = []
project_id = request.GET.get('project')
spider_id = request.GET.get('spider')
job_id = request.GET.get('job')
job = Job.objects.get(id=job_id)
if job:
log = _get_log(project_id, job.spider, job.id, job.status)
if log:
result = list(generateDicts(log.content))
return JSONResponse({
"project": project_id,
"spider": spider_id,
"job": job_id,
"log": result
})
def schedule_list(request):
result = []
schedules = Schedule.objects.all()
for schedule in schedules:
result.append({
'id':
schedule.id,
'project':
schedule.project,
'spider':
schedule.spider,
'start_time':
_get_string_from_timestamp(schedule.start_time),
'update_time':
_get_string_from_timestamp(schedule.date_update),
'interval':
schedule.interval,
'times':
schedule.times
})
return JSONResponse(result)
def job_list(request):
result = {}
project_id = request.GET.get('project')
spider = request.GET.get('spider', '')
_update_jobs_model(project_id)
for status in ['pending', 'running', 'finished']:
res_jobs = []
jobs = Job.objects.filter(status=status).order_by('-start_time')
for job in jobs:
if (spider == '' or spider == job.spider):
res_jobs.append({
'id':
job.id,
'index':
job.index,
'project':
project_id,
'spider':
job.spider,
'start_time':
_get_string_from_timestamp(job.start_time),
'end_time':
_get_string_from_timestamp(job.end_time),
'error_count':
job.error_count,
'warning_count':
job.warning_count,
'critical_count':
job.critical_count
})
result[status] = res_jobs
return JSONResponse(result)