def index_status(request): """ index statistics :param request: request object :return: json """ if request.method == 'GET': clients = Client.objects.all() data = { 'success': 0, 'error': 0, 'project': 0, } # clients info for client in clients: try: requests.get(scrapyd_url(client.ip, client.port), timeout=1) data['success'] += 1 except ConnectionError: data['error'] += 1 path = os.path.abspath(join(os.getcwd(), PROJECTS_FOLDER)) files = os.listdir(path) # projects info for file in files: if os.path.isdir(join(path, file)) and not file in IGNORES: data['project'] += 1 return JsonResponse(data)
def project_deploy(request, client_id, project_name): """ deploy project operation :param request: request object :param client_id: client id :param project_name: project name :return: json of deploy result """ if request.method == 'POST': # get project folder path = os.path.abspath(join(os.getcwd(), PROJECTS_FOLDER)) project_path = join(path, project_name) # find egg file egg = find_egg(project_path) egg_file = open(join(project_path, egg), 'rb') # get client and project model client = Client.objects.get(id=client_id) project = Project.objects.get(name=project_name) # execute deploy operation scrapyd = ScrapydAPI(scrapyd_url(client.ip, client.port)) try: scrapyd.add_version(project_name, int(time.time()), egg_file.read()) # update deploy info deployed_at = timezone.now() Deploy.objects.filter(client=client, project=project).delete() deploy, result = Deploy.objects.update_or_create(client=client, project=project, deployed_at=deployed_at, description=project.description) return JsonResponse(model_to_dict(deploy)) except Exception: return JsonResponse({'message': get_traceback()}, status=500)
def project_version(request, client_id, project_name): """ get project deploy version :param request: request object :param client_id: client id :param project_name: project name :return: deploy version of project """ if request.method == 'GET': # get client and project model client = Client.objects.get(id=client_id) project = Project.objects.get(name=project_name) scrapyd = ScrapydAPI(scrapyd_url(client.ip, client.port)) # if deploy info exists in db, return it if Deploy.objects.filter(client=client, project=project): deploy = Deploy.objects.get(client=client, project=project) # if deploy info does not exists in db, create deploy info else: try: versions = scrapyd.list_versions(project_name) except ConnectionError: return JsonResponse({'message': 'Connect Error'}, status=500) if len(versions) > 0: version = versions[-1] deployed_at = timezone.datetime.fromtimestamp(int(version), tz=pytz.timezone(TIME_ZONE)) else: deployed_at = None deploy, result = Deploy.objects.update_or_create(client=client, project=project, deployed_at=deployed_at) # return deploy json info return JsonResponse(model_to_dict(deploy))
def project_list(request, client_id): """ project deployed list on one client :param request: request object :param client_id: client id :return: json """ if request.method == 'GET': client = Client.objects.get(id=client_id) scrapyd = ScrapydAPI(scrapyd_url(client.ip, client.port)) try: projects = scrapyd.list_projects() return JsonResponse(projects) except ConnectionError: return JsonResponse({'message': 'Connect Error'}, status=500)
def client_status(request, client_id): """ get client status :param request: request object :param client_id: client id :return: json """ if request.method == 'GET': # get client object client = Client.objects.get(id=client_id) try: requests.get(scrapyd_url(client.ip, client.port), timeout=3) return JsonResponse({'result': '1'}) except ConnectionError: return JsonResponse({'message': 'Connect Error'}, status=500)
def job_cancel(request, client_id, project_name, job_id): """ cancel a job :param request: request object :param client_id: client id :param project_name: project name :param job_id: job id :return: json of cancel """ if request.method == 'GET': client = Client.objects.get(id=client_id) try: scrapyd = ScrapydAPI(scrapyd_url(client.ip, client.port)) result = scrapyd.cancel(project_name, job_id) return JsonResponse(result) except ConnectionError: return JsonResponse({'message': 'Connect Error'})
def spider_start(request, client_id, project_name, spider_name): """ start a spider :param request: request object :param client_id: client id :param project_name: project name :param spider_name: spider name :return: json """ if request.method == 'GET': client = Client.objects.get(id=client_id) scrapyd = ScrapydAPI(scrapyd_url(client.ip, client.port)) try: job = scrapyd.schedule(project_name, spider_name) return JsonResponse({'job': job}) except ConnectionError: return JsonResponse({'message': 'Connect Error'}, status=500)
def spider_list(request, client_id, project_name): """ get spider list from one client :param request: request Object :param client_id: client id :param project_name: project name :return: json """ if request.method == 'GET': client = Client.objects.get(id=client_id) scrapyd = ScrapydAPI(scrapyd_url(client.ip, client.port)) try: spiders = scrapyd.list_spiders(project_name) spiders = [{'name': spider, 'id': index + 1} for index, spider in enumerate(spiders)] return JsonResponse(spiders) except ConnectionError: return JsonResponse({'message': 'Connect Error'}, status=500)
def project_deploy(request, id, project): if request.method == 'GET': path = os.path.abspath(merge(os.getcwd(), PROJECTS_FOLDER)) project_path = merge(path, project) egg = find_egg(project_path) egg_file = open(merge(project_path, egg), 'rb') deploy_version = time.time() client_model = Client.objects.get(id=id) project_model = Project.objects.get(name=project) Deploy.objects.filter(client=client_model, project=project_model).delete() deploy = Deploy.objects.update_or_create( client=client_model, project=project_model, description=project_model.description) scrapyd = ScrapydAPI(scrapyd_url(client_model.ip, client_model.port)) result = scrapyd.add_version(project, int(deploy_version), egg_file.read()) return HttpResponse(result)
def index_status(request): if request.method == 'GET': clients = Client.objects.all() data = { 'success': 0, 'error': 0, 'project': 0, } for client in clients: try: requests.get(scrapyd_url(client.ip, client.port), timeout=1) data['success'] += 1 except ConnectionError: data['error'] += 1 path = os.path.abspath(merge(os.getcwd(), PROJECTS_FOLDER)) files = os.listdir(path) for file in files: if os.path.isdir(merge(path, file)) and not file in IGNORES: data['project'] += 1 return HttpResponse(json.dumps(data))
def project_versions(request, id, project): if request.method == 'GET': client_model = Client.objects.get(id=id) project_model = Project.objects.get(name=project) scrapyd = ScrapydAPI(scrapyd_url(client_model.ip, client_model.port)) if Deploy.objects.filter(client=client_model, project=project_model): deploy = Deploy.objects.get(client=client_model, project=project_model) timestamp = deploy.deployed_at.timestamp() localtime = time.localtime(timestamp) datetime = time.strftime('%Y-%m-%d %H:%M:%S', localtime) return HttpResponse(json.dumps({'datetime': datetime, 'description': deploy.description})) else: versions = scrapyd.list_versions(project) if len(versions) > 0: version = versions[-1] localtime = time.localtime(int(version)) datetime = time.strftime('%Y-%m-%d %H:%M:%S', localtime) else: datetime = None return HttpResponse(json.dumps({'datetime': datetime}))
def job_list(request, client_id, project_name): """ get job list of project from one client :param request: request object :param client_id: client id :param project_name: project name :return: list of jobs """ if request.method == 'GET': client = Client.objects.get(id=client_id) scrapyd = ScrapydAPI(scrapyd_url(client.ip, client.port)) try: result = scrapyd.list_jobs(project_name) jobs = [] statuses = ['pending', 'running', 'finished'] for status in statuses: for job in result.get(status): job['status'] = status jobs.append(job) return JsonResponse(jobs) except ConnectionError: return JsonResponse({'message': 'Connect Error'}, status=500)
def scheduler_job(): """ 每分钟检查一次定时任务 :return: """ models = Task.objects.all() for model in models: scheduler_at = model.scheduler_at updated_at = model.updated_at scheduler_at_time_stamp = scheduler_at * 60 updated_at_time_stamp = time.mktime(updated_at.timetuple()) if time.time() - updated_at_time_stamp > scheduler_at_time_stamp: client_id = model.client_id project_name = model.project_name spider_name = model.spider_name client = Client.objects.get(id=client_id) scrapyd = ScrapydAPI(scrapyd_url(client.ip, client.port)) try: job = scrapyd.schedule(project_name, spider_name) model.success = 1 except ConnectionError: model.success = 0 finally: model.save()
def work_func(client, project, spider): ip_port = Client.objects.get(id=client) scrapyd = ScrapydAPI(scrapyd_url(ip_port.ip, ip_port.port)) scrapyd.schedule(project, spider)
def job_cancel(request, id, project, job): if request.method == 'GET': client = Client.objects.get(id=id) scrapyd = ScrapydAPI(scrapyd_url(client.ip, client.port)) result = scrapyd.cancel(project, job) return HttpResponse(json.dumps(result))
def project_list(request, id): if request.method == 'GET': client = Client.objects.get(id=id) scrapyd = ScrapydAPI(scrapyd_url(client.ip, client.port)) projects = scrapyd.list_projects() return HttpResponse(json.dumps(projects))
def spider_start(request, id, project, spider): if request.method == 'GET': client = Client.objects.get(id=id) scrapyd = ScrapydAPI(scrapyd_url(client.ip, client.port)) result = scrapyd.schedule(project, spider) return HttpResponse(json.dumps({'job': result}))