def project_versions(request, id, project):
if request.method == 'GET':
client_model = Client.objects.get(id=id)
project_model = Project.objects.get(name=project)
scrapyd = ScrapydAPI(scrapyd_url(client_model.ip, client_model.port))
if Deploy.objects.filter(client=client_model, project=project_model):
deploy = Deploy.objects.get(client=client_model,
project=project_model)
timestamp = deploy.deployed_at.timestamp()
localtime = time.localtime(timestamp)
datetime = time.strftime('%Y-%m-%d %H:%M:%S', localtime)
return HttpResponse(
json.dumps({
'datetime': datetime,
'description': deploy.description
}))
else:
versions = scrapyd.list_versions(project)
if len(versions) > 0:
version = versions[-1]
localtime = time.localtime(int(version))
datetime = time.strftime('%Y-%m-%d %H:%M:%S', localtime)
else:
datetime = None
return HttpResponse(json.dumps({'datetime': datetime}))
def project_version(request, client_id, project_name):
"""
get project deploy version
:param request: request object
:param client_id: client id
:param project_name: project name
:return: deploy version of project
"""
if request.method == 'GET':
# get client and project model
client = Client.objects.get(id=client_id)
project = Project.objects.get(name=project_name)
scrapyd = ScrapydAPI(scrapyd_url(client.ip, client.port))
# if deploy info exists in db, return it
if Deploy.objects.filter(client=client, project=project):
deploy = Deploy.objects.get(client=client, project=project)
# if deploy info does not exists in db, create deploy info
else:
try:
versions = scrapyd.list_versions(project_name)
except ConnectionError:
return JsonResponse({'message': 'Connect Error'}, status=500)
if len(versions) > 0:
version = versions[-1]
deployed_at = timezone.datetime.fromtimestamp(int(version), tz=pytz.timezone(TIME_ZONE))
else:
deployed_at = None
deploy, result = Deploy.objects.update_or_create(client=client, project=project, deployed_at=deployed_at)
# return deploy json info
return JsonResponse(model_to_dict(deploy))
def version(client, project):
url = 'http://{ip}:{port}'.format(ip=client.ip, port=client.port)
try:
scrapyd = ScrapydAPI(url)
versions = scrapyd.list_versions(project)
if (len(versions) > 0):
return versions[-1]
return ''
except (ConnectionError, InvalidURL, UnicodeError):
return ''
def get_project_version(request,project,client_id):
if request.method == 'GET':
client = Node.objects.get(id=client_id)
scrapyd = ScrapydAPI(scrapyd_url(client.ip,client.port))
try:
versions = scrapyd.list_versions(project)
versions = [{'name': version, 'id': index + 1} for index, version in enumerate(versions)]
return JsonResponse(versions)
except ConnectionError:
return JsonResponse({'message': 'Connect Error'}, status=500)
class Scrapyd_Control(object):
def __init__(self):
scrapyd_url = input('请输入scrapyd地址： ')
project = input('请输入项目名称： ')
self.project = project
self.scrapyd = ScrapydAPI(scrapyd_url)
# 启动爬虫
def schedule(self):
spider = input('请输入爬虫名称： ')
return {
'project': self.project,
'spider': spider,
'jobid': self.scrapyd.schedule(self.project, spider)
}
start, run = schedule, schedule
# 取消爬虫
def cancel(self):
jobid = input('请粘贴要取消的爬虫jobid： ')
return self.scrapyd.cancel(self.project, jobid)
# 查看项目
def listprojects(self):
return self.scrapyd.list_projects()
# 查看爬虫
def listspiders(self):
return self.scrapyd.list_spiders(self.project)
# 列出所有jobs
def listjobs(self):
return self.scrapyd.list_jobs(self.project)
# 查看job状态
def jobstatus(self):
jobid = input('请粘贴要查看的jobid: ')
return self.scrapyd.job_status(self.project, jobid)
# 查看版本
def listversions(self):
return self.scrapyd.list_versions(self.project)
# 删除版本
def delversion(self):
version_name = input('请粘贴要删除的版本： ')
yes = input('是否确认删除该版本{}，请输yes否则回车跳过删除\n'.format(version_name))
if yes == 'yes':
return self.scrapyd.delete_version(self.project, version_name)
else:
pass
# 删除项目
def delproject(self):
yes = input('是否确认删除该项目{}，请输yes否则回车跳过删除\n'.format(self.project))
if yes == 'yes':
return self.scrapyd.delete_project(self.project)
else:
pass
# 列出所有命令
def help(self):
print("""
启动爬虫 schedule|start|run
取消爬虫 cancel
查看项目 listprojects
查看爬虫 listspiders
列出所有jobs listjobs
查看job状态 jobstatus
查看版本 listversions
删除版本 delversion
删除项目 deleproject
列出所有命令 help
""")
