def post(self):
project_name = request.form['project_name']
project = Project()
project.project_name = project_name
db.session.add(project)
db.session.commit()
return project.to_dict()
def project_create():
project_name = request.form['project_name']
project = Project()
project.project_name = project_name
db.session.add(project)
db.session.commit()
return redirect("/project/%s/spider/deploy" % project.id, code=302)
def get_project_list(self):
data = request("get", self._scrapyd_url() + "/listprojects.json", return_type="json")
result = []
if data:
for project_name in data['projects']:
project = Project()
project.project_name = project_name
result.append(project)
return result
def post(self):
project_name = request.form['project_name']
project = Project()
project.project_name = project_name
try:
db.session.add(project)
db.session.commit()
except:
db.session.rollback()
raise
return project.to_dict()
def get_project_list(self):
"""
功能: 获取所有的爬虫工程列表
:return: 返回工程对象列表
"""
data = self.scrapyd_api.list_projects() # 获取scrapyd上的所有工程列表
result = []
if data:
for project_name in data:
project = Project() # 实例化工程对象
project.project_name = project_name
result.append(project)
return result
def job_periodic(project_id):
project = Project.find_project_by_id(project_id)
job_instance_list = [job_instance.to_dict() for job_instance in
JobInstance.query.filter_by(run_type="periodic", project_id=project_id).order_by(
JobInstance.id).all()] # ADD order_by
return render_template("job_periodic.html",
job_instance_list=job_instance_list)
def project_update():
stat_url = request.form['stat_url']
project_id = request.form['project_id']
project = Project.find_project_by_id(project_id)
project.stat_url = stat_url
db.session.commit()
return redirect("/project/manage", code=302)
def log_url_slave(self, job_execution):
"""
功能: 获取从爬虫的日志,只要获取一个
:param job_execution: job_execution对象
:return: 返回log的url
"""
job_instance = JobInstance.find_job_instance_by_id(job_execution.job_instance_id)
project = Project.find_project_by_id(job_instance.project_id)
# 主从爬虫运行的服务器字符串
service_job_execution_id = job_execution.service_job_execution_id.split('>')
# 从爬虫服务器列表
slave_service_job_execution_id = service_job_execution_id[1].split(',')
# 爬虫运行的服务器地址
running_on = job_execution.running_on.split('>')
slave_running_on = running_on[1].split(',')
# 调用从爬虫的日志
spider_name_slave_obj = SpiderInstance.query.filter_by(
spider_name=job_instance.spider_name,
project_id=job_instance.project_id).first()
spider_name_slave = spider_name_slave_obj.spider_name_slave
for spider_service_instance in self.spider_service_instances_slave:
for job_execution_id, running_on_ in zip(slave_service_job_execution_id, slave_running_on):
if spider_service_instance.server == running_on_:
slave_log_url = spider_service_instance.log_url(
project.project_name, spider_name_slave,
job_execution_id)
return slave_log_url
def job_add(project_id):
project = Project.find_project_by_id(project_id)
job_instance = JobInstance()
job_instance.spider_name = request.form['spider_name']
job_instance.project_id = project_id
job_instance.spider_arguments = request.form['spider_arguments']
job_instance.priority = request.form.get('priority', 0)
job_instance.run_type = request.form['run_type']
# chose daemon manually
if request.form['daemon'] != 'auto':
spider_args = []
if request.form['spider_arguments']:
spider_args = request.form['spider_arguments'].split(",")
spider_args.append("daemon={}".format(request.form['daemon']))
job_instance.spider_arguments = ','.join(spider_args)
if job_instance.run_type == JobRunType.ONETIME:
job_instance.enabled = -1
db.session.add(job_instance)
db.session.commit()
agent.start_spider(job_instance)
if job_instance.run_type == JobRunType.PERIODIC:
job_instance.cron_minutes = request.form.get('cron_minutes') or '0'
job_instance.cron_hour = request.form.get('cron_hour') or '*'
job_instance.cron_day_of_month = request.form.get(
'cron_day_of_month') or '*'
job_instance.cron_day_of_week = request.form.get(
'cron_day_of_week') or '*'
job_instance.cron_month = request.form.get('cron_month') or '*'
# set cron exp manually
if request.form.get('cron_exp'):
job_instance.cron_minutes, job_instance.cron_hour, job_instance.cron_day_of_month, job_instance.cron_day_of_week, job_instance.cron_month = \
request.form['cron_exp'].split(' ')
db.session.add(job_instance)
db.session.commit()
return redirect(request.referrer, code=302)
def download_items(project_id, job_exec_id):
format = request.args.get('format')
if not format in ['json', 'csv']:
abort(404)
job_execution = JobExecution.query.filter_by(project_id=project_id,
id=job_exec_id).first()
job_instance = JobInstance.find_job_instance_by_id(
job_execution.job_instance_id)
project = Project.find_project_by_id(job_instance.project_id)
res = requests.get(agent.items_url(job_execution))
res.encoding = 'utf8'
json_data = [json.loads(s) for s in filter(None, res.text.split('\n'))]
filename = '{}-{}.{}'.format(project.project_name,
job_instance.spider_name, format)
if format == 'json':
open(os.path.join(app.static_folder, filename),
'w').write(json.dumps(json_data))
elif format == 'csv':
f = open(os.path.join(app.static_folder, filename), 'w')
csvwriter = csv.writer(f)
count = 0
for item in json_data:
if count == 0:
header = item.keys()
csvwriter.writerow(header)
count += 1
csvwriter.writerow(item.values())
f.close()
return send_from_directory(app.static_folder, filename, as_attachment=True)
def get(self, project_id):
project = Project.find_project_by_id(project_id)
return [
spider_instance.to_dict()
for spider_instance in SpiderInstance.query.filter_by(
project_id=project_id).all()
]
def log_url(self, job_execution):
job_instance = JobInstance.find_job_instance_by_id(job_execution.job_instance_id)
project = Project.find_project_by_id(job_instance.project_id)
for spider_service_instance in self.spider_service_instances:
if spider_service_instance.server == job_execution.running_on:
return spider_service_instance.log_url(project.project_name, job_instance.spider_name,
job_execution.service_job_execution_id)
def start_spider(self, job_instance):
project = Project.find_project_by_id(job_instance.project_id)
spider_name = job_instance.spider_name
arguments = {}
if job_instance.spider_arguments:
arguments = dict(
map(lambda x: x.split("="),
job_instance.spider_arguments.split(",")))
threshold = 0
daemon_size = len(self.spider_service_instances)
if job_instance.priority == JobPriority.HIGH:
threshold = int(daemon_size / 2)
if job_instance.priority == JobPriority.HIGHEST:
threshold = int(daemon_size)
threshold = 1 if threshold == 0 else threshold
candidates = self.spider_service_instances
leaders = []
# TODO optimize some better func to vote the leader
for i in range(threshold):
leaders.append(random.choice(candidates))
for leader in leaders:
serviec_job_id = leader.start_spider(project.project_name,
spider_name, arguments)
job_execution = JobExecution()
job_execution.project_id = job_instance.project_id
job_execution.service_job_execution_id = serviec_job_id
job_execution.job_instance_id = job_instance.id
job_execution.create_time = datetime.datetime.now()
job_execution.running_on = leader.server
db.session.add(job_execution)
db.session.commit()
def log_url(self, job_execution):
job_instance = JobInstance.find_job_instance_by_id(job_execution.job_instance_id)
project = Project.find_project_by_id(job_instance.project_id)
for spider_service_instance in self.spider_service_instances:
if spider_service_instance.server == job_execution.running_on:
return spider_service_instance.log_url(project.project_name, job_instance.spider_name,
job_execution.service_job_execution_id)
def project_allprojects():
project_instance_list = [
project_instance.to_dict()
for project_instance in Project.all_project()
]
return render_template("all_projects.html",
project_instance_list=project_instance_list)
def job_add(project_id):
# Save the upload file, and save the file path to the
# job_instance.spider_arguments
dst = ''
if 'file' in request.files:
file = request.files['file']
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '':
pass
if file and allowed_seed(file.filename):
filename = secure_filename(file.filename)
dst = os.path.join(
app.config['UPLOAD_DIR'],
datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S-") +
filename)
file.save(dst)
project = Project.find_project_by_id(project_id)
job_instance = JobInstance()
job_instance.spider_name = request.form['spider_name']
job_instance.project_id = project_id
job_instance.spider_arguments = request.form['spider_arguments']
if dst:
if job_instance.spider_arguments:
job_instance.spider_arguments += (",seed={}".format(dst))
else:
job_instance.spider_arguments = "seed={}".format(dst)
job_instance.priority = request.form.get('priority', 0)
job_instance.run_type = request.form['run_type']
# chose daemon manually
if request.form['daemon'] != 'auto':
spider_args = []
if request.form['spider_arguments']:
spider_args = request.form['spider_arguments'].split(",")
spider_args.append("daemon={}".format(request.form['daemon']))
job_instance.spider_arguments = ','.join(spider_args)
if job_instance.run_type == JobRunType.ONETIME:
job_instance.enabled = -1
db.session.add(job_instance)
db.session.commit()
agent.start_spider(job_instance)
if job_instance.run_type == JobRunType.PERIODIC:
job_instance.cron_minutes = request.form.get('cron_minutes') or '0'
job_instance.cron_hour = request.form.get('cron_hour') or '*'
job_instance.cron_day_of_month = request.form.get(
'cron_day_of_month') or '*'
job_instance.cron_day_of_week = request.form.get(
'cron_day_of_week') or '*'
job_instance.cron_month = request.form.get('cron_month') or '*'
# set cron exp manually
if request.form.get('cron_exp'):
job_instance.cron_minutes, job_instance.cron_hour, job_instance.cron_day_of_month, job_instance.cron_day_of_week, job_instance.cron_month = \
request.form['cron_exp'].split(' ')
db.session.add(job_instance)
db.session.commit()
return redirect(request.referrer, code=302)
def project_delete(project_id):
project = Project.find_project_by_id(project_id)
agent.delete_project(project)
try:
db.session.delete(project)
db.session.commit()
except:
db.session.rollback()
raise
return redirect("/project/manage", code=302)
def cancel_spider(self, job_execution):
job_instance = JobInstance.find_job_instance_by_id(job_execution.job_instance_id)
project = Project.find_project_by_id(job_instance.project_id)
for spider_service_instance in self.spider_service_instances:
if spider_service_instance.server == job_execution.running_on:
if spider_service_instance.cancel_spider(project.project_name, job_execution.service_job_execution_id):
job_execution.end_time = datetime.datetime.now()
job_execution.running_status = SpiderStatus.CANCELED
db.session.commit()
break
def cancel_spider(self, job_execution):
job_instance = JobInstance.find_job_instance_by_id(job_execution.job_instance_id)
project = Project.find_project_by_id(job_instance.project_id)
# TODO multi service
for spider_service_instance in self.spider_service_instances:
if spider_service_instance.server == job_execution.running_on:
if spider_service_instance.cancel_spider(project.project_name, job_execution.service_job_execution_id):
job_execution.running_status = SpiderStatus.CANCELED
db.session.commit()
break
def get_all_spiders_info():
"""
功能: 返回所有的爬虫的相关信息
:return: 返回数据格式: json, 样式如下：
{
"last_run_status": "success",
"last_run_time": "2018-09-19 03:30:45",
"project_alias": "深圳市共享开放平台",
"project_id": 1,
"project_name": "opendata_sz",
"spider_alias": "所有数据",
"spider_id": 1,
"spider_name": "list"
}
"""
# 保存数据的临时列表
data = []
# 遍历实例数据库, 获取爬虫信息
job_instance = JobInstance.query.order_by(
JobInstance.date_created).group_by(JobInstance.project_id).all()
for spider in job_instance:
# 得到实例的字典信息
_temp = spider.to_dict()
# 依据工程id查找Project数据库, 获取工程名以及备注信息
project_base_info = Project.find_project_by_id(_temp['project_id'])
instance_to_job_execution = JobExecution.query.filter_by(
job_instance_id=_temp['job_instance_id']).all()
if instance_to_job_execution:
# 获取状态信息
_status = instance_to_job_execution[-1].running_status
# 状态信息格式转变
status = switcher.get(_status, "CANCELED")
# 获取实例的上一次运行时间
last_run_time = instance_to_job_execution[-1].end_time
service_job_execution_id = instance_to_job_execution[
-1].service_job_execution_id
else:
status = 'PENDING'
last_run_time = None
service_job_execution_id = None
# 将信息封装成字典格式
_dict = dict(project_id=_temp['project_id'],
project_name=project_base_info.project_name,
project_alias=project_base_info.project_alias,
spider_id=SpiderInstance.query.filter_by(
project_id=_temp['project_id']).first().id,
spider_name=_temp['spider_name'],
spider_alias=_temp['desc'],
last_run_status=status,
last_run_time=str(last_run_time).split('.')[0],
run_type=_temp['run_type'],
job_exec_id=service_job_execution_id,
is_msd=project_base_info.is_msd)
data.append(_dict)
return json.dumps({"code": 200, 'data': data})
def inject_project():
project_context = {}
project_context['project_list'] = Project.query.all()
if project_context['project_list'] and (not session.get('project_id')):
project = Project.query.first()
session['project_id'] = project.id
if session.get('project_id'):
project_context['project'] = Project.find_project_by_id(session['project_id'])
project_context['spider_list'] = [spider_instance.to_dict() for spider_instance in
SpiderInstance.query.filter_by(project_id=session['project_id']).all()]
else:
project_context['project'] = {}
return project_context
def cancel_spider(self, job_execution):
job_instance = JobInstance.find_job_instance_by_id(job_execution.job_instance_id)
project = Project.find_project_by_id(job_instance.project_id)
for spider_service_instance in self.spider_service_instances:
if spider_service_instance.server == job_execution.running_on:
if spider_service_instance.cancel_spider(project.project_name, job_execution.service_job_execution_id):
job_execution.end_time = datetime.datetime.now()
job_execution.running_status = SpiderStatus.CANCELED
try:
db.session.commit()
except:
db.session.rollback()
raise
break
def start_spider(self, job_instance):
project = Project.find_project_by_id(job_instance.project_id)
# job_execution = JobExecution.find_job_by_service_id(job_instance.project_id)
spider_name = job_instance.spider_name
arguments = {}
if job_instance.spider_arguments:
arguments = dict(
map(lambda x: x.split("="),
job_instance.spider_arguments.split(",")))
threshold = 0
daemon_size = len(self.spider_service_instances)
if job_instance.priority == JobPriority.HIGH:
threshold = int(daemon_size / 2)
if job_instance.priority == JobPriority.HIGHEST:
threshold = int(daemon_size)
threshold = 1 if threshold == 0 else threshold
candidates = self.spider_service_instances
leaders = []
if 'daemon' in arguments:
for candidate in candidates:
if candidate.server == arguments['daemon']:
leaders = [candidate]
else:
# TODO optimize some better func to vote the leader
for i in range(threshold):
leaders.append(random.choice(candidates))
for leader in leaders:
# add more arguments to scrapyd to run a spider
arguments['project_id'] = job_instance.project_id
arguments['project_name'] = project.project_name
arguments['job_instance_id'] = job_instance.job_instance_id
arguments['priority'] = job_instance.priority
arguments['args'] = job_instance.spider_arguments
arguments['execute_ip'] = leader.server
arguments['create_time'] = datetime.datetime.now()
serviec_job_id = leader.start_spider(project.project_name,
spider_name, arguments)
job_execution = JobExecution()
job_execution.project_id = job_instance.project_id
job_execution.service_job_execution_id = serviec_job_id
job_execution.job_instance_id = job_instance.job_instance_id
job_execution.create_time = arguments['create_time']
job_execution.running_on = leader.server
db.session.add(job_execution)
db.session.commit()
def clear_jobexecution(job_execution):
"""
clear_jobexecution
check JobExecution still existed on scrapyd servers
delete it if didn't existed anymore.
:param job_execution:
:return:
"""
job_instance = JobInstance.find_job_instance_by_id(
job_execution.job_instance_id)
project = Project.find_project_by_id(job_instance.project_id)
if not check_job_existed(running_on=job_execution.running_on,
project_name=project.project_name,
spider_name=job_instance.spider_name,
job_id=job_execution.service_job_execution_id):
db.session.delete(job_execution)
db.session.commit()
def spider_egg_upload(project_id):
project = Project.find_project_by_id(project_id)
if 'file' not in request.files:
flash('No file part')
return redirect(request.referrer)
file = request.files['file']
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '':
flash('No selected file')
return redirect(request.referrer)
if file:
filename = secure_filename(file.filename)
dst = os.path.join(tempfile.gettempdir(), filename)
file.save(dst)
agent.deploy(project, dst)
flash('deploy success!')
return redirect(request.referrer)
def start_spider(self, job_instance):
project = Project.find_project_by_id(job_instance.project_id)
spider_name = job_instance.spider_name
#arguments = {}
#if job_instance.spider_arguments:
# arguments = dict(map(lambda x: x.split("="), job_instance.spider_arguments.split(",")))
from collections import defaultdict
arguments = defaultdict(list)
if job_instance.spider_arguments:
for k, v in list(map(lambda x: x.split('=', 1), job_instance.spider_arguments.split(','))):
arguments[k].append(v)
# threshold = 0
# daemon_size = len(self.spider_service_instances)
# if job_instance.priority == JobPriority.HIGH:
# threshold = int(daemon_size / 2)
# if job_instance.priority == JobPriority.HIGHEST:
# threshold = int(daemon_size)
# threshold = 1 if threshold == 0 else threshold
threshold = 1
candidates = self.spider_service_instances
leaders = []
if 'daemon' in arguments:
for candidate in candidates:
if candidate.server == arguments['daemon']:
leaders = [candidate]
else:
# TODO optimize some better func to vote the leader
for i in range(threshold):
leaders.append(random.choice(candidates))
for leader in leaders:
serviec_job_id = leader.start_spider(project.project_name, spider_name, arguments)
job_execution = JobExecution()
job_execution.project_id = job_instance.project_id
job_execution.service_job_execution_id = serviec_job_id
job_execution.job_instance_id = job_instance.id
job_execution.create_time = datetime.datetime.now()
job_execution.running_on = leader.server
try:
db.session.add(job_execution)
db.session.commit()
except:
db.session.rollback()
raise
def log_url_master(self, job_execution):
"""
功能: 获取主爬虫的日志
:param job_execution: job_execution对象
:return: 返回log的url
"""
job_instance = JobInstance.find_job_instance_by_id(job_execution.job_instance_id)
project = Project.find_project_by_id(job_instance.project_id)
# 主从爬虫运行的服务器字符串
service_job_execution_id = job_execution.service_job_execution_id.split('>')
# 主爬虫服务器
master_service_job_execution_id = service_job_execution_id[0]
# 爬虫运行的服务器地址
running_on = job_execution.running_on.split('>')
master_running_on = running_on[0]
# 调用主爬虫的日志
for spider_service_instance in self.spider_service_instances_master:
if spider_service_instance.server == master_running_on:
master_log_url = spider_service_instance.log_url(
project.project_name, job_instance.spider_name,
master_service_job_execution_id)
return master_log_url
def start_spider(self, job_instance):
project = Project.find_project_by_id(job_instance.project_id)
spider_name = job_instance.spider_name
#arguments = {}
#if job_instance.spider_arguments:
# arguments = dict(map(lambda x: x.split("="), job_instance.spider_arguments.split(",")))
from collections import defaultdict
arguments = defaultdict(list)
if job_instance.spider_arguments:
for k, v in list(map(lambda x: x.split('=', 1), job_instance.spider_arguments.split(','))):
arguments[k].append(v)
threshold = 0
daemon_size = len(self.spider_service_instances)
if job_instance.priority == JobPriority.HIGH:
threshold = int(daemon_size / 2)
if job_instance.priority == JobPriority.HIGHEST:
threshold = int(daemon_size)
threshold = 1 if threshold == 0 else threshold
candidates = self.spider_service_instances
leaders = []
if 'daemon' in arguments:
for candidate in candidates:
if candidate.server == arguments['daemon']:
leaders = [candidate]
else:
# TODO optimize some better func to vote the leader
for i in range(threshold):
leaders.append(random.choice(candidates))
for leader in leaders:
serviec_job_id = leader.start_spider(project.project_name, spider_name, arguments)
job_execution = JobExecution()
job_execution.project_id = job_instance.project_id
job_execution.service_job_execution_id = serviec_job_id
job_execution.job_instance_id = job_instance.id
job_execution.create_time = datetime.datetime.now()
job_execution.running_on = leader.server
db.session.add(job_execution)
db.session.commit()
def job_add(project_id):
project = Project.find_project_by_id(project_id)
job_instance = JobInstance()
job_instance.spider_name = request.form['spider_name']
job_instance.project_id = project_id
job_instance.spider_arguments = request.form['spider_arguments']
job_instance.priority = request.form.get('priority', 0)
job_instance.run_type = request.form['run_type']
if job_instance.run_type == JobRunType.ONETIME:
job_instance.enabled = -1
db.session.add(job_instance)
db.session.commit()
agent.start_spider(job_instance)
if job_instance.run_type == JobRunType.PERIODIC:
job_instance.cron_minutes = request.form.get('cron_minutes') or '0'
job_instance.cron_hour = request.form.get('cron_hour') or '*'
job_instance.cron_day_of_month = request.form.get(
'cron_day_of_month') or '*'
job_instance.cron_day_of_week = request.form.get(
'cron_day_of_week') or '*'
job_instance.cron_month = request.form.get('cron_month') or '*'
db.session.add(job_instance)
db.session.commit()
return redirect(request.referrer, code=302)
def job_add(project_id):
project = Project.find_project_by_id(project_id)
job_instance = JobInstance()
job_instance.spider_name = request.form['spider_name']
job_instance.project_id = project_id
job_instance.spider_arguments = request.form['spider_arguments']
job_instance.priority = request.form.get('priority', 0)
job_instance.run_type = request.form['run_type']
spider_url = request.form['spider_url']
spider_models = request.form['spider_models']
if job_instance.spider_name == "news":
allowed_domains = spider_url.split('/')[2]
a = "allowed_domains=" + allowed_domains
a = a + "," + "model=" + spider_models
job_instance.spider_arguments = a
r = CRedis()
r.lpush(allowed_domains + ':start_urls', spider_url)
elif job_instance.spider_name == 'jd':
r = CRedis()
r.lpush('jd:start_urls', spider_url)
if job_instance.run_type == JobRunType.ONETIME:
job_instance.enabled = -1
db.session.add(job_instance)
db.session.commit()
agent.start_spider(job_instance)
if job_instance.run_type == JobRunType.PERIODIC:
job_instance.cron_minutes = request.form.get('cron_minutes') or '0'
job_instance.cron_hour = request.form.get('cron_hour') or '*'
job_instance.cron_day_of_month = request.form.get(
'cron_day_of_month') or '*'
job_instance.cron_day_of_week = request.form.get(
'cron_day_of_week') or '*'
job_instance.cron_month = request.form.get('cron_month') or '*'
db.session.add(job_instance)
db.session.commit()
return redirect(request.referrer, code=302)
def service_stats(project_id):
project = Project.find_project_by_id(project_id)
run_stats = JobExecution.list_run_stats_by_hours(project_id)
return render_template("server_stats.html", run_stats=run_stats)
def spider_deploy(project_id):
project = Project.find_project_by_id(project_id)
return render_template("spider_deploy.html")
def get_project_list(self):
project_list = self.spider_service_instances[0].get_project_list()
Project.load_project(project_list)
return [project.to_dict() for project in Project.query.all()]
def project_delete(project_id):
project = Project.find_project_by_id(project_id)
agent.delete_project(project)
db.session.delete(project)
db.session.commit()
return redirect("/project/manage", code=302)