def __init__(self, host_port=None, exc_time=None, **kwargs):
super().__init__(**kwargs)
if exc_time and isinstance(exc_time, str):
exc_time = datetime.datetime.strptime(exc_time, '%Y-%m-%d %H:%M:%S')
if host_port:
if exc_time:
self.vc_md5 = get_md5(
host_port, exc_time.strftime('%Y-%m-%d %H:%M:%S'))
self.node_md5 = get_md5(host_port)
self.host_port = host_port
if exc_time:
self.exc_time = exc_time
pass
def __init__(self, name=None, **kwargs):
super().__init__(**kwargs)
self.name = name
if self.permissions is None:
self.permissions = 0
if name is not None:
self.vc_md5 = get_md5(name)
pass
def init_settings(cls):
"""初始系统配置"""
exists = {model.key for model in cls.query.filter(cls.vc_md5.in_(
[get_md5(k) for k in cls.default_settings])).all()}
for key in set(cls.default_settings.keys()).difference(exists):
db.session.add(cls(key=key, value=cls.default_settings[key]))
db.session.commit()
pass
def __init__(self, plan_name=None, project_name=None, version_name=None,
spider_name=None, **kwargs):
super().__init__(**kwargs)
if plan_name:
self.vc_md5 = get_md5(plan_name)
self.plan_name = plan_name
if project_name:
self.project_md5 = get_md5(project_name)
self.project_name = project_name
if version_name and spider_name:
self.spider_md5 = get_md5(
project_name, version_name, spider_name)
if version_name:
self.version_name = version_name
if spider_name:
self.spider_name = spider_name
pass
def __init__(self, host_port=None, project_name=None, job_id=None,
exc_time=None, **kwargs):
super().__init__(**kwargs)
if exc_time and isinstance(exc_time, str):
exc_time = datetime.datetime.strptime(exc_time, '%Y-%m-%d %H:%M:%S')
if host_port and project_name and job_id:
if exc_time:
self.vc_md5 = get_md5(
host_port, project_name, job_id,
exc_time.strftime("%Y-%m-%d %H:%M:%S"))
self.job_md5 = get_md5(host_port, project_name, job_id)
if host_port:
self.host_port = host_port
if project_name:
self.project_name = project_name
if job_id:
self.job_id = job_id
if exc_time:
self.exc_time = exc_time
pass
def __init__(self, host_port=None, project_name=None, job_id=None,
version_name=None, spider_name=None, **kwargs):
super().__init__(**kwargs)
if host_port:
if project_name and job_id:
self.vc_md5 = get_md5(host_port, project_name, job_id)
self.node_md5 = get_md5(host_port)
self.host_port = host_port
if project_name:
self.project_md5 = get_md5(project_name)
if version_name and spider_name:
self.spider_md5 = get_md5(
project_name, version_name, spider_name)
self.project_name = project_name
if version_name:
self.version_name = version_name
if spider_name:
self.spider_name = spider_name
if job_id:
self.job_id = job_id
pass
def cancel_spider(self, host_port, project_name, job_id):
proxies_detail = self.proxies.get(host_port, {})
if not proxies_detail.get("status"):
return
instance = proxies_detail.get("instance")
if not isinstance(instance, ProxySpider):
return
if instance.cancel_spider(project_name, job_id):
JobsModel.update_one(vc_md5=get_md5(host_port, project_name,
job_id),
end_time=datetime.datetime.now(),
job_status=JobStatus.CANCELED.value)
return True
pass
def __init__(self, username=None, **kwargs):
super().__init__(**kwargs)
self.username = username
if self.username is not None:
self.vc_md5 = get_md5(self.username)
if self.role is None:
if self.email == current_app.config['MAIL_ADMIN']:
self.role = RolesModel.query.filter_by(
name='Administrator').first()
if self.role is None:
self.role = RolesModel.query.filter_by(default=True).first()
if self.email is not None and self.avatar_hash is None:
self.avatar_hash = self.gravatar_hash()
self.logger = current_app.logger
pass
def get_versions(self):
if not self.versions:
return []
tmp = dict()
for s in self.spiders:
version_name = s.version_name
if version_name not in tmp:
tmp[version_name] = []
tmp[version_name].append(s.to_dict())
return [{
"vc_md5": get_md5(self.project_name, v),
"project_name": self.project_name,
"version_name": v,
"spiders": tmp.get(v, []), } for v in json.loads(self.versions)]
pass
def sync_spiders(cls, project_name, version_name, *spider_names):
md5_set = set()
# 新增或更新 add
for spider_name in spider_names:
model = cls(project_md5=get_md5(project_name),
project_name=project_name,
version_name=version_name,
spider_name=spider_name)
md5_set.add(model.vc_md5)
db.session.merge(model)
db.session.commit()
# 删除 synchronize_session=False
query = cls.query.filter(cls.vc_md5.notin_(md5_set),
cls.version_name == version_name,
cls.project_name == project_name)
query.delete(synchronize_session=False)
db.session.commit()
pass
def delete_project(self, project_name, version_name=None):
"""
删除项目
:param project_name: 项目名称
:param version_name: 版本名称
:return:
"""
for slave_name, proxies_detail in self.proxies.items():
if not proxies_detail.get("status"):
continue
instance = proxies_detail.get("instance")
if not isinstance(instance, ProxySpider):
continue
if version_name:
instance.del_version(project_name, version_name)
self.del_slaves_value(slave_name, project_name, version_name)
else:
instance.del_project(project_name)
self.del_slaves_value(slave_name, project_name)
pass
try:
self.egg_storage.delete(project_name, version_name)
except Exception as e:
print(e)
versions = self.egg_storage.list(project_name)
if versions:
default_version = versions[0]
if project_name in self.projects:
if version_name in self.projects[project_name]:
self.projects[project_name].pop(version_name)
SpidersModel.del_many(version_name=version_name,
project_name=project_name)
ProjectsModel.merge_one(
project_name=project_name,
version_name=default_version,
versions=json.dumps(versions),
)
else:
if project_name in self.projects:
self.projects.pop(project_name)
SpidersModel.del_many(project_name=project_name)
ProjectsModel.del_one(vc_md5=get_md5(project_name))
pass
def sync_slaves_status(self):
"""
检查运行节点
:return:
"""
for host_port, proxies_detail in self.proxies.items():
instance = proxies_detail.get("instance")
if not isinstance(instance, ProxySpider):
continue
daemon_status = instance.get_daemon_status() or {}
status = daemon_status.get("status", False)
proxies_detail["status"] = status
NodesModel.update_one(vc_md5=get_md5(host_port),
node_name=daemon_status.get(
"node_name", False),
status='运行正常' if status else "链接失败",
pending=daemon_status.get("pending"),
running=daemon_status.get("running"),
finished=daemon_status.get("finished"))
pass
def __init__(self, key=None, **kwargs):
super().__init__(**kwargs)
if key:
self.vc_md5 = get_md5(key)
self.key = key
pass
def __init__(self, project_name=None, **kwargs):
super().__init__(**kwargs)
if project_name:
self.vc_md5 = get_md5(project_name)
self.project_name = project_name
pass
def __init__(self, host_port=None, **kwargs):
super().__init__(**kwargs)
if host_port:
self.vc_md5 = get_md5(host_port)
self.host_port = host_port
pass