def post(self, headers):
data = self.parser.parse_args()
print(data)
# json_data = request.get_json(force=True)
# if not json_data:
# return {'message': 'No input data provided'}, 400
# data, errors = taskTemplatesSchema().load(json_data)
# if errors:
# return errors,422
# sop_name = taskTemplates.query.filter_by(sop_name=json_data['sop_name']).first()
# if sop_name:
# return {'message': 'taskTemplate already exists'}, 400
#变量如果没有会怎样,需要指定阶段文件？
template = request.files['template']
if not template:
return {'message': 'No input template provided'}, 400
template_name = Template.query.filter_by(templateName=data.get('templateName')).first()
template_URL = Template.query.filter_by(templateDownloadURL=template.filename).first()
templateCreatorRealName = session.query(User).filter_by(userID=headers["userID"]).first().userRealName
taskTemplate = Template(
templateName=data.get('templateName'),
templateDescription = data.get('templateDescription'),
templateCreateDate = time.ctime(time.time()),
templateCreatorID = headers["userID"],
templateStatus = data.get('templateStatus'),
templateRemoveDate = None,
templateRemoveExecutorID = None,
templateDeleteDate = None,
templateDeleteExecutorID = None,
templateDownloadURL = template.filename,
templateCreatorRealName = templateCreatorRealName
)
if template_name:
return {'message': 'taskTemplate already exists'}, 400
if template_URL:
return {'message': 'Template file already exists, please rename'}, 400
#略掉中文字符,文件不要有中文字符
# templatename = secure_templatename(template.templatename)
template.save(os.path.join('./static/', template.filename))
session.add(taskTemplate)
session.commit()
#return {'message': 'template save error'}, 400
return {"statusCode": "1"}
def post(self, headers):
json_data = request.get_json(force=True)
if not json_data:
return {'message': 'No input data provided'}, 400
print(json_data)
#接收的日期格式为2014-08-11T05:26:03.869245
# data, errors = TaskSchema().load(json_data, session=session)
# if errors:
# return errors,422
name = Task.query.filter_by(taskName=json_data['taskName']).first()
if name:
return {'message': 'name already exists'}, 400
#变量如果没有会怎样,需要指定阶段文件？
taskBelongedToProjectName = session.query(Project).filter_by(
projectID=json_data['taskBelongedToProjectID']).first().projectName
taskExecutorRealName = session.query(User).filter_by(
userID=headers['userID']).first().userRealName
task = Task(
taskName=json_data['taskName'],
taskBelongedToProjectID=json_data['taskBelongedToProjectID'],
#belongedToTrialName = json_data['belongedToTrialName'],
taskCreatorID=headers['userID'],
#taskCreatorName = json_data['taskCreatorName'],
taskCreatedTime=time.strftime("%Y-%m-%d", time.localtime()),
taskExecutorID=json_data['taskExecutorID'],
#taskExecutorName = json_data['taskExecutorName'],
taskReceivedStatus=json_data['taskReceivedStatus'],
taskDueTime=json_data['taskDueTime'],
taskProgress=json_data['taskProgress'],
taskCompletedStatus=None,
taskDescription=json_data['taskDescription'],
taskActualCompletedTime=None,
taskBelongedToProjectName=taskBelongedToProjectName,
taskExecutorRealName=taskExecutorRealName)
session.add(task)
session.commit()
executor = User.query.filter(
User.userID == task.taskExecutorID).first()
taskBelongedToProject = Project.query.filter(
Project.projectID == task.taskBelongedToProjectID).first()
sendMail("任务分配通知", executor.userEmail, executor.userRealName,
taskBelongedToProject.projectName, task.taskName,
task.taskDescription, task.taskDueTime.strftime("%Y-%m-%d"))
return {"statusCode": "1", "taskID": task.taskID}
def post(self, headers):
print(headers['userID'])
#需要判断该用户是否有增加新用户的权限
json_data = request.get_json(force=True)
if not json_data:
return {'message': 'No input data provided'}, 400
print(json_data)
# data, errors = UserSchema().load(json_data, session=session)
# if errors:
# return errors,422
user = User.query.filter_by(username=json_data['username']).first()
if user:
return {'message': 'user already exists'}, 400
#变量如果没有会怎样
user = User(username=json_data['username'],
userRealName=json_data['userRealName'],
password=json_data['password'],
userEmail=json_data['userEmail'],
isAdmin=False,
userAccountStatus=json_data['userAccountStatus'],
userLastLoginTime=None)
session.add(user)
session.commit()
if "userInvolvedProjectsID" in json_data:
db.session.execute(
userProject.__table__.insert(),
[{
"userID": user.userID,
"projectID": json_data["userInvolvedProjectsID"][i],
"userType": "2"
} for i in range(len(json_data["userInvolvedProjectsID"]))])
db.session.commit()
if 'userCanManageProjectsID' in json_data:
db.session.execute(
userProject.__table__.insert(),
[{
"userID": user.userID,
"projectID": json_data['userCanManageProjectsID'][i],
"userType": "1"
} for i in range(len(json_data["userCanManageProjectsID"]))])
db.session.commit()
sendMail("新用户创建成功", user.userEmail, user.userRealName, user.username,
user.password)
return {"statusCode": "1"}
def put(self, headers):
# json_data = request.get_json(force=True)
data = self.parser.parse_args()
data_template_id = data.get('templateID')
update_template = session.query(Template).filter_by(templateID=data_template_id).first()
if data.get('templateName'):
update_template.templateName = data.get('templateName')
if data.get('templateDescription'):
update_template.templateDescription = data.get('templateDescription')
if data.get('templateStatus'):
update_template.templateStatus = data.get('templateStatus')
if data.get('templateStatus') == "2":
update_template.templateRemoveExecutorID = headers["userID"]
update_template.templateRemoveDate = time.ctime(time.time())
# 更新方法，变量json_data中的所有数据
# for k in json_data:
# update_user.update({k:json_data[k]})
session.commit()
return {"statusCode": "1"}
def put(self, headers):
json_data = request.get_json(force=True)
data_user_id = json_data['userID']
session.query(userProject).filter(
userProject.userID == data_user_id).delete()
session.commit()
update_user = session.query(User).filter_by(userID=data_user_id)
update_json = json_data.copy()
update_json.pop("userInvolvedProjectsID")
update_json.pop("userCanManageProjectsID")
print(update_json)
update_user.update(update_json)
# session.delete(del_by_id)
#更新方法，变量json_data中的所有数据
# for k in json_data:
# update_user.update({k:json_data[k]})
session.commit()
if json_data["userInvolvedProjectsID"]:
db.session.execute(
userProject.__table__.insert(),
[{
"userID": data_user_id,
"projectID": json_data["userInvolvedProjectsID"][i],
"userType": "2"
} for i in range(len(json_data["userInvolvedProjectsID"]))])
db.session.commit()
if json_data["userCanManageProjectsID"]:
db.session.execute(
userProject.__table__.insert(),
[{
"userID": data_user_id,
"projectID": json_data['userCanManageProjectsID'][i],
"userType": "1"
} for i in range(len(json_data["userCanManageProjectsID"]))])
db.session.commit()
return {"statusCode": "1"}
def update_user(self, _id, new_username, new_password, new_email):
update = session.query(User).filter(User.id == _id).first()
update.username = new_username
session.commit()
update.password = new_password
session.commit()
update.email = new_email
session.commit()
return
def put(self, headers):
# json_data = request.get_json(force=True)
data = self.parser.parse_args()
data_file_id = data.get("fileID")
# data_file_id = json_data['fileID']
update_file = session.query(File).filter_by(
fileID=data_file_id).first()
if data.get('fileName'):
update_file.fileName = data.get('fileName')
if data.get('fileDescription'):
update_file.fileDescription = data.get('fileDescription')
if data.get('fileStatus'):
update_file.fileStatus = data.get('fileStatus')
if data.get('fileStatus') == "2":
update_file.fileRemoveID = headers["userID"]
update_file.fileRemoveDate = time.ctime(time.time())
# update_file.update()
# 更新方法，变量json_data中的所有数据
# for k in json_data:
# update_user.update({k:json_data[k]})
session.commit()
return {"statusCode": "1"}
def change_email(self, id, email):
session.query(User).filter(User.id == id).update({'email': email})
session.commit()
return
def change_password(self, id, password):
session.query(User).filter(User.id == id).update(
{'password': password})
session.commit()
return
def change_username(self, id, username):
session.query(User).filter(User.id == id).update(
{'username': username})
session.commit()
return
def add_endpoint(self, id, data):
endp = Endpoint(uri=data['uri'], api_id=id)
session.add(endp)
session.commit()
return
def change_email(self, id, email):
user = User.query.get(id)
user.email = email
session.commit()
return
def post(self, headers):
data = self.parser.parse_args()
print(data)
# json_data = request.get_json(force=True)
# if not json_data:
# return {'message': 'No input data provided'}, 400
# data, errors = taskFilesSchema().load(json_data)
# if errors:
# return errors,422
# sop_name = taskFiles.query.filter_by(sop_name=json_data['sop_name']).first()
# if sop_name:
# return {'message': 'taskFile already exists'}, 400
#变量如果没有会怎样,需要指定阶段文件？
file = request.files['file']
if not file:
return {'message': 'No input file provided'}, 400
file_name = File.query.filter_by(fileName=data.get('fileName')).first()
file_URL = File.query.filter_by(fileDownloadURL=file.filename).first()
fileBelongedToTaskName = session.query(Task).filter_by(
taskID=data.get('fileBelongedToTaskID')).first().taskName
fileBelongedToProjectName = session.query(Project).filter_by(
projectID=data.get('fileBelongedToProjectID')).first().projectName
fileCreatorName = session.query(User).filter_by(
userID=headers["userID"]).first().username
taskFile = File(
fileName=data.get('fileName'),
fileDescription=data.get('fileDescription'),
fileBelongedToTaskID=data.get('fileBelongedToTaskID'),
fileBelongedToProjectID=data.get('fileBelongedToProjectID'),
fileCreateDate=time.ctime(time.time()),
fileCreatorID=headers["userID"],
fileStatus=data.get('fileStatus'),
fileRemoveDate=None,
fileRemoveExecutorID=None,
fileDeleteDate=None,
fileDeleteExecutorID=None,
fileDownloadURL=file.filename,
fileBelongedToTaskName=fileBelongedToTaskName,
fileBelongedToProjectName=fileBelongedToProjectName,
fileCreatorName=fileCreatorName
# fileName = data.get('fileName'),
# createDate = data.get('createDate'),
# creatorID = data.get('creatorID'),
# # self.creatorName = creatorName
# description = data.get('description'),
# deleteDate = data.get('deleteDate'),
# deleteExecutorID = data.get('deleteExecutorID'),
# # self.deleteExecutorName = deleteExecutorName
# downloadURL = file.filename
)
if file_name:
return {'message': 'taskFile already exists'}, 400
if file_URL:
return {'message': 'File already exists, please rename'}, 400
#略掉中文字符,文件不要有中文字符
# filename = secure_filename(file.filename)
file.save(os.path.join('./static/', file.filename))
session.add(taskFile)
session.commit()
#return {'message': 'file save error'}, 400
return {"statusCode": "1"}
def delete_api(self, id):
api = session.query(Api).get(id)
session.delete(api)
session.commit()
return
def add_api(self, id, data):
api = Api(api_name=data['api_name'], description=data['description'], user_id=id)
session.add(api)
session.commit()
return
def change_username(self, id, username):
user = User.query.get(id)
user.username = username
session.commit()
return
def change_password(self, id, password):
user = User.query.get(id)
user.password = password
session.commit()
return
def post(self):
# 看传递过来的是formdata数据还是json数据
try:
headers = jwt.decode(self.header["Authorization"],
'secret',
algorithms=['HS256'])
return {
"statusCode": "0",
"error": {
"message": "当前用户已经登录",
"errorCode": "400"
}
}
except:
json_data = self.parser.parse_args()
if not json_data:
return {
"statusCode": "0",
"error": {
"message": "内容为空",
"errorCode": "403"
}
}
# data, errors = UserSchema().load(json_data, session=session)
# if errors:
# return errors, 422
user = User.query.filter_by(username=json_data.get("username")).first()
print(user, json_data)
if not user:
return {
"statusCode": "0",
"error": {
"message": "用户不存在",
"errorCode": "403"
}
}
if (user.password != json_data.get("password")):
return {
"statusCode": "0",
"error": {
"message": "密码或用户名错误",
"errorCode": "403"
}
}
#更新登录时间
user.userLastLoginTime = time.ctime(time.time())
session.commit()
#添加token(密码加密后返回?)
#secret可以写入配置文件中, 'exp':int(time.time())+8000
token = jwt.encode({
'userID': user.userID,
'isAdmin': user.isAdmin
},
'secret',
algorithm='HS256')
return {
"statusCode": "1",
"userInfo": {
'username': user.username,
'userRealName': user.userRealName,
'isAdmin': user.isAdmin,
'userID': user.userID
},
"token": token.decode('utf-8')
}
def change_apis(self, id, apis):
user.apis = apis
# HUR UPPDATERA EN RELATION?
session.commit()
return
def get_page_message(self, href):
print(href)
self.driver.get(href)
try:
login_btn = self.driver.find_element_by_link_text("请登录")
if login_btn is not None:
self.login()
self.driver.get(href)
except Exception as e:
# print(e)
print()
bs4 = BeautifulSoup(self.driver.page_source, features="html.parser")
basic = bs4.select_one(".tb-property")
price = origin_price = score = 0
tags_str = labels_str = title = ""
if basic is not None:
title_box = basic.select_one(".tb-detail-hd > h1 > a")
if title_box is None:
return
# 标题
title = title_box.get_text()
# 标签
tag = basic.select_one(
"#J_DetailMeta > div.tm-clear > div.tb-property > div > div.tb-detail-hd > p"
)
if tag is not None:
tags = helper.get_char(tag.get_text())
tags_str = ",".join(tags)
# 当前售价
price = self.get_price(basic)
if isinstance(price, str) and price.find("-") != -1:
price = str.split(price, "-")[0]
# 原价
origin_price = self.get_origin_price(basic)
# 商家
merchant = bs4.select_one(
"#shopExtra > div.slogo > a > strong").get_text()
parameter_list = bs4.select(
".tm-tableAttr > tbody > tr:not(.tm-tableAttrSub)")
# 属性
parameter = {}
if parameter_list is not None:
for item in parameter_list:
parameter[item.select_one("th").get_text()] = item.select_one(
"td").get_text()
ignore = False
# 用户评价需要点击后加载
# 淘宝发爬虫, 一定情况下会弹出滑动验证码, 弹出后跳过
try:
self.driver.execute_script("window.scrollTo(0, 200)")
tags_button = self.wait.until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, "#J_TabBar > li:nth-child(3)")))
tags_button.click()
self.wait.until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, "#J_Reviews > div > "
"div.rate-header")))
except Exception as e:
print(e)
ignore = True
if ignore is False:
bs4 = BeautifulSoup(self.driver.page_source,
features="html.parser")
rate_box = bs4.select_one("#J_Reviews > div > div.rate-header")
if rate_box is None:
rate_box = bs4.select_one(
"#J_Reviews > div > div.rate-header.rate-header-tags")
if rate_box is not None:
# 好评率
score = rate_box.select_one(".rate-score > strong")
if score is not None:
# 跟苏宁评分规则不一致, 转化成百分制
score = self.__transToCentesimal(score.get_text())
tag_list = rate_box.select(".rate-tag-inner > span > a")
if tag_list is not None:
# 评价标签, 部分商品不存在评价标签
labels = [
helper.get_char(tag.get_text())[0] for tag in tag_list
]
labels_str = ",".join(labels)
else:
print("not found tag list")
model = Conditioner.airConditioner(tag_str=tags_str,
link=self.driver.current_url,
title=title,
merchant=merchant,
property=parameter,
price=float(price) * 100,
origin_price=float(origin_price) *
100,
feedback=score,
labels=labels_str,
platform=self.platform) # 转化成百分制
session.add(model)
try:
session.commit()
redis_client.sadd(self.spider_key, href)
except exc.SQLAlchemyError as e:
print("insert taobao data failed.", e)
return
def delete_user(self, id):
user = session.query(User).get(id)
session.delete(user)
session.commit()
return
def getSuNing(page=0, offset=0):
while True:
ins = sn.sn()
url = ins.get_list_url(page, offset)
response = requests.get(url)
if response.status_code != 200:
print("获取苏宁家电异常, url:%s, code:%d" % (url, response.status_code))
return
ct = response.headers["Content-type"].split("charset=")[1].lower()
bs = BeautifulSoup(response.content,
features="html.parser",
from_encoding=ct)
data = bs.find_all("li", class_="basic")
if not data:
break
for item in data:
box = item.select_one(".product-box")
img_box = box.select_one(".res-img > .img-block > a")
store_box = box.select_one(".store-stock > a")
href = img_box["href"]
if not href.startswith("http"):
href = "https:" + href
if redis_client.sismember(REDIS_KEY, href) > 0:
print("the url is scraped. url:%s" % href)
continue
print("fetch href: %s" % href)
# 空调相关的参数
parameter = ins.get_parameter(href)
# 获取售卖价格
price = ins.get_price()
# 获取原价
origin_price = ins.get_origin_price()
# 获取好评率
score = ins.get_evaluate_score()
# 获取评价标签
labels = ins.get_evaluate_labels()
# 插入DB
model = Conditioner.airConditioner(
tag_str=img_box["title"],
link=href,
title=img_box.select_one("img")["alt"],
merchant=store_box.get_text(),
property=parameter,
price=price * 100,
origin_price=origin_price * 100,
feedback=score,
labels=",".join(labels),
platform=ins.platform)
session.add(model)
redis_client.sadd(REDIS_KEY, href)
try:
session.commit()
except exc.SQLAlchemyError as e:
print("sql failed", e)
continue
# 苏宁的web规则, paging的值为0~3
if offset <= 3:
offset = offset + 1
else:
page = page + 1
offset = 0
if page > 50:
break
