def execute_cleanCrawlHuaban():
"""执行清理花瓣网插件目录下过期的压缩文件"""
huabandir = os.path.join(basedir, "plugins", "CrawlHuaban")
for root in os.listdir(huabandir):
board_id, root = root, os.path.join(huabandir, root)
if os.path.isdir(root):
# 判断是否锁目录中
lock = False
if os.path.exists(os.path.join(root, "board.lock")):
lock = True
logger.cli.info("Locking for {}".format(root))
for f in os.listdir(root):
filepath = os.path.join(root, f)
if ".zip" == os.path.splitext(f)[-1]:
timestamp = int(os.path.splitext(f)[0].split("_")[-1])
if timestamp_after_timestamp(
timestamp, hours=24) <= get_current_timestamp():
logger.cli.info("Remove zip file: {}".format(filepath))
try:
os.remove(filepath)
_sb.mysql_write.update(
"update plugin_crawlhuaban set status=2,mtime=%s where board_id=%s and filename=%s",
get_current_timestamp(), board_id, f)
except Exception, e:
logger.cli.error(e, exc_info=True)
else:
if lock is False:
os.remove(filepath)
try:
os.rmdir(root)
except OSError:
pass
def AddExecuteTask(self, sqlContent, dbId, inception, **kwargs):
"""添加记录并执行任务，流程：
通过代理器执行Check获取结果，code为0并且检查参数通过则继续；判断立即执行或定时执行
# 立即执行
# 以下是准备写入到MySQL的数据
1. 解析Check结果，判断errlevel，如果有2，status=1(自动审核失败)；如果有1，且不忽略警告status=1，忽略警告则status=2
2. status=2时写入mysql，成功后提交给rq去执行，
3. status=1不允许提交
# 定时执行
1. 定时执行时，解析定时时间放到rqscheduler执行，所需函数及参数同2
"""
# 获取其他需要参数，必须项有sd、dbId、inception、applicant，可选默认项
res = self.AutoCheck(sqlContent, dbId, inception)
if res["code"] == 0:
autoviewResult = res["data"]
res = dict(code=1, msg=None)
sd = kwargs["sd"]
applicant = kwargs["applicant"]
enableRemoteBackup = int(kwargs.get("enableRemoteBackup", 1))
enableIgnoreWarnings = int(kwargs.get("enableIgnoreWarnings", 0))
executeNow = int(kwargs.pop("executeNow", 1))
timer = kwargs.pop("timer") or 0
if sd and applicant and autoviewResult and isinstance(
autoviewResult, (list, tuple)):
# 参数验证完毕，解析Check结果获取sqlsha1、errlevel
pir, status = parse_inception_result(autoviewResult, 1), 0
if 2 in pir["errlevel"]:
status = 1
elif 1 in pir["errlevel"]:
status = 1 if enableIgnoreWarnings == 0 else 2
else:
status = 2
if status == 2:
# 立即执行与定时执行
oldtimer = timer
if executeNow == 0:
# timer指秒数，且不能小于5分钟，表示当前时间多少秒后执行
status = 7
try:
timer = datetime_to_timestamp(
timer) - get_current_timestamp()
if timer < 300:
raise TimingLimitError
except TimingLimitError:
res.update(msg="Invaild timing")
return res
except Exception:
res.update(msg="Invaild timer")
return res
logger.debug(
"now:{}, timer param string: {}, seconds: {}, execute at {}, utc: {}"
.format(
timestamp_datetime(get_current_timestamp()),
oldtimer, timer,
timestamp_datetime(
timestamp_after_timestamp(seconds=timer)),
timestamp_to_utcdatetime(
timestamp_after_timestamp(seconds=timer))))
else:
timer = 0
try:
sql = 'insert into incetops_task (dbId,sd,inception,applicant,ctime,status,timer,sqlContent,autoviewResult,autoviewSQLSHA1,enableIgnoreWarnings,enableRemoteBackup) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
taskId = self.mysql.insert(sql, dbId, sd, inception,
applicant,
get_current_timestamp(),
status, timer, sqlContent,
json.dumps(autoviewResult),
",".join(pir['sqlsha1']),
enableIgnoreWarnings,
enableRemoteBackup)
except Exception, e:
logger.error(e, exc_info=True)
res.update(msg="System is abnormal")
else:
kwargs["taskId"] = taskId
kwargs["timeout"] = 3600
if executeNow == 1:
self.asyncQueueHigh.enqueue_call(
func=InceptionProxy,
args=("Execute", sqlContent,
self.dbService.Get(dbId), inception),
kwargs=kwargs,
timeout=kwargs["timeout"])
else:
job = self.asyncScheduler.enqueue_at(
timestamp_to_utcdatetime(
timestamp_after_timestamp(seconds=timer)),
InceptionProxy, "Execute", sqlContent,
self.dbService.Get(dbId), inception, **kwargs)
self.mysql.update(
"update incetops_task set timer_id=%s where taskId=%s",
job.id, taskId)
logger.debug(self.asyncScheduler.get_jobs())
logger.debug(job.to_dict())
res.update(code=0, taskId=taskId)
else:
res.update(msg="Automatic review rejection")
else:
res.update(msg="There are invalid parameters")
def updateUserProfile(self, uid, **profiles):
"""更新用户基本资料
@param uid str: 用户id
@param profiles dict: 资料列表
"""
#: TODO 决不能拼接SQL，先获取当前资料，用提交更新的部分资料覆盖
res = dict(msg=None, code=1)
logger.debug(profiles)
nick_name = profiles.get("nick_name")
domain_name = profiles.get("domain_name")
birthday = profiles.get("birthday")
location = profiles.get("location")
gender = profiles.get("gender")
signature = profiles.get("signature")
# 定义
checked = True
invalid = []
can_lock_nick_name = False
can_lock_domain_name = False
# 防止sql注入的安全检测
for key, value in profiles.iteritems():
checked = sql_safestring_check(value)
logger.debug("check {}, value: {}, result: {}".format(
key, value, checked))
if checked is False:
invalid.append(key)
break
# 至少有一项资料修改才通过检测
if checked:
if not nick_name and \
not domain_name and \
not birthday and \
not location and \
not gender and \
not signature:
checked = False
invalid.append("all")
if checked and uid and isinstance(uid, basestring) and len(uid) == 22:
# 获取用户资料对比nick_name、domain_name是否修改过
user_old_data = self.getUserProfile(uid)
if user_old_data["code"] == 0:
# 开始检测并组建sql
sql = "UPDATE user_profile SET "
# 用户旧数据
user_old_data = user_old_data["data"]
# 昵称
if nick_name:
if len(nick_name) <= 49:
# nick_name可修改时拼接上sql
if user_old_data["lock"]["nick_name"] is False:
sql += "nick_name='%s'," % nick_name
if nick_name != user_old_data["nick_name"]:
# 判断昵称更改，加锁，设置其24小时后过期
can_lock_nick_name = True
sql += "lock_nick_name={},".format(
timestamp_after_timestamp(hours=24))
else:
checked = False
invalid.append("nick_name")
# 个性域名
if domain_name:
if domain_name_pat.match(
domain_name) and not domain_name.endswith(
'_') and not domain_name in (
"admin", "system", "root", "administrator",
"null", "none", "true", "false", "user"):
# domain_name可修改时拼接上sql
if user_old_data["lock"]["domain_name"] is False:
sql += "domain_name='%s'," % domain_name
if domain_name != user_old_data["domain_name"]:
# 判断域名更改，加锁，设置其永久有效
can_lock_domain_name = True
sql += "lock_domain_name=0,"
else:
checked = False
invalid.append("domain_name")
# 生日
if birthday:
try:
birthday = timestring_to_timestamp(birthday,
format="%Y-%m-%d")
except:
checked = False
invalid.append("birthday")
else:
sql += "birthday=%d," % birthday
# 性别
if gender:
try:
gender = int(gender)
except:
checked = False
invalid.append("gender")
else:
if gender in (0, 1, 2):
sql += "gender=%d," % gender
else:
checked = False
invalid.append("gender")
# 签名
if signature:
sql += "signature='%s'," % signature
# 城市
if location:
sql += "location='%s'," % location
# 更新sql
if checked:
# 资料更新，设置mtime
sql += "mtime={}".format(get_current_timestamp())
sql += " WHERE uid='%s'" % uid
logger.debug("update profile for {}, sql is: {}".format(
uid, sql))
try:
self.mysql.update(sql)
except IntegrityError, e:
logger.warn(e, exc_info=True)
res.update(msg="Personal domain has been occupied",
code=2)
except Exception, e:
logger.error(e, exc_info=True)
res.update(msg="System is abnormal", code=3)
else:
res.update(code=0,
refreshCache=self.refreshUserProfile(uid))
# 更新成功后设置锁
res.update(lock=dict(nick_name=can_lock_nick_name,
domain_name=can_lock_domain_name))
else:
res.update(msg="There are invalid parameters", code=5)
key = "EauDouce:CrawlHuaban:{site}:{board_id}".format(
site=site, board_id=board_id)
hasKey = False
try:
hasKey = pb.redis.exists(key)
except Exception, e:
logger.sys.error(e, exc_info=True)
if hasKey:
data = pb.redis.hgetall(key)
res.update(
msg=
'当前画板下载中，链接是<a href="{}" target="_blank" title="请点击打开新页面或手动复制链接">{}</a>。温馨提示，5分钟内请勿重复对同一个画板使用远程下载服务！'
.format(data["downloadUrl"], data["downloadUrl"]))
else:
ctime = get_current_timestamp()
etime = timestamp_after_timestamp(hours=24)
filename = "{}_{}.zip".format(site, ctime)
expireTime = timestamp_to_timestring(etime)
downloadUrl = url_for("CrawlHuaban.index",
board_id=board_id,
filename=filename,
_external=True)
pipe = pb.redis.pipeline()
pipe.hmset(
key, dict(downloadUrl=downloadUrl, expireTime=expireTime))
pipe.expire(key, 300)
try:
pipe.execute()
except Exception, e:
logger.sys.error(e, exc_info=True)
finally: