def get_raw_kind_batch(site, kind):
try:
now = int(time.time())
connect = db.connect()
sql = text(
'select task_schedule.key, task_schedule.value from task_schedule where site=:site and kind=:kind and status in (0,1) and dealtime < :now limit 10000;'
)
cursor = connect.execute(sql, site=site, kind=kind, now=now)
result = []
dealtime = int(time.time()) + TaskSchedule.STEP
for item in cursor:
sql = text(
'update task_schedule set status=1, dealtime=:dealtime where task_schedule.key=:ts_key and kind=:kind and site=:site;'
)
u_cursor = connect.execute(sql,
dealtime=dealtime,
ts_key=item[0],
kind=kind,
site=site)
u_cursor.close()
result.append({
"key":
item[0].encode("utf8")
if isinstance(item[0], unicode) else item[0],
"value":
item[1].encode("utf8") if
(item[1] and isinstance(item[1], unicode)) else
(item[1] or "")
})
cursor.close()
connect.close()
return result
except:
logger.error(traceback.format_exc())
return []
def batch_upsert(cls, session, **kwargs):
try:
for k_lst in kwargs["keys"]:
if isinstance(k_lst, list):
for k in k_lst:
ts = cls.find_by_no(session, k, kwargs["kind"],
kwargs["site"]) or cls()
ts.key = k
for _k, _v in kwargs.items():
if _k != "keys":
setattr(ts, _k, _v)
session.add(ts)
session.commit()
else:
ts = cls.find_by_no(session, k_lst, kwargs["kind"],
kwargs["site"]) or cls()
ts.key = k_lst
for _k, _v in kwargs.items():
if _k != "keys":
setattr(ts, _k, _v)
session.add(ts)
session.commit()
return True
except:
logger.error(traceback.format_exc())
return None
def raw_upsert(connect, **kwargs):
try:
name = kwargs.get("name", "") or ""
pro_no = kwargs["pro_no"]
shop_no = kwargs["shop_no"]
category_id = kwargs.get("category_id", "") or ""
image = kwargs.get("image", "") or ""
rate = kwargs.get("rate", 0) or 0
msrp = kwargs.get("msrp", 0) or 0
discount = kwargs.get("discount", 0) or 0
real_price = kwargs.get("real_price", 0) or 0
reviews_count = kwargs.get("reviews_count", 0) or 0
create_time = kwargs["create_time"]
update_time = kwargs["update_time"]
sql = text(
'insert into joom_pro (joom_pro.name,pro_no,shop_no,category_id,image,rate,msrp,discount,real_price,reviews_count,create_time,update_time,cate_id1,cate_id2,cate_id3,cate_id4,cate_id5,origin_price,r_count_30,r_count_7,r_count_7_14,growth_rate,save_count) values (:jp_name,:pro_no,:shop_no,:category_id,:image,:rate,:msrp,:discount,:real_price,:reviews_count,:create_time,:update_time,"","","","","",0,0,0,0,0,0) on duplicate key update joom_pro.name=:jp_name,category_id=:category_id,rate=:rate,msrp=:msrp,discount=:discount,real_price=:real_price,reviews_count=:reviews_count,update_time=:update_time;'
)
cursor = connect.execute(sql,
jp_name=name,
pro_no=pro_no,
shop_no=shop_no,
category_id=category_id,
image=image,
rate=rate,
msrp=msrp,
discount=discount,
real_price=real_price,
reviews_count=reviews_count,
create_time=create_time,
update_time=update_time)
cursor.close()
return True
except:
logger.error(traceback.format_exc())
return False
class MonitorControl(object):
def __init__(self, public_ip, session):
self.public_ip = public_ip
self.session = session
def get_record(self):
return Monitor.find_by_public_ip(self.session, self.public_ip)
def add_to_record(self,
user_name,
password,
name,
area,
tags,
service,
description="",
ssh_client=None):
if not ssh_client:
ssh_client = paramiko.SSHClient()
# 设置为接受不在known_hosts 列表的主机可以进行ssh连接
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh_client.connect(hostname=self.public_ip,
port=22,
username=user_name,
password=password)
memory_capacity = self.exec_command(
ssh_client, "free -m | grep Mem | awk '{print $2}'")
disk_capacity = self.exec_command(
ssh_client, "df -m | awk '{print $2}'| head -n 2 | tail -n 1")
core_count = self.exec_command(
ssh_client, "cat /proc/cpuinfo | grep 'cpu cores' | wc -l")
cpu_count = self.exec_command(
ssh_client, "cat /proc/cpuinfo | grep 'physical id' | wc -l")
inner_ip = self.exec_command(
ssh_client,
"/sbin/ifconfig -a | grep 'inet addr' | awk -F: '{print $2}' | awk '{print $1}' | head -n 1"
)
Monitor.create(
self.session, **{
"name": name,
"inner_ip": inner_ip,
"public_ip": self.public_ip,
"cpu_count": cpu_count,
"core_count": core_count,
"disk_capacity": disk_capacity,
"memory_capacity": memory_capacity,
"user_name": user_name,
"pwd": password,
"area": area,
"description": description,
"tags": tags,
"service": service
})
except socket.error, e:
logger.error("添加机器失败:%s" % e.message)
logger.error(traceback.format_exc(e))
except paramiko.ssh_exception.AuthenticationException, e:
logger.error("添加机器失败:%s" % e.message)
logger.error(traceback.format_exc(e))
def upsert(cls, session, **kwargs):
try:
joom_user = cls.find_by_no(
session, kwargs["user_no"]) or cls(user_no=kwargs["user_no"])
joom_user.full_name = kwargs.get("full_name", "")
joom_user.images = kwargs.get("images", "")
session.merge(joom_user)
session.commit()
return joom_user
except:
logger.error(traceback.format_exc())
return None
def upsert(cls, session, **kwargs):
try:
ts = cls.find_by_no(session, kwargs["key"], kwargs["kind"],
kwargs["site"]) or cls()
for k, v in kwargs.items():
setattr(ts, k, v)
session.merge(ts)
session.commit()
return ts
except:
logger.error(traceback.format_exc())
return None
def raw_upsert(connect, **kwargs):
sql = text(
'insert into joom_user (user_no, full_name, images) values (:user_no, :full_name, :images) on duplicate key update full_name=:full_name, images = :images;'
)
cursor = None
try:
cursor = connect.execute(sql, **kwargs)
except:
logger.error("joom user upsert error: %s" % sql)
finally:
if cursor:
cursor.close()
return True
def get(self, *args, **kwargs):
logger_dict = {"args": args, "kwargs": kwargs, "params": self.params, "method": "POST"}
interface = args[0]
method_settings = {
}
if interface not in method_settings:
raise HTTPError(404)
try:
response = yield method_settings[interface]()
self.write(response)
except Exception, e:
logger_dict["traceback"] = traceback.format_exc(e)
logger.error(logger_dict)
self.write({"status": 0, "message": "获取失败"})
def post(self, *args, **kwargs):
try:
interface = args[0]
method_settings = {
"generate/gif": self.generate_gif
}
response = yield method_settings[interface]()
self.write(response)
self.finish()
except Exception, e:
logger_dict = {"args": args, "kwargs": kwargs, "params": self.params, "method": "POST"}
logger_dict["traceback"] = traceback.format_exc(e)
logger.error(logger_dict)
self.write({"status": 0, "message": "操作失败"})
def main():
try:
options.parse_command_line()
port = options.options.port
settings.configure('PORT', port)
SETTINGS.update(session=session_settings)
app = web.Application(handlers=urls, debug=True, **SETTINGS)
server = httpserver.HTTPServer(app)
server.listen(settings.port)
print "the server is going to start..."
print "http://localhost:%s/" % options.options.port
ioloop.IOLoop().instance().start()
except Exception, e:
print traceback.format_exc(e)
logger.error(traceback.format_exc(e))
def post(self, *args, **kwargs):
logger_dict = {"args": args, "kwargs": kwargs, "params": self.params, "method": "POST"}
try:
interface = args[0]
method_settings = {
"jx3/info": self.update_jx3_info
}
response = yield method_settings[interface]()
self.write(response)
self.finish()
except Exception, e:
logger_dict["traceback"] = traceback.format_exc(e)
logger.error(logger_dict)
send_to_master("API出错", json.dumps(logger_dict))
self.write({"status": 0, "message": "获取失败"})
def fetch_review(tag, token, page_token=None):
url = "https://api.joom.com/1.1/products/%s/reviews?=all&count=1000&sort=top&language=en-US¤cy=USD&_=jfs3%s" % (tag, random_key(4))
params = {
"filter_id": "all",
"count": 200,
"sort": "top"
}
if page_token:
params["pageToken"] = page_token
logger.info(u"正在第%s次抓取产品%s的评论, 参数为%s" % (1, tag, params))
try:
res = requests.get(url, params=params, headers={"authorization": token}, timeout=20)
except Exception:
res = requests.get(url, params=params, headers={"authorization": token}, timeout=20)
if "unauthorized" in res.content:
token = get_joom_token()
fetch_review.delay(tag, token, page_token)
return
content = res.json()
if content.get("payload"):
reviews = content["payload"]["items"]
review_datas, review_users, review_count = retrieve_review(reviews)
# if len(review_datas):
# session.execute(JoomReview.__table__.insert(), review_datas)
with futures.ThreadPoolExecutor(max_workers=32) as executor:
future_to_user = {
executor.submit(upsert_review, review=review_data): review_data for review_data in review_datas
}
for future in futures.as_completed(future_to_user):
rev_pro = future_to_user[future]
try:
rp = future.result()
except Exception as exc:
logger.error("%s generated an exception: %s" % (rev_pro, exc))
with futures.ThreadPoolExecutor(max_workers=32) as executor:
future_to_user = {
executor.submit(upsert_user, user=rev_user): rev_user for rev_user in review_users
}
for future in futures.as_completed(future_to_user):
rev_pro = future_to_user[future]
try:
rp = future.result()
except Exception as exc:
logger.error("%s generated an exception: %s" % (rev_pro, exc))
if content["payload"].get("nextPageToken") and len(reviews):
return fetch_review.delay(tag, token, page_token=content["payload"]["nextPageToken"])
else:
logger.info(u"抓取产品%s的评论失败, 参数为%s" % (tag, params))
class LoginHandler(BaseHandler):
def post(self, *args, **kwargs):
option = args[0]
methods = {"in": self.log_in, "out": self.log_out}
methods[option]()
def log_in(self):
mobile = self.params.get("mobile")
pwd = self.params.get("pwd")
mess = ""
try:
if not mobile or not pwd:
raise NullArgumentException
with sessionCM() as session:
user = User.find_by_mobile(session, mobile)
if not user:
mess = {"status": 0, "message": "用户不存在!"}
else:
reg = user.check_password(pwd)
if not reg:
mess = {"status": 0, "message": "密码错误!"}
else:
self.session["user_id"] = user.id
self.set_cookie("MY_WEB", "true")
mess = {"status": 1, "message": "登陆成功!"}
except NullArgumentException, e:
mess = {"status": 0, "message": e.msg}
except Exception, e:
logger.error(traceback.format_exc(e))
mess = {"status": 0, "message": "未知错误"}
def retrieve_review(reviews_info):
try:
review_users = []
review_datas = []
now_dt = datetime.datetime.now()
moment_30 = int(time.mktime((now_dt - datetime.timedelta(days=30)).timetuple())) * 1000
moment_14 = int(time.mktime((now_dt - datetime.timedelta(days=14)).timetuple())) * 1000
moment_7 = int(time.mktime((now_dt - datetime.timedelta(days=7)).timetuple())) * 1000
rc30 = len(filter(lambda a: a["createdTimeMs"] > moment_30, reviews_info))
rc7 = len(filter(lambda a: a["createdTimeMs"] > moment_7, reviews_info))
rc714 = len(filter(lambda a: moment_14 < a["createdTimeMs"] < moment_7, reviews_info))
if rc714:
growth_rate = (float(float(rc7) / rc714) - 1) * 100
else:
growth_rate = 0
review_count = {"r_count_30": rc30, "r_count_7": rc7, "r_count_7_14": rc714, "growth_rate": growth_rate}
for review in reviews_info:
review_data = {
"review_no": review["id"],
"create_time": datetime.datetime.fromtimestamp(review["createdTimeMs"] / 1000),
"update_time": datetime.datetime.fromtimestamp(review["updatedTimeMs"] / 1000),
"pro_no": review["productId"],
"variation_id": review["productVariantId"],
"user_no": review["user"]["id"] if review.get("user") else "",
"language": review.get("originalLanguage", ""),
"origin_text": review.get("originalText", ""),
"new_text": review.get("text", ""),
"order_id": review.get("orderId", ""),
"is_anonymous": review["isAnonymous"],
"colors": json.dumps(review["productVariant"]["colors"]) if review["productVariant"].get("colors") else "",
"star": review["starRating"],
"shop_no": review["productLite"]["storeId"],
"photos": ""
}
if review.get("user"):
user_data = {
"full_name": review["user"].get("fullName", ""),
"user_no": review["user"]["id"],
"images": review["user"]["avatar"]["images"][0]["url"] if review["user"].get("avatar") else ""
}
review_users.append(user_data)
review_datas.append(review_data)
return review_datas, review_users, review_count
except Exception as e:
logger.error(traceback.format_exc(e))
raise Exception("reviewer error")
def create(cls, session, mobile, password, email=""):
try:
session.query(cls).filter(cls.mobile == mobile).one()
except NoResultFound:
user = User()
user.mobile = mobile
user.email = email
user.set_password(password)
session.add(user)
session.commit()
return user
except MultipleResultsFound:
logger.error(
"create user error with the multiple user founded in the user table."
)
else:
return True
def get(self, *args, **kwargs):
logger_dict = {
"args": args,
"kwargs": kwargs,
"params": self.params,
"method": "POST"
}
try:
interface = args[0]
method_settings = {"jx3info": self.get_jx3info}
response = yield method_settings[interface]()
self.write(response)
self.finish()
except Exception, e:
logger_dict["traceback"] = traceback.format_exc(e)
logger.error(logger_dict)
self.write({"status": 0, "message": "获取失败"})
def raw_update(site, kind, key, value="", status=0):
try:
sql = text(
'update task_schedule set task_schedule.value=:ts_value, status=:status where task_schedule.key=:ts_key and kind=:kind and site=:site;'
)
connect = db.connect()
cursor = connect.execute(sql,
ts_value=value,
status=status,
ts_key=key,
kind=kind,
site=site)
cursor.close()
connect.close()
return True
except:
logger.error(traceback.format_exc())
logger.error("key: %s, value: %s" % (key, value))
return False
def raw_upsert(connect, **kwargs):
try:
review_no = kwargs["review_no"]
create_time = kwargs["create_time"]
update_time = kwargs["update_time"]
pro_no = kwargs["pro_no"]
variation_id = kwargs["variation_id"] or ""
user_no = kwargs["user_no"]
language = kwargs["language"] or "en"
origin_text = kwargs["origin_text"] or ""
new_text = kwargs["new_text"] or ""
order_id = kwargs["order_id"]
is_anonymous = kwargs["is_anonymous"] or "0"
colors = kwargs["colors"]
star = kwargs["star"]
shop_no = kwargs["shop_no"]
photos = kwargs.get("photos", "") or ""
sql = text(
'insert into joom_review (review_no,create_time,update_time,pro_no,variation_id,user_no,joom_review.language,origin_text,new_text,order_id,is_anonymous,colors,star,shop_no,photos) VALUES (:review_no,:create_time,:update_time,:pro_no,:variation_id,:user_no,:rev_language,:origin_text,:new_text,:order_id,:is_anonymous,:colors,:star,:shop_no,:photos) on duplicate key update star=:star;'
)
cursor = connect.execute(sql,
review_no=review_no,
create_time=create_time,
update_time=update_time,
pro_no=pro_no,
variation_id=variation_id,
user_no=user_no,
rev_language=language,
origin_text=origin_text,
new_text=new_text,
order_id=order_id,
is_anonymous=is_anonymous,
colors=colors,
star=star,
shop_no=shop_no,
photos=photos)
cursor.close()
return True
except:
logger.error(traceback.format_exc())
return False
class RegisterHandler(BaseHandler):
def post(self, *args, **kwargs):
self.write(self.add_new_user())
def add_new_user(self):
try:
mobile = self.params.get("mobile")
password = self.params.get("password")
password2 = self.params.get("password2")
if not mobile or not password:
raise NullArgumentException
if password != password2:
raise ErrorArgumentError
with sessionCM() as session:
user = User.find_by_mobile(session, mobile)
if user:
return {"status": 0, "message": "用户已存在"}
else:
user = User.create(session, mobile, password)
self.session["user_id"] = user.id
return {"status": 1, "message": "注册成功!"}
except NullArgumentException, e:
return {"status": 0, "message": e.msg}
except ErrorArgumentError, e:
return {"status": 0, "message": e.msg}
except Exception, e:
logger.error(traceback.format_exc(e))
return {"status": 0, "message": "these is something wrong!"}
def get_usage(self):
with paramiko.SSHClient() as ssh:
record = self.get_record()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(hostname=self.public_ip,
port=22,
username=record.user_name,
password=record.pwd)
cpu_reset = self.exec_command(
ssh,
"top -bn 1 -i -c|grep Cpu|awk -F, '{print $4}'| awk -Fi '{print $1}'"
)
server_load = self.exec_command(
ssh,
"uptime | sed 's/.*average://g' | sed 's/\s*//g'| awk -F, '{print $1}'"
)
memory_rest = self.exec_command(
ssh, "free -m | grep Mem | awk '{print $7}'")
disk_usage = self.exec_command(
ssh, "df -m | awk '{print $3}'| head -n 2 | tail -n 1")
mysql_cpu = self.exec_command(
ssh,
"top -b -n 1 | grep mysqld | awk '{b+=$9}END{print b}'")
mongo_cpu = self.exec_command(
ssh,
"top -b -n 1 | grep mongod | awk '{b+=$9}END{print b}'")
UsageRecord.create(
self.session, **{
"cpu_usage":
1 - float(cpu_reset.strip().strip("%")) / 100,
"server_load":
server_load,
"memory_usage":
1 - float(float(memory_rest) / record.memory_capacity),
"disk_usage":
float(float(disk_usage) / record.disk_capacity),
"mysql_cpu":
float(mysql_cpu or 0),
"mongo_cpu":
float(mongo_cpu or 0),
"robot_id":
record.id
})
record.error_times = 0
record.status = STATUS["ACTIVE"]
if mysql_cpu and float(mysql_cpu) > LINE["mysql"]:
raise OverloadError
if mongo_cpu and float(mongo_cpu) > LINE["mysql"]:
raise OverloadError
except OverloadError, e:
send_to_master(
"服务器%s负载过高" % record.name,
"%s机器数据库负载过高当前机器状态为\n系统负载:%f\nCPU使用率:%f\n内存使用率:%f\n硬盘使用率:%f\nmysql CPU占用:%f\nmongo CPU占用:%f"
% (record.name, record.server_load, record.cpu_usage,
record.memory_usage, record.disk_usage,
record.mysql_cpu, record.mongo_cpu))
except socket.error, e:
record.error_times += 1
record.status = STATUS["CONNECT_ERROR"]
logger.error("获取状态信息失败:%s" % e.message)
logger.error(traceback.format_exc(e))
"memory_capacity": memory_capacity,
"user_name": user_name,
"pwd": password,
"area": area,
"description": description,
"tags": tags,
"service": service
})
except socket.error, e:
logger.error("添加机器失败:%s" % e.message)
logger.error(traceback.format_exc(e))
except paramiko.ssh_exception.AuthenticationException, e:
logger.error("添加机器失败:%s" % e.message)
logger.error(traceback.format_exc(e))
except StdError, e:
logger.error("添加机器失败:%s" % e.message)
logger.error(traceback.format_exc(e))
finally:
ssh_client.close()
def get_usage(self):
with paramiko.SSHClient() as ssh:
record = self.get_record()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(hostname=self.public_ip,
port=22,
username=record.user_name,
password=record.pwd)
cpu_reset = self.exec_command(
ssh,
if "unauthorized" in res.content:
token = get_joom_token()
fetch_cate_pro.delay(token, cate_id, pgToken, times)
return
content = json.loads(res.content)
items = content["payload"]["items"]
if len(items) == 0:
logger.info(u"分类%s抓取完成!" % cate_id)
else:
for item in items:
logger.info(u'产品id为%s' % item["id"])
fetch_review.delay(item["id"], token)
with futures.ThreadPoolExecutor(max_workers=16) as executor:
future_to_user = {
executor.submit(fetch_pro, tag=item["id"], token=token):
item["id"]
for item in items
}
for future in futures.as_completed(future_to_user):
rev_pro = future_to_user[future]
try:
rp = future.result()
except Exception as exc:
logger.error("%s generated an exception: %s" %
(rev_pro, exc))
if "nextPageToken" in content["payload"]:
fetch_cate_pro.delay(token, cate_id,
content["payload"]["nextPageToken"],
times + 1)
def add_my_job(self,
trigger,
res_type,
res_url,
job_id,
job_name,
schedule_args,
schedule_type,
func_args,
user_id=0,
remark="",
job_store="default"):
if trigger not in TRIGGER_LIST:
raise ErrorArgumentError
job_id = job_id or str(uuid.uuid1())
callback_args = [res_type, res_url, func_args, job_id]
try:
job = self.add_job_to_scheduler(trigger, job_id, job_name,
schedule_args, callback_args,
job_store)
with sessionCM() as session:
_scheduler = Scheduler.find_by_scheduler_id(session, job.id)
info = {
"request_url":
res_url,
"trigger":
trigger,
"action":
job_name,
"args":
json.dumps(func_args), # json 序列化后的参数
"mold":
schedule_type,
"type":
schedule_type, # sys
"next_run_time":
job.next_run_time.strftime("%Y-%m-%d %H:%M:%S"),
"user_id":
user_id,
"scheduler_id":
job.id,
"extra":
json.dumps(schedule_args), # 不同的trigger的不同参数
"remark":
remark,
"status":
ACTIVE
}
if not _scheduler:
Scheduler.create(session, **info)
else:
_scheduler.update(session, **info)
mess = "job_id为%s的任务添加成功" % job_id
logger.info(mess)
return {"status": 1, "message": mess}
except ConflictingIdError:
mess = "job_id为%s的任务已经存在" % job_id
logger.error(mess)
return {"status": 0, "message": mess}
except Exception, e:
logger.error(traceback.format_exc(e))
return {"status": 0, "message": "添加任务失败"}
