def get_raw_kind_batch(site, kind): try: now = int(time.time()) connect = db.connect() sql = text( 'select task_schedule.key, task_schedule.value from task_schedule where site=:site and kind=:kind and status in (0,1) and dealtime < :now limit 10000;' ) cursor = connect.execute(sql, site=site, kind=kind, now=now) result = [] dealtime = int(time.time()) + TaskSchedule.STEP for item in cursor: sql = text( 'update task_schedule set status=1, dealtime=:dealtime where task_schedule.key=:ts_key and kind=:kind and site=:site;' ) u_cursor = connect.execute(sql, dealtime=dealtime, ts_key=item[0], kind=kind, site=site) u_cursor.close() result.append({ "key": item[0].encode("utf8") if isinstance(item[0], unicode) else item[0], "value": item[1].encode("utf8") if (item[1] and isinstance(item[1], unicode)) else (item[1] or "") }) cursor.close() connect.close() return result except: logger.error(traceback.format_exc()) return []
def batch_upsert(cls, session, **kwargs): try: for k_lst in kwargs["keys"]: if isinstance(k_lst, list): for k in k_lst: ts = cls.find_by_no(session, k, kwargs["kind"], kwargs["site"]) or cls() ts.key = k for _k, _v in kwargs.items(): if _k != "keys": setattr(ts, _k, _v) session.add(ts) session.commit() else: ts = cls.find_by_no(session, k_lst, kwargs["kind"], kwargs["site"]) or cls() ts.key = k_lst for _k, _v in kwargs.items(): if _k != "keys": setattr(ts, _k, _v) session.add(ts) session.commit() return True except: logger.error(traceback.format_exc()) return None
def raw_upsert(connect, **kwargs): try: name = kwargs.get("name", "") or "" pro_no = kwargs["pro_no"] shop_no = kwargs["shop_no"] category_id = kwargs.get("category_id", "") or "" image = kwargs.get("image", "") or "" rate = kwargs.get("rate", 0) or 0 msrp = kwargs.get("msrp", 0) or 0 discount = kwargs.get("discount", 0) or 0 real_price = kwargs.get("real_price", 0) or 0 reviews_count = kwargs.get("reviews_count", 0) or 0 create_time = kwargs["create_time"] update_time = kwargs["update_time"] sql = text( 'insert into joom_pro (joom_pro.name,pro_no,shop_no,category_id,image,rate,msrp,discount,real_price,reviews_count,create_time,update_time,cate_id1,cate_id2,cate_id3,cate_id4,cate_id5,origin_price,r_count_30,r_count_7,r_count_7_14,growth_rate,save_count) values (:jp_name,:pro_no,:shop_no,:category_id,:image,:rate,:msrp,:discount,:real_price,:reviews_count,:create_time,:update_time,"","","","","",0,0,0,0,0,0) on duplicate key update joom_pro.name=:jp_name,category_id=:category_id,rate=:rate,msrp=:msrp,discount=:discount,real_price=:real_price,reviews_count=:reviews_count,update_time=:update_time;' ) cursor = connect.execute(sql, jp_name=name, pro_no=pro_no, shop_no=shop_no, category_id=category_id, image=image, rate=rate, msrp=msrp, discount=discount, real_price=real_price, reviews_count=reviews_count, create_time=create_time, update_time=update_time) cursor.close() return True except: logger.error(traceback.format_exc()) return False
class MonitorControl(object): def __init__(self, public_ip, session): self.public_ip = public_ip self.session = session def get_record(self): return Monitor.find_by_public_ip(self.session, self.public_ip) def add_to_record(self, user_name, password, name, area, tags, service, description="", ssh_client=None): if not ssh_client: ssh_client = paramiko.SSHClient() # 设置为接受不在known_hosts 列表的主机可以进行ssh连接 ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: ssh_client.connect(hostname=self.public_ip, port=22, username=user_name, password=password) memory_capacity = self.exec_command( ssh_client, "free -m | grep Mem | awk '{print $2}'") disk_capacity = self.exec_command( ssh_client, "df -m | awk '{print $2}'| head -n 2 | tail -n 1") core_count = self.exec_command( ssh_client, "cat /proc/cpuinfo | grep 'cpu cores' | wc -l") cpu_count = self.exec_command( ssh_client, "cat /proc/cpuinfo | grep 'physical id' | wc -l") inner_ip = self.exec_command( ssh_client, "/sbin/ifconfig -a | grep 'inet addr' | awk -F: '{print $2}' | awk '{print $1}' | head -n 1" ) Monitor.create( self.session, **{ "name": name, "inner_ip": inner_ip, "public_ip": self.public_ip, "cpu_count": cpu_count, "core_count": core_count, "disk_capacity": disk_capacity, "memory_capacity": memory_capacity, "user_name": user_name, "pwd": password, "area": area, "description": description, "tags": tags, "service": service }) except socket.error, e: logger.error("添加机器失败:%s" % e.message) logger.error(traceback.format_exc(e)) except paramiko.ssh_exception.AuthenticationException, e: logger.error("添加机器失败:%s" % e.message) logger.error(traceback.format_exc(e))
def upsert(cls, session, **kwargs): try: joom_user = cls.find_by_no( session, kwargs["user_no"]) or cls(user_no=kwargs["user_no"]) joom_user.full_name = kwargs.get("full_name", "") joom_user.images = kwargs.get("images", "") session.merge(joom_user) session.commit() return joom_user except: logger.error(traceback.format_exc()) return None
def upsert(cls, session, **kwargs): try: ts = cls.find_by_no(session, kwargs["key"], kwargs["kind"], kwargs["site"]) or cls() for k, v in kwargs.items(): setattr(ts, k, v) session.merge(ts) session.commit() return ts except: logger.error(traceback.format_exc()) return None
def raw_upsert(connect, **kwargs): sql = text( 'insert into joom_user (user_no, full_name, images) values (:user_no, :full_name, :images) on duplicate key update full_name=:full_name, images = :images;' ) cursor = None try: cursor = connect.execute(sql, **kwargs) except: logger.error("joom user upsert error: %s" % sql) finally: if cursor: cursor.close() return True
def get(self, *args, **kwargs): logger_dict = {"args": args, "kwargs": kwargs, "params": self.params, "method": "POST"} interface = args[0] method_settings = { } if interface not in method_settings: raise HTTPError(404) try: response = yield method_settings[interface]() self.write(response) except Exception, e: logger_dict["traceback"] = traceback.format_exc(e) logger.error(logger_dict) self.write({"status": 0, "message": "获取失败"})
def post(self, *args, **kwargs): try: interface = args[0] method_settings = { "generate/gif": self.generate_gif } response = yield method_settings[interface]() self.write(response) self.finish() except Exception, e: logger_dict = {"args": args, "kwargs": kwargs, "params": self.params, "method": "POST"} logger_dict["traceback"] = traceback.format_exc(e) logger.error(logger_dict) self.write({"status": 0, "message": "操作失败"})
def main(): try: options.parse_command_line() port = options.options.port settings.configure('PORT', port) SETTINGS.update(session=session_settings) app = web.Application(handlers=urls, debug=True, **SETTINGS) server = httpserver.HTTPServer(app) server.listen(settings.port) print "the server is going to start..." print "http://localhost:%s/" % options.options.port ioloop.IOLoop().instance().start() except Exception, e: print traceback.format_exc(e) logger.error(traceback.format_exc(e))
def post(self, *args, **kwargs): logger_dict = {"args": args, "kwargs": kwargs, "params": self.params, "method": "POST"} try: interface = args[0] method_settings = { "jx3/info": self.update_jx3_info } response = yield method_settings[interface]() self.write(response) self.finish() except Exception, e: logger_dict["traceback"] = traceback.format_exc(e) logger.error(logger_dict) send_to_master("API出错", json.dumps(logger_dict)) self.write({"status": 0, "message": "获取失败"})
def fetch_review(tag, token, page_token=None): url = "https://api.joom.com/1.1/products/%s/reviews?=all&count=1000&sort=top&language=en-US¤cy=USD&_=jfs3%s" % (tag, random_key(4)) params = { "filter_id": "all", "count": 200, "sort": "top" } if page_token: params["pageToken"] = page_token logger.info(u"正在第%s次抓取产品%s的评论, 参数为%s" % (1, tag, params)) try: res = requests.get(url, params=params, headers={"authorization": token}, timeout=20) except Exception: res = requests.get(url, params=params, headers={"authorization": token}, timeout=20) if "unauthorized" in res.content: token = get_joom_token() fetch_review.delay(tag, token, page_token) return content = res.json() if content.get("payload"): reviews = content["payload"]["items"] review_datas, review_users, review_count = retrieve_review(reviews) # if len(review_datas): # session.execute(JoomReview.__table__.insert(), review_datas) with futures.ThreadPoolExecutor(max_workers=32) as executor: future_to_user = { executor.submit(upsert_review, review=review_data): review_data for review_data in review_datas } for future in futures.as_completed(future_to_user): rev_pro = future_to_user[future] try: rp = future.result() except Exception as exc: logger.error("%s generated an exception: %s" % (rev_pro, exc)) with futures.ThreadPoolExecutor(max_workers=32) as executor: future_to_user = { executor.submit(upsert_user, user=rev_user): rev_user for rev_user in review_users } for future in futures.as_completed(future_to_user): rev_pro = future_to_user[future] try: rp = future.result() except Exception as exc: logger.error("%s generated an exception: %s" % (rev_pro, exc)) if content["payload"].get("nextPageToken") and len(reviews): return fetch_review.delay(tag, token, page_token=content["payload"]["nextPageToken"]) else: logger.info(u"抓取产品%s的评论失败, 参数为%s" % (tag, params))
class LoginHandler(BaseHandler): def post(self, *args, **kwargs): option = args[0] methods = {"in": self.log_in, "out": self.log_out} methods[option]() def log_in(self): mobile = self.params.get("mobile") pwd = self.params.get("pwd") mess = "" try: if not mobile or not pwd: raise NullArgumentException with sessionCM() as session: user = User.find_by_mobile(session, mobile) if not user: mess = {"status": 0, "message": "用户不存在!"} else: reg = user.check_password(pwd) if not reg: mess = {"status": 0, "message": "密码错误!"} else: self.session["user_id"] = user.id self.set_cookie("MY_WEB", "true") mess = {"status": 1, "message": "登陆成功!"} except NullArgumentException, e: mess = {"status": 0, "message": e.msg} except Exception, e: logger.error(traceback.format_exc(e)) mess = {"status": 0, "message": "未知错误"}
def retrieve_review(reviews_info): try: review_users = [] review_datas = [] now_dt = datetime.datetime.now() moment_30 = int(time.mktime((now_dt - datetime.timedelta(days=30)).timetuple())) * 1000 moment_14 = int(time.mktime((now_dt - datetime.timedelta(days=14)).timetuple())) * 1000 moment_7 = int(time.mktime((now_dt - datetime.timedelta(days=7)).timetuple())) * 1000 rc30 = len(filter(lambda a: a["createdTimeMs"] > moment_30, reviews_info)) rc7 = len(filter(lambda a: a["createdTimeMs"] > moment_7, reviews_info)) rc714 = len(filter(lambda a: moment_14 < a["createdTimeMs"] < moment_7, reviews_info)) if rc714: growth_rate = (float(float(rc7) / rc714) - 1) * 100 else: growth_rate = 0 review_count = {"r_count_30": rc30, "r_count_7": rc7, "r_count_7_14": rc714, "growth_rate": growth_rate} for review in reviews_info: review_data = { "review_no": review["id"], "create_time": datetime.datetime.fromtimestamp(review["createdTimeMs"] / 1000), "update_time": datetime.datetime.fromtimestamp(review["updatedTimeMs"] / 1000), "pro_no": review["productId"], "variation_id": review["productVariantId"], "user_no": review["user"]["id"] if review.get("user") else "", "language": review.get("originalLanguage", ""), "origin_text": review.get("originalText", ""), "new_text": review.get("text", ""), "order_id": review.get("orderId", ""), "is_anonymous": review["isAnonymous"], "colors": json.dumps(review["productVariant"]["colors"]) if review["productVariant"].get("colors") else "", "star": review["starRating"], "shop_no": review["productLite"]["storeId"], "photos": "" } if review.get("user"): user_data = { "full_name": review["user"].get("fullName", ""), "user_no": review["user"]["id"], "images": review["user"]["avatar"]["images"][0]["url"] if review["user"].get("avatar") else "" } review_users.append(user_data) review_datas.append(review_data) return review_datas, review_users, review_count except Exception as e: logger.error(traceback.format_exc(e)) raise Exception("reviewer error")
def create(cls, session, mobile, password, email=""): try: session.query(cls).filter(cls.mobile == mobile).one() except NoResultFound: user = User() user.mobile = mobile user.email = email user.set_password(password) session.add(user) session.commit() return user except MultipleResultsFound: logger.error( "create user error with the multiple user founded in the user table." ) else: return True
def get(self, *args, **kwargs): logger_dict = { "args": args, "kwargs": kwargs, "params": self.params, "method": "POST" } try: interface = args[0] method_settings = {"jx3info": self.get_jx3info} response = yield method_settings[interface]() self.write(response) self.finish() except Exception, e: logger_dict["traceback"] = traceback.format_exc(e) logger.error(logger_dict) self.write({"status": 0, "message": "获取失败"})
def raw_update(site, kind, key, value="", status=0): try: sql = text( 'update task_schedule set task_schedule.value=:ts_value, status=:status where task_schedule.key=:ts_key and kind=:kind and site=:site;' ) connect = db.connect() cursor = connect.execute(sql, ts_value=value, status=status, ts_key=key, kind=kind, site=site) cursor.close() connect.close() return True except: logger.error(traceback.format_exc()) logger.error("key: %s, value: %s" % (key, value)) return False
def raw_upsert(connect, **kwargs): try: review_no = kwargs["review_no"] create_time = kwargs["create_time"] update_time = kwargs["update_time"] pro_no = kwargs["pro_no"] variation_id = kwargs["variation_id"] or "" user_no = kwargs["user_no"] language = kwargs["language"] or "en" origin_text = kwargs["origin_text"] or "" new_text = kwargs["new_text"] or "" order_id = kwargs["order_id"] is_anonymous = kwargs["is_anonymous"] or "0" colors = kwargs["colors"] star = kwargs["star"] shop_no = kwargs["shop_no"] photos = kwargs.get("photos", "") or "" sql = text( 'insert into joom_review (review_no,create_time,update_time,pro_no,variation_id,user_no,joom_review.language,origin_text,new_text,order_id,is_anonymous,colors,star,shop_no,photos) VALUES (:review_no,:create_time,:update_time,:pro_no,:variation_id,:user_no,:rev_language,:origin_text,:new_text,:order_id,:is_anonymous,:colors,:star,:shop_no,:photos) on duplicate key update star=:star;' ) cursor = connect.execute(sql, review_no=review_no, create_time=create_time, update_time=update_time, pro_no=pro_no, variation_id=variation_id, user_no=user_no, rev_language=language, origin_text=origin_text, new_text=new_text, order_id=order_id, is_anonymous=is_anonymous, colors=colors, star=star, shop_no=shop_no, photos=photos) cursor.close() return True except: logger.error(traceback.format_exc()) return False
class RegisterHandler(BaseHandler): def post(self, *args, **kwargs): self.write(self.add_new_user()) def add_new_user(self): try: mobile = self.params.get("mobile") password = self.params.get("password") password2 = self.params.get("password2") if not mobile or not password: raise NullArgumentException if password != password2: raise ErrorArgumentError with sessionCM() as session: user = User.find_by_mobile(session, mobile) if user: return {"status": 0, "message": "用户已存在"} else: user = User.create(session, mobile, password) self.session["user_id"] = user.id return {"status": 1, "message": "注册成功!"} except NullArgumentException, e: return {"status": 0, "message": e.msg} except ErrorArgumentError, e: return {"status": 0, "message": e.msg} except Exception, e: logger.error(traceback.format_exc(e)) return {"status": 0, "message": "these is something wrong!"}
def get_usage(self): with paramiko.SSHClient() as ssh: record = self.get_record() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: ssh.connect(hostname=self.public_ip, port=22, username=record.user_name, password=record.pwd) cpu_reset = self.exec_command( ssh, "top -bn 1 -i -c|grep Cpu|awk -F, '{print $4}'| awk -Fi '{print $1}'" ) server_load = self.exec_command( ssh, "uptime | sed 's/.*average://g' | sed 's/\s*//g'| awk -F, '{print $1}'" ) memory_rest = self.exec_command( ssh, "free -m | grep Mem | awk '{print $7}'") disk_usage = self.exec_command( ssh, "df -m | awk '{print $3}'| head -n 2 | tail -n 1") mysql_cpu = self.exec_command( ssh, "top -b -n 1 | grep mysqld | awk '{b+=$9}END{print b}'") mongo_cpu = self.exec_command( ssh, "top -b -n 1 | grep mongod | awk '{b+=$9}END{print b}'") UsageRecord.create( self.session, **{ "cpu_usage": 1 - float(cpu_reset.strip().strip("%")) / 100, "server_load": server_load, "memory_usage": 1 - float(float(memory_rest) / record.memory_capacity), "disk_usage": float(float(disk_usage) / record.disk_capacity), "mysql_cpu": float(mysql_cpu or 0), "mongo_cpu": float(mongo_cpu or 0), "robot_id": record.id }) record.error_times = 0 record.status = STATUS["ACTIVE"] if mysql_cpu and float(mysql_cpu) > LINE["mysql"]: raise OverloadError if mongo_cpu and float(mongo_cpu) > LINE["mysql"]: raise OverloadError except OverloadError, e: send_to_master( "服务器%s负载过高" % record.name, "%s机器数据库负载过高当前机器状态为\n系统负载:%f\nCPU使用率:%f\n内存使用率:%f\n硬盘使用率:%f\nmysql CPU占用:%f\nmongo CPU占用:%f" % (record.name, record.server_load, record.cpu_usage, record.memory_usage, record.disk_usage, record.mysql_cpu, record.mongo_cpu)) except socket.error, e: record.error_times += 1 record.status = STATUS["CONNECT_ERROR"] logger.error("获取状态信息失败:%s" % e.message) logger.error(traceback.format_exc(e))
"memory_capacity": memory_capacity, "user_name": user_name, "pwd": password, "area": area, "description": description, "tags": tags, "service": service }) except socket.error, e: logger.error("添加机器失败:%s" % e.message) logger.error(traceback.format_exc(e)) except paramiko.ssh_exception.AuthenticationException, e: logger.error("添加机器失败:%s" % e.message) logger.error(traceback.format_exc(e)) except StdError, e: logger.error("添加机器失败:%s" % e.message) logger.error(traceback.format_exc(e)) finally: ssh_client.close() def get_usage(self): with paramiko.SSHClient() as ssh: record = self.get_record() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: ssh.connect(hostname=self.public_ip, port=22, username=record.user_name, password=record.pwd) cpu_reset = self.exec_command( ssh,
if "unauthorized" in res.content: token = get_joom_token() fetch_cate_pro.delay(token, cate_id, pgToken, times) return content = json.loads(res.content) items = content["payload"]["items"] if len(items) == 0: logger.info(u"分类%s抓取完成!" % cate_id) else: for item in items: logger.info(u'产品id为%s' % item["id"]) fetch_review.delay(item["id"], token) with futures.ThreadPoolExecutor(max_workers=16) as executor: future_to_user = { executor.submit(fetch_pro, tag=item["id"], token=token): item["id"] for item in items } for future in futures.as_completed(future_to_user): rev_pro = future_to_user[future] try: rp = future.result() except Exception as exc: logger.error("%s generated an exception: %s" % (rev_pro, exc)) if "nextPageToken" in content["payload"]: fetch_cate_pro.delay(token, cate_id, content["payload"]["nextPageToken"], times + 1)
def add_my_job(self, trigger, res_type, res_url, job_id, job_name, schedule_args, schedule_type, func_args, user_id=0, remark="", job_store="default"): if trigger not in TRIGGER_LIST: raise ErrorArgumentError job_id = job_id or str(uuid.uuid1()) callback_args = [res_type, res_url, func_args, job_id] try: job = self.add_job_to_scheduler(trigger, job_id, job_name, schedule_args, callback_args, job_store) with sessionCM() as session: _scheduler = Scheduler.find_by_scheduler_id(session, job.id) info = { "request_url": res_url, "trigger": trigger, "action": job_name, "args": json.dumps(func_args), # json 序列化后的参数 "mold": schedule_type, "type": schedule_type, # sys "next_run_time": job.next_run_time.strftime("%Y-%m-%d %H:%M:%S"), "user_id": user_id, "scheduler_id": job.id, "extra": json.dumps(schedule_args), # 不同的trigger的不同参数 "remark": remark, "status": ACTIVE } if not _scheduler: Scheduler.create(session, **info) else: _scheduler.update(session, **info) mess = "job_id为%s的任务添加成功" % job_id logger.info(mess) return {"status": 1, "message": mess} except ConflictingIdError: mess = "job_id为%s的任务已经存在" % job_id logger.error(mess) return {"status": 0, "message": mess} except Exception, e: logger.error(traceback.format_exc(e)) return {"status": 0, "message": "添加任务失败"}