def check(self):
"""Check our monitoring target"""
result = False
result_log = "{url}: {result}"
try:
start = time()
contents = self._fetch_contents()
end = time()
elapsed = "{0:.3f}".format(end - start)
if self._check_contents(contents, self.settings):
log.info(result_log.format(
url=self.url,
result="GOOD {0} ms response time".format(elapsed)
))
result = elapsed
else:
log.warning(result_log.format(
url=self.url,
result="BAD, Content mismatch, {0} ms response time".format(
elapsed
)
))
except IOError as e:
log.error(result_log.format(
url=self.url,
result="BAD, Connection error: {0}".format(e)
))
return result
def get_token(self):
values = {'code': MAIN_USER, 'password': MAIN_PASSWD}
data = self.post("/thirdpart/login", values)
if data :
if data["resultCode"] == 0:
self.token = data["token"]
return self.token
else:
log.error("get token error")
return None
def get_alarm(self,id):
values = {'lastid': id, 'token': self.token}
data = self.post("/thirdpart/event/queryevent", values)
if data:
if data["resultCode"] == 0:
if data.has_key("events"):
return data["events"]
else:
return []
else:
log.error( "get event error")
return None
def del_file(path, ignore=[]):
files = get_file_list(path, ignore)
for file in files:
try:
os.remove(file)
except Exception as e:
log.error('''
删除出错: %s
Exception : %s
''' % (file, str(e)))
else:
log.debug(file + " 删除成功")
finally:
pass
def is_html(url):
if not url:
return False
try:
content_type = request.urlopen(url).info().get('Content-Type', '')
if 'text/html' in content_type:
return True
else:
return False
except Exception as e:
log.error(e)
return False
def inner_run_safe_model(func):
try:
@functools.wraps(func) #将函数的原来属性付给新函数
def run_func(*args, **kw):
callfunc = ''
try:
callfunc = func(*args, **kw)
except Exception as e:
log.error(module_name + ": " + func.__name__ + " - " + str(e))
return callfunc
return run_func
except Exception as e:
log.error(module_name + ": " + func.__name__ + " - " + str(e))
return func
def set_primary_key(self, table, key = "ID"):
if STOP_ORCL:
return
try:
sql = 'alter table {table_name} add constraint pk_{key} primary key ({key})'.format(table_name = table, key = key)
print(sql)
self.cursor.execute(sql)
self.conn.commit()
except Exception as e:
log.error(table + ' ' + str(e) + ' key = '+ key)
else:
log.debug('%s表创建主键成功 主键为 %s'%(table, key))
def clean(model, session, entry_limit=DB_ENTRY_LIMIT):
"""
'Cleans' a specific database by deleting the oldest entries
Keeps entry_limit number of entries still in the database
Model is a sqlalchemyModel
A sqlalchemy session (session) must be passed in
"""
try:
entries = session.query(databases_dict.get(model)).all()
number_to_delete = len(entries) - entry_limit
for x in range(number_to_delete):
session.delete(entries[x])
except:
logger.error("error during clean")
def get_json(json_str):
'''
@summary: 取json对象
---------
@param json_str: json格式的字符串
---------
@result: 返回json对象
'''
try:
return json.loads(json_str) if json_str else {}
except Exception as e:
log.error(e)
return {}
def do_requst(self, data=None):
if self.method == "get":
try:
log.info(f"发送GET请求,请求的路径为{self.url},请求参数{data}")
r = requests.get(self.url, params=data)
result_data = r.json()
log.info(f"服务器响应结果{result_data}")
return r
except Exception as e:
log.error(f"服务器发生异常，请检查错误信息: {e}")
elif self.method == "post":
log.info(f"发送POST请求,请求的路径为{self.url},请求参数{data}")
return requests.post(self.url, data=data)
def is_file(url, file_type):
if not url:
return False
try:
content_type = request.urlopen(url).info().get('Content-Type', '')
if file_type in content_type:
return True
else:
return False
except Exception as e:
log.error(e)
return False
def get_release_time(release_time):
try:
data = tools.time.time()
ltime = tools.time.localtime(data)
timeStr = tools.time.strftime("%Y-%m-%d", ltime)
if '年前' in release_time:
years = tools.re.compile('(\d+)年前').findall(release_time)
years_ago = (tools.datetime.datetime.now() -
tools.datetime.timedelta(days=int(years[0]) * 365))
release_time = years_ago.strftime("%Y-%m-%d")
elif '月前' in release_time:
months = tools.re.compile('(\d+)月前').findall(release_time)
months_ago = (tools.datetime.datetime.now() -
tools.datetime.timedelta(days=int(months[0]) * 30))
release_time = months_ago.strftime("%Y-%m-%d")
elif '周前' in release_time:
weeks = tools.re.compile('(\d+)周前').findall(release_time)
weeks_ago = (tools.datetime.datetime.now() -
tools.datetime.timedelta(days=int(weeks[0]) * 7))
release_time = weeks_ago.strftime("%Y-%m-%d")
elif '天前' in release_time:
ndays = tools.re.compile('(\d+)天前').findall(release_time)
days_ago = (tools.datetime.datetime.now() -
tools.datetime.timedelta(days=int(ndays[0])))
release_time = days_ago.strftime("%Y-%m-%d")
elif '小时前' in release_time:
nhours = tools.re.compile('(\d+)小时前').findall(release_time)
hours_ago = (tools.datetime.datetime.now() -
tools.datetime.timedelta(hours=int(nhours[0])))
release_time = hours_ago.strftime("%Y-%m-%d %H:%M")
elif tools.re.compile('分钟前').findall(release_time):
nminutes = tools.re.compile('(\d+)分钟前').findall(release_time)
minutes_ago = (tools.datetime.datetime.now() -
tools.datetime.timedelta(minutes=int(nminutes[0])))
release_time = minutes_ago.strftime("%Y-%m-%d %H:%M")
else:
if len(release_time) < 10:
release_time = '%s-%s' % (timeStr[0:4], release_time)
except Exception as e:
log.error(e)
release_time = ''
finally:
return release_time
def get_html_auto_deal_code(url):
html = ''
try:
page = request.urlopen(quote(url,safe='/:?=&'), timeout = 3)
html = page.read()
html_code_utf8 = html.decode('utf-8','ignore')
html_code_gb2312 = html.decode('gb2312','ignore')
html = len(html_code_gb2312) > len(html_code_utf8) and html_code_gb2312 or html_code_utf8
page.close()
except Exception as e:
log.error(e)
return html
def add(self, sql, exception_callfunc=''):
if STOP_ORCL:
return True
try:
self.cursor.execute(sql)
self.conn.commit()
except Exception as e:
log.error(e)
if exception_callfunc:
exception_callfunc(e)
return False
else:
return True
def getHtmlByGet(url, code='utf-8'):
html = None
if not url.endswith('.exe') and not url.endswith('.EXE'):
r = None
try:
r = requests.get(url, timeout=TIME_OUT)
if code:
r.encoding = code
html = r.text
except Exception as e:
log.error(e)
finally:
r and r.close()
return html if len(html) < 1024 * 1024 else None
def get(self):
# 最多请求5词
count = 5
while count > 0:
try:
resp = requests.get(self.url, headers=headers)
detect_res = detect(resp.content)
resp.encoding = detect_res.get('encoding', 'utf-8')
return resp.text
except Exception as e:
log.error("请求百度热点异常:{}".format(str(e)))
count -= 1
continue
return None
class AsyncTestTask(Task):
'''test case'''
def __init__(self, **kargs):
super(AsyncTestTask, self).__init__()
self.kargs = kargs
@status
def run(self):
'''it is just a test case'''
log.debug("test task start...")
import time
print "run befor try", time.time()
try:
import time
time.sleep(5)
print "run in run", time.time()
return 0, "test success"
except AgentError, e:
msg = "test failed. detail: %s" % e
log.error(msg)
import traceback
traceback.print_exc()
return e.code(), msg
except Exception, e:
msg = "test failed. detail: %s" % e
log.error(msg)
import traceback
traceback.print_exc()
return OtherError.code(), msg
class AsyncTaskHandler(tornado.web.RequestHandler):
def set_cross_domain(self):
self.set_header('Access-Control-Allow-Origin', "*")
self.set_header('Access-Control-Allow-Credentials', 'true')
self.set_header('Access-Control-Allow-Methods', 'POST, GET, OPTIONS')
@classmethod
def do(cls, **data):
result = dict()
try:
d = cls.task_cls(**data)
ret = TaskManager().add_task(d)
log.debug("==============Task %s start=============" %
cls.__name__)
log.debug("Task param: %s" % data)
if ret[0] == 0:
log.debug("task start...")
d.start()
result["ret"], result["msg"] = ret
result["task_id"] = d.task_id
except VtransError, e:
msg = "%s request failed. detail: %s" % (cls.__name__, e)
result['ret'], result['msg'] = e.code(), msg
log.error(msg)
traceback.print_exc()
except Exception, e:
msg = "%s request failed. detail: %s" % (cls.__name__, e)
log.error(msg)
result['ret'], result['msg'] = OtherError.code(), str(e)
def execute(self, sql):
try:
conn, cursor = self.get_connection()
cursor.execute(sql)
conn.commit()
except Exception as e:
log.error('''
error:%s
sql: %s
''' % (e, sql))
return False
else:
return True
finally:
self.close_connection(conn, cursor)
def set_unique_key(self, table, key):
try:
sql = 'alter table %s add unique (%s)' % (table, key)
conn, cursor = self.get_connection()
cursor.execute(sql)
conn.commit()
except Exception as e:
log.error(table + ' ' + str(e) + ' key = ' + key)
return False
else:
log.debug('%s表创建唯一索引成功 索引为 %s' % (table, key))
return True
finally:
self.close_connection(conn, cursor)
def run(self):
'''it is just a test case'''
log.debug("test task start...")
import time
print "run befor try", time.time()
try:
import time
time.sleep(5)
print "run in run", time.time()
return 0, "test success"
except AgentError, e:
msg = "test failed. detail: %s" % e
log.error(msg)
import traceback
traceback.print_exc()
return e.code(), msg
def delete(self, table, condition={}):
'''
@summary: 删除数据
---------
@param table: 表名
@param condition: 删除条件 {}删除所有
---------
@result:
'''
try:
self._db[table].remove(condition)
except Exception as e:
log.error(e)
return False
else:
return True
def get_detail(self, keyword, link):
# 最多请求5次
count = 5
while count > 0:
try:
resp = requests.get(link, headers=headers)
detect_res = detect(resp.content)
resp.encoding = detect_res.get('encoding', 'utf-8')
return resp.text
except Exception as e:
log.error("请求资讯具体内容异常:keyword{}\tlink:{}\t{}".format(
keyword, link, str(e)))
count -= 1
continue
return None
def get_json_by_requests(url, params=None, headers=''):
json = {}
try:
#response = requests.get(url, params = params)
if headers:
response = requests.get(url,
params=params,
headers=headers,
timeout=TIME_OUT)
else:
response = requests.get(url, params=params, timeout=TIME_OUT)
json = response.json()
except Exception as e:
log.error(e)
return json
def deal_request(self):
web.header('Content-Type', 'text/html;charset=UTF-8')
print(str(web.input()))
data = json.loads(json.dumps(web.input()))
# 文章信息
article_id = data.get('article_id')
may_invalid = data.get('may_invalid') or 0
# 热点信息
hot_id = data.get('hot_id')
hot_value = data.get('hot_value') or 0
# 通用参数
clues_ids = data.get('clues_ids') or ''
article_count = data.get('article_count') or 0
vip_count = data.get('vip_count') or 0
negative_emotion_count = data.get('negative_emotion_count') or 0
zero_ids = data.get('zero_ids') or ''
status = 0 # 0 处理失败 1 处理成功
weight = -1
try:
if hot_id:
status, weight = RelatedSortAction._related_sort_service.deal_hot(
hot_id, float(hot_value), clues_ids, zero_ids,
int(article_count), int(vip_count),
int(negative_emotion_count))
elif article_id:
status, weight = RelatedSortAction._related_sort_service.deal_article(
article_id, clues_ids, zero_ids, int(may_invalid),
int(vip_count), int(negative_emotion_count))
except Exception as e:
log.error(e)
result = {
"status": 1 if status else 0,
"message": "处理成功" if status else "处理失败",
"id": hot_id or article_id,
"weight": weight
}
return tools.dumps_json(result)
def add(self, table, key_value):
'''
@summary: 添加 表不存在自动创建表
---------
@param table: 表名
@param key_value: 要添加的值 字典格式
---------
@result: True / False
'''
try:
self._db[table].save(key_value)
except Exception as e:
log.error(e)
return False
else:
return True
def get_metadata(full_name):
metadata = None
try:
parser = createParser(full_name)
metadata = extractMetadata(parser)
if parser:
parser.stream._input.close()
del parser
except hachoir.stream.input.InputStreamError:
## is directory
metadata = None
except Exception as err:
log.exception(err)
log.error("Cannot extract metadata")
metadata = None
finally:
return metadata
def set_fsattrs(self):
log.debug("Getting filesystem attributes {0}".format(self.path))
try:
me = json.loads(
str(
sh.lsblk("-pJ", "-o", self.LSBLK_KEYS, self.path).stdout,
'utf8'))["blockdevices"][0]
try:
self.uuid = me["uuid"]
log.debug("GOT UUID: {0}".format(self.uuid))
except KeyError:
self.uuid = None
try:
self.fstype = me["fstype"]
log.debug("GOT FSTYPE: {0}".format(self.fstype))
except KeyError:
self.fstype = None
try:
self.label = me["label"]
except KeyError:
self.label = None
except sh.ErrorReturnCode_32:
# Not a block device falling back to blkid
cmd = str(sh.blkid("-o", "export", self.path).stdout, 'utf8')
try:
self.uuid = re.search("UUID=(.*)", cmd).group(1)
except (TypeError, AttributeError):
self.uuid = None
try:
self.fstype = re.search("TYPE=(.*)", cmd).group(1)
except (TypeError, AttributeError):
self.fstype = None
try:
self.label = re.search("LABEL=(.*)", cmd).group(1)
except (TypeError, AttributeError):
self.label = None
if str(self.fstype) not in self.SUPPORTED_FS:
log.error("{0} is out of {1} : {2}".format(self.fstype,
self.SUPPORTED_FS,
self.path))
raise IncompatibleFS("Not working with {0} fstype {1}".format(
self.path, self.fstype))
return True
def get_json_by_requests(url, params = None, headers = '', data = None, proxies = {}):
json = {}
response = None
try:
#response = requests.get(url, params = params)
if data:
response = requests.post(url, headers = headers, data = data, params = params, timeout = TIME_OUT, proxies = proxies)
else:
response = requests.get(url, headers=headers, params = params, timeout=TIME_OUT, proxies = proxies)
response.encoding = 'utf-8'
json = response.json()
except Exception as e:
log.error(e)
finally:
response and response.close()
return json
def run(self):
is_show_tip = False
while True:
try:
datas = self.get_data_from_redis(SYNC_STEP)
if not datas:
if not is_show_tip:
print('\n{time} 无数据 休眠... '.format(
time=tools.get_current_date()))
is_show_tip = True
elif self.add_data_to_es(datas):
is_show_tip = False
self._sync_count += len(datas)
tools.print_one_line('已同步 %d 条数据' % self._sync_count)
tools.delay_time(1)
except Exception as e:
log.error(e)
def do(cls, **data):
result = dict()
try:
d = cls.task_cls(**data)
ret = TaskManager().add_task(d)
log.debug("==============Task %s start=============" %
cls.__name__)
log.debug("Task param: %s" % data)
if ret[0] == 0:
log.debug("task start...")
d.start()
result["ret"], result["msg"] = ret
result["task_id"] = d.task_id
except VtransError, e:
msg = "%s request failed. detail: %s" % (cls.__name__, e)
result['ret'], result['msg'] = e.code(), msg
log.error(msg)
traceback.print_exc()
def mount_compat(self, mode="ro"):
status = True
try:
sh.mount("-o", "{0},{1}".format(mode, self.MOUNT_OPTIONS),
self.path, self.output_path)
except sh.ErrorReturnCode as e:
log.debug("Legacy re-mount opts for {0}".format(self))
try:
sh.mount("-o", "{0}".format(mode), self.path, self.output_path)
except:
try:
sh.mount(self.path, self.output_path)
except Exception as e:
log.error("Cannot mount : {0}".format(self))
log.exception(e)
status = False
return status
def read_file(filename, readlines=False, encoding='utf-8'):
'''
@summary: 读文件
---------
@param filename: 文件名（有路径）
@param readlines: 按行读取 （默认False）
---------
@result: 按行读取返回List，否则返回字符串
'''
content = ''
try:
with open(filename, 'r', encoding=encoding) as file:
content = file.readlines() if readlines else file.read()
except Exception as e:
log.error(e)
return content
def post(self):
log.debug("ManageHandler post in")
data = self.get_data()
if data:
log.debug(data)
if data.has_key("ManageTelephone") and data.has_key("ManagePassword") \
and data.has_key("ManageName") :
if data["ManageName"] == "":
log.error("ManageName is NULL")
else:
db.insert_manage(data["ManageName"], data["ManageTelephone"],
data["ManagePassword"])
result ={}
result["result"] = "ok"
self.send_data(result)
str = "add manage name %s telephone %s"%(data["ManageName"].encode('utf-8'),data["ManageTelephone"].encode('utf-8') )
save_record(self.login_user, "manage", 0, "add", str)
return
else:
log.error("Manaage data key is not right")
else:
log.error("data is none")
result = {}
result["result"] = "error"
result["message"] = "Manaage info is error"
self.send_data(result)