コード例 #1
0
ファイル: monitor.py プロジェクト: lietu/pywebstatmon
    def check(self):
        """Check our monitoring target"""

        result = False

        result_log = "{url}: {result}"

        try:
            start = time()
            contents = self._fetch_contents()
            end = time()
            elapsed = "{0:.3f}".format(end - start)

            if self._check_contents(contents, self.settings):
                log.info(result_log.format(
                    url=self.url,
                    result="GOOD {0} ms response time".format(elapsed)
                ))

                result = elapsed
            else:
                log.warning(result_log.format(
                    url=self.url,
                    result="BAD, Content mismatch, {0} ms response time".format(
                        elapsed
                    )
                ))

        except IOError as e:
            log.error(result_log.format(
                url=self.url,
                result="BAD, Connection error: {0}".format(e)
            ))

        return result
コード例 #2
0
ファイル: httpclient.py プロジェクト: ssdxiao/alarm_platform
    def get_token(self):
        values = {'code': MAIN_USER, 'password': MAIN_PASSWD}
        data = self.post("/thirdpart/login", values)
        if data :
            if data["resultCode"] == 0:
                self.token = data["token"]
                return self.token
            else:
                log.error("get token error")

        return None
コード例 #3
0
ファイル: httpclient.py プロジェクト: ssdxiao/alarm_platform
    def get_alarm(self,id):
        values = {'lastid': id, 'token': self.token}
        data = self.post("/thirdpart/event/queryevent", values)
        if data:
            if data["resultCode"] == 0:
                if data.has_key("events"):
                    return data["events"]
                else:
                    return []
            else:
                log.error( "get event error")

        return None
コード例 #4
0
ファイル: tools.py プロジェクト: BoyYongXin/proxy_pool
def del_file(path, ignore=[]):
    files = get_file_list(path, ignore)
    for file in files:
        try:
            os.remove(file)
        except Exception as e:
            log.error('''
                删除出错: %s
                Exception : %s
                ''' % (file, str(e)))
        else:
            log.debug(file + " 删除成功")
        finally:
            pass
コード例 #5
0
ファイル: tools.py プロジェクト: lxshen/try
def is_html(url):
    if not url:
        return False

    try:
        content_type = request.urlopen(url).info().get('Content-Type', '')

        if 'text/html' in content_type:
            return True
        else:
            return False
    except Exception as e:
        log.error(e)
        return False
コード例 #6
0
ファイル: tools.py プロジェクト: zzzz123321/wechat-spider-1
 def inner_run_safe_model(func):
     try:
         @functools.wraps(func)  #将函数的原来属性付给新函数
         def run_func(*args, **kw):
             callfunc = ''
             try:
                 callfunc = func(*args, **kw)
             except Exception as e:
                 log.error(module_name + ": " + func.__name__ + " - " + str(e))
             return callfunc
         return run_func
     except Exception as e:
         log.error(module_name + ": " + func.__name__ + " - " + str(e))
         return func
コード例 #7
0
    def set_primary_key(self, table, key = "ID"):
        if STOP_ORCL:
            return

        try:
            sql = 'alter table {table_name} add constraint pk_{key} primary key ({key})'.format(table_name = table, key = key)
            print(sql)
            self.cursor.execute(sql)
            self.conn.commit()

        except Exception as e:
            log.error(table + ' ' + str(e) + ' key = '+ key)
        else:
            log.debug('%s表创建主键成功 主键为 %s'%(table, key))
コード例 #8
0
def clean(model, session, entry_limit=DB_ENTRY_LIMIT):
    """
    'Cleans' a specific database by deleting the oldest entries
    Keeps entry_limit number of entries still in the database
    Model is a sqlalchemyModel
    A sqlalchemy session (session) must be passed in
    """
    try:
        entries = session.query(databases_dict.get(model)).all()
        number_to_delete = len(entries) - entry_limit
        for x in range(number_to_delete):
            session.delete(entries[x])
    except:
        logger.error("error during clean")
コード例 #9
0
ファイル: tools.py プロジェクト: striver-ing/weibo_news
def get_json(json_str):
    '''
    @summary: 取json对象
    ---------
    @param json_str: json格式的字符串
    ---------
    @result: 返回json对象
    '''

    try:
        return json.loads(json_str) if json_str else {}
    except Exception as e:
        log.error(e)
        return {}
コード例 #10
0
ファイル: do_requets.py プロジェクト: imzengyang/pythonapi
    def do_requst(self, data=None):
        if self.method == "get":
            try:
                log.info(f"发送GET请求,请求的路径为{self.url},请求参数{data}")
                r = requests.get(self.url, params=data)
                result_data = r.json()
                log.info(f"服务器响应结果{result_data}")
                return r
            except Exception as e:
                log.error(f"服务器发生异常,请检查错误信息: {e}")

        elif self.method == "post":
            log.info(f"发送POST请求,请求的路径为{self.url},请求参数{data}")
            return requests.post(self.url, data=data)
コード例 #11
0
ファイル: tools.py プロジェクト: striver-ing/weibo_news
def is_file(url, file_type):
    if not url:
        return False

    try:
        content_type = request.urlopen(url).info().get('Content-Type', '')

        if file_type in content_type:
            return True
        else:
            return False
    except Exception as e:
        log.error(e)
        return False
コード例 #12
0
def get_release_time(release_time):
    try:
        data = tools.time.time()
        ltime = tools.time.localtime(data)
        timeStr = tools.time.strftime("%Y-%m-%d", ltime)
        if '年前' in release_time:
            years = tools.re.compile('(\d+)年前').findall(release_time)
            years_ago = (tools.datetime.datetime.now() -
                         tools.datetime.timedelta(days=int(years[0]) * 365))
            release_time = years_ago.strftime("%Y-%m-%d")

        elif '月前' in release_time:
            months = tools.re.compile('(\d+)月前').findall(release_time)
            months_ago = (tools.datetime.datetime.now() -
                          tools.datetime.timedelta(days=int(months[0]) * 30))
            release_time = months_ago.strftime("%Y-%m-%d")

        elif '周前' in release_time:
            weeks = tools.re.compile('(\d+)周前').findall(release_time)
            weeks_ago = (tools.datetime.datetime.now() -
                         tools.datetime.timedelta(days=int(weeks[0]) * 7))
            release_time = weeks_ago.strftime("%Y-%m-%d")

        elif '天前' in release_time:
            ndays = tools.re.compile('(\d+)天前').findall(release_time)
            days_ago = (tools.datetime.datetime.now() -
                        tools.datetime.timedelta(days=int(ndays[0])))
            release_time = days_ago.strftime("%Y-%m-%d")

        elif '小时前' in release_time:
            nhours = tools.re.compile('(\d+)小时前').findall(release_time)
            hours_ago = (tools.datetime.datetime.now() -
                         tools.datetime.timedelta(hours=int(nhours[0])))
            release_time = hours_ago.strftime("%Y-%m-%d %H:%M")

        elif tools.re.compile('分钟前').findall(release_time):
            nminutes = tools.re.compile('(\d+)分钟前').findall(release_time)
            minutes_ago = (tools.datetime.datetime.now() -
                           tools.datetime.timedelta(minutes=int(nminutes[0])))
            release_time = minutes_ago.strftime("%Y-%m-%d %H:%M")

        else:
            if len(release_time) < 10:
                release_time = '%s-%s' % (timeStr[0:4], release_time)
    except Exception as e:
        log.error(e)
        release_time = ''
    finally:
        return release_time
コード例 #13
0
ファイル: tools.py プロジェクト: zzzz123321/wechat-spider-1
def get_html_auto_deal_code(url):
    html = ''
    try:
        page = request.urlopen(quote(url,safe='/:?=&'), timeout = 3)
        html = page.read()

        html_code_utf8 = html.decode('utf-8','ignore')
        html_code_gb2312 = html.decode('gb2312','ignore')

        html = len(html_code_gb2312) > len(html_code_utf8) and html_code_gb2312 or html_code_utf8
        page.close()

    except Exception as e:
        log.error(e)
    return html
コード例 #14
0
    def add(self, sql, exception_callfunc=''):
        if STOP_ORCL:
            return True

        try:
            self.cursor.execute(sql)
            self.conn.commit()
        except Exception as e:
            log.error(e)
            if exception_callfunc:
                exception_callfunc(e)

            return False
        else:
            return True
コード例 #15
0
def getHtmlByGet(url, code='utf-8'):
    html = None
    if not url.endswith('.exe') and not url.endswith('.EXE'):
        r = None
        try:
            r = requests.get(url, timeout=TIME_OUT)
            if code:
                r.encoding = code
            html = r.text

        except Exception as e:
            log.error(e)
        finally:
            r and r.close()
    return html if len(html) < 1024 * 1024 else None
コード例 #16
0
    def get(self):
        # 最多请求5词
        count = 5

        while count > 0:
            try:
                resp = requests.get(self.url, headers=headers)
                detect_res = detect(resp.content)
                resp.encoding = detect_res.get('encoding', 'utf-8')
                return resp.text
            except Exception as e:
                log.error("请求百度热点异常:{}".format(str(e)))
                count -= 1
                continue

        return None
コード例 #17
0
class AsyncTestTask(Task):
    '''test case'''
    def __init__(self, **kargs):
        super(AsyncTestTask, self).__init__()
        self.kargs = kargs

    @status
    def run(self):
        '''it is just a test case'''
        log.debug("test task start...")
        import time
        print "run befor try", time.time()
        try:
            import time
            time.sleep(5)
            print "run in run", time.time()
            return 0, "test success"
        except AgentError, e:
            msg = "test failed. detail: %s" % e
            log.error(msg)
            import traceback
            traceback.print_exc()
            return e.code(), msg
        except Exception, e:
            msg = "test failed. detail: %s" % e
            log.error(msg)
            import traceback
            traceback.print_exc()
            return OtherError.code(), msg
コード例 #18
0
ファイル: handler.py プロジェクト: staticzeng/hermes
class AsyncTaskHandler(tornado.web.RequestHandler):
    def set_cross_domain(self):
        self.set_header('Access-Control-Allow-Origin', "*")
        self.set_header('Access-Control-Allow-Credentials', 'true')
        self.set_header('Access-Control-Allow-Methods', 'POST, GET, OPTIONS')

    @classmethod
    def do(cls, **data):
        result = dict()
        try:
            d = cls.task_cls(**data)
            ret = TaskManager().add_task(d)
            log.debug("==============Task %s start=============" %
                      cls.__name__)
            log.debug("Task param: %s" % data)
            if ret[0] == 0:
                log.debug("task start...")
                d.start()
            result["ret"], result["msg"] = ret
            result["task_id"] = d.task_id
        except VtransError, e:
            msg = "%s request failed. detail: %s" % (cls.__name__, e)
            result['ret'], result['msg'] = e.code(), msg
            log.error(msg)
            traceback.print_exc()
        except Exception, e:
            msg = "%s request failed. detail: %s" % (cls.__name__, e)
            log.error(msg)
            result['ret'], result['msg'] = OtherError.code(), str(e)
コード例 #19
0
ファイル: mysqldb.py プロジェクト: 18141922515/wechat-spider
    def execute(self, sql):
        try:
            conn, cursor = self.get_connection()
            cursor.execute(sql)
            conn.commit()

        except Exception as e:
            log.error('''
                error:%s
                sql:  %s
                ''' % (e, sql))
            return False
        else:
            return True
        finally:
            self.close_connection(conn, cursor)
コード例 #20
0
ファイル: mysqldb.py プロジェクト: 18141922515/wechat-spider
    def set_unique_key(self, table, key):
        try:
            sql = 'alter table %s add unique (%s)' % (table, key)

            conn, cursor = self.get_connection()
            cursor.execute(sql)
            conn.commit()

        except Exception as e:
            log.error(table + ' ' + str(e) + ' key = ' + key)
            return False
        else:
            log.debug('%s表创建唯一索引成功 索引为 %s' % (table, key))
            return True
        finally:
            self.close_connection(conn, cursor)
コード例 #21
0
 def run(self):
     '''it is just a test case'''
     log.debug("test task start...")
     import time
     print "run befor try", time.time()
     try:
         import time
         time.sleep(5)
         print "run in run", time.time()
         return 0, "test success"
     except AgentError, e:
         msg = "test failed. detail: %s" % e
         log.error(msg)
         import traceback
         traceback.print_exc()
         return e.code(), msg
コード例 #22
0
 def delete(self, table, condition={}):
     '''
     @summary: 删除数据
     ---------
     @param table: 表名
     @param condition: 删除条件 {}删除所有
     ---------
     @result:
     '''
     try:
         self._db[table].remove(condition)
     except Exception as e:
         log.error(e)
         return False
     else:
         return True
コード例 #23
0
    def get_detail(self, keyword, link):
        # 最多请求5次
        count = 5

        while count > 0:
            try:
                resp = requests.get(link, headers=headers)
                detect_res = detect(resp.content)
                resp.encoding = detect_res.get('encoding', 'utf-8')
                return resp.text
            except Exception as e:
                log.error("请求资讯具体内容异常:keyword{}\tlink:{}\t{}".format(
                    keyword, link, str(e)))
                count -= 1
                continue

        return None
コード例 #24
0
def get_json_by_requests(url, params=None, headers=''):
    json = {}
    try:
        #response = requests.get(url, params = params)
        if headers:
            response = requests.get(url,
                                    params=params,
                                    headers=headers,
                                    timeout=TIME_OUT)
        else:
            response = requests.get(url, params=params, timeout=TIME_OUT)

        json = response.json()
    except Exception as e:
        log.error(e)

    return json
コード例 #25
0
    def deal_request(self):
        web.header('Content-Type', 'text/html;charset=UTF-8')
        print(str(web.input()))
        data = json.loads(json.dumps(web.input()))

        # 文章信息
        article_id = data.get('article_id')
        may_invalid = data.get('may_invalid') or 0

        # 热点信息
        hot_id = data.get('hot_id')
        hot_value = data.get('hot_value') or 0

        # 通用参数
        clues_ids = data.get('clues_ids') or ''
        article_count = data.get('article_count') or 0
        vip_count = data.get('vip_count') or 0
        negative_emotion_count = data.get('negative_emotion_count') or 0
        zero_ids = data.get('zero_ids') or ''

        status = 0  # 0 处理失败 1 处理成功
        weight = -1

        try:
            if hot_id:
                status, weight = RelatedSortAction._related_sort_service.deal_hot(
                    hot_id, float(hot_value), clues_ids, zero_ids,
                    int(article_count), int(vip_count),
                    int(negative_emotion_count))

            elif article_id:
                status, weight = RelatedSortAction._related_sort_service.deal_article(
                    article_id, clues_ids, zero_ids, int(may_invalid),
                    int(vip_count), int(negative_emotion_count))

        except Exception as e:
            log.error(e)

        result = {
            "status": 1 if status else 0,
            "message": "处理成功" if status else "处理失败",
            "id": hot_id or article_id,
            "weight": weight
        }

        return tools.dumps_json(result)
コード例 #26
0
    def add(self, table, key_value):
        '''
        @summary: 添加 表不存在自动创建表
        ---------
        @param table: 表名
        @param key_value: 要添加的值 字典格式
        ---------
        @result: True / False
        '''

        try:
            self._db[table].save(key_value)
        except Exception as e:
            log.error(e)
            return False
        else:
            return True
コード例 #27
0
def get_metadata(full_name):
    metadata = None
    try:
        parser = createParser(full_name)
        metadata = extractMetadata(parser)
        if parser:
            parser.stream._input.close()
            del parser
    except hachoir.stream.input.InputStreamError:
        ## is directory
        metadata = None
    except Exception as err:
        log.exception(err)
        log.error("Cannot extract metadata")
        metadata = None
    finally:
        return metadata
コード例 #28
0
    def set_fsattrs(self):
        log.debug("Getting filesystem attributes {0}".format(self.path))
        try:
            me = json.loads(
                str(
                    sh.lsblk("-pJ", "-o", self.LSBLK_KEYS, self.path).stdout,
                    'utf8'))["blockdevices"][0]
            try:
                self.uuid = me["uuid"]
                log.debug("GOT UUID: {0}".format(self.uuid))
            except KeyError:
                self.uuid = None
            try:
                self.fstype = me["fstype"]
                log.debug("GOT FSTYPE: {0}".format(self.fstype))
            except KeyError:
                self.fstype = None

            try:
                self.label = me["label"]
            except KeyError:
                self.label = None

        except sh.ErrorReturnCode_32:
            # Not a block device falling back to blkid
            cmd = str(sh.blkid("-o", "export", self.path).stdout, 'utf8')
            try:
                self.uuid = re.search("UUID=(.*)", cmd).group(1)
            except (TypeError, AttributeError):
                self.uuid = None
            try:
                self.fstype = re.search("TYPE=(.*)", cmd).group(1)
            except (TypeError, AttributeError):
                self.fstype = None
            try:
                self.label = re.search("LABEL=(.*)", cmd).group(1)
            except (TypeError, AttributeError):
                self.label = None

        if str(self.fstype) not in self.SUPPORTED_FS:
            log.error("{0} is out of {1} : {2}".format(self.fstype,
                                                       self.SUPPORTED_FS,
                                                       self.path))
            raise IncompatibleFS("Not working with {0} fstype {1}".format(
                self.path, self.fstype))
        return True
コード例 #29
0
ファイル: tools.py プロジェクト: zzzz123321/wechat-spider-1
def get_json_by_requests(url, params = None, headers = '', data = None, proxies = {}):
    json = {}
    response = None
    try:
        #response = requests.get(url, params = params)
        if data:
            response = requests.post(url, headers = headers, data = data, params = params, timeout = TIME_OUT, proxies = proxies)
        else:
            response = requests.get(url, headers=headers, params = params, timeout=TIME_OUT, proxies = proxies)
        response.encoding = 'utf-8'
        json = response.json()
    except Exception as e:
        log.error(e)
    finally:
        response and response.close()

    return json
コード例 #30
0
 def run(self):
     is_show_tip = False
     while True:
         try:
             datas = self.get_data_from_redis(SYNC_STEP)
             if not datas:
                 if not is_show_tip:
                     print('\n{time} 无数据 休眠...    '.format(
                         time=tools.get_current_date()))
                     is_show_tip = True
             elif self.add_data_to_es(datas):
                 is_show_tip = False
                 self._sync_count += len(datas)
                 tools.print_one_line('已同步 %d 条数据' % self._sync_count)
             tools.delay_time(1)
         except Exception as e:
             log.error(e)
コード例 #31
0
ファイル: handler.py プロジェクト: staticzeng/hermes
 def do(cls, **data):
     result = dict()
     try:
         d = cls.task_cls(**data)
         ret = TaskManager().add_task(d)
         log.debug("==============Task %s start=============" %
                   cls.__name__)
         log.debug("Task param: %s" % data)
         if ret[0] == 0:
             log.debug("task start...")
             d.start()
         result["ret"], result["msg"] = ret
         result["task_id"] = d.task_id
     except VtransError, e:
         msg = "%s request failed. detail: %s" % (cls.__name__, e)
         result['ret'], result['msg'] = e.code(), msg
         log.error(msg)
         traceback.print_exc()
コード例 #32
0
    def mount_compat(self, mode="ro"):
        status = True

        try:
            sh.mount("-o", "{0},{1}".format(mode, self.MOUNT_OPTIONS),
                     self.path, self.output_path)
        except sh.ErrorReturnCode as e:
            log.debug("Legacy re-mount opts for {0}".format(self))
            try:
                sh.mount("-o", "{0}".format(mode), self.path, self.output_path)
            except:
                try:
                    sh.mount(self.path, self.output_path)
                except Exception as e:
                    log.error("Cannot mount : {0}".format(self))
                    log.exception(e)
                    status = False
        return status
コード例 #33
0
def read_file(filename, readlines=False, encoding='utf-8'):
    '''
    @summary: 读文件
    ---------
    @param filename: 文件名(有路径)
    @param readlines: 按行读取 (默认False)
    ---------
    @result: 按行读取返回List,否则返回字符串
    '''

    content = ''
    try:
        with open(filename, 'r', encoding=encoding) as file:
            content = file.readlines() if readlines else file.read()
    except Exception as e:
        log.error(e)

    return content
コード例 #34
0
ファイル: manage.py プロジェクト: ssdxiao/alarm_platform
    def post(self):
        log.debug("ManageHandler post in")
        data = self.get_data()
        if data:
            log.debug(data)
            if data.has_key("ManageTelephone") and data.has_key("ManagePassword") \
                    and data.has_key("ManageName") :

                if data["ManageName"] == "":
                    log.error("ManageName is NULL")
                else:
                    db.insert_manage(data["ManageName"], data["ManageTelephone"],
                                   data["ManagePassword"])


                    result ={}
                    result["result"] = "ok"
                    self.send_data(result)
                    str = "add manage name %s telephone %s"%(data["ManageName"].encode('utf-8'),data["ManageTelephone"].encode('utf-8') )
                    save_record(self.login_user, "manage", 0, "add", str)
                    return

            else:
                log.error("Manaage data key is not right")
        else:
            log.error("data is none")

        result = {}
        result["result"] = "error"
        result["message"] = "Manaage info is error"
        self.send_data(result)