Ejemplo n.º 1
0
    def load_config(self, config: configparser, proxy: dict = None) -> dict:
        self.cfg['proxy'] = proxy
        log.info(f"Connecting to {self.name} webhook")

        try:
            self.cfg["url"] = config.get(self.name, "url")
        except Exception as e:
            e = f"Bad {self.name} configuration: {e}"
            log.error(e)
            raise Exception(e)

        try:
            r = requests.get(self.cfg["url"], timeout=10, proxies=proxy)
            if r.status_code != 200:
                raise ConnectionError

            j = r.json()
            self.cfg['token'] = j['token']
            self.cfg['name'] = j['name']
            log.info(f"Connected to Discord webhook: {self.cfg['name']}")

        except ConnectionError:
            raise Exception("Bad answer from Discord. Check WEBHOOK URL")
        except KeyError:
            raise Exception("WEBHOOK doesn't return token")

        if 'token' not in self.cfg:
            e = f"Discord WEBHOOK didn't return token"
            log.error(e)
            raise Exception(e)

        return self.cfg
Ejemplo n.º 2
0
    def send_notify(self, app:str, event:str, body:str) -> bool:
        try:
            # Формирует заголовок письма
            msg = MIMEMultipart('mixed')
            msg['Subject'] = templater.tmpl_fill(self.name, 'subject')
            msg['From'] = self.cfg['fromHeader']
            msg['To'] = self.cfg['sendTo']
            msg.attach(MIMEText(body))
        except Exception as e:
            log.error(str(e))

        log.debug(f"Connecting to email server {self.cfg['server']}")
        try:
            if self.cfg["useSSL"]:
                s = smtplib.SMTP_SSL(host=self.cfg['server'], port=self.cfg['port'])
                s.ehlo()
                s.login(self.cfg['user'], self.cfg['password'])
                s.auth_plain()
            else:
                s = smtplib.SMTP(self.cfg['server'], self.cfg['port'])
                s.ehlo().starttls().ehlo().login(self.cfg['user'], self.cfg['password']) # Рукопожатие, обязательно

            log.debug(f"Sending report")
            s.sendmail(self.cfg["fromHeader"], self.cfg["sendTo"], msg.as_string())

            log.info(f"Report of an event {app} sent")
            return True
        except Exception as e:
            if e.errno == 11004:
                log.error("Fail to connect to email server")
            else:
                log.error("Fail to send report: %s" % e)
            return False
Ejemplo n.º 3
0
 def delay_calc(taskStartTime):
     startTime = dtime.datetime.now()
     if taskStartTime.lower() != 'now':
         now = dtime.datetime.now()
         now = now.hour * 3600 + now.minute * 60 + now.second
         try:
             nextStart = dtime.datetime.strptime(taskStartTime, '%H:%M:%S')
             nextStart = nextStart.hour * 3600 + nextStart.minute * 60 + nextStart.second
             if now > nextStart:
                 delay = 86400 - now + nextStart  # сегодня = что прошло+время завтра до старта
                 startTime += dtime.timedelta(seconds=delay)
                 if onStart:
                     log.info(f"Tasks will start at {taskStartTime}")
             else:
                 delay = nextStart - now
                 startTime += dtime.timedelta(seconds=delay)
                 if onStart:
                     log.info(f"Tasks will start today at {taskStartTime}")
         except Exception as e:
             log.error(
                 f'Check parameter taskStartTime: {e}. Correct format used HH:MM:SS'
             )
             time.sleep(2)
             shutdown_me(1, '')
     return startTime
Ejemplo n.º 4
0
def send_notify(taskName: str, event: str, body: str):
    try:
        now = dtime.datetime.now()
        if taskName not in sendedNotify:
            sendedNotify[taskName] = {}

        if event not in sendedNotify[taskName]:
            sendedNotify[taskName][event] = {"dtm": now, "body": body}
        else:
            delta = now - sendedNotify[taskName][event]['dtm']
            if delta < resendTime:
                log.info(
                    f"Reject report of an event {event}: is already sent.")
                return

            if cfg['notify']['onlyChanges'] and sendedNotify[taskName][event][
                    'body'] == body:
                log.info(
                    f"Reject report of an event {event}: is not changed from last sent"
                )
                return

        log.debug(f"New report of an event {taskName}: {event}")

        if notify.send_notify(taskName, event, body):  # update
            sendedNotify[taskName][event] = {"dtm": now, "body": body}
        else:
            del sendedNotify[taskName][event]

    except Exception as e:
        log.error(f"Fail send notify: {e}")
        del sendedNotify[taskName][event]
Ejemplo n.º 5
0
    def distribute_request(self,http_req_handler):
        '''
        根据URL匹配规则路由请求到相应地处理器
        '''
        path = urlparse(http_req_handler.path).path
        handled = False

        #代理支持
        if C('enable_proxy') and utils.isDict(C('proxy')):
            for reg,target in C('proxy').items():
                target_path = get_proxy_url(http_req_handler.path,reg,target)
                if target_path:
                     log.info('[proxy](%s) to (%s)'%(http_req_handler.path,target_path))
                     return proxy(target_path,Request(http_req_handler),Response(http_req_handler))

        for h in self.handlers:
            if 'ALL' == h.get('method') or h.get('method') == http_req_handler.command and re.findall(h.get('pattern'),path):
                handled = True
                ret = (h.get('handler'))(Request(http_req_handler),Response(http_req_handler))
                if True == ret:
                    continue
                else:
                    break
        #if not handled by any handlers,405
        if not handled:
            log.error('%s is not handled'%path)
            http_req_handler.send_header(405,'%s not supported'%path)
            http_req_handler.end_headers()
            self.http_req_handler.wfile.close()
Ejemplo n.º 6
0
 def dir_to_uri(self, module_name):
     """将路径转换为uri
     Args:
         module_name: 路径名称
     """
     assert self._uri.startswith('/'), "URI must startswith '/'"
     log.info('request uri auto generate: {} ---> {}'.format(module_name, self._uri))
     return self._uri
Ejemplo n.º 7
0
def post_delete_handler(sender, instance):
    '''所有的model删除后回调函数
    '''
    if Database.current_user is None:
        current_user = '******'
    else:
        current_user = Database.current_user['account']
    log.info('account=%s,command=delete,table=%s,record_id=%s',
             current_user, instance._meta.db_table, instance.id)
Ejemplo n.º 8
0
def shutdown_me(signum, frame, appServerSvc=None):
    # ловит ctrl-C. Останавливает модули в нужном порядке
    log.warning('Stopping...')

    if appServerSvc:
        appServerSvc.daemon.exit()

    log.info("Shutdown is successful")
    os._exit(0)
Ejemplo n.º 9
0
def post_delete_handler(sender, instance):
    '''所有的model删除后回调函数
    '''
    if Database.current_user is None:
        current_user = '******'
    else:
        current_user = Database.current_user['account']
    log.info('account=%s,command=delete,table=%s,record_id=%s', current_user,
             instance._meta.db_table, instance.id)
Ejemplo n.º 10
0
def shutdown_me(signum=1, frame=1):
    """
    Останавливает модули в нужном порядке
    """
    log.warning(
        f'Lootnika stopping on {cfg["rest"]["host"]}:{cfg["rest"]["port"]}')
    if selfControl.exit:
        return

    selfControl.exit = True
    selfControl.rate = 0.3
    n = 0
    try:
        while True:
            time.sleep(0.3)
            if not bool(selfControl.myThreads):
                break

            if selfControl.myThreads['RestServer']:
                if n < 1:
                    log.debug("Stopping REST server")
                    try:
                        if cfg["rest"]["host"] in ['::1', '0.0.0.0']:
                            host = '127.0.0.1'
                        else:
                            host = cfg["rest"]["host"]

                        cnx = httpClient.HTTPConnection(host,
                                                        cfg["rest"]["port"],
                                                        timeout=12)
                        cnx.request(method="GET", url='/a=stop?stop')
                        cnx.getresponse()
                    except Exception:
                        pass
                    n = 1
                    continue
            elif selfControl.myThreads['Scheduler']:
                if n < 2:
                    log.debug("Stopping Scheduler thread")
                    scheduler.cmd = 'stop'
                    n = 2
            elif selfControl.myThreads['Datastore']:
                if n < 3:
                    log.debug("Stopping Datastore thread")
                    ds.close()
                    n = 3
            else:
                break

    except Exception as e:
        log.error(f'Shutdown failed: {traceback.format_exc()}')
    finally:
        selfControl.stop = True
        log.info("Lootnika stopped")
        if not stillWork:
            os._exit(1)
Ejemplo n.º 11
0
def store_run(storeDate):
    try:
        log.info("[mal_dns] Download started.")

        merge_blacklist.main(storeDate)

        log.info("[mal_dns] Download done.")

    except Exception, e:
        log.error("[mal_dns] Download failed.\n{0}".format(e))
Ejemplo n.º 12
0
    def _work_manager(self, taskName: str = '', lastTask: str = '', cmd=False):
        """
        Обёртка исполнителя задания (Picker). Работает как таймер чтобы
        отложить запуск до заданного времени. Потому проверка статуса
        планировщика так же выполняется здесь.

        NOTE: сейчас планировщик сам проверяет время старта и запускает
        задание сразу
        """
        self._update_startTime()
        if not (self.status == 'ready' or self.status == 'wait'):
            log.warning(
                f'Previous task is still running. Next start will be at {self.startTime}'
            )
            return

        self.status = 'work'  # он должен работатьт только при ready
        if self.taskCycles > 0:
            self.taskCycles -= 1
        # if self.taskCycles==0: self.startTime = None

        # task может быть только при cmd=True
        if taskName != '':
            self._start_task(taskName)
        else:
            if not cmd:
                log.info('New tasks cycle')
            else:
                log.info('Start all tasks')

            for taskName in self.taskList:
                # в случае отмены не продолжать
                if self.status == 'cancel':  # далее уже сам воркер следит даже если пауза
                    self.curTask = ''
                    self.status = 'ready'
                    return
                else:
                    self._start_task(taskName)

        self.curTask = ''
        if self.taskCycles > 0:
            self.status = 'wait'
        else:
            self.status = 'ready'

        if cmd:
            log.info('All tasks completed')
        else:
            if self.startTime is None:
                log.info('Tasks cycle done. Task replays are over')
            else:
                log.info(f'Tasks cycle done. Left: {self.taskCycles}')
Ejemplo n.º 13
0
    def load_config(self, config: configparser, proxy: dict = None) -> dict:
        self.cfg['proxy'] = proxy
        log.info(f"Connecting to {self.name} webhook")

        try:
            self.cfg["url"] = config.get(self.name, "url")
            log.info(f"Slack using WEBHOOK {self.cfg['url']}")
        except Exception as e:
            e = f"Bad {self.name} configuration: {e}"
            log.error(e)
            raise Exception(e)

        return self.cfg
Ejemplo n.º 14
0
    def send_notify(self, app: str, event: str, body: str) -> bool:
        try:
            data = {"username": "******", "content": body}
            res = requests.post(self.cfg['url'],
                                json=data,
                                timeout=10,
                                proxies=self.cfg['proxy'])

            if res.ok:
                log.info(f"Report sent")
                return True
            else:
                raise Exception("Server return status %s" % res.status_code)
        except Exception as e:
            log.error("Fail sent report by Discord  %s" % e)
            return False
Ejemplo n.º 15
0
    def run(self, Picker):
        log.debug("Starting Scheduler thread")
        self.Picker = Picker

        while not selfControl.started and self.cmd != 'stop':
            time.sleep(0.2)

        while self.cmd != 'stop':
            self._get_workers()
            # print('%s >= %s is %s' %(dtime.datetime.now(), self.startTime, self._isTaskTime()))
            # message(('self.status',  self.status), clrSun)

            if self.status == 'ready':
                # если расписание не включено или всё выполнилось,
                # то переходит в ждущий режим
                if self.taskCycles > 0:
                    self.status = 'wait'
                    if self._isTaskTime():
                        ht = Thread(name='work_manager',
                                    target=self._work_manager)
                        ht.start()
                        self.workers.append(ht)
                    # все последующие повторы отсчитываются от первого
            else:
                if self.taskCycles > 0:
                    if self._isTaskTime():
                        ht = Thread(name='work_manager',
                                    target=self._work_manager)
                        ht.start()
                        self.workers.append(ht)

                elif self.taskCycles == 0:  # -1 означает выкл. расписание
                    if self.status == 'ready':
                        log.info('Tasks cycle done')

            # print('!#cycle Status', self.status)
            time.sleep(1)

        # при выходе из цикла ждёт завершения работы рабочих и отменяет таймеры
        self.status = 'stop'
        self._get_workers()
        for ht in self.workers:
            # message(ht,clrSun)
            ht.join()

        log.debug("Stopped Scheduler thread")
        return
Ejemplo n.º 16
0
def log_inspector():
    log.debug("log_inspector started")
    selfName = 'log_inspector'
    while True:
        try:
            for taskName, task in cfg['tasks']['logTask'].items():
                log.info(f"Check log {taskName}")
                logFile = task['file']
                templates = task['tmpl']

                try:
                    # TODO open if file is changed
                    with open(logFile, encoding='utf-8') as f:
                        cnt = f.read()

                    for tmplName in templates:
                        tmpl = templater.get_tmpl(selfName, tmplName)
                        if tmpl in cnt:
                            ev = f"Found log expression {taskName}: {tmplName}"
                            log.warning(ev)
                            body = templater.tmpl_fill(selfName,
                                                       'error').replace(
                                                           '{{taskName}}',
                                                           taskName, -1)
                            event = 'error'

                            new_toast('log_inspector', event)
                            if 'eventScript' in task:
                                allowSend, body = execute_event_script(
                                    log, task['eventScript'], taskName, event,
                                    body)
                            else:
                                allowSend = True

                            if allowSend:
                                send_notify(taskName, event, body)

                except FileNotFoundError:
                    log.error(f"Not found log file {taskName}")
                except Exception as e:
                    log.error(f"Fail to parse log file {taskName}: {e}")

            sleep(intervalCheckMin * 2)
        except Exception:
            e = traceback.format_exc()
            log.critical(str(e))
            break
Ejemplo n.º 17
0
    def _start_task(self, taskName: str):
        self.curTask = taskName
        log.info(f'Start task {taskName}')
        try:
            lg = create_task_logger(taskName, console)
            ts = TaskStore(taskName, lg,
                           self.taskList[taskName]['overwriteTaskstore'])
            taskId = self._mark_task_start(taskName)

            # [total ,seen, new, differ, delete, task error, export error, last doc id]
            self.syncCount[taskId] = [-1, 0, 0, 0, 0, 0, 0, '']
            cf = self.taskList[taskName]

            fc = Factory(taskName, lg, cfg['exporters'][cf['exporter']],
                         self.syncCount[taskId])
            picker = self.Picker(taskId, taskName, cf, lg, ts, fc,
                                 self.syncCount[taskId])
            picker.run()

            tab = '\n' + '\t' * 5
            lg.info(f"Task done"
                    f"{tab}Total objects: {self.syncCount[taskId][0]}"
                    f"{tab}Seen: {self.syncCount[taskId][1]}"
                    f"{tab}New: {self.syncCount[taskId][2]}"
                    f"{tab}Differ: {self.syncCount[taskId][3]}"
                    f"{tab}Deleted: {self.syncCount[taskId][4]}"
                    f"{tab}Task errors: {self.syncCount[taskId][5]}"
                    f"{tab}Export errors: {self.syncCount[taskId][6]}")

            if self.syncCount[taskId][5] != 0:
                lg.warning('Task done with some errors. Check logs')
            if self.syncCount[taskId][6] != 0:
                log.warning(
                    'Task had errors with sending documents. '
                    f'Documents that were not sent are saved in a folder {picker.factory.failPath}'
                )

            self.check_point(taskId, 'complete')
        except Exception as e:
            if log.level == 10:
                e = traceback.format_exc()
            log.error(f"Fail with task {taskName}: {e}")
Ejemplo n.º 18
0
def proxy(target_url,req,res):
    '''
    '''
    if not target_url:
        return res.send(code = 500,content = 'Empty url not supported')

    #二进制资源直接重定向
    parsed_url = urlparse(target_url)
    
    if utils.isBinary(parsed_url.path,strict = True):
        return res.redirect(target_url)

    if 'GET' == req.method:
        request = R.get
    elif 'POST' == req.method:
        request = R.post

    try:
        #通知远端服务器不要压缩
        if req.headers.get('accept-encoding'):
            del req.headers['accept-encoding']
        if req.headers.get('host'):
            del req.headers['host']
        log.info('[proxy]requesting %s'%target_url)

        r = request(target_url,headers = req.headers)
        #本地服务器覆写Date和Server
        if r.headers.get('date'):
            del r.headers['date']
        if r.headers.get('server'):
            del r.headers['server']
        if r.headers.get('transfer-encoding'):
            del r.headers['transfer-encoding']

        log.info('[proxy] status=%d'%r.status_code)

        return res.send(code = r.status_code,content = r.content or '',headers =  r.headers)
    except Exception, e:
        log.error('[proxy]%s'%e)
        return res.send(code = 500,content = '%s'%e)
Ejemplo n.º 19
0
    def send_notify(self, app: str, event: str, body: str) -> bool:
        try:
            data = json.dumps({"text": body})
            headers = {
                "Content-type": "application/json",
                'Content-Length': len(body)
            }

            res = requests.post(self.cfg['url'],
                                data,
                                headers,
                                timeout=10,
                                proxies=self.cfg['proxy'])
            if res.status_code != 200:
                raise Exception("Server return status %s" % res.status_code)

            log.info(f"Report sent")
            return True

        except Exception as e:
            log.error("Fail sent report by Slack  %s" % e)
            return False
Ejemplo n.º 20
0
    def load_config(self, config: configparser, proxy:dict = None) -> dict:
        self.cfg['proxy'] = proxy
        try:
            self.cfg["sendTo"] = config.get(self.name, "sendTo")
            self.cfg["server"] = config.get(self.name, "server")
            self.cfg["port"] = config.getint(self.name, "port")
            self.cfg["useSSL"] = config.getboolean(self.name, "useSSL")
            self.cfg["user"] = config.get(self.name, "user")
            self.cfg["password"] = config.get(self.name, "password")
            self.cfg["fromHeader"] = config.get(self.name, "fromHeader")
            log.info(f'Recipient mail address {self.cfg["sendTo"]}')
        except Exception as e:
            e = f"Bad {self.name} configuration: {e}"
            log.error(e)
            raise Exception(e)

        if re.findall(r'\w+@\w+.\w+', self.cfg["sendTo"]):
            log.debug(f'Recipient mail address: {self.cfg["sendTo"]}')
        else:
            log.error("Wrong email sendTo.")
            raise SystemExit(1)

        return self.cfg
Ejemplo n.º 21
0
    def run(self):
        n = 1
        crash = False  # при краше нужно только обновлтяь статусы потоков
        while True:
            self.resources_usage()
            self.threads_names()

            # for i in get_threads():
            #     sout.print(f'{i} {i.isAlive()}', 'green')
            # print('------')

            for i in self.myThreads:
                if i in self.allThreads:
                    # message(i+ ' Run',clrSun)
                    self.myThreads[i] = True
                else:
                    self.myThreads[i] = False

            # отмечает какие модули запустились
            self.isVerified = True
            for i in self.myThreads:
                if not self.myThreads[i]:
                    self.isVerified = False

            if not crash:
                # ждёт запуска всех модулей
                if not self.started:
                    if self.isVerified:
                        if sys.argv[0].lower().endswith('.exe'):
                            log.info(
                                f"Lootnika started - Executable version: {__version__}_{platform}"
                            )
                        else:
                            log.info(
                                f"Lootnika started - Source version: {__version__}_{platform}"
                            )

                        log.info(
                            f"Welcome to http://localhost:{cfg['rest']['port']}/admin"
                        )
                        ds.execute("UPDATE lootnika SET self_status='working'")
                        self.started = True
                        self.rate = 2  # уже можно реже смотреть
                    else:
                        n += 1
                        if n == 20:  # ограничение времени запуска
                            crash = self.crash(
                                'One of the modules does not work correctly')
                        elif n == 10:
                            log.warning("detected slow Lootnika startup")
                # иначе следит за их работой
                else:
                    if not self.isVerified and not self.exit:
                        crash = self.crash(
                            "One of the modules does not work correctly")

            time.sleep(self.rate)
Ejemplo n.º 22
0
        def _deco(ch, method, properties, body):
            try:
                context.RequestContext()
                log.info("开始执行业务方法")
                func(ch, method, properties, body)
            except Exception as e:
                print(e)
                if conf.env == 'conf':
                    mail_title = Utils.currentTime() + " " + sys.argv[0] + "执行异常,异常内容见邮件"
                    Utils.sendMail(mail_title, traceback.format_exc(), [
                        '邮箱地址'
                    ])
                else:
                    traceback.print_exc()
                log.info("业务方法异常")

            finally:
                log.info("队列消息确认消费")
                ch.basic_ack(delivery_tag=method.delivery_tag)
Ejemplo n.º 23
0
    # ht = Thread(target=tray_icon, name='tray_icon')
    # ht.start()

    if cfg['tasks']['diskTask'] != {}:
        ht3 = Thread(target=disk_inspector, name='disk_inspector')
        ht3.start()

    if cfg['tasks']['logTask'] != {}:
        ht2 = Thread(target=log_inspector, name='log_inspector')
        ht2.start()

    if cfg['tasks']['jobList'] != {}:
        jobList = cfg['tasks']['jobList']
        ht1 = Thread(target=process_inspector, name='process_inspector')
        ht1.start()

    log.info(
        f"AppWatch started. Version: {__version__}_{PLATFORM}. "
        f"Python version: {sys.version_info[0]}.{sys.version_info[1]}.{sys.version_info[2]}"
    )

    if 'run' in sys.argv:
        signal.signal(signal.SIGINT, shutdown_me)
        signal.signal(signal.SIGTERM, shutdown_me)
        if PLATFORM != 'nt':
            signal.signal(signal.SIGQUIT, shutdown_me)

        input()
        while True:
            input('Use Ctrl+C to stop me\n')
Ejemplo n.º 24
0
                    pass

                for i in os.listdir(f'{dataDir}docs'):
                    shutil.copy2(f'{dataDir}docs/{i}', f'{homeDir}docs/{i}')

            elif 'help' in sys.argv:
                raise Exception('Show help')
            elif 'run' in sys.argv:
                from conf import log, logRest, console
                if devMode:
                    print('\n!#RUNNING IN DEVELOPER MODE\n')
                    log.setLevel(10)
                    logRest.setLevel(10)
                    console.setLevel(10)

                log.info(f"Starting...")
                import core

            elif 'make-doc' in sys.argv:
                from conf import log, logRest, console
                log.info(f"Compile documentation")
                sys.argv = sys.argv[:1]
                import sphinxbuilder
                sphinxbuilder.build()
            else:
                raise Exception('Show help')

    except Exception as e:
        e = traceback.format_exc()
        print(f'Fail to start main thread: {e}')
Ejemplo n.º 25
0
 def __init__(self, name: str):
     log.info(f"lucky-slacky v{__version__}")
     self.name = name
     self.cfg = {}
     self.defaultCfg = {"url": "YOUR_WEBHOOK_URL_HERE"}
Ejemplo n.º 26
0
 def _deco(*args, **kwargs):
     with timeutils.StopWatch() as w:
         func(*args, **kwargs)
     log.info("脚本运行结束,共计耗时 : {expense} 秒".format(expense=int(w.elapsed())))
     sys.stdout.flush()
Ejemplo n.º 27
0
def reconnect():
    """重连数据库
    """
    log.info("重连数据库...")
    ZKDASH_DB.connect()
Ejemplo n.º 28
0
def reconnect():
    '''重连数据库
    '''
    log.info('重连数据库...')
    ZKDASH_DB.connect()
Ejemplo n.º 29
0
def process_inspector():
    def get_pid(exe: str, exePath: str, workDir: str = None) -> int:
        # if give workDir, will check only it

        for p in psutil.process_iter(["name", 'exe', 'cwd']):
            # if 'calc1' in p.info['name']:
            # sout(f"{p.pid} | {p.info['name']} | {p.info['cwd']} | {p.info['exe']}", 'violet' )

            if exe == p.info['name'].lower():
                if workDir:
                    if not p.info['cwd'].endswith('/'):
                        p.info['cwd'] = f"{p.info['cwd']}/"

                    if workDir.lower() == p.info['cwd'].replace('\\', '/',
                                                                -1).lower():
                        return p.pid
                else:
                    if PLATFORM == 'nt':
                        exePath = f"{exePath}{exe}"
                    else:
                        exePath = exePath[:-1]

                    if exePath.lower() == p.info['exe'].replace('\\', '/',
                                                                -1).lower():
                        return p.pid

    def restart(job: dict,
                exePid: int = None,
                killRecursive: bool = False) -> str:
        data = ""
        status = 0
        failList[taskName]['attemp'] += 1
        if exePid:
            try:
                assert exePid != os.getpid(), "won't kill myself"
                parent = psutil.Process(exePid)
                children = parent.children(killRecursive)
                children.append(parent)

                # TODO try soft kill before hard
                for p in children:
                    try:
                        # p.send_signal(signal.SIGTERM)
                        p.kill()
                    except psutil.NoSuchProcess:
                        pass

                _, alive = psutil.wait_procs(children, timeout=60)
                if alive:
                    raise Exception(
                        f"Fail to kill process {exe} (PID {exePid})")
            except Exception as e:
                data = f'Fail to restart process {exe}: {e}\n'
                log.error(data)
                status = 2

        if status == 0:
            log.debug(f"Launch application {taskName}")
            whatStart = job['whatStart']

            if whatStart == 'command':
                target = job['command']
            elif whatStart == 'exe':
                target = f"{job['exePath']}{exe} {job['exeKey']}"
            else:
                target = None

            if target:
                log.info(f"Starting {taskName}")
                try:
                    if PLATFORM == 'nt':
                        os.system(f"start cmd /c {target}")
                    else:
                        os.system(f"command {target} &")
                except Exception as e:
                    data = f"Fail to restart application: {exe} ({taskName}): {e}\n"
                    status = 3
            else:
                log.info(f"Starting service {job['service']}")
                try:
                    if PLATFORM == 'nt':
                        win32serviceutil.StartService(job['service'])
                    else:
                        os.system(f"systemctl start {job['service']}")

                except Exception as e:
                    e = traceback.format_exc()
                    log.error(str(e))
                    status = 3
                    data = f"Fail to start service: {job['service']} ({taskName}): {e}\n"

            # проверка что он снова не упал
            # TODO отсчёт времени падения после старта
            if status == 0:
                sleep(restartTime)
                if get_pid(exe, checkPath, workDir):
                    data += 'Successfully restarted application'
                    failList[taskName]['isAlive'] = False
                    failList[taskName]['attemp'] -= 1
                    log.info(data)
                else:
                    data += f'Fail to start {taskName}'
                    log.error(data)
            else:
                log.error(data)

        new_toast(taskName, data)
        return data

    sleep(3)
    selfName = "process_inspector"
    failList = {}
    for job in jobList:
        failList[job] = {'isAlive': False, "attemp": 0}

    while True:
        try:
            for job in jobList.values():
                taskName = job['task']
                exe = job['exe'].lower()
                checkPath = job['checkPath']
                exePath = job['exePath']
                workDir = job['workDir']
                doRestart = job['doRestart']
                alwaysWork = job['alwaysWork']
                restartTime = job['restartTime']
                respTime = job['respTime']
                status = 0
                body = ''

                log.info(f'Check app {taskName}')
                exePid = get_pid(exe, checkPath, workDir)

                if exePid and not job['checkUrl']:
                    log.debug(f"{taskName} is fine.")
                elif exePid and job['checkUrl']:
                    log.debug(f"Found {taskName}. Check http status")
                    try:
                        res = requests.get(job['url'], timeout=respTime)
                        if res.status_code != 200:
                            raise Exception(
                                f"Server return status {res.status_code}")

                        log.debug(f"{taskName} is fine.")

                        if not failList[taskName]['isAlive']:
                            continue
                        else:
                            failList[taskName]['isAlive'] = False
                            data = templater.tmpl_fill(selfName, 'alive')
                    except Exception:
                        status = 1
                        data = f"{taskName} didn't respond or return wrong answer. Trying to restart application\n"
                        new_toast(f'Restarting {taskName}', data)
                        log.warning(data)

                        body = templater.tmpl_fill(selfName,
                                                   "badAnswer").replace(
                                                       "{{taskName}}",
                                                       taskName, -1)
                        failList[taskName]['isAlive'] = True

                    if status != 0 and doRestart:
                        data += restart(job, exePid)
                        body += data

                    if 'eventScript' in job:
                        allowSend, body = execute_event_script(
                            log, job['eventScript'], taskName, 'badAnswer',
                            body)
                    else:
                        allowSend = True

                    if allowSend:
                        send_notify(taskName, 'badAnswer', body)

                elif not exePid and alwaysWork:
                    body = templater.tmpl_fill(selfName, 'notFound').replace(
                        "{{taskName}}", taskName, -1)
                    data = f"Not found required application {taskName}. Trying to restart\n"
                    log.warning(data)
                    new_toast(f'Starting {taskName}', data)

                    data += restart(job, exePid)
                    body += data

                    new_toast('log_inspector', 'notFound')
                    if 'eventScript' in job:
                        allowSend, body = execute_event_script(
                            log, job['eventScript'], taskName, 'notFound',
                            body)
                    else:
                        allowSend = True

                    if allowSend:
                        send_notify(taskName, 'notFound', body)

            sleep(intervalCheckMin)
        except Exception:
            e = traceback.format_exc()
            log.critical(str(e))
            break
Ejemplo n.º 30
0
 def build(self):
     '''
     do build
     '''
     tmbegin = time.time()
     self._check();
     log.info ('copying directories')
     self._dir();
     log.info('handling less...');
     self._less();
     log.info ('handling css...')
     self._css();
     log.info ('handling  javascript...')
     self._js();
     log.info ('handling template...')
     self._tpl();
     if self._generate_html:
         log.info ('handling html...')
         self._html();
     log.info ('replacing all token')
     self._replace();
     log.info ('Time cost %s s.' % (time.time()-tmbegin) )
Ejemplo n.º 31
0
def disk_inspector():
    def fill_tmpl(event: str) -> str:
        body = templater.tmpl_fill(selfName, event)
        body = body.replace('{{critFree}}', str(critFree), -1)
        body = body.replace('{{diskFree}}', str(diskFree), -1)
        body = body.replace('{{diskUsage}}', diskUsage, -1)
        body = body.replace('{{taskName}}', taskName, -1)
        return body.replace('{{diskWarn}}', str(diskWarn), -1)

    log.debug("disk_inspector started")
    selfName = 'disk_inspector'

    while True:
        for taskName, task in cfg['tasks']['diskTask'].items():
            critFree = task['critFree']
            diskUsage = task['diskUsage']
            diskWarn = task['diskWarn']

            try:
                diskFree = round(
                    shutil.disk_usage(diskUsage).free / 1073741824, 2)
                if diskFree < critFree:
                    log.error(
                        f"Free disk space is critically small on {diskUsage}: {diskFree}"
                    )
                    event = 'critFree'
                    body = fill_tmpl(event)

                    new_toast(
                        diskUsage,
                        f"Free disk space is critically small: {diskFree}")
                    if 'eventScript' in task:
                        allowSend, body = execute_event_script(
                            log, task['eventScript'], taskName, event, body)
                    else:
                        allowSend = True

                    if allowSend:
                        send_notify(taskName, event, body)

                elif diskFree < diskWarn:
                    log.warning(
                        f"Free disk space is ends {diskUsage}: {diskFree}GB")
                    event = 'diskWarn'
                    body = fill_tmpl(event)

                    new_toast(diskUsage,
                              f"Free disk space is ends: {diskFree}GB")
                    if 'eventScript' in task:
                        allowSend, body = execute_event_script(
                            log, task['eventScript'], taskName, event, body)
                    else:
                        allowSend = True

                    if allowSend:
                        send_notify(taskName, event, body)
                elif diskFree > diskWarn:
                    log.info(f"disk {diskUsage}: {diskFree}GB free")

            except FileNotFoundError:
                log.error(f'disk_inspector: wrong path: {diskUsage}')
            except Exception as e:
                log.critical(f'disk_inspector: {traceback.format_exc()}')
                shutdown_me(9, 9)
        sleep(intervalCheckMin)
Ejemplo n.º 32
0
 def __init__(self, name: str):
     log.info(f"Discord connector v{__version__}")
     self.name = name
     self.cfg = {}
     self.defaultCfg = {"url": "YOUR_WEBHOOK_URL_HERE"}
Ejemplo n.º 33
0
def check_rst(ds: Datastore) -> dict:
    #TODO need refactoring
    log.debug("Check documentation sources")
    changed = False
    try:
        rows = ds.select('SELECT * FROM sphinxbuilder', )
        oldRst = {
            'lootnika': {
                'path': "docs/rst/",
                'type': 'lootnika',
                'rst': {}
            }
        }
        for row in rows:
            if row[1] not in oldRst:
                oldRst[row[1]] = {'rst': {}}

            oldRst[row[1]]['rst'][row[3]] = {'file': row[2], 'hash': row[4]}

        newRst = {
            'lootnika': {
                'path': "docs/rst/",
                'type': 'lootnika',
                'rst': {}
            }
        }
        for exporter in os.listdir(f'{homeDir}exporters'):
            path = f"exporters/{exporter}/docs/rst/"
            ls = os.listdir(f"{homeDir}{path}")
            if ls == []:
                log.warning(f"No documentation sources found for {exporter}")
                continue

            if exporter not in oldRst:
                log.info(f"Found new exporter docs: {exporter}")
                oldRst[exporter] = {
                    'path': path,
                    'type': 'exporter',
                    'rst': {}
                }

            newRst[exporter] = {'path': path, 'type': 'exporter', 'rst': {}}
            for file in ls:
                rst = f"{path}{file}"
                with open(f"{homeDir}{rst}", encoding='utf-8',
                          mode='r') as cnt:
                    hsh = f"{cityhash.CityHash64(cnt.read())}"

                newRst[exporter]['rst'][rst] = {'file': file, 'hash': hsh}
                if rst in oldRst[exporter]['rst']:
                    if not oldRst[exporter]['rst'][rst]['hash'] == hsh:
                        changed = True
                else:
                    changed = True

        for picker in os.listdir(f'{homeDir}pickers'):
            path = f"pickers/{picker}/docs/rst/"
            ls = os.listdir(f"{homeDir}{path}")
            if ls == []:
                log.warning(f"No documentation sources found for {picker}")
                continue

            if picker not in oldRst:
                log.info(f"Found new picker docs: {picker}")
                oldRst[picker] = {'path': path, 'type': 'exporter', 'rst': {}}

            newRst[picker] = {'path': path, 'type': 'picker', 'rst': {}}
            for file in ls:
                rst = f"{path}{file}"
                with open(f"{homeDir}{rst}", encoding='utf-8',
                          mode='r') as cnt:
                    hsh = f"{cityhash.CityHash64(cnt.read())}"

                newRst[picker]['rst'][rst] = {'file': file, 'hash': hsh}
                if rst in oldRst[picker]['rst']:
                    if not oldRst[picker]['rst'][rst]['hash'] == hsh:
                        changed = True
                else:
                    changed = True

        exporter = "lootnika"
        path = newRst[exporter]['path']
        ls = os.listdir(f"{homeDir}{path}")
        for file in ls:
            rst = f"{path}{file}"
            with open(f"{homeDir}{rst}", encoding='utf-8', mode='r') as cnt:
                hsh = f"{cityhash.CityHash64(cnt.read())}"

            newRst[exporter]['rst'][rst] = {'file': file, 'hash': hsh}
            if rst in oldRst[exporter]['rst']:
                if not oldRst[exporter]['rst'][rst]['hash'] == hsh:
                    changed = True
            else:
                changed = True

        if changed:
            log.warning(
                "Found changes in documentations. Start me with <make-doc> key."
            )

        return newRst
    except Exception as e:
        raise Exception(
            f"Fail check sources for help documentation: {traceback.format_exc()}"
        )
Ejemplo n.º 34
0
    def restart(job: dict,
                exePid: int = None,
                killRecursive: bool = False) -> str:
        data = ""
        status = 0
        failList[taskName]['attemp'] += 1
        if exePid:
            try:
                assert exePid != os.getpid(), "won't kill myself"
                parent = psutil.Process(exePid)
                children = parent.children(killRecursive)
                children.append(parent)

                # TODO try soft kill before hard
                for p in children:
                    try:
                        # p.send_signal(signal.SIGTERM)
                        p.kill()
                    except psutil.NoSuchProcess:
                        pass

                _, alive = psutil.wait_procs(children, timeout=60)
                if alive:
                    raise Exception(
                        f"Fail to kill process {exe} (PID {exePid})")
            except Exception as e:
                data = f'Fail to restart process {exe}: {e}\n'
                log.error(data)
                status = 2

        if status == 0:
            log.debug(f"Launch application {taskName}")
            whatStart = job['whatStart']

            if whatStart == 'command':
                target = job['command']
            elif whatStart == 'exe':
                target = f"{job['exePath']}{exe} {job['exeKey']}"
            else:
                target = None

            if target:
                log.info(f"Starting {taskName}")
                try:
                    if PLATFORM == 'nt':
                        os.system(f"start cmd /c {target}")
                    else:
                        os.system(f"command {target} &")
                except Exception as e:
                    data = f"Fail to restart application: {exe} ({taskName}): {e}\n"
                    status = 3
            else:
                log.info(f"Starting service {job['service']}")
                try:
                    if PLATFORM == 'nt':
                        win32serviceutil.StartService(job['service'])
                    else:
                        os.system(f"systemctl start {job['service']}")

                except Exception as e:
                    e = traceback.format_exc()
                    log.error(str(e))
                    status = 3
                    data = f"Fail to start service: {job['service']} ({taskName}): {e}\n"

            # проверка что он снова не упал
            # TODO отсчёт времени падения после старта
            if status == 0:
                sleep(restartTime)
                if get_pid(exe, checkPath, workDir):
                    data += 'Successfully restarted application'
                    failList[taskName]['isAlive'] = False
                    failList[taskName]['attemp'] -= 1
                    log.info(data)
                else:
                    data += f'Fail to start {taskName}'
                    log.error(data)
            else:
                log.error(data)

        new_toast(taskName, data)
        return data
Ejemplo n.º 35
0
# -*- coding: utf-8 -*-

from app import create_app
from conf import SERVER, log

application = create_app()

if __name__ == '__main__':
    log.info("start_i_manager_server")
    application.run(host=SERVER.get('host', '0.0.0.0'),
                    port=SERVER.get('port', 8919),
                    debug=SERVER.get('debug', True))