def taskFunc(schedid, searchcond, startime, endtime, warnornot, warncondop, warncondval, warnlevel, saveresult): """ 计划任务的实际功能 """ if postdata != None and postdata.is_master() == False: return execTime = datetime.datetime.now() #print 'Task start time: %s' % execTime.strftime( '%Y-%m-%d %H:%M:%S' ) # 要将searchstart和searchend转换为绝对时间;原类型为字符串 searchstart = scheduletime.fmtRelTime(startime, execTime) #print 'Condition start time: %s' % searchstart.strftime( '%Y-%m-%d %H:%M:%S' ) searchend = scheduletime.fmtRelTime(endtime, execTime) #print 'Conditon end time: %s' % searchend.strftime( '%Y-%m-%d %H:%M:%S' ) try: es = esdata.ESData(configFile) # 查询符合条件的数据量 cnt = es.Count(searchcond, searchstart, searchend) #print 'Count of search: %s' % cnt if saveresult > 0: #lst = es.Search( searchcond, searchstart, searchend, fields = DEFAULTFIELDS, size = saveresult ) lst = es.Search(searchcond, searchstart, searchend, fields=None, size=saveresult) #print 'Length of search list: %s' % len( lst ) # 向t_schedresult中插入一条数据,并返回插入数据的id(resultid) resultid = postdata.newSchedResult(schedid, execTime, searchcond, searchstart, searchend, cnt, saveresult) warnid = 0 if warnornot.upper() != 'N': if ( warncondop == '>' and cnt > warncondval ) or \ ( warncondop == '>=' and cnt >= warncondval ) or \ ( warncondop == '<=' and cnt <= warncondval ) or \ ( warncondop == '<' and cnt < warncondval ) or \ ( warncondop == '=' and cnt == warncondval ): # 向t_warn中插入报警数据,并返回插入数据的id(warnid) warnid = postdata.newWarn(schedid, execTime, warnlevel, searchcond, searchstart, searchend, cnt) # 向t_resultdetail中插入数据 if saveresult > 0 and len(lst) > 0: postdata.newResultDetail(warnid, resultid, lst) except Exception, e: conf = sys_config.SysConfig(configFile) _logFile = conf.getConfig("tasksched", "logFile") _instance = conf.getConfig("tasksched", "instanceName") sys_log.SysLog(_logFile, _instance).writeLog("error", str(traceback.format_exc()))
def run(start_type = None): global dicTask, dicJob, postdata conf = sys_config.SysConfig(configFile) # 进程号文件名 pidFile = conf.getConfig("pullfile", "pidFile") if start_type == None: if os.path.exists(pidFile): os.remove(pidFile) pFile = open(pidFile, "w") pFile.write(str(os.getpid())) pFile.close() # 生成postdata对象 postdata = postgrespullfile.PostgresData( configFile ) # 清空表t_pfnew、t_pfupd、t_pfdel postdata.clearSchedule() # 读取活动的文件提取任务 lstTask = postdata.getTask() if lstTask != None and len( lstTask ) > 0: # 创建并启动文件提取任务 #print "Length of lstTask is: %s" % len( lstTask ) createTask( lstTask ) while True: # 延时60秒 time.sleep( LOOPTIME ) # 读取变化过的文件提取任务 lstNew = postdata.getNewTask() lstUpd = postdata.getUpdTask() lstDel = postdata.getDelTask() # 清空表t_pfnew、t_pfupd、t_pfdel postdata. clearSchedule() # 创建新的文件提取任务 if lstNew != None and len( lstNew ) > 0: createTask( lstNew ) # 修改已存在的文件提取任务 if lstUpd != None and len( lstUpd ) > 0: # 先停止 deleteTask( lstUpd ) # 再重建 createTask( lstUpd ) # 删除文件提取任务 if lstDel != None and len( lstDel ) > 0: deleteTask( lstDel )
def run(start_type=None): global dicTask, postdata conf = sys_config.SysConfig(configFile) # 进程号文件名 pidFile = conf.getConfig("dataextract", "pidFile") if start_type == None: if os.path.exists(pidFile): os.remove(pidFile) pFile = open(pidFile, "w") pFile.write(str(os.getpid())) pFile.close() # 生成postdata对象 postdata = postgresdataextract.PostgresData(configFile) # 清空表t_dbnew、t_dbupd、t_dbdel postdata.clearDataExtraction() # 读取活动的数据库数据提取任务 在这个地方把数据提取任务直接传过来 lstTask = postdata.getDataExtractions() if lstTask != None and len(lstTask) > 0: # 创建并启动数据库数据提取任务 createTask(lstTask) while True: # 延时 time.sleep(LOOPTIME) # 读取变化过的数据抽取任务 lstNew = postdata.getNewDataExtraction() lstUpd = postdata.getUpdDataExtraction() lstDel = postdata.getDelDataExtraction() # 清空表t_dbnew、t_dbupd、t_dbdel postdata.clearDataExtraction() # 创建新的数据抽取任务 if lstNew != None and len(lstNew) > 0: createTask(lstNew) # 修改已存在的数据抽取任务 if lstUpd != None and len(lstUpd) > 0: # 先停止 deleteTask(lstUpd) # 再重建 createTask(lstUpd) # 删除数据抽取任务 if lstDel != None and len(lstDel) > 0: deleteTask(lstDel)
def run(start_type = None): global postdata conf = sys_config.SysConfig(configFile) # 进程号文件名 pidFile = conf.getConfig("datastats", "pidFile") if start_type == None: if os.path.exists(pidFile): os.remove(pidFile) pFile = open(pidFile, "w") pFile.write(str(os.getpid())) pFile.close() # 生成postdata对象 postdata = postgrestats.PostgresData(configFile) argus = [ postdata ] sched = Scheduler(standalone = True) sched.add_cron_job(funcNoon, year=cron_noon[0], month=cron_noon[1], \ week=cron_noon[2], day_of_week=cron_noon[3], day=cron_noon[4], \ hour=cron_noon[5], minute=cron_noon[6], second=cron_noon[7], args=argus) sched.add_cron_job(funcHour, year=cron_hour[0], month=cron_hour[1], \ week=cron_hour[2], day_of_week=cron_hour[3], day=cron_hour[4], \ hour=cron_hour[5], minute=cron_hour[6], second=cron_hour[7], args=argus) sched.add_cron_job(funcMidnight, year=cron_midnight[0], month=cron_midnight[1], \ week=cron_midnight[2], day_of_week=cron_midnight[3], day=cron_midnight[4], \ hour=cron_midnight[5], minute=cron_midnight[6], second=cron_midnight[7], args=argus) # 自定义dashboard统计服务 sched.add_cron_job(stats_dashboard.stats_dashboard, year=cron_dashboard[0], month=cron_dashboard[1], \ week=cron_dashboard[2], day_of_week=cron_dashboard[3], day=cron_dashboard[4], \ hour=cron_dashboard[5], minute=cron_dashboard[6], second=cron_dashboard[7], args=argus) # 每隔几分钟(默认5分钟)检查是否需要删除原始pcap文件 interval_chkdevice = conf.getConfig("datastats", "intervalCheckDevice") if interval_chkdevice == None: interval_chkdevice = 5 else: interval_chkdevice = int(interval_chkdevice) sched.add_interval_job(chkdevice.checkDevice, weeks=0, days=0, hours=0, minutes=interval_chkdevice, seconds=0, args=argus) try: sched.start() except (KeyboardInterrupt, SystemExit): pass
def run(start_type=None): # 进程号文件名及配置 conf = sys_config.SysConfig(configFile) pidFile = conf.getConfig("dataextract", "pidFile") #把进程号写进一上文件 if start_type is None: if os.path.exists(pidFile): os.remove(pidFile) pFile = open(pidFile, "w") pFile.write(str(os.getpid())) pFile.close() #传进任务列表(列表形式), 数据库连接 (字典形式) dic_tasklist = globalvariable.dic_tasklist dic_connection = globalvariable.dic_connection #如果任务列表不为空以及列表元素大小零,执行任务 if dic_tasklist is not None and len(dic_tasklist) > 0: createTask(dic_tasklist, dic_connection)
def __init__(self, cfgFile=None): self._configFile = cfgFile if self._configFile == None: #self._configFile = sys_config.getDir() + "/conf/" + CONFIG_FILE self._configFile = os.path.split( os.path.realpath(__file__))[0] + "/conf/" + CONFIG_FILE if os.path.exists(self._configFile) == False: self._configFile = "/opt/mdstack/conf/mdstackd/" + CONFIG_FILE conf = sys_config.SysConfig(self._configFile) eshost = conf.getConfig("elasticsearch", "esHost") esport = conf.getConfig("elasticsearch", "esPort") self._config = eshost + ':' + esport # 日志文件 self._logFile = conf.getConfig("dataextract", "logFile") # 实例名 self._instance = conf.getConfig("dataextract", "instanceName")
def __init__( self, cfgFile = None ): if cfgFile == None: #cfgFile = sys_config.getDir() + "/conf/" + CONFIG_FILE cfgFile = os.path.split(os.path.realpath(__file__))[0] + "/conf/" + CONFIG_FILE if os.path.exists(cfgFile) == False: cfgFile = "/opt/mdstack/conf/mdstackd/" + CONFIG_FILE conf = sys_config.SysConfig( cfgFile ) # 数据库连接相关 self._dbhost = conf.getConfig( "postgresql", "dbHost" ) self._dbport = conf.getConfig( "postgresql", "dbPort" ) self._dbname = conf.getConfig( "postgresql", "dbName" ) self._dbuser = conf.getConfig( "postgresql", "dbUser" ) self._dbpwd = conf.getConfig( "postgresql", "dbPwd" ) # 日志文件 self._logFile = conf.getConfig( "dataextract", "logFile" ) # 实例名 self._instance = conf.getConfig( "dataextract", "instanceName" )
def taskFunc( pfid, groupid, configpath, logsource, protocol, port, username, userpass, fpath, files ): """ 文件提取任务的实际功能 """ execTime = datetime.datetime.now() #print 'Task start time: %s' % execTime.strftime( '%Y-%m-%d %H:%M:%S' ) #print 'Task function parameters: %s' %( [pfid, groupid, configpath, logsource, protocol, port, username, userpass, fpath, files] ) # 读取配置 conf = sys_config.SysConfig( configFile ) rootPath = conf.getConfig( "pullfile", "rootPath" ) tmpPath = os.path.join( rootPath, 'tmp' ) # 读取组名 groupName = postdata.getGroupName( groupid ) if username == None: username = '' if userpass == None: userpass = '' else: #userpass = userpass userpass = base64.decodestring( userpass ) obj = None if protocol.lower() == 'ftp': # FTP if port == None: port = 21 obj = fileftp.FileFtp( username, userpass, logsource, port ) elif protocol.lower() == 'samba': # SMB if port == None: port = 139 obj = filesmb.FileSMB( username, userpass, logsource, port ) elif protocol.lower() == 'ssh': # SSH if port == None: port = 22 obj = filessh.FileSSH( {'username': username, 'password': userpass}, logsource, port ) if obj != None: lstFile = obj.GetFileList( os.path.join( fpath, files ) ) for remoteFile in lstFile: lstName = os.path.split( remoteFile ) localFile = os.path.join( tmpPath, lstName[1] ) obj.DownLoadFile( localFile, remoteFile ) if os.path.exists( localFile ): hashvalue = getFileHash( localFile ) if postdata.existsFileHash( hashvalue ): os.remove( localFile ) else: #lastmoditime = datetime.datetime.now() lastmoditime = obj.getFileModiTime( remoteFile ) postdata.insFileHash( hashvalue, lstName[1], lastmoditime, configpath, logsource ) fileName = hashvalue lstExt = lstName[1].split('.') if len(lstExt) > 1: fileName = hashvalue + '.' + lstExt[len(lstExt) - 1] dstPath = os.path.join( rootPath, 'pull', groupName, execTime.strftime('%Y%m%d'), logsource.replace(':','-'), configpath ) if os.path.isdir( dstPath ) == False: os.makedirs( dstPath ) os.rename( localFile, os.path.join( dstPath, fileName ) ) obj.close()
def taskFunc(dbid, conname, tbname, idxname, reclimit, inctype, timefld, incrfld, msgfld, curpos, curpos_stime, curpos_etime): """ 数据提取进程实际功能 """ # 提取数据库连接相关信息 # 返回值为None或有以下键的字典( conname, conntype, hostname, port, dbname, username, userpass, usepooling ) connInfo = postdata.getConnInfo(conname) es = esdataextract.ESData(configFile) dbDic = {} dbDic['dbhost'] = connInfo["hostname"] dbDic['dbport'] = str(connInfo["port"]) dbDic['dbname'] = connInfo["dbname"] dbDic['dbuser'] = connInfo["username"] #dbDic['dbpass'] = connInfo["userpass"] dbDic['dbpass'] = base64.decodestring(connInfo["userpass"]) dbDic['tbname'] = tbname dbDic['reclimit'] = str(reclimit) dbDic['inctype'] = inctype dbDic['timefld'] = timefld dbDic['incrfld'] = incrfld dbDic['msgfld'] = msgfld dbfac = dbfactory.DbFactory(dbDic) db = dbfac.factory(connInfo["conntype"]) # 读取要提取的数据库字段 lstFlds = [] n = 0 while len(lstFlds) == 0: # 返回值为None或有以下键的字典列表( fldid, fldsrc, fldout ) lstFlds = db.getTableStructure() n += 1 if n == 12: n = 0 conf = sys_config.SysConfig(configFile) _logFile = conf.getConfig("dataextract", "logFile") _instance = conf.getConfig("dataextract", "instanceName") sys_log.SysLog(_logFile, _instance).writeLog( "error", "tbname: " + tbname + " Not found fields!") time.sleep(5) # 保存字段及数据类型,此功能以后应改为由web页面实现 postdata.saveDbFieldList(dbid, lstFlds) # 如果需要修改导入的字段,则手工改t_dbfields表,并用下面的方法 #lstFlds = postdata.getDbFieldList(dbid) flds = [] for fld in lstFlds: flds.append(fld["fldsrc"]) # 判断自增字段数据类型:number、string、time if (fld["fldsrc"]).lower() == incrfld.lower(): incrfld_type = fld["fldout"] if curpos == None or curpos == "": if incrfld_type == "number": curpos = "0" elif incrfld_type == "string": curpos = " " elif incrfld_type == "time": curpos = "1980-01-01 00:00:00" else: curpos = " " if db != None: # 循环读取日志数据 while True: try: lst = db.getData(flds, incrfld_type, curpos, curpos_stime, curpos_etime) print "Start fetch data(t_database.dbid=" + str( dbid) + "): records=" + str( len(lst)), datetime.datetime.now() if lst == None or len(lst) == 0: # 1 主键增量; 2 时间增量; 0 一次性导入数据 if inctype == 2: if incrfld_type == "number": curpos = "0" elif incrfld_type == "string": curpos = " " elif incrfld_type == "time": curpos = "1980-01-01 00:00:00" else: curpos = " " if curpos_stime == '1980-01-01 00:00:00': curpos_stime = curpos_etime curpos_etime = db.getMinTime() else: curpos_stime = curpos_etime curpos_etime = db.getMaxTime() #如果时间差距过大,为防止数据量太大导致排序速度慢,那么让日期逐步增加 t1 = datetime.datetime.strptime( curpos_stime, '%Y-%m-%d %H:%M:%S') t2 = datetime.datetime.strptime( curpos_etime, '%Y-%m-%d %H:%M:%S') delta = t2 - t1 if delta.days > 2: curpos_etime = (t1 + datetime.timedelta(1) ).strftime('%Y-%m-%d %H:%M:%S') elif inctype == 0: return time.sleep(DELAYTIME) else: # 取最后一行数据 row = lst[len(lst) - 1] # 取最后一行数据的主键值 curpos = row[len(flds)] # 更新下次要提取数据的位置 postdata.updCurPosition(dbid, curpos, curpos_stime, curpos_etime) # 向ES中写数据 es.bulkInsData(lst, inctype, timefld, lstFlds, incrfld, tbname, idxname) #if len(lst) < reclimit: # time.sleep( DELAYTIME ) # 卡口测试 if tbname == "b_bk_jgcl": conf = sys_config.SysConfig(configFile) _logFile = conf.getConfig("dataextract", "logFile") _instance = conf.getConfig("dataextract", "instanceName") sys_log.SysLog(_logFile, _instance).writeLog( "error", "=====|" + tbname + "|" + str(time.time()) + "|" + str(curpos) + "|" + str(curpos_stime) + "|" + str(curpos_etime)) except Exception, e: conf = sys_config.SysConfig(configFile) _logFile = conf.getConfig("dataextract", "logFile") _instance = conf.getConfig("dataextract", "instanceName") sys_log.SysLog(_logFile, _instance).writeLog( "error", "tbname" + " --- " + str(traceback.format_exc()))