def speech_to_text(audio_file_path):
data = None
cnt = 0
# if data is None HTTP Request failed,
# The cause is probably because there is a limit of how many Request can be sent,
# but since it looks random we'll just try again when it happens...
while data is None and cnt < 5:
data = _send_request(audio_file_path)
cnt += 1
if data is None:
log.warning(f'try number {cnt} failed for {audio_file_path}')
time.sleep(0.5)
if data is None:
log.warning(f'{audio_file_path} had an HTTP Request failure')
return
if data.get('RecognitionStatus') != 'Success':
logwarning = f'{audio_file_path} has an empty transcript'
if data.get('RecognitionStatus') is not None:
logwarning += ', RecognitionStatus = ' + data.get(
'RecognitionStatus')
log.warning(logwarning)
return
transcript = data.get('NBest')[0].get('Display')
log.info("Transcript: %s" % transcript)
return transcript
def start(self):
"""
启动调度器和配置文件监视器
"""
log.info('apollocrond start')
self._ap_sched.start()
self._notifier.loop()
def start(self):
"""
启动调度器和配置文件监视器
"""
log.info('apollocrond start')
self._ap_sched.start()
self._notifier.loop()
def refreshJobs(self):
"""
purpose:计划任务随配置的变化而变化
"""
self.host = getHostname()
cron_jobs, status = parseCrontab(self._cronfile)
if status:
running_jobs = self._ap_sched.get_jobs()
new_jobs = set(cron_jobs.keys()) - set(running_jobs.keys())
old_jobs = set(running_jobs.keys()) - set(cron_jobs.keys())
samename_jobs = set(cron_jobs.keys()) & set(running_jobs.keys())
for key in old_jobs:
self._ap_sched.unschedule_job(running_jobs[key])
for key in new_jobs:
self._ap_sched.add_job(key, cron_jobs[key]['name'], cron_jobs[key]['trigger'], self.execCmd,
args=cron_jobs[key]['args'], kwargs=None)
for key in samename_jobs:
trigger = None
args = None
new_name = None
if str(cron_jobs[key]['trigger']) != str(running_jobs[key].trigger):
trigger = cron_jobs[key]['trigger']
if cron_jobs[key]['args'] != running_jobs[key].args:
args = cron_jobs[key]['args']
if cron_jobs[key]['name'] != running_jobs[key].name:
new_name = cron_jobs[key]['name']
if trigger or args or new_name:
self._ap_sched.update_job(running_jobs[key], trigger, args, new_name)
log.info('job[%s] changed' % key)
def process_IN_MODIFY(self, event):
"""
当cron配置文件发生变化时作相应的更新
"""
if os.path.realpath(os.path.abspath(self._crond._cronfile)) == event.pathname:
log.info('crontab配置文件发生变化，重新载入')
self._crond.refreshJobs()
def checkActiveTask():
'''检查是否有任务需要执行,负责调配任务的执行
'''
global next_conf_check, task_list
now = int(time())
if now >= next_conf_check:
global conf_last_change
mtime = int(os.stat(DEFAULT['task_file_config']).st_mtime) + int(os.stat(DEFAULT['host_file_config']).st_mtime)
if mtime != conf_last_change:
log.info('任务和主机配置文件发生变化，重新载入配置文件')
parseConf()
conf_last_change = mtime
log.info('当前任务数:%d,[%s]'%(len(task_list), ','.join([task['task_name'] for task in task_list])))
next_conf_check += DEFAULT['config_check_delay']
for task in task_list:
try:
if now >= task['nextcheck']:
dt = threads.deferToThread(doTask, task)
#if 256 == int(task.get('return_type', 256)):
# #dt.addCallback((lambda x: pass))
# continue
#else:
dt.addCallback(sender_manager.sendData)
if task.has_key('task_interval'):
task['nextcheck'] += task['task_interval']
elif task.has_key('task_time_specify'):
task['nextcheck'] += 3600*24
except:
continue
reactor.callLater(1,checkActiveTask)
def checkActiveTask():
'''检查是否有任务需要执行,负责调配任务的执行
'''
global next_conf_check, task_list
now = int(time())
if now >= next_conf_check:
global conf_last_change
mtime = int(os.stat(DEFAULT['task_file_config']).st_mtime) + int(
os.stat(DEFAULT['host_file_config']).st_mtime)
if mtime != conf_last_change:
log.info('任务和主机配置文件发生变化，重新载入配置文件')
parseConf()
conf_last_change = mtime
log.info('当前任务数:%d,[%s]' % (len(task_list), ','.join(
[task['task_name'] for task in task_list])))
next_conf_check += DEFAULT['config_check_delay']
for task in task_list:
try:
if now >= task['nextcheck']:
dt = threads.deferToThread(doTask, task)
#if 256 == int(task.get('return_type', 256)):
# #dt.addCallback((lambda x: pass))
# continue
#else:
dt.addCallback(sender_manager.sendData)
if task.has_key('task_interval'):
task['nextcheck'] += task['task_interval']
elif task.has_key('task_time_specify'):
task['nextcheck'] += 3600 * 24
except:
continue
reactor.callLater(1, checkActiveTask)
def refreshJobs(self):
"""
purpose:计划任务随配置的变化而变化
"""
self.host = getHostname()
cron_jobs, status = parseCrontab(self._cronfile)
if status:
running_jobs = self._ap_sched.get_jobs()
new_jobs = set(cron_jobs.keys()) - set(running_jobs.keys())
old_jobs = set(running_jobs.keys()) - set(cron_jobs.keys())
samename_jobs = set(cron_jobs.keys()) & set(running_jobs.keys())
for key in old_jobs:
self._ap_sched.unschedule_job(running_jobs[key])
for key in new_jobs:
self._ap_sched.add_job(key,
cron_jobs[key]['name'],
cron_jobs[key]['trigger'],
self.execCmd,
args=cron_jobs[key]['args'],
kwargs=None)
for key in samename_jobs:
trigger = None
args = None
new_name = None
if str(cron_jobs[key]['trigger']) != str(
running_jobs[key].trigger):
trigger = cron_jobs[key]['trigger']
if cron_jobs[key]['args'] != running_jobs[key].args:
args = cron_jobs[key]['args']
if cron_jobs[key]['name'] != running_jobs[key].name:
new_name = cron_jobs[key]['name']
if trigger or args or new_name:
self._ap_sched.update_job(running_jobs[key], trigger, args,
new_name)
log.info('job[%s] changed' % key)
def find_5_results_per_percentage_step():
accuracy_file_path = join(config.accuracy_dir(),
f'{config.provider()}_accuracy.txt')
accuracy_info = load_accuracy_info(accuracy_file_path)
interesting_results = {}
log.info("Looking for some test data results...")
for line in accuracy_info:
file_name = line[:line.index('\t')]
result = float(line[line.index('\t') + 1:].replace('\n', ''))
original_text_file_path = join(config.clean_data_text_dir(), file_name)
custom_text_file_path = join(config.provider_accuracy_dir(), file_name)
original_words = extract_clean_words(
open(original_text_file_path, 'r').read())
custom_words = extract_clean_words(
open(custom_text_file_path, 'r').read())
for error_rate in range(0, 101, 5):
if result == error_rate and len(
[x for x in interesting_results.values() if x == result]) < 5:
interesting_results.update({
f'{" ".join(original_words)}\n{" ".join(custom_words)}':
result
})
for k, v in sorted(interesting_results.items(), key=lambda x: x[1]):
log.info(f'{v}% word error rate:\n{k}\n')
def __op_new(self, data_loaded, connection):
item_id = int(data_loaded['id'])
item = UrlItem.load_with_content(
id=item_id,file_path=config.path_judge_inbox)
feature = self.__fe.extract_item(item)
if 'decision' not in data_loaded.keys():
decision, confidence = self.__auto_judge(feature)
log.info("[%s]: [%s] # %s # %s%%" % (item_id, FeatureExtract.str_feature(feature), decision, confidence))
else:
decision, confidence = data_loaded['decision'],100
log.info("[%s]: back from Extractor # %s # %s%%" % (item_id, decision, confidence))
self.__relearn_clf(feature,decision)
if confidence > config.const_CONFIDENCE_THRESHOLD:
item['is_target'] = decision
item.save()
if int(item['is_target']) in [config.const_IS_TARGET_MULTIPLE, config.const_IS_TARGET_SIGNLE]:
self.__send_to_extractor(item)
else:
os.remove(config.path_judge_inbox + "/%s" % item.filename())
else:
item['is_target'] = config.const_IS_TARGET_UNKNOW
item.save()
self.__judge_queue[item_id] = {
"title": item['title'],
"url": item['url'],
"filename": item.filename(),
"confidence": round(confidence,2),
"decision": decision,
"feature": feature
}
pass
def __view_update(self, bot, job):
channel_id = job if isinstance(job, str) else job.context
result = self.__check_more_than_max_numbers_of_find_things()
last_message = ''
for find_thing in FIND_THINGS:
if result[find_thing]:
export_message = '--- {0} ---\n'.format(find_thing)
count = 0
for notice in result[find_thing]:
count += 1
export_message += '{title}(<a href="{url}">링크</a>)\n'\
.format(title=notice['title'], url=notice['url'])
bot.sendMessage(channel_id,
text=export_message,
disable_notification=True,
parse_mode=ParseMode.HTML)
last_message += '{0}({1}개)/'.format(find_thing, count)
if last_message:
message_text = last_message[:-1]
bot.sendMessage(channel_id,
text=NEW_NOTICE + message_text,
parse_mode=ParseMode.HTML)
else:
message_text = '공지사항 없음'
log.info(message_text)
def main():
for audio_file_path in glob.glob(
join(config.clean_data_audio_dir(), '*.wav')):
spoken_text = speech_to_text(audio_file_path)
clean_text = clean_up_text(spoken_text)
flight_data = analyse_text(clean_text)
log.info(flight_data)
def __list_of_files(self,
dir_name,
dir_name_as_string,
count,
root_dir=False):
if count > ZIP_DIR_DEPTH:
return [], []
list_of_file = os.listdir(dir_name)
all_files_as_string = list()
all_files = list()
for entry in list_of_file:
full_path = os.path.join(dir_name, entry)
current_dir_name_as_string = os.path.basename(dir_name)
if current_dir_name_as_string == "__MACOSX":
log.info(f'Skipping __MACOSX folder')
continue
if not root_dir:
new_dir_as_string = f'{dir_name_as_string}/{current_dir_name_as_string}'
else:
new_dir_as_string = ""
if os.path.isdir(full_path):
files, strings = self.__list_of_files(full_path,
new_dir_as_string,
count + 1)
all_files = all_files + files
all_files_as_string = all_files_as_string + strings
elif os.path.isfile(full_path) and self.__check_file_type(
full_path):
all_files.append(full_path)
all_files_as_string.append(new_dir_as_string)
return all_files, all_files_as_string
def handle_thread_inner(self, thread_num):
last_smpl = None
begin = thread_num * self.num_frames / THREAD_NUM
end = (thread_num + 1) * self.num_frames / THREAD_NUM
for i in xrange(begin, end):
if i == begin:
current_frame = self.base_frame
else:
current_frame = self.create_frame(i, last_smpl)
log.info('Fit frame {}'.format(i))
# re-init if necessary
self.reinit_frame(current_frame, self.prior_data['mean'],
self.nohands, self.debug_rn)
# final fit
self.fit_pose(current_frame, last_smpl, self.frustum, self.nohands,
self.debug_rn)
self.temp_poses_dset[i] = current_frame.smpl.pose.r
self.temp_trans_dset[i] = current_frame.smpl.trans.r
if i == begin:
self.temp_betas_dset[:] = current_frame.smpl.betas.r
last_smpl = current_frame.smpl
def process_IN_MODIFY(self, event):
"""
当cron配置文件发生变化时作相应的更新
"""
if os.path.realpath(os.path.abspath(
self._crond._cronfile)) == event.pathname:
log.info('crontab配置文件发生变化，重新载入')
self._crond.refreshJobs()
def stringReceived(self, data):
if data.startswith('PING'):
info = data.split(":", 2)
self.sendString('PINGBACK:%s' % info[1])
#log.info('收到%s的心跳包'%self._peer)
return
log.info('收到%s发回的数据' % self._peer)
self.service.dealData(data)
def stringReceived(self, data):
if data.startswith('PING'):
info = data.split(":", 2)
self.sendString('PINGBACK:%s' % info[1])
#log.info('收到%s的心跳包'%self._peer)
return
log.info('收到%s发回的数据'%self._peer)
self.service.dealData(data)
def convert_rml_and_determine_accuracy(suffix="", override=False):
log.info(suffix)
convert_rml(config.language_understanding_result_dir(suffix='_RML' +
suffix),
suffix=suffix,
override=override)
# Run LUIS Accuracy Check
determine_accuracy(suffix="_RML" + suffix)
def command_close(self, bot, update, args):
if len(args) < 1:
return bot.sendMessage(update.message.chat_id, text=TEXT_NOT_INPUT)
elif args[0] in FIND_THINGS:
self.db.set_enable(args[0], False)
log.info('[/close] {0} {1} '.format(args[0],
TEXT_DONE.format(False)))
return bot.sendMessage(update.message.chat_id,
text=TEXT_DONE.format(False))
def checkConfUpdate():
global conf_last_change
cur = int(os.stat(DEFAULT['realtime_task_config']).st_mtime)
if cur != conf_last_change:
log.info('listener configuration has changed,reloading...')
parseConf()
conf_last_change = cur
reactor.callLater(DEFAULT['config_check_delay'],checkConfUpdate)
def printWERThreshold(accuracy_map):
h = sorted(accuracy_map.values())
log.info("\nWER (%)" + "\n0 %d" % (len([x for x in h if x == 0])) +
"\n>0-20 %d" % (len([x for x in h if x > 0 and x <= 20])) +
"\n>20-40 %d" % (len([x for x in h if x > 20 and x <= 40])) +
"\n>40-60 %d" % (len([x for x in h if x > 40 and x <= 60])) +
"\n>60-80 %d" % (len([x for x in h if x > 60 and x <= 80])) +
"\n>80-100 %d" % (len([x for x in h if x > 80 and x <= 100])) +
"\n>80 %d" % (len([x for x in h if x > 80])))
def run(self):
# get betas from 5 frames
log.info('Initial fit')
# num_init = 5
# indices_init = np.ceil(np.arange(num_init) * self.num_frames * 1. / num_init).astype(np.int)
#
# init_frames = [self.base_frame]
# for i in indices_init[1:]:
# init_frames.append(self.create_frame(i, self.base_smpl))
#
# init(init_frames, self.body_height, self.b2m, self.debug_rn)
# fp = h5py.File(self.out, 'w')
# self.poses_dset = fp.create_dataset("pose", (self.num_frames, 72), 'f', chunks=True, compression="lzf")
# self.trans_dset = fp.create_dataset("trans", (self.num_frames, 3), 'f', chunks=True, compression="lzf")
# self.betas_dset = fp.create_dataset("betas", (10,), 'f', chunks=True, compression="lzf")
# pool = multiprocessing.Pool(THREAD_NUM)
# pool.map(handle_thread, [(self, i) for i in range(THREAD_NUM)])
# pool.join()
ps = []
for i in range(THREAD_NUM):
p = Process(target=self.handle_thread_inner, args=(i, ))
p.start()
ps.append(p)
for p in ps:
p.join()
log.info('Done.')
log.info('Write hdf5 begin')
with h5py.File(self.out, 'w') as fp:
poses_dset = fp.create_dataset("pose", (self.num_frames, 72),
'f',
chunks=True,
compression="lzf")
trans_dset = fp.create_dataset("trans", (self.num_frames, 3),
'f',
chunks=True,
compression="lzf")
betas_dset = fp.create_dataset("betas", (10, ),
'f',
chunks=True,
compression="lzf")
for i in range(self.num_frames):
log.info('writing frame {}'.format(i))
# assert self.temp_poses_dset[i] is not None
# assert self.temp_trans_dset[i] is not None
# log.info('frame poses info: {}'.format(self.temp_poses_dset[i]))
# log.info('frame trans info: {}'.format(self.temp_trans_dset[i]))
poses_dset[i] = self.temp_poses_dset[i]
trans_dset[i] = self.temp_trans_dset[i]
betas_dset[:] = self.temp_betas_dset[:]
log.info('Finally done!!')
def _create_window(self):
if self.window:
del self.window
if self.is_login:
log.info('Token {} found, show main window'.format(
type(self.is_login)))
return MainWindow(self)
else:
log.info('No token found, show login')
return LoginWindow(self)
def test_insert(self):
mon = StockMonitor(stock_id="sz000068",
monitoring_type="top",
threshold_value="top",
status="0",
user_id="default")
db.session.add(mon)
db.session.commit()
mons = StockMonitor.query.all()
log.info(json.dumps(mons, cls=AutoJSONEncoder, indent=2))
def command_setting(self, bot, update, args):
if len(args) < 1:
return bot.sendMessage(update.message.chat_id, text=TEXT_NOT_INPUT)
elif args[0] in FIND_THINGS:
enable = not self.db.get_enable(args[0])
self.db.set_enable(args[0], enable)
log.info('[/setting] {0} {1} '.format(args[0],
TEXT_DONE.format(enable)))
return bot.sendMessage(update.message.chat_id,
text=TEXT_DONE.format(enable))
def command_set(self, bot, update, job_queue):
channel_id = CHANNEL_ID
when = FREQUENCY.split(' ')
self.set_alarms(channel_id, when, job_queue)
text = ''
for w in when:
text += '{0}시, '.format(w)
log.info('[/set] {0}'.format(SET_ALARM.format(text[:-2])))
return bot.sendMessage(update.message.chat_id,
text=SET_ALARM.format(text[:-2]))
def start_cluster_generation_thread(data, settings) -> Tuple[str, List, List]:
log.info('Starting request: {0}'.format(threading.active_count()))
cluster_generation_lock.acquire()
log.info('Enter cluster generation: ID {0}'.format(
threading.current_thread().ident))
try:
c = cached_cluster(data, settings)
finally:
cluster_generation_lock.release()
return c
def __init__(self, language):
log.info('Setup NLTK')
nltk.download('punkt', quiet=True)
nltk.download('averaged_perceptron_tagger', quiet=True)
nltk.download('maxent_ne_chunker', quiet=True)
nltk.download('words', quiet=True)
nltk.download('wordnet', quiet=True)
nltk.download('stopwords', quiet=True)
self.language = language
self.bigrams = False
def test_best_plan(self):
grid = Grid()
grid.init_generate_grid()
grid.show_detail()
result = grid.swap_by_strategy(StrategyType.HIGH_ORDER_FIRST)
if len(result) == 0:
log.error('no swap')
for i in result:
log.info('swap %s' % i)
def prepare_clustered_data_structure(data_handler, k_cluster) -> List[RestDisplayStructure]:
log.info(f'Generating Prediction')
result = [RestDisplayStructure(label, meta_info, term, cluster_id)
for label, meta_info, term, cluster_id in
zip(data_handler.display_labels(),
data_handler.meta_info(),
remove_rare_terms(k_cluster.get_terms()),
k_cluster.get_cluster_id())]
if data_handler.SHUFFLE_DATA:
random.shuffle(result)
return result
def stringReceived(self,data):
"""接受心跳包回复 """
if data.startswith('PINGBACK'):
suretime = data.split(":", 2)[1]
deltatime = time() - self._heartsendtime
if int(suretime) != self._htime or deltatime > self.factory._connectiontimeout:
return
log.info('收到心跳回复,时间差:%f' % deltatime)
if self.factory._paused:
self.factory._paused = False
self.sendQueue()
self._heartreceivetime = time()
def show_soldier(self):
result = []
for (ref_id, order, level) in self.soldiers:
result.append(self.soldiers[(ref_id, order, level)])
# log.info('%s' % self.soldiers[(ref_id, order, level)])
result = sorted(result, key=lambda soldiers: soldiers.order.value, reverse=True)
count = 0
for soldier in result:
log.info('%s' % soldier)
count += soldier.count
log.info('count=%s' % count)
def generate_cluster(selected_data, settings) -> Tuple[str, List, List]:
data_handler = initialize_data(selected_data, settings)
pre_load_uuid = data_handler.PRE_LOAD_UUID
if data_handler.HAS_MULTIPLE_DATA_SOURCES:
selected_data = data_handler.DATA_SOURCE
if pre_load_uuid and storage_io.is_model_present(pre_load_uuid, selected_data):
log.info(f'Preload model: {pre_load_uuid}')
return load_cluster(data_handler.PRE_LOAD_UUID, "", selected_data, settings, data_handler=data_handler)
return generate_k_means(data_handler, selected_data)
def __get_var(self, var):
if not self.__config_file:
try:
with open(self.__application_config_path, 'r') as stream:
self.__config_file = yaml.safe_load(stream)
log.info("Config Loaded")
except FileNotFoundError:
log.info("Config not found, using ENV Var")
return os.environ.get(var)
try:
return os.environ.get(var) or self.__config_file[var]
except KeyError:
log.error('Can not find ENV var: %s' % var)
def connectionMade(self):
"""当连接建立后启动心跳机制 """
self._peer = self.transport.getPeer()
log.info('connect to %s successfully' % self._peer)
self.factory._paused = False
self._connected = True
self._heartsendtime = 0
self._heartreceivetime = 0
self.factory.resetDelay()
self.startHeartbeat()
self.factory._connectionMade.callback(self)
self.factory._connectionMade = Deferred()
self.resumeQueue()
def _convert_audio_with_normalisation(audio_file_path, speed):
suffix = f'_speed-{speed}-normal'
file_name = _add_suffix(audio_file_path, suffix)
output_path = os.path.join(AUDIO_DIR(suffix), file_name)
if os.path.exists(output_path):
log.info(
f'already exists... skipping:\t {file_name}-{speed}-normalisation')
return
log.info(f'Convert {file_name} with 1.1 normalisation')
os.system(
f'ffmpeg -i {audio_file_path} -filter:a "atempo={speed}" {output_path}'
)
def __op_new(self, data_loaded, connection):
item_id = int(data_loaded['id'])
item = UrlItem.load_with_content(
id=item_id,file_path=config.path_extractor_inbox)
count, maps = db.get_url_with_same_layout_hash(item['layout_hash'])
log.info(str(maps))
log.info(count)
if len(maps) > 0:
import operator
tar_ext = max(maps.iteritems(), key=operator.itemgetter(1))
log.info(float(tar_ext[1]) / len(maps))
if tar_ext[1] > config.extractor_same_layout_number:
extractor = tool.str2extractor(tar_ext[0])
self.__extract(item,extractor)
return
extractor = config.const_RULE_UNKNOW
self.__ext_queue[item_id] = {
"title": item['title'],
"url": item['url'],
"filename": item.filename(),
"decision": item['is_target'],
"extractor": extractor
}
log.info("[%s]: # %s " % (item_id, extractor))
pass
def __op_preview(self, data_loaded, connection):
log.info(data_loaded['extractor'])
if data_loaded['extractor'] == config.const_RULE_UNKNOW:
result = {x: "" for x in xrange(1, self.__ie.num_attr() + 1)}
else:
item_id = int(data_loaded['id'])
item = UrlItem.load_with_content(
item_id, file_path=config.path_extractor_inbox)
extractor = data_loaded['extractor']
result = self.__ie.extract(item, extractor)
preview = list()
for att, str in result.iteritems():
preview.insert(att, dict(name=self.__ie.name(att), value=str))
log.info(preview)
tool.send_msg(connection, pickle.dumps(preview, -1))
pass
def stringReceived(self, data):
commands = data.split(None, 1)
command_name = commands[0]
args = ''
if len(commands) > 1:
args = commands[1]
command = task_list.get(command_name.strip())
do_cmd = '%s %s' % (command, args)
#实时任务未在任务白名单中
if command is None:
result = {'s': -1, 'o': 'invalid command'}
log.error('不合法的远程请求:%s'%data)
self.connectionEnd(result)
else:
log.info('远程请求:%s'%(do_cmd))
#异步执行实时任务
defer = self.factory.getStatusOutput(do_cmd)
defer.addCallback(self.action)
defer.addErrback(self.errorAction)
def __extract(self,item,extractor):
result = self.__ie.extract(item, extractor)
log.info(extractor)
log.info(result)
info = dict()
for att, str in result.iteritems():
info[self.__ie.db_col(att)] = str[0:self.__ie.max_len(att)]
log.info(info)
db.new_sem_with_map(item['id'], info)
item['extractor'] = extractor
item.save()
os.remove(config.path_extractor_inbox + "/%s" % item.filename())
def execCmd(self, cmd, ctid):
"""crontab 的实际执行过程"""
pscmd = "ps -A -opid,ppid,state,user,cmd|grep '\-\-ctid %s'|grep -v 'grep '" % ctid
rets, retv = getstatusoutput(pscmd)
retv = retv.strip()
log.info('ps 结果: \n%s' % retv)
if retv:
log.info('进程还在执行')
return {'seq': -1,
'exec_time': 0,
'stats': 0,
'output': '检测到相同任务正在执行,*未被调度*',
}
start = time.time()
status, output = getstatusoutput(cmd)
log.info('EXEC %s: %d' % (cmd, status))
if len(output) > 60000:
output = '===large output===\n...%s' % output.decode('utf-8')[-10240:].encode('utf-8')
end = time.time()
exec_time = end - start
return {'seq': 3, 'exec_time': '%.3f' % exec_time, 'stats': status, 'output': output}
def __op_refresh(self,data_loaded,connection):
delete_ids = []
for key, ent in self.__ext_queue.iteritems():
item_id = int(key)
if not os.path.isfile(
config.path_extractor_inbox+"/"+str(item_id)+".html"):
delete_ids.append(item_id)
continue
item = UrlItem.load_with_content(
id=item_id,file_path=config.path_extractor_inbox)
count, maps = db.get_url_with_same_layout_hash(item['layout_hash'])
log.info(str(maps))
log.info(count)
if len(maps) > 0:
import operator
tar_ext = max(maps.iteritems(), key=operator.itemgetter(1))
log.info(float(tar_ext[1]) / len(maps))
if tar_ext[1] > config.extractor_same_layout_number:
extractor = tool.str2extractor(tar_ext[0])
self.__extract(item,extractor)
delete_ids.append(item_id)
for ent_id in delete_ids:
del self.__ext_queue[ent_id]
parseConf()
conf_last_change = mtime
log.info('当前任务数:%d,[%s]'%(len(task_list), ','.join([task['task_name'] for task in task_list])))
next_conf_check += DEFAULT['config_check_delay']
for task in task_list:
try:
if now >= task['nextcheck']:
dt = threads.deferToThread(doTask, task)
#if 256 == int(task.get('return_type', 256)):
# #dt.addCallback((lambda x: pass))
# continue
#else:
dt.addCallback(sender_manager.sendData)
if task.has_key('task_interval'):
task['nextcheck'] += task['task_interval']
elif task.has_key('task_time_specify'):
task['nextcheck'] += 3600*24
except:
continue
reactor.callLater(1,checkActiveTask)
log.setOwn('apollocollecter')
log.info('apollocollecter程序启动,读取任务和主机配置文件')
parseConf()
conf_last_change = int(os.stat(DEFAULT['task_file_config']).st_mtime) + int(os.stat(DEFAULT['host_file_config']).st_mtime)
log.info('当前任务数:%d,[%s]'%(len(task_list), ','.join([task['task_name'] for task in task_list])))
reactor.callLater(1,checkActiveTask)
log.info('apollo agent starting...')
reactor.run()
__author__ = 'LeoDong'
import socket
import sys
from util import config
from judge.SAEJudge import SAEJudge
from util.logger import log
#TODO unique id in queue, store to file and reload.
# Create a TCP/IP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
log.info('start listening on %s' % str(config.socket_addr_judge))
sock.bind(config.socket_addr_judge)
# Listen for incoming connections
sock.listen(10)
judge = SAEJudge(config.path_judge_dtree,config.dtree_param)
try:
while True:
# Wait for a connection
connection, client_address = sock.accept()
judge.process(connection, client_address)
finally:
log.info("Saving")
judge.save()
log.error('配置文件格式错误[%s]'%str(e))
return
global task_list
task_list = {}
for sec in task_cfg.sections():
items = task_cfg.items(sec)
key = None
value = None
for k, v in items:
if 'task_name' == k and v.strip != '':
key = v
elif 'task_content' == k and v.strip != '':
value = v
if key and value:
task_list[key] = value
log.info('realtime task:[%s]' % ','.join(task_list.keys()))
def checkConfUpdate():
global conf_last_change
cur = int(os.stat(DEFAULT['realtime_task_config']).st_mtime)
if cur != conf_last_change:
log.info('listener configuration has changed,reloading...')
parseConf()
conf_last_change = cur
reactor.callLater(DEFAULT['config_check_delay'],checkConfUpdate)
class AgentListenerProtocol(NetstringReceiver):
def connectionMade(self):
self._peer = self.transport.getPeer()
log.info('收到来自%s的连接'%self._peer)
def connectionLost(self, reason):
log.info('%s断开连接' % self._peer)
def connectionMade(self):
self._peer = self.transport.getPeer()
log.info('收到来自%s的连接'%self._peer)
def dealJobEvent(self, event):
"""
处理调度过程中产生的事件
- event : 事件的相关信息对象
"""
def_ret = {
'jn' : self._ap_sched._jobs_num,
'host_name' : self.host,
}
if EVENT_JOB_STARTED == event.code or EVENT_JOB_MISSED == event.code:
log.info('[%s] %s, 任务数 %d' % (event.job.name, self.code_string[event.code], def_ret['jn']))
ret = {
'cid' : event.job.key,
'name' : event.job.name,
'seq' : 1 if EVENT_JOB_STARTED == event.code else -1,
'stats' : 0,
'exec_time' : 0,
'stime' : str(event.scheduled_run_time),
'output': self.code_string[event.code],
}
ret.update(def_ret)
info = {
'req_type': 'RESPONSE',
'res_type': 'CRON',
'retval' : json.dumps(ret),
}
self.sendMessage(json.dumps(info))
elif EVENT_JOB_EXECUTED == event.code:
retval = event.retval
if -1 == retval['seq']:
log.info('[%s]异常,未被调度' % event.job.name)
else:
log.info('[%s] 执行完成, 任务数 %d' % (event.job.name, def_ret['jn']))
retval.update(def_ret)
retval.update(name=event.job.name, cid = event.job.key, stime=str(event.scheduled_run_time))
info = {
'req_type': 'RESPONSE',
'res_type': 'CRON',
'retval': json.dumps(retval),
}
self.sendMessage(json.dumps(info))
elif EVENT_JOB_MAXINSTANCE == event.code:
log.info('job[%s] reach max instance, 不被调度' % event.job.name)
elif EVENT_JOBSTORE_JOB_ADDED == event.code:
log.info('job[%s] added, 任务数 %d' % (event.job.name, def_ret['jn']))
elif EVENT_JOBSTORE_JOB_REMOVED == event.code:
log.info('job[%s] removed, 任务数 %d' % (event.job.name, def_ret['jn']))
elif EVENT_JOB_ERROR == event.code:
log.info('job[%s] error:%s' % (event.job.name,event.exception))
def updateBeat(self, hostname):
self.factory.beats[hostname] = time()
class ApolloReceiverFactory(protocol.ServerFactory):
"""实现心跳检测"""
protocol = ApolloReceiverProtocol
def __init__(self, serve):
self._clientcount = 0
self.beats = {}
def checkBeats(self):
"""检测每个连到服务器端的client的心跳状态是否正常"""
now = time()
for host, last in self.beats:
time_diff = now - last
from twisted.internet import epollreactor
epollreactor.install()
from twisted.internet import reactor
log.setOwn('apolloreceiver')
service = ApolloService()
factory = ApolloReceiverFactory(service)
reactor.listenTCP(DEFAULT['receiver_port'], factory)
log.info('apolloserver starting...')
reactor.run()
def connectionMade(self):
self.factory._clientcount += 1
self._peer = self.transport.getPeer()
log.info('%s connected, 当前连接数: %d' % (self._peer, self.factory._clientcount))
def connectionLost(self, reason):
self.factory._clientcount -= 1
log.info('%s lost connection, 当前连接数: %d' % (self._peer, self.factory._clientcount))
