def run_job(argv_list=sys.argv):
# 取得SQL文件路径
job_conf_file = argv_list[1]
(file_path,conf_file_name) = os.path.split(job_conf_file)
(shot_name,extension) = os.path.splitext(conf_file_name)
job_log_file = os.path.split(os.path.abspath(argv_list[0]))[0]+'/../../Logs/'+shot_name
argv_dict = {}
job_sql_file = file_path + '/' + shot_name + '.HQL'
# 取得配置文件中的参数
if len(argv_list) >= 3:
global_parser = ConfigParser.SafeConfigParser()
global_parser.read('/data/etl_home/ETLScripts/SQLScripts/jobs_global.conf') # 针对现场进行修改
for item in global_parser.items('DEFAULT'):
argv_dict[item[0]]=item[1]
job_parser = ConfigParser.SafeConfigParser()
print(job_conf_file)
job_parser.read(job_conf_file)
for item in job_parser.items('CUSTOM'):
argv_dict[item[0]]=item[1]
for input_str in argv_list[2:]:
if not input_str.find('='):
show_usage_msg()
(item_name,item_val) = input_str.split('=')
argv_dict[item_name]=item_val
argv_dict['parallel_id'] = argv_dict['lan_id']
print(u'脚本参数列表如下：')
for each_argv in argv_dict:
print(u' ' + each_argv.decode('utf-8') + ' = ' + argv_dict[each_argv].decode('utf-8'))
else:
show_usage_msg()
logger = CustomLogger(job_log_file,argv_dict['cycle_id'],argv_dict['parallel_id'])
job = JobParser(job_sql_file,argv_dict['cycle_id'],argv_dict['parallel_id'],logger)
#job.check_job()
job.run(argv_dict)
job.del_tmp_tabs(argv_dict['del_tmp_flag'])
job.merge_tab()
job.write_log()
preprocess_time = time.time() - start_time # --------------------------------┘
start_time = time.time() # -------------------------------------------------┐
model = Sequential()
model.add(Flatten(input_shape=(28, 28)))
for hidden in options.hidden:
model.add(Dense(hidden, activation='relu'))
model.add(Dense(10))
model.compile(optimizer='adam',
loss=SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
startup_time = time.time() - start_time # -----------------------------------┘
model.summary()
custom_logger = CustomLogger(options.log_path)
start_time = time.time() # -------------------------------------------------┐
model.fit(train_images, train_labels,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(test_images, test_labels),
callbacks=[custom_logger])
train_time = time.time() - start_time # -------------------------------------┘
start_time = time.time() # -------------------------------------------------┐
pred = model.predict(test_images)
val_time = time.time() - start_time # ---------------------------------------┘
pred = np.argmax(pred, axis=1)
from rtmidi.midiutil import open_midioutput, open_midiinput
from MidiUtilities import get_velocity_symbol, calculate_base_note_octave, \
parse_note, is_valid_midi_message, is_valid_sysex, \
NOTE_MIDI_TO_SYMBOL, NOTE_SYMBOL_TO_MIDI, \
FIRST_OCTAVE, LAST_OCTAVE
from MidiInputHandler import MidiInputHandler
from ByteUtilities import convert_unicode_from_7_bit_bytes
import codecs
#By default, file logging is enabled
file_log_level = logging.DEBUG
#Disable file logging as follows:
#file_log_level = logging.NOTSET
CustomLogger.init_logging(file_log_level = file_log_level)
#Creates a logger for this module.
logger = logging.getLogger(CustomLogger.get_module_name())
#Setups the logger with the standard settings
logger.setup()
#Register the logger with this class
@logged(logger)
class MyMidiInputHandler(MidiInputHandler):
"""
Subclass from MidiInputHandler. Here you should only refefine the methods
you want to modify, ie: _on_note_on. Never overwrite __init__ or
__call__. If you want to overwrite __init__, then make sure that you
also call the superclass method
import traceback
import sys
import fnmatch
from rtmidi import MidiIn, MidiOut
from rtmidi.midiutil import open_midiport
from MidiProcessor import MidiProcessor
from CustomLogger import CustomLogger, PrettyFormat
import logging
from autologging import logged
import xmlschema
import platform
VIRTUAL_PREFFIX = "Virtual:"
#Creates a logger for this module.
logger = logging.getLogger(CustomLogger.get_module_name())
#Setups the logger with default settings
logger.setup()
#Register the logger with this class
@logged(logger)
class MidiConnector:
"""
Opens the MIDI ports and process the incomming connections
"""
def __init__(self, args, xsd_schema='conf/MidiBassPedalController.xsd'):
"""
Initializes the MidiConnector class
Parameters:
data_db_oper.close_connection() #关闭数据库
return (msg_code,msg_info)
sql_str = u"""\
DROP TABLE IF EXISTS tmp.{table_name}_rollback_{parallel_id}_tmp1
"""
sql_str = sql_str.format(table_name = self.job_name ,\
parallel_id = self.parallel_id)
(msg_code,msg_info) = data_db_oper.excute_sql(sql_str,log_flag='1')
if msg_code != 0 :
data_db_oper.close_connection() #关闭数据库
return (msg_code,msg_info)
data_db_oper.close_connection() #关闭数据库
return (msg_code,msg_info)
if __name__ == '__main__':
sql_file_path = '/var/lib/hive/liuym/SQLScripts/a.HQL'
logger = CustomLogger(sql_file_path,'2018-10-10','410105')
check_dict = {}
check_dict['job_name'] = sys.argv[1]
check_dict['cycle_id'] = sys.argv[2]
check_dict['parallel_id'] = '410105'
check_dict['script_path'] = '/var/lib/hive/liuym/A.HQL'
(msg_code,msg_info) = PreOperator(logger).run(check_dict)
print('msg_code=%s,msg_info=%s' %(msg_code,msg_info))
cycle_id = self.cycle_id,\
parallel_id = self.parallel_id ,\
job_name = self.job_name,\
run_status = self.run_status)
self.logger.debug(u'删除原有的失败信息，执行SQL语句：\n' + sql_str)
(msg_code, msg_info) = self.db_oper.excute_sql(sql_str,
log_flag=self.log_flag)
if msg_code != 0:
self.logger.error(u"脚本写日志出错，请核实写日志脚本是否正确。错误信息如下:\n" + msg_info)
return -1
return 0
if __name__ == '__main__':
sql_file_path = '/var/lib/hive/liuym/SQLScripts/TMP_TEST_LOG.HQL'
logger = CustomLogger(sql_file_path, '2018-10-10')
log_dict = {}
log_dict['job_name'] = 'test_job'
log_dict['cycle_id'] = '2018-10-10'
log_dict['parallel_id'] = '410105'
log_dict['run_status'] = '1' #运行状态：0-成功，-1-失败，-2-同一个任务多个脚本运行，1-依赖未完成
log_dict['log_msg'] = 'test message..'
log_dict['script_path'] = '/var/lib/hive/liuym/test_job'
log_dict['start_time'] = '20181016 19:00:00'
job = DBLogger(logger).run(log_dict)
create table tmp.lym_tmp (
str1 int
,str2 varchar(200)
)
"""
sql_str2 = """\
insert into tmp.lym_tmp values(333,'这是一条测试数据2222')
"""
sql_str3 = """\
select concat(str1,'串'),str2 from tmp.lym_tmp
"""
sql_file_path = '/var/lib/hive/liuym/SQLScripts/TMP_STD_BASIC_INFO_D.HQL'
logger = CustomLogger(sql_file_path, '2018-10-10')
pgsql_oper = PostgresqlOperator('PGSQL', logger)
#(msg_code,msg_info)=pgsql_oper.excute_sql(sql_str0)
#logger.info("==================================================")
#logger.info("msg_code:"+str(msg_code))
#logger.info("msg_info:"+msg_info)
#(msg_code,msg_info)=pgsql_oper.excute_sql(sql_str1)
#logger.info("==================================================")
#logger.info("msg_code:"+str(msg_code))
#logger.info("msg_info:"+msg_info)
#(msg_code,msg_info)=pgsql_oper.excute_sql(sql_str2)
#logger.info("==================================================")
#logger.info("msg_code:"+str(msg_code))
"Experience is a wonderful thing. It enables you to recognise a mistake when you make it again.",
"You can't have everything, where would you put it?",
"Don't you wish they made a clap on clap off device for some peoples mouths?",
"If your parents never had children, chances are you won't either.",
]
LOG_LEVELS = [
logging.DEBUG,
logging.INFO,
logging.WARNING,
logging.ERROR,
logging.CRITICAL,
]
SLEEP_TIME = int(os.environ.get("SLEEP_TIME", "60"))
# initialize logger
logger = CustomLogger()
async def main():
try:
if not sys.version >= "3.5.3":
raise Exception(
"The sample requires python 3.5.3+. Current version of Python: %s"
% sys.version)
print("IoT Hub Client for Python")
# The client object is used to interact with your Azure IoT hub.
module_client = IoTHubModuleClient.create_from_edge_environment()
# connect the client.
await module_client.connect()