def __init__(self, os_properties):
# Read OS properties
self.os_properties = os_properties
micropython.alloc_emergency_exception_buf(100)
# Logger
self.logger = Logger(self.os_properties['log_level'])
def execute(self):
text = self.form.txtEdit.toPlainText()
filewriter = FileWriter(self.form.userfile)
filewriter.open()
filewriter.writeLine(text)
filewriter.close()
logger = Logger()
logger.log("saveText()")
def execute(self):
reply = QMessageBox.question(self.form, 'Message',
"Are you sure to clear text?",
QMessageBox.Yes | QMessageBox.No,
QMessageBox.No)
if reply == QMessageBox.Yes:
self.form.txtEdit.setText('')
logger = Logger()
logger.log("clearText()")
def execute(self):
filereader = FileReader(self.form.userfile)
filereader.open()
text = ''
while True:
temp_str = filereader.readLine()
if not temp_str:
break
text += temp_str
self.form.txtEdit.setText(text)
logger = Logger()
logger.log("readTextFromFile()")
filereader.close()
def dqn_run_experiments():
for i in range(NUM_EXP):
# Make environment
env = rlcard.make('blackjack', config={'seed': i})
eval_env = rlcard.make('blackjack', config={'seed': i})
# Set the iterations numbers and how frequently we evaluate/save plot
# The initial memory size
memory_init_size = 100
# Train the agent every X steps
train_every = 1
# The paths for saving the logs and learning curves
log_dir = f"{DQN_RES_DIR}/{i}"
# Set up the agents
agent = DQNAgent('dqn',
action_num=env.action_num,
replay_memory_init_size=memory_init_size,
train_every=train_every,
state_shape=env.state_shape,
mlp_layers=[128, 256, 512],
debug=True)
env.set_agents([agent])
eval_env.set_agents([agent])
# Init a Logger to plot the learning curve
logger = Logger(log_dir, debug=True)
for episode in range(DQN_TRAINING_DURATION):
# Generate data from the environment
trajectories, _ = env.run(is_training=True)
# Feed transitions into agent memory, and train the agent
for ts in trajectories[0]:
agent.feed(ts)
# Evaluate the performance. Play with random agents.
if episode % EVALUATE_EVERY == 0:
logger.log_performance(env.timestep, tournament(eval_env, EVALUATE_NUM_OF_HANDS)[0])
# Close files in the logger
# logger.close_files()
# Plot the learning curve
logger.plot(f"DQN_{i}")
BaseAgent.plot_avg(DQN_RES_DIR, "DQN")
def __init__(self):
"""
Initializes and sets required default global variables by
collecting metadata from information.yaml
"""
self.ru_yaml = YAML()
self.ru_yaml.width = 4096
self.logger_object = Logger()
self.webex_wrapper = WebexNotifierWrapper()
self.logger = logging.getLogger('chatbot_logger')
self.logger.info("Starting default initialization...")
self.webex_auth_headers = {'content-type': 'application/json'}
try:
# Collecting metadata from information.yaml
with open('information.yaml', 'r') as ifh:
self.doc = yaml.safe_load(ifh)
# Getting Webex Teams Room details
self.webex_url = str(
self.doc["webex_teams_details"]["webex_url"])
self.webex_room_id = str(
self.doc["webex_teams_details"]["webex_room_id"])
# Getting Webex Teams Bot details
self.auth_token = str(
self.doc["webex_teams_details"]["auth_token"])
self.webex_bot_name = str(
self.doc["webex_teams_details"]["webex_bot_name"])
# Setting Webex connectivity parameters
self.webex_auth_headers[
'authorization'] = 'Bearer ' + self.auth_token
self.webex_auth_headers['Accept'] = 'application/json'
# Removing '+ "&max=1"' for now from webex_teams_get_url bcoz of
# the webex list messages API issue -- including max parameter
# impacts the sort order of the message results when bot calls
# list messages api
self.webex_teams_get_url = self.webex_url + "?mentionedPeople=me" \
+ "&roomId=" + self.webex_room_id
self.logger.info("Default initialization complete!!!")
except Exception as e:
self.logger.error("Error -- {}".format(e))
def __init__(self, name, ticker, period, live_mode, periods_needed=200):
"""
- name: string, the name of the bot
- ticker: string, the ticker formatted like that: ASSET1/ASSET2
- period: string, the period on which the loop will be set, and the resolution of the candles
- live_mode: bool, should we launch the live loop and start trading live
- periods_needed: int, the number of candles you will get every loop, optional
"""
self.live_mode = live_mode
self.name = name
self.ticker = ticker
self.period_text = period
self.periods_needed = periods_needed
self.offset_seconds = 10
if (not self.name in config.get_config()):
print("❌ Cannot instantiate bot: no config entry")
exit(1)
self.config = config.get_config()[self.name]
if (not "capitalAllowed" in self.config):
print("❌ Cannot instantiate bot: no 'capitalAllowed' property")
exit(1)
try:
self.logger = Logger(self.name, live_mode)
except:
print("❌ Cannot connect to the log DB, are you sure it's running?")
raise
if (self.live_mode):
self.data = Data(self.name)
else:
self.data = Data(self.name + "-test")
self.exchange = Exchange(self.logger, self.data, self.config['capitalAllowed'], live_mode, self.ticker, self.period_text)
try:
self.period = period_matching[period]
except:
print("Available periods: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 3h, 4h, 1d, 1w")
raise
self.logger.log("ℹ️", f"Bot {self.name} started with a period of {period}")
self.logger.log("ℹ️", f"Capital allowed: {self.config['capitalAllowed']}%")
self.setup()
if (self.live_mode):
self.preloop()
def __init__(self, env):
"""
Parameters
----------
env : str
the environment that the current instance is running
"""
print("[ENDPOINTS] Initializing...")
# initialize libraries
self._env = env
self._db = DB(self._env, self._workers)
self._logger = Logger(self._db, self._env)
self._crypto = Crypto()
# initialize Flask
self._app = Flask(__name__)
self._app.json_encoder = CustomJSONEncoder
self._api = Api(self._app)
self._app.before_request(self.detectAuthorization)
self._app.after_request(self.finishRequest)
for url in self._endpoints: self.addResource(self._endpoints[url], url)
print("[ENDPOINTS] Done.")
def __init__(self, env=gym.make('Blackjack-v0'), log_dir=None):
self._env = env
self.q = defaultdict(lambda: np.zeros(self._env.action_space.n))
self.policy = None
self.eval_policy = None
self.log_dir = log_dir
self.logger = Logger(self.log_dir, debug=False)
# Adding run function to Gym env
if isinstance(self._env, BlackjackEnv):
def run(is_training=False):
observation = self._env.reset()
while True:
if (self.eval_policy is None) or \
(observation not in self.eval_policy):
action = np.random.choice(
np.arange(env.action_space.n))
else:
action = np.argmax(self.eval_policy[observation])
observation, reward, done, _ = self._env.step(action)
if done:
return _, np.asarray([int(reward)])
self._env.run = run
self._env.player_num = 1
help='date to continue for',
default=MODEL_DATE)
parser.add_argument('--paper-note-path',
default='../paper-notes/data/words')
parser.add_argument('--model-epoch',
help='epoch to continue for',
default=MODEL_EPOCH,
type=int)
args = parser.parse_args()
# TRAINING
LOG_NAME = '{}-{}'.format("otf-iam-paper", args.model_date)
model_folder = os.path.join(Constants.MODELS_PATH, LOG_NAME)
models_path = os.path.join(model_folder,
'model-{}'.format(args.model_epoch))
logger = Logger()
config = Configuration.load(model_folder, "algorithm")
algorithm = HtrNet(config['algo_config'])
dataset = PreparedDataset.PreparedDataset(config['dataset'], False,
config['data_config'])
algorithm.configure(batch_size=config['batch'],
learning_rate=config['learning_rate'],
sequence_length=dataset.max_length,
image_height=dataset.meta["height"],
image_width=dataset.meta["width"],
vocab_length=dataset.vocab_length,
channels=dataset.channels,
class_learning_rate=config.default(
'class_learning_rate', config['learning_rate']))
executor = Executor(algorithm, True, config, logger=logger)
def main(n, config, seasonality, log_dir, log_prefix):
"""
Main loop that runs a simulation. This simulation can be configured by passing
a configuration dictionary, and specifying where all logs will be written to.
Parameters
----------
n: int
The Nth simulation.
config: dict
Configuration for the simulation. Should contain the following keys:
- servers: List of dictionaries, describing a server pool.
- process: Sequence of kinds of servers, describing how a process within
the simulation runs.
- runtime: Until when the simulation should run.
- max_volumne: Maximum number of events.
seasonality: Seasonality
Seasonality object to use for the simulation. This defines the intervals
between events.
log_dir: string
Path pointing to where all logs should be written.
log_prefix: string
Prefix of every log file.
Returns
-------
bool
"""
# we need a new environment which we can run.
environment = Environment()
# we need a server pool
servers = MultiServers()
# iterate over all of the servers that need to be configured that
# we received from the client
for server in config['servers']:
# append a new server pool to the multiserver system
servers.append(
Servers(environment,
size=server['size'],
capacity=server['capacity'],
kind=server['kind']))
# we need a logger that will log all events that happen in the simulation
name = "{0}_{1:04d}_{2}".format(log_prefix, n,
datetime.now().strftime("%Y-%m-%d_%H-%M"))
logger = Logger(name, directory=log_dir, show_stdout=False, usequeue=False)
# we also need a logger for all error events that happen in the simulation
error_logger = Logger(f"error-{name}",
directory=log_dir,
show_stdout=False)
# Start QueueListener
if hasattr(logger, "listener"):
logger.listener.start()
# Enter first line for correct .csv headers
logger.log(
'Time;Server;Message_type;CPU Usage;Memory Usage;Latency;Transaction_ID;To_Server;Message'
)
error_logger.log('Time;Server;Error type;Start-Stop')
# we can use the logger for the simulation, so we know where all logs will be written
environment.logger(logger)
environment.logger(error_logger, type="error")
# we need a new form of seasonality
seasonality = Seasonality(seasonality, max_volume=config["max_volume"])
# now, we can put the process in the simulation
Processor(environment,
servers,
seasonality=seasonality,
kinds=config['process'])
# run the simulation with a certain runtime (runtime). this runtime is not equivalent
# to the current time (measurements). this should be the seasonality of the system.
# for example, day or week.
environment.run(until=int(config['runtime']))
# Start QueueListener
if hasattr(logger, "listener"):
logger.listener.stop()
return name
# ----------------------------------------------------------------------------------------------------------------------
# Initialize OS
# ----------------------------------------------------------------------------------------------------------------------
from lib.Kernel import Kernel
from lib.toolkit import load_properties, determine_preferred_wifi
# Start-up Kernel
kernel = Kernel(load_properties("conf/os.properties"))
log = kernel.logger
# ----------------------------------------------------------------------------------------------------------------------
# Logger
# ----------------------------------------------------------------------------------------------------------------------
import gc
from lib.Logger import Logger
log = Logger("DEBUG")
log.info("Hello!")
log.error("Critical Issue!!")
log.debug("Free memory: " + str(gc.mem_free()))
# ----------------------------------------------------------------------------------------------------------------------
# Update DuckDNS
# ----------------------------------------------------------------------------------------------------------------------
from lib.toolkit import update_duck_dns
# Update DuckDNS service
update_duck_dns("mydomain", "mytoken", "myip")
# ----------------------------------------------------------------------------------------------------------------------
#
def main():
banner()
opts = CmdLineParser().cmd_parser()
python_version = platform.python_version()
if not python_version.startswith('3'):
ColorFormatter.fatal('此脚本只能运行在 Python3 之中')
sys.exit(1)
if opts.show_config:
space = "-"
line_len_list = []
line_1 = "| # ['database']"
line_2 = "| DATABASE_USER = '******'".format(DATABASE_USER)
line_3 = "| DATABASE_PASSWORD = '******'".format(DATABASE_PASSWORD)
line_4 = "| DATABASE = '{}'".format(DATABASE)
line_5 = "| TABLE_NAME = '{}'".format(TABLE_NAME)
line_len_list.append(len(line_2))
line_len_list.append(len(line_3))
line_len_list.append(len(line_4))
line_len_list.append(len(line_5))
line_len_list.sort(reverse=True)
longest_line = line_len_list[0]
print(colored('Show database config', 'white'))
print(colored(space * (longest_line + 2), 'white'))
chajia = longest_line - len(line_1)
print(colored("{}{} |".format(line_1, chajia * ' '), 'white'))
chajia = longest_line - len(line_2)
print(colored("{}{} |".format(line_2, chajia * ' '), 'white'))
chajia = longest_line - len(line_3)
print(colored("{}{} |".format(line_3, chajia * ' '), 'white'))
chajia = longest_line - len(line_4)
print(colored("{}{} |".format(line_4, chajia * ' '), 'white'))
chajia = longest_line - len(line_5)
print(colored("{}{} |".format(line_5, chajia * ' '), 'white'))
print(colored(space * (longest_line + 2), 'white'))
# 安全退出程序
sys.exit(0)
# control log level, default is 1
log_level = str(opts.log_level)
if log_level == '1':
log_level = 'WARNING'
elif log_level == '2':
log_level = 'INFO'
elif log_level == '3':
log_level = 'DEBUG'
else:
log_level = 'DEBUG'
# setting Logger object
logger = Logger(log_level=log_level)
# debug.log 应该包含所有的日志信息, 如果命令行没有指定 log_level 为3 的话, debug信息 不会出现在 debug.log 中
debug_logger = logger.get_logger(logger_name='debug_logger')
# error.log 只包含错误信息
error_logger = logger.get_logger(logger_name='error_logger')
# 初始化数据库
try:
database = Database(host='127.0.0.1',
database=opts.database_name,
logger=logger)
except pymysql.err.InternalError as e:
ColorFormatter.error('Init database InternalError: {}'.format(e))
debug_logger.error('Init database InternalError: {}'.format(e))
error_logger.error('Init database InternalError: {}'.format(e))
if 'Unknown database' in str(e):
ColorFormatter.fatal(
'Please open the lib/settings.py file and move to database section content, modify '
'DATABASE parameter and try again, for example see below: ')
error_string = """
----------------------------------------
| # ['database'] |
| DATABASE_USER = '******' |
| DATABASE_PASSWORD = '******' |
| DATABASE = 'testdb' |
| TABLE_NAME = 'test_table' |
----------------------------------------
"""
ColorFormatter.fatal(error_string)
sys.exit(1)
except Exception as e:
ColorFormatter.error('Init database Unknow Error: {}'.format(e))
debug_logger.error('Init database Unknow Error: {}'.format(e))
error_logger.error('Init database Unknow Error: {}'.format(e))
sys.exit(1)
# 运行程序
try:
if not len(sys.argv[1:]):
ColorFormatter.fatal(
"You failed to provide an option, redirecting to help menu")
debug_logger.debug(
'You failed to provide an option, redirecting to help menu')
# 停顿2秒之后再显示 help banner
time.sleep(2)
print()
CmdLineParser().cmd_parser(get_help=True)
else:
if opts.drop_table:
prompt_result = ColorFormatter.prompt(
'Are you sure to drop "{}" table?(y/N)'.format(
opts.drop_table),
opts='y/n')
debug_logger.debug('type --drop-table option')
if prompt_result == 'y':
# result = db.create_table('大爷', ['id', 'name', 'age'])
result = database.drop_table(opts.drop_table)
debug_logger.warning('input y')
if result:
ColorFormatter.info('Drop table {} successful'.format(
opts.drop_table))
debug_logger.warning('Drop table {} successful'.format(
opts.drop_table))
else:
ColorFormatter.error('Drop table {} failed'.format(
opts.drop_table))
error_logger.error('Drop table {} failed'.format(
opts.drop_table))
debug_logger.error('Drop table {} failed'.format(
opts.drop_table))
# 执行完 drop table 就正常退出程序
sys.exit(0)
if opts.csv_to_database:
csv_file_path = opts.csv_to_database
if not os.path.isfile(csv_file_path):
# 不存在 csv 文件
debug_logger.warning(
'File "{}" not exists'.format(csv_file_path))
ColorFormatter.fatal(
'File "{}" not exists'.format(csv_file_path))
time.sleep(1)
ColorFormatter.fatal('Program exit')
debug_logger.warning('Program exist')
sys.exit(1)
else:
if opts.fast:
start_time = time.time()
# 使用多核前, 将大文件进行分割, 分割成 4 份, 再加上不足的那一份
ColorFormatter.info('启用多进程导入数据库')
ColorFormatter.info('开始分割原数据文件')
# 这里需要再分割前 删除 split 目录下的所有文件
cmd = 'rm -rf split && mkdir split'
debug_logger.debug('开始运行 {} 命令'.format(cmd))
p = subprocess.Popen(cmd,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
shell=True)
(stdout, stderr) = p.communicate()
return_code = p.returncode
if return_code != 0 or stderr:
ColorFormatter.error(
'删除split文件时出错, 出错信息为: {}'.format(
stderr.strip()))
debug_logger.error(
'删除split文件时出错, 出错信息为: {}'.format(
stderr.strip()))
error_logger.error(
'删除split文件时出错, 出错信息为: {}'.format(
stderr.strip()))
# 异常退出
sys.exit(1)
else:
ColorFormatter.success('初始化split文件夹成功')
debug_logger.debug(
'初始化split文件夹成功, 返回码: {}, 输出为: {}'.format(
return_code, stdout))
debug_logger.info('初始化split文件夹成功')
# 开始分割
split_result = split_file(csv_file_path, log_level)
if split_result:
ColorFormatter.success('原数据文件分割完成')
debug_logger.info('原数据文件分割完成')
else:
ColorFormatter.error('原数据文件分割失败')
ColorFormatter.fatal('异常, 退出程序')
debug_logger.error('原数据文件分割失败')
error_logger.error('原数据文件分割失败')
sys.exit(1)
# 分割完成
# 多进程, 使用多核一起干
table_name = TABLE_NAME
process_list = []
for i in range(1, os.cpu_count() + 1):
process = Process(target=handle_data_to_database,
args=(
i,
database,
table_name,
debug_logger,
error_logger,
opts.skip_error,
))
process_list.append(process)
process.start()
# 计算 csv 文件的总行
cmd = "wc -l split/split_file_* | grep total | awk '{print $1}'"
debug_logger.debug('开始运行 {} 命令'.format(cmd))
p = subprocess.Popen(cmd,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
shell=True)
(stdout, stderr) = p.communicate()
try:
total_lines = int(stdout.strip())
except:
total_lines = 0
return_code = p.returncode
if return_code != 0 or stderr:
ColorFormatter.error(
'计算文件 "{}" 总行数 时出错, 出错信息为: {}'.format(
csv_file_path, stderr.strip()))
debug_logger.error(
'计算文件 "{}" 总行数 时出错, 出错信息为: {}'.format(
csv_file_path, stderr.strip()))
error_logger.error(
'计算文件 "{}" 总行数 时出错, 出错信息为: {}'.format(
csv_file_path, stderr.strip()))
ColorFormatter.fatal(
'Please contact author or look through logs/error.log file to examine error place'
)
# 异常退出
sys.exit(1)
else:
ColorFormatter.info(
'计算文件 "{}" 总行数结束, 一共 {} 行'.format(
csv_file_path, str(total_lines)))
debug_logger.debug(
'计算文件 "{}" 总行数命令结果, 返回码: {}, 输出为: {}'.format(
csv_file_path, return_code,
str(total_lines)))
debug_logger.info(
'计算需要分割文件 "{}" 行数结束, 一共 {} 行'.format(
csv_file_path, str(total_lines)))
# 进度条 对象
progress_bar = ProgressBar(total_line=total_lines,
description='正在插入数据: ')
while True:
# 每 2 秒读一次总数文件
time.sleep(2)
all_progress = 0
for suffix_name in range(1, os.cpu_count() + 1):
record_line_file = os.path.join(
SPLIT_PATH, 'record_line_{}.txt'.format(
str(suffix_name)))
try:
with open(record_line_file, 'r') as r:
per_line = r.readline()
all_progress += int(per_line)
except:
# 有可能开始还没有读到文件
pass
# 打印进度条
progress_bar.handle_multiprocessing_progress(
current_progress=all_progress)
if all_progress >= total_lines:
# 有可能这个判断不准确, 然后卡在这里
break
for _ in process_list:
_.join()
end_time = time.time()
print(
colored('总共用时: {}'.format(end_time - start_time),
'white'))
# 清除 split 文件夹
cmd = "rm -rf split"
debug_logger.debug('开始运行 {} 命令'.format(cmd))
p = subprocess.Popen(cmd,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
shell=True)
(stdout, stderr) = p.communicate()
return_code = p.returncode
if return_code != 0 or stderr:
ColorFormatter.error(
'清除 split 文件夹时出错, 出错信息为: {}'.format(
csv_file_path, stderr.strip()))
debug_logger.error(
'清除 split 文件夹时出错, 出错信息为: {}'.format(
csv_file_path, stderr.strip()))
error_logger.error(
'清除 split 文件夹时出错, 出错信息为: {}'.format(
csv_file_path, stderr.strip()))
ColorFormatter.fatal(
'Please contact author or look through logs/error.log file to examine error place'
)
# 异常退出
sys.exit(1)
else:
ColorFormatter.info('清除 split 文件夹成功')
debug_logger.debug('清除 split 文件夹成功')
ColorFormatter.success('插入数据完成')
debug_logger.info('insert data to database done')
# 显示汇总信息
# 多进程无法显示 失败 和 成功 的条数, 暂时丢着吧
else:
# 单进程, 速度很慢
start_time = time.time()
table_name = TABLE_NAME
with open(csv_file_path, 'r') as r:
# 这里是用, 分隔的 csv文件
line = csv.reader(r,
delimiter=',',
quoting=csv.QUOTE_NONE)
csv_file_path = '\ '.join(csv_file_path.split())
columns_str = ''
ColorFormatter.info(
'开始计算csv文件 {} 的行数'.format(csv_file_path))
debug_logger.info(
'开始计算csv文件 {} 的行数'.format(csv_file_path))
# 计算 csv 文件的行
cmd = "wc -l %s | awk '{print $1}'" % csv_file_path
debug_logger.debug('开始运行 {} 命令'.format(cmd))
p = subprocess.Popen(cmd,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
shell=True)
(stdout, stderr) = p.communicate()
return_code = p.returncode
if return_code != 0 or stderr:
ColorFormatter.error(
'计算csv文件 "{}" 行数时出错, 出错信息为: {}'.format(
csv_file_path, stderr.strip()))
debug_logger.error(
'计算csv文件 "{}" 行数时出错, 出错信息为: {}'.format(
csv_file_path, stderr.strip()))
error_logger.error(
'计算csv文件 "{}" 行数时出错, 出错信息为: {}'.format(
csv_file_path, stderr.strip()))
# 异常退出
sys.exit(1)
else:
ColorFormatter.info(
'计算 "{}" 文件行数结束, 一共 "{}" 行'.format(
csv_file_path, stdout.strip()))
debug_logger.debug(
'计算 "{}" 文件行数命令结果, 返回码: {}, 输出为: {}'.
format(csv_file_path, return_code,
stdout.strip()))
csv_file_line = int(stdout.strip())
debug_logger.info(
'计算 "{}" 文件行数结束, 一共 "{}" 行'.format(
csv_file_path, csv_file_line))
# 进度条 对象
progress_bar = ProgressBar(
total_line=csv_file_line,
description='正在插入数据: ')
count = 1
for all_cols in line:
# 跳过空行
if all_cols:
if count == 1:
# 第一行，获取所有数据的列名, 然后创建数据库表
columns_str = ", ".join(all_cols)
debug_logger.debug(
'create table {}'.format(
table_name))
create_result = database.create_table(
table_name=table_name,
table_column_list=all_cols)
if create_result == 'True':
debug_logger.info(
'create table {} successful'.
format(table_name))
ColorFormatter.success(
'创建表 "{}" 成功'.format(
table_name))
elif 'already exists' or 'Duplicate column name' in create_result:
debug_logger.warning(
'table "{}" has been existed'.
format(table_name))
ColorFormatter.info(
'表 "{}" 已经存在'.format(
table_name))
else:
debug_logger.error(
'create table "{}" failed'.
format(table_name))
error_logger.error(
'create table "{}" failed'.
format(table_name))
ColorFormatter.error(
'创建表 "{}" 失败'.format(
table_name))
else:
# 不为第一行
# 剔除 \r\n 的列
all_cols = escape_file_string(all_cols)
# 将每一列的数据拼成一行
data_list = ', '.join([
'"{}"'.format(_) for _ in all_cols
])
database.insert_data(
table_name=table_name,
column_list=columns_str,
data_list=data_list,
skip_error=opts.skip_error)
progress_bar.handle_progress()
count += +1
# 事务提交, 插入数据
debug_logger.debug('execute insert sql commit')
database.execute_commit()
debug_logger.info(
'execute insert sql commit successful')
# 输出插入数据总共用时
end_time = time.time()
print(
colored(
'总共用时: {:.2f}秒'.format(end_time -
start_time),
'white'))
r.close()
ColorFormatter.success('插入数据完成')
debug_logger.info('insert data to database done')
# 显示汇总信息
ColorFormatter.info(
'总共插入 "{}" 条数据, 成功 "{}" 条, 失败 "{}" 条'.format(
database.insert_total_count,
database.insert_success_count,
database.insert_failed_count))
debug_logger.info(
'总共插入 "{}" 条数据, 成功 "{}" 条, 失败 "{}" 条'.format(
database.insert_total_count,
database.insert_success_count,
database.insert_failed_count))
# 安全退出
sys.exit(0)
if opts.txt_to_database:
txt_files = opts.txt_to_database
table_name = opts.table_name
skip_error = opts.skip_error
# 用来分隔 txt 文件的列的
separator = opts.separator
column_list = opts.column_list
if not column_list:
ColorFormatter.error(
'Must with -c option if you want to use txt-to-database option'
)
ColorFormatter.fatal('sleep 2s redirect help page')
time.sleep(2)
print()
CmdLineParser().cmd_parser(get_help=True)
if not table_name:
table_name = TABLE_NAME
if separator == '\\t':
separator = '\t'
insert_txt_to_database(txt_files=txt_files,
separator=separator,
database_obj=database,
column_list=column_list,
debug_logger=debug_logger,
error_logger=error_logger,
table_name=table_name,
skip_error=skip_error)
# 正常退出程序
sys.exit(0)
if opts.mysql_command:
cmd = opts.mysql_command
dict_result = database.execute_command(cmd)
for i in dict_result:
print(i)
if opts.clean_cache:
ColorFormatter.warning('Clean log cache')
debug_logger.warning('Clean log cache')
try:
os.remove(os.path.join(LOG_PATH, 'error.log'))
os.remove(os.path.join(LOG_PATH, 'debug.log'))
except:
pass
ColorFormatter.success('Clean log cache successful')
debug_logger.info('Clean log cache success')
# 执行完 clean log cache 就正常的退出程序
sys.exit(0)
except KeyboardInterrupt as e:
ColorFormatter.fatal('user abort')
debug_logger.error('user abort: {}'.format(e))
error_logger.error('user abort: {}'.format(e))
except Exception as e:
ColorFormatter.error('Total program Unknow error: {}, type: {}'.format(
e, type(e)))
ColorFormatter.fatal('Please contact author')
debug_logger.error('Total program Unknow error: {}, type: {}'.format(
e, type(e)))
error_logger.error('Total program Unknow error: {}, type: {}'.format(
e, type(e)))
from lib.Exchange import Exchange
from lib.Logger import Logger
import time
log = Logger("MACD")
log.balance(100, "USDT")
# exchange = Exchange(log, 10, "BTC/USDT", "1m")
# exchange.buy()
# exchange.sell()
# log.order('buy', 60123, 31)
# time.sleep(10)
# log.order('sell', 61003, 36)
# time.sleep(20)
# log.order('buy', 61129, 33)
# time.sleep(13)
# log.order('sell', 60890, 29)
# log.pnl(0.1, 20)
# time.sleep(10)
# log.pnl(-0.4, -6)
# time.sleep(20)
# log.pnl(1.4, 130)
#!/usr/bin/env python
# author: samren
import logging
import traceback
import unittest
import HTMLTestRunner
import time
from lib.Logger import Logger
from testcases.cases_login_logout.admin_login_logout import Bugfree管理员登录退出
if __name__ == '__main__':
logger = Logger('./log/logger.log', logging.INFO)
logging.info("本次测试开始执行，以下是详细日志")
try:
suite = unittest.TestSuite() # 新建一个suite，测试套件
loader = unittest.TestLoader() # 新建一个加载器，自定义的方式把测试用例加载到suite里
suite.addTests(loader.loadTestsFromTestCase(
Bugfree管理员登录退出)) # 把测试类所有的方法都加载到suite里
# unittest.TextTestRunner(verbosity=2).run(suite) # unittest运行suite
fp = open(
'reports/report_bugfree_{0}.html'.format(
time.strftime("%Y-%m-%d %H-%M-%S")), 'wb')
runner = HTMLTestRunner.HTMLTestRunner(
stream=fp, title='Bugfree的测试报告', description='Bugfree的所有测试用例执行细节')
runner.run(suite)
logging.info("测试顺利结束^_^ ")
except Exception:
"""print_exc() 把异常输出到屏幕上，而format_exc() 把异常返回成字符串"""
traceback.print_exc()
logging.error(traceback.format_exc())
logging.error("测试异常终止")
def __init__(self, config={}, globalConfig={}):
self.config = Configuration(config)
self.globalConfig = Configuration(globalConfig)
self._parse_config()
self.logger = Logger()
self.config()
def simulation():
"""
Function to install handlers on the /simulation path. This allows for
requesting simulation data or starting a new simulation.
Parameters
----------
POST:
servers: list
List containing configurations for a server pool as dicts.
{ capacity: int, size: int, kind: string }
For example, { size: 10, capacity: 10, kind: 'regular' }.
process: list
List specifying how a process should go (from server to server).
This should contain a sequence of server kinds.
For example, ["regular", "balance", "pay"].
runtime: int
Runtime of the simulation (defined by simpy package).
Returns
-------
GET: dict
POST: int
"""
if request.method == "POST":
# nonlocal use of the simulation count
nonlocal simc
# increment the simulation count
simc += 1
# we need a new environment which we can run.
environment = Environment()
# we need a server pool
servers = MultiServers()
# iterate over all of the servers that need to be configured that
# we received from the client
for kind in request.form['kinds'].split(','):
# append a new server pool to the multiserver system
servers.append(
Servers(environment,
size=int(request.form['size']),
capacity=int(request.form['capacity']),
kind=kind.strip()))
# Get the current date and time to append to the logger file name
log_timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M")
# now that we have an output dir, we can construct our logger which
# we can use for the simulation
name = "{0}_{1:04d}_{2}".format(file_prefix, simc, log_timestamp)
logger = Logger(name, directory=LOG_PATH)
# we also need a logger for all error events that happen in the simulation
error_logger = Logger(f"error-{name}", directory=LOG_PATH)
# Enter first line for correct .csv headers
logger.info(
'Time;Server;Message_type;CPU Usage;Memory Usage;Latency;Transaction_ID;To_Server;Message'
)
error_logger.info('Time;Server;Error type;Start-Stop')
# we can use the logger for the simulation, so we know where all logs will be written
environment.logger(logger)
environment.logger(error_logger, type="error")
# we need a new form of seasonality
seasonality = Seasonality(
join(Seasonality_folder, Seasonality_file),
max_volume=int(request.form['max_volume']))
# now, we can put the process in the simulation
Processor(environment,
servers,
seasonality=seasonality,
kinds=[
kind.strip()
for kind in request.form['process'].split(',')
])
# run the simulation with a certain runtime (runtime). this runtime is not equivalent
# to the current time (measurements). this should be the seasonality of the system.
# for example, day or week.
environment.run(until=int(request.form['runtime']))
# expose the id of the simulation
return jsonify(simc)
if request.method == "GET":
if 'id' in request.args:
logfile_id = "{:04d}".format(int(request.args.get('id')))
# Scan the logfile directory
list_of_files = glob.glob(os.path.join(LOG_PATH, 'log_*.csv'))
# Return only the filename to get no errors with old functions
log_filenames = [
os.path.basename(filename) for filename in list_of_files
]
if log_filenames:
logfile_ids = [f.split('_')[1] for f in log_filenames]
name_id_dict = dict(zip(logfile_ids, log_filenames))
if logfile_id in logfile_ids:
# Logfile associated to given ID was successfully found
return jsonify({
"data": name_id_dict[logfile_id],
"message": "success"
})
else:
# No logfile associated to given ID was found
return jsonify(
{"message": "No logfile (.csv) with given ID exists."})
else:
# No logfiles found (/logs is empty)
return jsonify({"message": "No logfiles were found in /logs."})
# ----------------------------------------------------------------------------------------------------------------------
# Initialize OS
# ----------------------------------------------------------------------------------------------------------------------
from lib.Kernel import Kernel
from lib.toolkit import load_properties
# Start-up Kernel
kernel = Kernel(load_properties("conf/os.properties"))
log = kernel.logger
# ----------------------------------------------------------------------------------------------------------------------
# Logger
# ----------------------------------------------------------------------------------------------------------------------
import gc
from lib.Logger import Logger
log = Logger()
log.info("Hello!")
log.error("Critical Issue!!")
log.debug("Free memory: " + str(gc.free_mem()))
# ----------------------------------------------------------------------------------------------------------------------
# Update DuckDNS
# ----------------------------------------------------------------------------------------------------------------------
from lib.toolkit import update_duck_dns
# Update DuckDNS service
update_duck_dns("mydomain", "mytoken", "192.168.0.10")
# ----------------------------------------------------------------------------------------------------------------------
#
from lib.EncryptUtil import EncryptUtil
pub_key_path = dir_sign + '/lib/pub.key'
pri_key_path = dir_sign + '/lib/pri.key'
enc = EncryptUtil(pub_key_path, pri_key_path)
from django.shortcuts import render
import json
from django.http import HttpResponse
from lib.Logger import Logger
import time
######1、接收到post请求，解密
########2、提取custId、custNm、recevieUrl，新生成traceNo并组装返回给pjs的post报文
########3、根据recevieUrl，post相应的 pjs环境
########4、组装xml报文，发送给pjs前置机 这个暂时不开发
log_debug = Logger('all.log', level='debug')
log_error = Logger('error.log', level='error')
dict_t036 = {
"orgCode": "105584099990002",
"custId": "",
"traceNo": "",
"reqSn": '',
"retcode": "00000",
"errCode": "0000",
"errMsg": "处理成功",
"settleDay": "",
"retMsg": "充值确认交易成功."
}
def job51_shoot(username, password):
"""智联招聘简历投递"""
print username + " " + password
#智能等待页面完成加载
zl = Job51(username, password)
#zl.login()
zl.search_job()
zl.custom_select_job()
if __name__ == "__main__":
d1 = datetime.datetime.now()
#生成日志文件
ZHAOPIN_SITE = {'zhilian': zhilian_shoot}
now = time.strftime('%Y-%m-%d %H-%M-%S', time.localtime(time.time()))
logger = Logger("./log/test_shoot_%s.log" % now,
loglevel=logging.INFO).getlog()
count = len(open('stu_data.txt', 'rU').readlines())
user_dict = get_user_dict_from_file('stu_data.txt')
for k, v in user_dict.items():
print k, v
try:
#cjol_shoot(k, v)
#zhilian_shoot(k, v)
job51_shoot(k, v)
logger.info(k + ' is success!!!')
except Exception as e:
traceback.print_exc()
logger.error(k + ' is failed!!!')
exit(0)
from lib.HTMLTestRunner import HTMLTestRunner
from testcase.admin_login_logout.admin_login_correction import AdminLoginCorrection
from lib.SendEmail import send_email
FROM_ADDR = u"*****@*****.**"
FROM_PSWD = u"qwer1234" # 163设置的第三方授权码
TO_ADDR = u"*****@*****.**"
def suites():
suite = unittest.TestSuite()
loader = unittest.TestLoader()
suite.addTests(loader.loadTestsFromTestCase(AdminLoginCorrection))
return suite
if __name__ == "__main__":
logger = Logger().getlog()
logger.info('start testcase...')
report_path = 'result/test_result_%s.html' % time.strftime(
"%Y-%m-%d %H-%M-%S")
fp = open(report_path, 'wb')
runner = HTMLTestRunner(stream=fp,
title=u"测试报告",
description=u"测试用例执行情况: ")
runner.run(suites())
fp.close()
send_email(FROM_ADDR, FROM_PSWD, TO_ADDR, u"测试报告", report_path)
logger.info('stop testcase...')
from lib.Logger import Logger
import logging
def suites():
suite = unittest.TestSuite()
loader = unittest.TestLoader()
suite.addTests(loader.loadTestsFromTestCase(WeatherTest))
#suite.addTests(loader.loadTestsFromTestCase(BugfreeImportFile))
# suite.addTests(loader.loadTestsFromTestCase(ProductAdd))
#suite.addTests(loader.loadTestsFromTestCase(LoginLogoutTest))
return suite
if __name__ == "__main__":
logger = Logger(loglevel=logging.ERROR).getlog()
logger.info('日志开始')
try:
suite = suites()
fp = open(
'./reports/results_%s.html' % time.strftime("%Y-%m-%d %H-%M-%S"),
'wb')
runner = HTMLTestRunner(stream=fp,
title=u'接口测试报告',
description=u"测试用例执行情况：")
runner.run(suite)
except Exception, e:
raise e
finally:
fp.close()
from testcases.weather import WeatherTest
from lib.Logger import Logger
import logging
def suites():
suite=unittest.TestSuite()
loader=unittest.TestLoader()
suite.addTests(loader.loadTestsFromTestCase(WeatherTest))
#suite.addTests(loader.loadTestsFromTestCase(BugfreeImportFile))
# suite.addTests(loader.loadTestsFromTestCase(ProductAdd))
#suite.addTests(loader.loadTestsFromTestCase(LoginLogoutTest))
return suite
if __name__ == "__main__":
logger = Logger(loglevel=logging.INFO).getlog()
logger.info(u'日志开始')
try:
suite = suites()
fp = open('./reports/results_%s.html' % time.strftime("%Y-%m-%d %H-%M-%S"), 'wb')
runner = HTMLTestRunner(
stream=fp,
title=u'接口测试报告',
description=u"测试用例执行情况：")
runner.run(suite)
except Exception, e:
raise e
finally:
fp.close()
logging.info(u'日志结束')
def split_file(file_path, log_level, split_count=os.cpu_count()):
"""
有可能每个系统的换行符不一样, 可能会造成数据分割之后的数据"变多"
例如:
如果有一行的数据为: CHEN^M GUIDE,,,VSA,USER,NAME, Python 会把 ^M 当成换行符, 从而数据变成两行:
1. CHEN
2. GUIDE,,,VSA,USER,NAME,
这个只能当做坏数据处理, 使用 error.log 手动导入即可, 不过这种情况应该不会很多, so do not worry about it.
:param file_path: 需要分割文件的文件路径
:param log_level: log level
:param split_count: 需要分成多少份
:return: Boolean, 成功或者失败
"""
try:
# setting Logger object
logger = Logger(log_level=log_level)
# debug.log 应该包含所有的日志信息, 如果命令行没有指定 log_level 为3 的话, debug信息 不会出现在 debug.log 中
debug_logger = logger.get_logger(logger_name='debug_logger')
# error.log 只包含错误信息
error_logger = logger.get_logger(logger_name='error_logger')
# normalization file path
file_path = '\ '.join(file_path.split())
ColorFormatter.info('开始计算需要分割的文件 "{}" 的行数'.format(file_path))
debug_logger.info('开始计算需要分割的文件 "{}" 的行数'.format(file_path))
# 计算 csv 文件的行
cmd = "wc -l %s | awk '{print $1}'" % file_path
debug_logger.debug('开始运行 {} 命令'.format(cmd))
p = subprocess.Popen(cmd,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
shell=True)
(stdout, stderr) = p.communicate()
return_code = p.returncode
if return_code != 0:
ColorFormatter.error('计算需要分割文件 "{}" 行数时出错, 出错信息为: {}'.format(
file_path, stderr.strip()))
debug_logger.error('计算需要分割文件 "{}" 行数时出错, 出错信息为: {}'.format(
file_path, stderr.strip()))
error_logger.error('计算需要分割文件 "{}" 行数时出错, 出错信息为: {}'.format(
file_path, stderr.strip()))
ColorFormatter.fatal(
'Please contact author or look through logs/error.log file to examine error place'
)
# 异常退出
sys.exit(1)
else:
ColorFormatter.info('计算需要分割文件 "{}" 行数结束, 一共 {} 行'.format(
file_path, str(stdout.strip())))
debug_logger.debug('计算需要分割文件 "{}" 行数命令结果, 返回码: {}, 输出为: {}'.format(
file_path, return_code, stdout.strip()))
file_line = int(stdout.strip())
debug_logger.info('计算需要分割文件 "{}" 行数结束, 一共 {} 行'.format(
file_path, str(file_line)))
if file_line > 0:
per_file_line = int(file_line / split_count)
else:
per_file_line = 1
# 计数器
flag = 0
# 文件名
name = 1
# 存放数据
data_list = []
ColorFormatter.info('开始分割文件')
debug_logger.info('开始分割文件')
file_path = ''.join(file_path.split('\\'))
try:
with open(file_path, 'r') as r:
for line in r:
flag += 1
data_list.append(line)
# 如果等于设置的每个文件的行数, 就分一个文件, 直到全部分完
if flag == per_file_line:
split_file = os.path.join(
SPLIT_PATH, 'split_file_{}.txt'.format(str(name)))
with open(split_file, 'w+') as f:
for data in data_list:
f.write(data)
ColorFormatter.info("文件{}: {} 分割完成".format(
name, split_file))
debug_logger.info("文件{}: {} 分割完成".format(
name, split_file))
# 分割一次 重新初始化一次
name += 1
flag = 0
data_list = []
except Exception as e:
ColorFormatter.error('分割文件 {} 时候出错, 出错信息为: {}'.format(name, e))
ColorFormatter.fatal('Please contact author')
debug_logger.error('分割文件 {} 时候出错, 出错信息为: {}'.format(name, e))
error_logger.error('分割文件 {} 时候出错, 出错信息为: {}'.format(name, e))
try:
if name != split_count:
# 如果不相等, 就是还有多出来的数据
last_split_file = os.path.join(
SPLIT_PATH, 'split_file_{}.txt'.format(str(name - 1)))
# 处理最后一批
with open(last_split_file, 'a+') as f_target:
for data in data_list:
f_target.write(data)
except Exception as e:
ColorFormatter.error('分割文件 合并最后一个文件时出错, 出错信息为: {}'.format(e))
ColorFormatter.fatal('Please contact author')
debug_logger.error('分割文件 合并最后一个文件时出错, 出错信息为: {}'.format(e))
error_logger.error('分割文件 合并最后一个文件时出错, 出错信息为: {}'.format(e))
ColorFormatter.success("文件 {} 分割完成".format(file_path))
debug_logger.info("文件 {} 分割完成".format(file_path))
return True
except:
return False
