def pool_parallel(destination, p_num=4):
# generate child process
with MyPool(p_num) as pool:
pool.map(
Worker(load=load_dataset(batch_size=64), destination=destination),
range(p_num))
# wait all child process to work done
pool.close()
pool.join()
# collect data
collect_df(destination=destination, num=p_num)
init_logger().info(
"success to collect data into '{dest}'".format(dest=destination))
def main():
# Parse the arguments for each run
args = parse_args()
# Init the logger
logger.init_logger(args['env'], args['solver'])
# Init the environment
env = getattr(env_module, args['env'])()
# Init the solver
solver = getattr(solver_module, args['solver'])(env, args)
# Run the experiment
run(solver, args)
def __init__(self,
x_train,
y_train,
bucket_name,
grid_params=None,
score=mean_squared_error,
estimator=ElasticNet):
if grid_params is None:
raise ValueError("grid params are needed")
self.x_train = x_train
self.y_train = y_train
self.scorer = score
self.error = None # pd.Series
self.metric = None
# s3
self.s3_manager = None
# logger
self.logger = init_logger()
super().__init__(estimator=estimator(),
param_grid=grid_params,
scoring=make_scorer(self.scorer,
greater_is_better=False))
def __init__(self, dataset_name):
self.logger = init_logger()
self.date = datetime.datetime.now().strftime("%m%Y")
self.PreProcessor = PreProcessor(filename=dataset_name)
self.dataset = self.PreProcessor.process(
) # self.build_dataset(filename=dataset_name)
def __init__(self,
x_train,
y_train,
bucket_name,
grid_params=None,
score=mean_squared_error):
if grid_params is None:
grid_params = {
"max_iter": [1, 5, 10],
"alpha": [0, 0.0001, 0.001, 0.01, 0.1, 1, 10, 100],
"l1_ratio": np.arange(0.0, 1.0, 0.1)
}
self.x_train = x_train
self.y_train = y_train
self.scorer = score
self.error = None # pd.Series
self.metric = None
# s3
self.s3_manager = None
# logger
self.logger = init_logger()
super().__init__(x_train, y_train, bucket_name, grid_params, score,
ElasticNet)
def __init__(self,
bucket_name,
x_train,
y_train,
params=None,
estimator=ElasticNet):
# logger
self.logger = init_logger()
# s3
self.s3_manager = None
if params is None:
self.model = estimator()
else:
self.model = estimator(**params)
self.x_train, self.y_train = x_train, y_train
# result
self.x_true = None
self.y_true = None
self.pred = None
self.error = None
self.err_ratio = None
self.metric = None
def __init__(self,
block_type,
input_channel,
num_layers,
dataset,
gpu_id=0,
parallel=False):
self.logger = init_logger()
# dataset
self.test_loader = dataset
self.model = LennaNet(num_blocks=[1],
num_layers=num_layers,
normal_ops=normal_ops,
reduction_ops=reduction_ops,
block_type=block_type,
input_channel=input_channel,
n_classes=10) # for cifar10
# allocate 4 processes to 4 gpu respectively
device = 'cuda:{}'.format(
gpu_id) if torch.cuda.is_available() else 'cuda'
self.logger.info("assign to {}".format(device))
self.device = torch.device(device)
self.model.to(self.device)
if device == 'cuda:{}'.format(gpu_id):
# self.p_model = torch.nn.DataParallel(module=self.model)
cudnn.benchmark = True
self.model.eval()
def __init__(self,
x_train,
y_train,
bucket_name,
grid_params=None,
score=mean_squared_error):
if grid_params is None:
raise ValueError("grid params are needed")
self.x_train = x_train
self.y_train = y_train
self.scorer = score
self.error = None # pd.Series
self.metric = None
# s3
self.s3_manager = None
# logger
self.logger = init_logger()
super().__init__(
estimator=MLPRegressor(learning_rate='adaptive'),
param_grid=grid_params,
scoring='neg_mean_absolute_error',
)
def __init__(self, counts=1000, size=(256, 16, 224, 224)):
self.logger = init_logger()
self.counts = counts
# batch_size(32 or 64) X depth X width X height
self.X = (torch.rand(size=size).uniform_() > 0.8).float()
self.models = [
MyModel1(), MyModel3()
] # [MyModel1(), MyModel2(), MyModel3()] # [MyModel4(), Parallel1(), Parallel2()]
def main(arg="parallel"):
destination = "training_data/data"
if arg is "parallel":
logger = init_logger()
logger.info("director id: %s" % (os.getpid()))
parallel(destination=destination, outer_loop=2, inner_loop=1, p_num=4)
else:
single(destination=destination)
def __init__(self, sub_pid=0, g_type="random", b_type=None, in_ch=None):
"""
block type: 0 -> reduction , 1-> normal
input_channel: 1~1000
num_layers: fix 5 or 6
"""
self.logger = init_logger()
self.sub_pid = sub_pid
def __init__(self, config, maze):
self._logger = init_logger(self)
self._config = config
self._maze = maze
self._training_mode = None
self.id = 1
self.location = None
self.before_move = Event()
self.after_move = Event()
def __init__(self, config, maze):
self._logger = init_logger(self)
self._config = config
self._maze = maze
self._training_mode = None
self.id = 1
self.location = None
self.before_move = Event()
self.after_move = Event()
def collect_df(destination, num):
# collect df from csv files
combined_df = pd.concat(
[pd.read_csv(destination + str(i)) for i in range(num)], axis=0)
# save
if os.path.isfile(destination) is True:
combined_df.to_csv(destination, mode='w', index=False, header=True)
else:
combined_df.to_csv(destination, mode='a', index=False, header=False)
# check
init_logger().info("final saved df's tail 5: \n{df}".format(
df=pd.read_csv(destination).tail(5)))
# delete
for i in range(num):
if os.path.exists(destination + str(i)):
os.remove(destination + str(i))
return 0
def global_wrapper(labor):
"""
wrapper: logger, database, json
"""
web.ctx.log = logger.init_logger()
web.ctx.db = engine.Session()
try:
ret = labor()
finally:
web.ctx.db.close()
web.header('Content-Type', 'application/json')
web.header('Access-Control-Allow-Origin', '*')
return json.dumps(ret, cls=engine.AlchemyEncoder)
def draw_hist(s, h_type: str = "dist", name: str = None):
plt.figure()
h_method = {
"dist": sns.distplot,
"count": sns.countplot,
}
try:
method = h_method[h_type]
except KeyError:
# TODO: handle exception
init_logger().critical(
"histogram type '{h_type}' is not supported".format(h_type=h_type))
sys.exit()
if isinstance(s, pd.Series):
plt.title('{name} histogram'.format(name=s.name))
method(s)
plt.show()
else:
# for jupyter notebook
plt.title('{name} histogram'.format(name=name))
return list(map(lambda series: method(series), s))
def __init__(self, model, dataset, gpu_id=0, parallel=False):
self.logger = init_logger()
# dataset
self.test_loader = dataset
# model
self.model = model
# allocate 4 processes to 4 gpu respectively
device = 'cuda:{}'.format(gpu_id) if torch.cuda.is_available() else 'cuda'
self.logger.info("assign to {}".format(device))
self.device = torch.device(device)
torch.cuda.set_device(self.device)
self.model.to(self.device)
if device == 'cuda:{}'.format(gpu_id):
# self.p_model = torch.nn.DataParallel(module=self.model)
cudnn.benchmark = True
self.model.eval()
def __init__(self, idx, destination, lock):
self.logger = init_logger()
self.sub_pid = idx
self.lock = lock
# constant
self.normal_ops = [
'3x3_Conv',
'5x5_Conv',
'3x3_ConvDW',
'5x5_ConvDW',
'3x3_dConv',
'5x5_dConv',
'3x3_dConvDW',
'5x5_dConvDW',
'3x3_maxpool',
'3x3_avgpool',
# 'Zero',
'Identity',
]
self.reduction_ops = [
'3x3_Conv',
'5x5_Conv',
'3x3_ConvDW',
'5x5_ConvDW',
'3x3_dConv',
'5x5_dConv',
'3x3_dConvDW',
'5x5_dConvDW',
'2x2_maxpool',
'2x2_avgpool',
]
self.destination = destination # + str(idx)
self.df = None
import json
from index import app
from util import logger
LOG = logger.init_logger()
def test_normal_scene():
parsec_url = 'http://www.parsec.com.cn'
ret = app.request('/add', 'POST', {'url': parsec_url})
LOG.info(ret)
ret_data = json.loads(ret.data)
assert ret_data['code'] == 0
key = ret_data['key']
ret = app.request('/_' + key)
LOG.info(ret)
assert ret.status == '303 See Other' and ret.headers['Location'] == parsec_url
ret = app.request('/manager/clear', 'POST')
ret_data = json.loads(ret.data)
assert ret_data['code'] == 0
ret = app.request('/_' + key)
ret_data = json.loads(ret.data)
assert ret_data['code'] == 1
def setup():
'frames_processed':
frames_processed,
'frame_rate':
processing_frame_rate,
'frames_left':
frames_count - frames_processed,
'percentage_processed':
round((frames_processed / frames_count) * 100, 2),
},
})
ret, frame = cap.read()
# end capture, close window, close log file and video object if any
cap.release()
if not headless:
cv2.destroyAllWindows()
if record:
output_video.release()
logger.info('Processing ended.', extra={'meta': {'label': 'END_PROCESS'}})
if __name__ == '__main__':
from dotenv import load_dotenv
load_dotenv()
from util.logger import init_logger
init_logger()
run()
OS: {}
Python: {}
System: {} on {}/{}
Working directory: {}
Commandline: {}
============================
Log file: {}
============================
""".format(version.__version__, version.__release_date__,
platform.platform(), sys.version, getpass.getuser(), hostname,
ip, get_runtime_path(), ' '.join(sys.argv),
logger.current_log_file_path)
return preamble
# Actual bootstrapping code, executed with this module is imported
try:
if not BOOTSTRAP_DONE:
import util.logger as logger
logger.init_logger(get_runtime_path())
_logger = logger.get_logger(__name__)
# This will print on the console as well as the log file, so we know when a new execution started
_logger.info(get_preamble())
BOOTSTRAP_DONE = True
else:
_logger.debug("Bootstrap already complete")
except:
traceback.print_exc()
sys.stderr.write('Failed to bootstrap the application.\n')
def __init__(self):
self._size_list = None
self.logger = init_logger()
self.latency_list = []
super(MyModule, self).__init__()
def test_init_logger():
assert job_id not in logging.root.manager.loggerDict, "job id is not initialized"
init_logger()
assert job_id in logging.root.manager.loggerDict, "job id is initialized"
def main():
# Parse the arguments for each run
args = {
'env': 'Tiger',
'solver': 'MEPOP',
'discount': 1,
'n_epochs': 1000,
'quiet': True,
'n_sims': 100,
'n_start_states': 200,
'min_particle_count': 100,
'max_particle_count': 200,
'max_depth': 100,
'uct_coefficient': 100,
'me_tau': 40,
'me_epsilon': 0
}
env = getattr(env_module, args['env'])()
# # Test the arguments of POMCP
# args['solver'] = 'POMCP'
# logger.init_logger(args['env'], args['solver'])
# for uct_c in range(0, 160, 10):
# args['uct_coefficient'] = uct_c
# solver = getattr(solver_module, args['solver'])(env, args)
# run(solver, args)
# # Test the arguments of MEPOP
# args['solver'] = 'MEPOP'
# logger.init_logger(args['env'], args['solver'])
# for me_tau in np.arange(0.2, 2.2, 0.2):
# for me_epsilon in [0.75]:
# args['me_tau'] = me_tau
# args['me_epsilon'] = me_epsilon
# solver = getattr(solver_module, args['solver'])(env, args)
# run(solver, args)
# Test the POMCP
args['solver'] = 'POMCP'
logger.init_logger(args['env'], args['solver'])
pomcp_history = []
for n_sims in range(50, 1050, 50):
args['n_sims'] = n_sims
solver = getattr(solver_module, args['solver'])(env, args)
result = run(solver, args)
pomcp_history.append(result.undiscounted_return.mean)
# Test the MEPOP
args['solver'] = 'MEPOP'
logger.init_logger(args['env'], args['solver'])
mepop_history = []
for n_sims in range(50, 1050, 50):
args['n_sims'] = n_sims
solver = getattr(solver_module, args['solver'])(env, args)
result = run(solver, args)
mepop_history.append(result.undiscounted_return.mean)
df = pd.DataFrame({'pomcp': pomcp_history, 'mepop': mepop_history})
with open('results/csv/tiger_pomcp_and_mepop.csv', 'w'):
df.to_csv('results/csv/tiger_pomcp_and_mepop.csv')
