def __init__(self, params_file):
""" Initializes an experiment.
"""
self.params = self.get_parameter(params_file)
self.exp_dir = self.set_exp_dir()
_logger = self.set_logger()
_logger.info("Initializing new experiment of type %s" %
str(self.params['type']))
_logger.info("Loading parameters from %s" % str(params_file))
_logger.info("Saving logs in %s" % str(self.exp_dir))
# self.set_s tatus('Initializing')
# copy parameter source
helper.copy_file(params_file, os.path.join(self.exp_dir,
'params.yaml'))
# Mersenne Twister pseudo-random number generator
self.rng = np.random.RandomState(self.params['random_seed'])
# set environment
self.env = Gridworld(grid=os.path.join(os.getcwd(), 'maps',
self.params['grid']),
max_steps=self.params['max_steps'],
visual=self.params['visual'],
rng=self.rng)
self.current_task = 'None'
self.current_run = 0
self.current_episode = 0
self.exp_steps = 0
def save_best_episode(self):
df = pd.read_csv(os.path.join(self.run_dir, 'stats_run.csv'))
least_steps_row = df.ix[df['steps_mean'].idxmin()]
run_best_file = os.path.join(self.run_dir, 'stats_run_best.csv')
headers = ['run']
content = [int(self.current_run)]
for column in df:
headers.append(str(column))
content.append(least_steps_row[column])
helper.write_stats_file(run_best_file, headers)
helper.write_stats_file(run_best_file, content)
helper.copy_file(
os.path.join(self.run_dir,
'episode_' + str(int(least_steps_row['episode'])),
'Qs.npy'), os.path.join(self.run_dir, 'best_Qs.npy'))
def CopyFileToDestinationDir(self):
"""
Copies computed file to mounted s3 location on ec2
"""
src_file_path = os.path.join(
BASE_DIR, ADS_COMPUTED_OUTPUT_FILE_PATH.format(self.client))
client_output_dir = DESTINATION_ROOT_PATH.format(self.client)
create_dir(client_output_dir)
dest_file_path = os.path.join(client_output_dir,
ADS_COMPUTED_OUTPUT_FILE)
if os.path.exists(src_file_path):
copy_file(src_file_path, dest_file_path)
return dest_file_path
else:
return None
def save_best_run(self):
# Save best Q-table for current task
df = pd.read_csv(
os.path.join(self.task_dir, 'run_' + str(1), 'stats_run_best.csv'))
for i in range(2, self.params['runs']):
df.append(pd.read_csv(
os.path.join(self.task_dir, 'run_' + str(i),
'stats_run_best.csv')),
ignore_index=True)
least_steps_row = df.ix[df['steps_mean'].idxmin()]
task_best_file = os.path.join(self.task_dir, 'stats_task_best.csv')
headers = ['task']
content = [str(self.current_task['name'])]
for column in df:
headers.append(str(column))
content.append(least_steps_row[column])
helper.write_stats_file(task_best_file, headers)
helper.write_stats_file(task_best_file, content)
helper.copy_file(
os.path.join(self.task_dir,
'run_' + str(int(least_steps_row['run'])),
'best_Qs.npy'),
os.path.join(self.task_dir, 'best_Qs.npy'))
def uncompress_file(self, logfile, dist_path):
self.logger.info("Start uncommpress file '%s' to '%s' ." %
(logfile, dist_path))
if self.log_compression_method == None:
try:
self.logger.debug("copy logfile to %s" % dist_path)
helper.copy_file(logfile, dist_path)
except Exception as e:
helper.delete_file(logfile)
# common.upload_with_local(logfile, self.logbee_workdir+self.log_work_name+"_error/"+os.path.basename(logfile))
self.logger.warning("Copy file fail.%s" % e)
elif self.log_compression_method in settings.UNCOMPRESSION_FUNCTION.keys(
):
try:
settings.UNCOMPRESSION_FUNCTION[self.log_compression_method](
logfile, dist_path)
except Exception as e:
helper.delete_file(logfile)
# common.upload_with_local(logfile, self.logbee_workdir+self.log_work_name+"_error/"+os.path.basename(logfile))
self.logger.warning("uncompress logfile fail .%s" % e)
else:
self.logger.error("Unsupported commpression method: %s" %
self.log_compression_method)
def test_add_media_folder_to_preference_as_watch_folder(self):
assert self.RT.does_exist_in_library(upload_video_item, 'Upload', default_wait_time), "Fail to find Upload.AVI in library"
# copy file to watch folder
log("Copy file %s to folder %s" %
(self.test_image_name, self.test_case_path))
assert helper.copy_file(os.path.join(test_content_path, "original", self.test_image_name),
self.test_case_path), "Fail to copy file from '" + self.test_image_name + "' to folder '" + self.test_case_path + "'"
# verify the shared item is in the view
if osx_version == "10.10":
assert region.exists(new_photo_found_notification_10_10,
default_wait_long_time), "The 'new photo found' notification doesn't show up."
elif osx_version == "10.9":
assert region.exists(new_photo_found_notification_10_9,
default_wait_long_time), "The 'new photo found' notification doesn't show up."
# verify new item is added to library view
assert self.RT.does_exist_in_library(media_scan_item, 'MediaScan', default_wait_time), "Cannot find the thumbnail of scanned media in library view"
def test_add_new_media_files_to_watch_folder(self):
# enable pictures from media library
log("select checkbox '" + picture_root_folder + "' from media library preferences dialog")
op.check_one_item_in_media_library_preferences(picture_root_folder)
log("wait one minute for library loading")
time.sleep(60)
# copy file to watch folder
log("Copy file %s to pictures watch folder %s" % (self.test_image_name, picture_root_folder))
assert helper.copy_file(
self.test_image_full_path, picture_root_folder
), "Fail to copy file from '%s' to pictures watch folder '%s'" % (self.test_image_name, picture_root_folder)
# verify the shared item is in the view
if osx_version == "10.10":
assert region.exists(
new_photo_found_notification_10_10, default_wait_long_time
), "The 'new photo found' notification doesn't show up."
assert region.waitVanish(
new_photo_found_notification_10_10, default_wait_long_time
), "The notification doesn't be dismissed after a while"
elif osx_version == "10.9":
assert region.exists(
new_photo_found_notification_10_9, default_wait_long_time
), "The 'new photo found' notification doesn't show up."
assert region.waitVanish(
new_photo_found_notification_10_9, default_wait_long_time
), "The notification doesn't be dismissed after a while"
time.sleep(2)
log("search scanned media in library")
rt = RT(region)
rt.search_library(self.test_image_name)
# verify new item is added to library view
assert self.RT.does_exist_in_library(
media_scan_item, "MediaScan", default_wait_time
), "Cannot find the thumbnail of scanned media in library view"
def persistClientDateTimeLogFile(self):
CRON_LOG_FILE_PATH = os.path.join(BASE_DIR, LOG_DIR, LOG_FILE)
DATE_TIME_STAMP_LOG_FILE_PATH = os.path.join(
BASE_DIR, LOG_DIR, '{}_{}.log'.format(self.client,
CURRENT_DATE_STAMP))
copy_file(CRON_LOG_FILE_PATH, DATE_TIME_STAMP_LOG_FILE_PATH)
