def main():
config = Config()
working_dir = os.path.join(config.working_dir, config.specs)
working_dir = os.path.join(working_dir,
dt.datetime.now().strftime("%Y-%m-%dT%H-%M-%S"))
make_sure_path_exists(working_dir)
with open(os.path.join(working_dir, 'log.txt'), 'a') as logfile:
sys.stdout = Tee(sys.stdout, logfile, sys.stdout)
bl_config = config.get_value('bl_config')
logger.configure(os.path.join(working_dir, 'baselines'),
['tensorboard', 'log', 'stdout'])
train(env_id=bl_config['env'],
num_timesteps=bl_config['num_timesteps'],
policy=config.get_value('policy'),
working_dir=working_dir,
config=config)
sys.stdout.flush()
def __setup_working_dir__(self):
# fix permissions of workspace root
make_sure_path_exists(self.workspace)
try:
chmod(self.workspace, 0o775)
except PermissionError:
print("PermissionError when trying to change permissions of workspace to 775")
# setup working directory
specs_dir = os.path.realpath("{}/{}".format(self.workspace, self.specs))
working_dir = os.path.realpath("{}/{}".format(specs_dir, self.timestamp))
# Set up result folder structure
results_path = "{}/results".format(working_dir, self.timestamp)
make_sure_path_exists(results_path)
# Set up tensorboard directory
tensorboard = "{}/tensorboard".format(working_dir, self.timestamp)
make_sure_path_exists(tensorboard)
# set path to kill file (if this file exists abort run)
kill_file_name = "ABORT_RUN"
kill_file = os.path.join(working_dir, kill_file_name)
# create plot file to plot by default
plot_file_name = "PLOT_ON"
plot_file = os.path.join(working_dir, plot_file_name)
touch(plot_file)
# remove kill file before starting the run (should not exist anyway)
if os.path.isfile(kill_file):
os.remove(kill_file)
# fix permissions to grant group write access (to allow kill_file creation and plot control)
try:
chmod(self.workspace, 0o775, recursive=False)
chmod(specs_dir, 0o775, recursive=False)
chmod(working_dir, 0o775, recursive=True)
chmod(plot_file, 0o664)
except PermissionError:
print("PermissionError when trying to change permissions of workspace to 775")
# compress and copy current script and dependencies to results dir
command = " ".join(sys.argv)
# copy current code to temp dir
script_dir = os.path.dirname(os.path.realpath(__main__.__file__))
tempdir = tempfile.mkdtemp("tell")
copydir(script_dir, tempdir,
exclude=[self.workspace, os.path.join(script_dir, ".git"), os.path.join(script_dir, ".idea"),
os.path.join(script_dir, "__pycache__")])
zipdir(dir=tempdir, zip=os.path.join(working_dir, '00-script.zip'), info=command,
exclude=[self.workspace, '.git'])
rmdir(tempdir)
return [working_dir, results_path, tensorboard, kill_file, plot_file, None]
def save_subplots(kwargs):
"""Plotting list of images as subplots
Parameters
-------
kwargs : dict
images : list of lists of numpy arrays or list of numpy arrays or numpy array of shape (x,y,3) or (x,y)
List of lists of images to plot; Images with shape (x,y,3) or (x,y);
List with shape [[image_11, image_21], [image_12, image_22]] for placement in subfigure (len(images) will be the
number of columns);
filename : str
Filename to save subplot as
title : str
Main title of plot
subfigtitles : list of str
Strings to be used as titles for the subfigures
colorbar : bool
True: plot colorbar
Returns
-------
Saves figure to filename
"""
images = kwargs.get('images')
filename = kwargs.get('filename')
title = kwargs.get('title', None)
subfigtitles = kwargs.get('subfigtitles', [])
subplotranges = kwargs.get('subplotranges', [])
colorbar = kwargs.get('colorbar', True)
# Check if filepath is valid
if len(os.path.dirname(filename)):
make_sure_path_exists(os.path.dirname(filename))
# Check if image is valid, nest list to 2D if necessary
if not isinstance(images, list):
images = [images]
if not isinstance(images[0], list):
images = [images]
# Create empty subplot
n_cols = len(images)
n_rows = max([len(l) for l in images])
if n_cols <= 1:
n_cols = 2
if n_rows <= 1:
n_rows = 2
f, ax = pl.subplots(n_rows, n_cols)
if title is not None:
f.suptitle(title)
for col in ax:
for row in col:
row.get_xaxis().set_visible(False)
row.get_yaxis().set_visible(False)
for col_i, col in enumerate(images):
for row_i, row in enumerate(col):
# Check dimensions of image, squeeze if necessary
if len(row.shape) == 3:
if row.shape[-1] == 1:
# If an image has shape (x, y, 1) flatten it to (x, y) to plot it as grayscale
row = row.reshape(row.shape[:-1])
# Plot image at position in subfigure with optional color-range
try:
if subplotranges[col_i][row_i]:
im = ax[row_i][col_i].imshow(
np.array(row, dtype=np.float),
interpolation='nearest',
vmin=subplotranges[col_i][row_i][0],
vmax=subplotranges[col_i][row_i][1])
else:
im = ax[row_i][col_i].imshow(np.array(row, dtype=np.float),
interpolation='nearest')
except IndexError:
im = ax[row_i][col_i].imshow(np.array(row, dtype=np.float),
interpolation='nearest')
# Try to add title
try:
ax[row_i][col_i].set_title(subfigtitles[col_i][row_i],
fontdict=dict(fontsize=6))
except IndexError:
pass
# Add colorbar to subplot
if colorbar:
# Create divider for existing axes instance
divider = make_axes_locatable(ax[row_i][col_i])
# Append axes to the right of ax, with 20% width of ax
cax = divider.append_axes("right", size="20%", pad=0.05)
# Create colorbar in the appended axes
# Tick locations can be set with the kwarg `ticks`
# and the format of the ticklabels with kwarg `format`
cbar = pl.colorbar(
im, cax=cax) #, ticks=MultipleLocator(0.2), format="%.2f")
cbar.ax.tick_params(labelsize=5)
ax[row_i][col_i].get_xaxis().set_visible(False)
ax[row_i][col_i].get_yaxis().set_visible(False)
ax[row_i][col_i].set_aspect('auto')
# for row_i in ax:
# for col_i in ax:
# col_i.get_xaxis().set_visible(False)
# col_i.get_xaxis().set_visible(False)
#pl.tight_layout()
f.savefig(filename)
pl.close()
def make_movie(kwargs):
""" vid, filename, title=None, subfigtitles=[], colorbar=True, interval=1, dpi=100"""
vid = kwargs.get('vid')
filename = kwargs.get('filename')
title = kwargs.get('title', None)
subfigtitles = kwargs.get('subfigtitles', [])
colorbar = kwargs.get('colorbar', True)
interval = kwargs.get('interval', 100)
dpi = kwargs.get('dpi', 100)
if len(os.path.dirname(filename)):
make_sure_path_exists(os.path.dirname(filename))
# create empty subplot
n_cols = len(vid[0])
n_rows = max([len(l) for l in vid[0]])
f, ax = pl.subplots(n_rows, n_cols)
im = []
# Initialize subplot
for col_i, col in enumerate(vid[0]):
for row_i, row in enumerate(col):
# plot image at position in subfigure
im.append(ax[row_i][col_i].imshow(np.array(row, dtype=np.float),
interpolation='nearest'))
# add title to subplot, if existent
try:
ax[row_i][col_i].set_title(subfigtitles[0][col_i][row_i])
except IndexError:
pass
if colorbar:
# Create divider for existing axes instance
divider = make_axes_locatable(ax[row_i][col_i])
# Append axes to the right of ax, with 20% width of ax
cax = divider.append_axes("right", size="20%", pad=0.05)
# Create colorbar in the appended axes
# Tick locations can be set with the kwarg `ticks`
# and the format of the ticklabels with kwarg `format`
pl.colorbar(
im[row_i + col_i * len(col)],
cax=cax) #, ticks=MultipleLocator(0.2), format="%.2f")
ax[row_i][col_i].get_xaxis().set_visible(False)
ax[row_i][col_i].get_yaxis().set_visible(False)
ax[row_i][col_i].set_aspect('auto')
#pl.tight_layout()
def make_frame(vid):
for f_i, frame in enumerate(len(vid)):
images = vid[frame]
for col_i, col in enumerate(images):
for row_i, row in enumerate(col):
# plot image at position in subfigure
im[row_i + col_i * len(col)].set_data(
np.array(row, dtype=np.float))
# add title to subplot, if existent
try:
ax[row_i][col_i].set_title(
subfigtitles[f_i][col_i][row_i])
except IndexError:
pass
if colorbar:
# Create divider for existing axes instance
divider = make_axes_locatable(ax[row_i][col_i])
# Append axes to the right of ax, with 20% width of ax
cax = divider.append_axes("right",
size="20%",
pad=0.05)
# Create colorbar in the appended axes
# Tick locations can be set with the kwarg `ticks`
# and the format of the ticklabels with kwarg `format`
pl.colorbar(
im[row_i + col_i * len(col)], cax=cax
) #, ticks=MultipleLocator(0.2), format="%.2f")
ax[row_i][col_i].get_xaxis().set_visible(False)
ax[row_i][col_i].get_yaxis().set_visible(False)
#pl.tight_layout()
yield im
animated = animation.FuncAnimation(f,
make_frame,
vid,
blit=False,
interval=interval,
repeat=False)
animated.save(filename, dpi=dpi)
pl.close()
def _save_subplots_line_plots(images,
filename,
title=None,
subfigtitles=(),
subplotranges=(),
automatic_positioning=False,
tight_layout=False,
aspect='auto',
resolution=None,
fontdict=None,
x_size=15,
y_size=15):
"""Plotting list of line plots as subplots
Parameters
-------
images : list of lists of numpy arrays or list of numpy arrays or numpy array of shape (x,y,3) or (x,y)
List of lists of images to plot; Images with shape (x,y,3) or (x,y);
List with shape [[image_11, image_21], [image_12, image_22]] for placement in subfigure (len(images) will be the
number of columns);
filename : str
Filename to save subplot as
title : str
Main title of plot
subfigtitles : list of lists of strings or list of strings or string
Strings to be used as titles for the subfigures
subplotranges : list of lists of tuples or list of tuples or tuple
Tuples defining the maximum and minimum values to plot in the figure
automatic_positioning : bool
True: Automatically position plots in subplot
tight_layout : bool
True: Create figure in tight_layout mode
fontdict : dict
fontdict of subfig .set_title function
Returns
-------
Saves figure to filename
"""
# Check if filepath is valid
if len(os.path.dirname(filename)):
make_sure_path_exists(os.path.dirname(filename))
# Check if images are valid, nest list to 2D if necessary
images = _make_2d_list(images)
if automatic_positioning:
images = quadratic_list_layout(images)
# Check if subfigtitles are valid, nest list to 2D if necessary
subfigtitles = _make_2d_list(subfigtitles)
if automatic_positioning:
subfigtitles = quadratic_list_layout(subfigtitles)
if fontdict is None:
fontdict = dict(fontsize=6)
# Create empty subplot
n_cols = len(images)
n_rows = max([len(l) for l in images])
if n_cols <= 1:
n_cols = 2
if n_rows <= 1:
n_rows = 2
f, ax = pl.subplots(n_rows, n_cols, figsize=(x_size, y_size))
if title is not None:
f.suptitle(title)
for col in ax:
for row in col:
row.get_xaxis().set_visible(False)
row.get_yaxis().set_visible(False)
for col_i, col in enumerate(images):
for row_i, row in enumerate(col):
if row is None:
continue
# Check dimensions of image, squeeze if necessary
if len(row.shape) == 3:
if row.shape[-1] == 1:
# If an image has shape (x, y, 1) flatten it to (x, y) to plot it as grayscale
row = row.reshape(row.shape[:-1])
data = np.array(row, dtype=np.float)
if len(data.shape) > 1:
for i in range(0, data.shape[1]):
im = ax[row_i][col_i].plot(data[:, i])
# Try to add title
try:
ax[row_i][col_i].set_title(subfigtitles[col_i][row_i],
fontdict=dict(fontsize=6))
except IndexError:
pass
ax[row_i][col_i].get_xaxis().set_visible(True)
ax[row_i][col_i].get_yaxis().set_visible(True)
ax[row_i][col_i].set_aspect(aspect)
else:
# For 1D data
im = ax[row_i][col_i].plot(data)
# Try to add title
try:
ax[row_i][col_i].set_title(subfigtitles[col_i][row_i],
fontdict=dict(fontsize=6))
except IndexError:
pass
ax[row_i][col_i].get_xaxis().set_visible(True)
ax[row_i][col_i].get_yaxis().set_visible(True)
ax[row_i][col_i].set_aspect(aspect)
if tight_layout:
pl.tight_layout()
if resolution is None:
f.savefig(filename)
else:
f.savefig(filename, dpi=resolution)
pl.close()
def _save_movie(images,
filename,
title=None,
subplotranges=(),
tight_layout=False,
interpolation='nearest',
aspect='auto',
resolution=None,
fontdict=None,
fps=30,
interval=200):
"""Plotting list of images as subplots
Parameters
-------
images : list of 2D or 3D numpy array of shape (x,y,3) or (x,y)
list of 2D or 3D numpy arrays of shape (x,y,3) or (x,y)
filename : str
Filename to save subplot as
title : str
Main title of plot
subplotranges : tuple(vmin, vmax) or None
Tuples defining the maximum and minimum values to plot in the figure
tight_layout : bool
True: Create figure in tight_layout mode
resolution : int
resolution in dpi
fontdict : dict
fontdict of subfig .set_title function
fps : int
framerate of writer
interval : int
Delay between frames in milliseconds
Returns
-------
Saves figure to filename
"""
# Check if filepath is valid
if len(os.path.dirname(filename)):
make_sure_path_exists(os.path.dirname(filename))
# Check if images are valid, i.d. list of 2D arrays
images = [
np.array(im, dtype=np.float)
if len(im.shape) == 2 or len(im.shape) == 3 else None for im in images
]
if np.any(np.isnan(images)):
raise ValueError(
"List of images must only contain 2D or 3D numpy arrays!")
if fontdict is None:
fontdict = dict(fontsize=6)
# Create empty subplot
fig = pl.figure()
ax = fig.add_subplot(111)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
ax.set_aspect(aspect)
try:
im = ax.imshow(images[0],
interpolation=interpolation,
vmin=subplotranges[0],
vmax=subplotranges[1])
except IndexError:
im = ax.imshow(images[0], interpolation=interpolation)
if title is not None:
fig.suptitle(title, fontdict=fontdict)
if tight_layout:
pl.tight_layout()
def update_img(n):
try:
im.set_data(images[n])
except IndexError:
im.set_data(images[n])
return im
ani = animation.FuncAnimation(fig,
update_img,
frames=len(images),
interval=interval)
writer = animation.writers['ffmpeg'](fps=fps)
ani.save(filename, writer=writer, dpi=resolution)
del ani
pl.close()
def _save_subplots(images,
filename,
title=None,
subfigtitles=(),
subplotranges=(),
colorbar=True,
automatic_positioning=False,
tight_layout=False,
interpolation='nearest',
aspect='auto',
resolution=None,
fontdict=None):
"""Plotting list of images as subplots
Parameters
-------
images : list of lists of numpy arrays or list of numpy arrays or numpy array of shape (x,y,3) or (x,y)
List of lists of images to plot; Images with shape (x,y,3) or (x,y);
List with shape [[image_11, image_21], [image_12, image_22]] for placement in subfigure (len(images) will be the
number of columns);
filename : str
Filename to save subplot as
title : str
Main title of plot
subfigtitles : list of lists of strings or list of strings or string
Strings to be used as titles for the subfigures
subplotranges : list of lists of tuples or list of tuples or tuple
Tuples defining the maximum and minimum values to plot in the figure
colorbar : bool
True: plot colorbar
automatic_positioning : bool
True: Automatically position plots in subplot
tight_layout : bool
True: Create figure in tight_layout mode
fontdict : dict
fontdict of subfig .set_title function
Returns
-------
Saves figure to filename
"""
# Check if filepath is valid
if len(os.path.dirname(filename)):
make_sure_path_exists(os.path.dirname(filename))
# Check if images are valid, nest list to 2D if necessary
images = _make_2d_list(images)
if automatic_positioning:
images = quadratic_list_layout(images)
# Check if subfigtitles are valid, nest list to 2D if necessary
subfigtitles = _make_2d_list(subfigtitles)
if automatic_positioning:
subfigtitles = quadratic_list_layout(subfigtitles)
if fontdict is None:
fontdict = dict(fontsize=6)
# Check if plotranges are valid, nest list to 2D if necessary
subplotranges = _make_2d_list(subplotranges)
if automatic_positioning:
subplotranges = quadratic_list_layout(subplotranges)
# Create empty subplot
n_cols = len(images)
n_rows = max([len(l) for l in images])
if n_cols <= 1:
n_cols = 2
if n_rows <= 1:
n_rows = 2
f, ax = pl.subplots(n_rows, n_cols)
if title is not None:
f.suptitle(title)
for col in ax:
for row in col:
row.get_xaxis().set_visible(False)
row.get_yaxis().set_visible(False)
for col_i, col in enumerate(images):
for row_i, row in enumerate(col):
if row is None:
continue
# Check dimensions of image, squeeze if necessary
if len(row.shape) == 3:
if row.shape[-1] == 1:
# If an image has shape (x, y, 1) flatten it to (x, y) to plot it as grayscale
row = row.reshape(row.shape[:-1])
data = np.array(row, dtype=np.float)
if len(data.shape) > 1:
# Plot image at position in subfigure with optional color-range
try:
if len(data.shape) > 1:
# For image data
if subplotranges[col_i][row_i]:
im = ax[row_i][col_i].imshow(
data,
interpolation=interpolation,
vmin=subplotranges[col_i][row_i][0],
vmax=subplotranges[col_i][row_i][1])
else:
im = ax[row_i][col_i].imshow(
data, interpolation=interpolation)
except IndexError:
im = ax[row_i][col_i].imshow(np.array(row, dtype=np.float),
interpolation=interpolation)
# Try to add title
try:
ax[row_i][col_i].set_title(subfigtitles[col_i][row_i],
fontdict=fontdict)
except IndexError:
pass
# Add colorbar to subplot
if colorbar:
# Create divider for existing axes instance
divider = make_axes_locatable(ax[row_i][col_i])
# Append axes to the right of ax, with 20% width of ax
cax = divider.append_axes("right", size="20%", pad=0.05)
# Create colorbar in the appended axes
# Tick locations can be set with the kwarg `ticks`
# and the format of the ticklabels with kwarg `format`
cbar = pl.colorbar(
im, cax=cax
) # , ticks=MultipleLocator(0.2), format="%.2f")
cbar.ax.tick_params(labelsize=5)
ax[row_i][col_i].get_xaxis().set_visible(False)
ax[row_i][col_i].get_yaxis().set_visible(False)
ax[row_i][col_i].set_aspect(aspect)
else:
# For 1D data
im = ax[row_i][col_i].plot(data)
# Try to add title
try:
ax[row_i][col_i].set_title(subfigtitles[col_i][row_i],
fontdict=dict(fontsize=6))
except IndexError:
pass
ax[row_i][col_i].get_xaxis().set_visible(True)
ax[row_i][col_i].get_yaxis().set_visible(True)
ax[row_i][col_i].set_aspect(aspect)
if tight_layout:
pl.tight_layout()
if resolution is None:
f.savefig(filename)
else:
f.savefig(filename, dpi=resolution)
pl.close()
#
# Train reward redistribution model and get integrated gradients signal
#
# Tensorflow configuration
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
tf_config = tf.ConfigProto(allow_soft_placement=True,
inter_op_parallelism_threads=8,
intra_op_parallelism_threads=8)
tf_config.gpu_options.allow_growth = True
tf_sess = tf.Session(config=tf_config)
tf.global_variables_initializer().run(session=tf_sess)
outputpath = os.path.join('results',
dt.datetime.now().strftime("%Y-%m-%dT%H-%M-%S"))
outputpath_training = os.path.join(outputpath, 'training')
make_sure_path_exists(outputpath)
make_sure_path_exists(outputpath_training)
avg_return = 0.
n_max_updates = 1e5
episode = 0
n_plotted = 0
avg_loss = 0
while episode < n_max_updates:
sample = generate_sample()
feed_dict = {
states_placeholder: sample['states'],
actions_placeholder: sample['actions'],
rewards_placeholder: sample['rewards'],
avg_reward_placeholder: avg_return
}
tf_session=tf_session,
working_dir=working_dir,
config=config,
plotting=dict(
save_subplots=save_subplots,
save_movie=save_movie,
save_subplots_line_plots=save_subplots_line_plots),
rnd_gen=rnd_gen)
if __name__ == '__main__':
config = Config()
working_dir = os.path.join(config.working_dir, config.specs)
working_dir = os.path.join(working_dir,
dt.datetime.now().strftime("%Y-%m-%dT%H-%M-%S"))
make_sure_path_exists(working_dir)
with open(os.path.join(working_dir, 'log.txt'), 'a') as logfile:
sys.stdout = Tee(sys.stdout, logfile, sys.stdout)
bl_config = config.get_value('bl_config')
logger.configure(os.path.join(working_dir, 'baselines'),
['tensorboard', 'log', 'stdout'])
train(env_id=bl_config['env'],
num_timesteps=bl_config['num_timesteps'],
policy=config.get_value('policy'),
working_dir=working_dir,
config=config)
sys.stdout.flush()