def plot(layer_output,path,layer_idx,batch_idx,img_plot_remaining,max_subplots=100):
#print layer_output;
matplotlib.use('Agg')
import matplotlib.pyplot as plt
num_examples = layer_output.__len__();
for eg_idx in xrange(num_examples):
if img_plot_remaining == 0:
break;
save_path = path%(layer_idx,batch_idx,eg_idx)
logger.debug('Plotting the feature map %d in %s'%(eg_idx,save_path));
eg_output = layer_output[eg_idx];
num_plots = min(max_subplots,eg_output.__len__());
cols = int(np.ceil(num_plots/10.0));
fig,plots = plt.subplots(10,cols);
plots = plots.flatten();
for idx in xrange(num_plots):
plots[idx].imshow(eg_output[idx],interpolation='nearest');
plots[idx].axis('off');
create_folder_structure_if_not_exists(save_path)
# Save the full figure...
fig.savefig(save_path,bbox_inches='tight')
plt.close()
img_plot_remaining-=1;
return img_plot_remaining;
def plot(layer_output, path, layer_idx, batch_idx, img_plot_remaining, max_subplots=100):
# print layer_output;
matplotlib.use("Agg")
import matplotlib.pyplot as plt
num_examples = layer_output.__len__()
for eg_idx in xrange(num_examples):
if img_plot_remaining == 0:
break
save_path = path % (layer_idx, batch_idx, eg_idx)
logger.debug("Plotting the feature map %d in %s" % (eg_idx, save_path))
eg_output = layer_output[eg_idx]
num_plots = min(max_subplots, eg_output.__len__())
cols = int(np.ceil(num_plots / 10.0))
fig, plots = plt.subplots(10, cols)
plots = plots.flatten()
for idx in xrange(num_plots):
plots[idx].imshow(eg_output[idx], interpolation="nearest")
plots[idx].axis("off")
create_folder_structure_if_not_exists(save_path)
# Save the full figure...
fig.savefig(save_path, bbox_inches="tight")
plt.close()
img_plot_remaining -= 1
return img_plot_remaining
def write_dataset(options):
file_path = options['base_path'] + os.sep + options['filename'];
logger.info("%s dataset will be initialized to write to %s",
options['writer_type'],file_path);
create_folder_structure_if_not_exists(file_path);
file_writer = FileWriter.get_instance(file_path,options);
file_writer.write_file_info()
return file_writer
def write_dataset(options):
file_path = options['base_path'] + os.sep + options['filename']
logger.info("%s dataset will be initialized to write to %s",
options['writer_type'], file_path)
create_folder_structure_if_not_exists(file_path)
file_writer = FileWriter.get_instance(file_path, options)
file_writer.write_file_info()
return file_writer
def dump_data(self,out_fn,out_featdim):
filepath = self.data_spec['base_path'] + os.sep + self.data_spec['filename']
copy_path = create_folder_structure_if_not_exists(self.export_path + os.sep + self.data_spec['filename'])
shutil.copy(filepath,copy_path); #copies the file directly
self.filehandle = open(filepath,'rb')
line = self.filehandle.readline(); # reading file header
header = line.split();
num_classes = int(header[1]);
for idx in xrange(num_classes):
level1_filename = self.filehandle.readline().strip();
level1_filepath = self.data_spec['base_path'] + os.sep + level1_filename #filename of individual classes
copy_path = create_folder_structure_if_not_exists(self.export_path + os.sep + level1_filename)
shutil.copy(level1_filepath,copy_path); #copies the index file directly from the source directly
self.level1FileHandle = open(level1_filepath,'rb');
level2_filepath = self.level1FileHandle.readline().strip();
while level2_filepath.__len__()!= 0:
in_child_options = self.data_spec.copy();
in_child_options['filename'] = level2_filepath #filename of individual classes
in_child_options['reader_type'] = "TD"
in_child_options['label'] = idx;
file_reader = read_dataset(in_child_options,pad_zeros=True) #taking only one reader
out_child_options = in_child_options.copy();
out_child_options['base_path'] = self.export_path; #updating the base_path
out_child_options['featdim'] = out_featdim;
out_child_options['writer_type'] = "TD"
file_writer = write_dataset(out_child_options);
batch_size=file_reader.batch_size
while not file_reader.is_finish():
for batch_index in xrange(file_reader.nBatches):
s_idx = batch_index * batch_size; e_idx = s_idx + batch_size
data = out_fn(file_reader.feat[s_idx:e_idx])
label = file_reader.label[s_idx:e_idx];
if ((batch_index == file_reader.nBatches-1) and (not file_reader.num_pad_frames == 0)) :
data=data[:-file_reader.num_pad_frames]
label = label[:-file_reader.num_pad_frames]
file_writer.write_data(data,label);
file_reader.read_next_partition_data(pad_zeros=True);
level2_filepath = self.level1FileHandle.readline().strip();
logger.debug('T2 Dataexporter : data is exported to %s' % self.export_path);
def dump_data(self, out_fn, out_featdim):
filepath = self.data_spec['base_path'] + os.sep + self.data_spec[
'filename']
copy_path = create_folder_structure_if_not_exists(
self.export_path + os.sep + self.data_spec['filename'])
shutil.copy(filepath, copy_path)
#copies the file directly
self.filehandle = open(filepath, 'rb')
line = self.filehandle.readline()
# reading file header
header = line.split()
num_classes = int(header[1])
for idx in xrange(num_classes):
in_child_options = self.data_spec.copy()
in_child_options['filename'] = self.filehandle.readline().strip(
) #filename of individual classes
in_child_options['reader_type'] = "TD"
in_child_options['label'] = idx
file_reader = read_dataset(in_child_options,
pad_zeros=True) #taking only one reader
out_child_options = in_child_options.copy()
out_child_options['base_path'] = self.export_path
out_child_options['featdim'] = out_featdim
out_child_options['writer_type'] = "TD"
file_writer = write_dataset(out_child_options)
batch_size = file_reader.batch_size
while (not file_reader.is_finish()):
for batch_index in xrange(file_reader.nBatches):
s_idx = batch_index * batch_size
e_idx = s_idx + batch_size
data = out_fn(file_reader.feat[s_idx:e_idx])
label = file_reader.label[s_idx:e_idx]
if ((batch_index == file_reader.nBatches - 1)
and (not file_reader.num_pad_frames == 0)):
data = data[:-file_reader.num_pad_frames]
label = label[:-file_reader.num_pad_frames]
file_writer.write_data(data, label)
file_reader.read_next_partition_data(pad_zeros=True)
logger.debug('T1 Dataexporter : data is exported to %s' %
self.export_path)
