def _clear_folder(self):
utils.clear_folder(self._output_path)
print(f"Logging to {self._log_file}\n")
sys.stdout = utils.StdOut(self._log_file)
print(f"PyTorch version: {torch.__version__}")
if self._cuda:
print(f"CUDA version: {torch.version.cuda}\n")
def show_picture(self, data):
"""
Show a picture.
@param data raw picture data.
"""
utils.clear_folder(self._TMP_DIR)
filename = 'picture%d.jpg' % int(time.time())
path = os.path.join(self._TMP_DIR, filename)
f = open(path, 'wb')
f.write(data)
f.close()
img = Image.open(path)
"""
The image is loaded and saved because Showtimes image viewer can't handle many of the image that come straight from an iPhone.
Since PIL does not retain the Exif data, we must rotate the image if needed instead.
"""
exif = img._getexif()
if exif != None:
for tag, value in exif.items():
decoded = TAGS.get(tag, tag)
if decoded == 'Orientation':
if value == 3: img = img.rotate(180)
if value == 6: img = img.rotate(270)
if value == 8: img = img.rotate(90)
break
img.save(path);
path = "file://" + path
self.log.debug("filename %s", path);
self.open_url(path)
def classify(self, path='C:\\Users\\Dudu\\Desktop\\Licenta\\VisualProcessing\\AudioVisualClip\\JK\\a7.avi',
temp_path=os.path.join(os.path.abspath(os.path.dirname(__file__)), 'temp')):
emotion = ['negative', 'neutral', 'positive']
ensure_dir(temp_path)
images = FrameExtractor.process_file(path)
for i in range(len(images)):
cv2.imwrite(os.path.join(temp_path, 'unlabeled', "%d.jpg" % i), images[i])
batches, test_data = self.get_data(temp_path)
print('Starting prediction...')
features = np.mean(self.model.predict(test_data, batch_size=1), axis=0)
print('The video expresses %s feelings.' % emotion[np.argmax(features)])
clear_folder(temp_path)
return features
def __init__(self,
sess_config,
model_path,
log_path,
vocab_size=1024,
learning_rate=0.0005,
batch_size=32,
embedding_size=64,
model_name='seq2seq_test',
hidden_units=32,
display_steps=200,
saving_steps=100,
eval_mode=False,
restore_model=False,
use_raw_rnn=False,
BiDirectional=False):
self.vocab_size = vocab_size
self.embedding_size = embedding_size
self.batch_size = batch_size
self.hidden_units = hidden_units
self.model_name = model_name
self.display_steps = display_steps
self.saving_steps = saving_steps
self.learning_rate = learning_rate
self.model_path = model_path
self.log_path = log_path
self.USE_RAW_RNN = use_raw_rnn
self.BiDirectional = BiDirectional
self.graph = tf.Graph()
self.sess = tf.Session(graph=self.graph, config=sess_config)
self.global_step = 0
if eval_mode:
self._restore_model(training_mode=False)
self._build_eval_graph()
elif restore_model:
self._restore_model()
else:
clear_folder(self.log_path)
clear_folder(self.model_path)
generate_tensorboard_script(
self.log_path
) # create the script to start a tensorboard session
self._build_graph()
def clean_up(self):
logger.info('Cleaning up...')
self.set_status('Preparing to generate more content...')
utils.clear_folder(config.VID_PATH)
utils.clear_folder(config.GIF_PATH)
utils.clear_folder(config.PIC_PATH)
utils.clear_folder(config.SFX_PATH)
utils.clear_file(self.result_path)
utils.clear_mp3_files()
time.sleep(5)
def show_picture(self, data):
"""
Show a picture.
@param data raw picture data.
Note I'm using the XBMC PlaySlideshow command here, giving the pictures path as an argument.
This is a workaround for the fact that calling the XBMC ShowPicture method more than once seems
to crash XBMC?
"""
utils.clear_folder(self._TMP_DIR)
filename = 'picture%d.jpg' % int(time.time())
path = os.path.join(self._TMP_DIR, filename)
"""
write mode 'b' is needed for Windows compatibility, since we're writing a binary file here.
"""
f = open(path, 'wb')
f.write(data)
f.close()
self._http_api_request('PlaySlideshow(%s)' % self._TMP_DIR)
def show_picture(self, data):
"""
Show a picture.
@param data raw picture data.
Note I'm using the XBMC PlaySlideshow command here, giving the pictures path as an argument.
This is a workaround for the fact that calling the XBMC ShowPicture method more than once seems
to crash XBMC?
"""
utils.clear_folder(self._TMP_DIR)
filename = 'picture%d.jpg' % int(time.time())
path = os.path.join(self._TMP_DIR, filename)
"""
write mode 'b' is needed for Windows compatibility, since we're writing a binary file here.
"""
f = open(path, 'wb')
f.write(data)
f.close()
self._http_api_request('PlaySlideshow(%s)' % self._TMP_DIR)
def generate_dataset():
# ------------------------------------------------------
# Generate a folder of images randomly chosen from the imagenet dataset
# Images are generated up to a certain folder dimension (specified in mb)
# ------------------------------------------------------
source_folder = 'imagenet/ILSVRC2017_CLS-LOC/ILSVRC/Data/CLS-LOC/train'
destination_folder = 'generated_dataset'
folder_list = next(os.walk(source_folder))[1]
clear_folder(destination_folder)
# The average size of an image is ~200kb
# print("Fetching images", sep=' ', end='')
total_size = 0 # current byte size of folder
print("Fetching dataset...")
pbar = tqdm(total=train_set_dim)
while total_size < (train_set_dim * 2**20):
chosen_one = random.choice(folder_list)
img_path = random.choice(glob(source_folder + '/' + chosen_one + '/*.jpeg'))
size = os.path.getsize(img_path)
total_size += size
pbar.update(size / 2**20)
shutil.copy(img_path, destination_folder)
pbar.close()
print("\nDone")
def main():
# ------------------------------------------------------
# Run predictor on some validation images
# ------------------------------------------------------
# Toggle to force prediction on cpu (if gpu is busy)
# os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
if tf.test.gpu_device_name():
print('GPU found')
else:
print("No GPU found")
# Latest model is loaded, as only improvements are saved
model_1_weights_path = max(glob('models/*.hdf5'), key=os.path.getctime)
model_2_weights_path = min(glob('models/*.hdf5'), key=os.path.getctime)
model_1 = build_model()
model_2 = build_model()
model_1.load_weights(model_1_weights_path)
model_2.load_weights(model_2_weights_path)
print(model_1.summary())
predict_folder = 'test_images/misc'
names = [f for f in os.listdir(predict_folder) if f.lower().endswith('.jpg')]
names_jpeg = [f for f in os.listdir(predict_folder) if f.lower().endswith('.jpeg')]
names_png = [f for f in os.listdir(predict_folder) if f.lower().endswith('.png')]
names = names + names_jpeg + names_png
# Pick 10 samples from validation set
# samples = random.sample(names, 10)
height, width = img_rows // 4, img_cols // 4
# Load the array of quantized ab value
q_ab = np.load(os.path.join(data_dir, "lab_gamut.npy"))
nb_q = q_ab.shape[0]
if not os.path.exists('output_images'):
os.makedirs('output_images')
clear_folder('output_images')
print("----------------------------------------\n"
"Prediction '1' based on " + model_1_weights_path + "\n"
"Prediction '2' based on " + model_2_weights_path + "\n"
"----------------------------------------")
for i in range(len(names)):
image_name = names[i]
filename = os.path.join(predict_folder, image_name)
print('Processing image: {}'.format(filename))
# b: 0 <=b<=255, g: 0 <=g<=255, r: 0 <=r<=255.
bgr = cv2.imread(filename)
gray = cv2.imread(filename, 0)
bgr = cv2.resize(bgr, (img_rows, img_cols), interpolation=cv2.INTER_CUBIC)
gray = cv2.resize(gray, (img_rows, img_cols), interpolation=cv2.INTER_CUBIC)
lab = cv2.cvtColor(bgr, cv2.COLOR_BGR2LAB)
x_test = np.empty((1, img_rows, img_cols, 1), dtype=np.float32)
x_test[0, :, :, 0] = gray / 255.
out_bgr_1 = colorize(model_1, x_test, height, width, nb_q, q_ab, lab)
out_bgr_2 = colorize(model_2, x_test, height, width, nb_q, q_ab, lab)
# cv2.imwrite('output_images/{}_bw.png'.format(i), gray)
cv2.imwrite('output_images/{}_gt.png'.format(i), bgr)
cv2.imwrite('output_images/{}_1.png'.format(i), out_bgr_1)
cv2.imwrite('output_images/{}_2.png'.format(i), out_bgr_2)
K.clear_session()
torch.manual_seed(FLAGS.seed)
if FLAGS.cuda:
torch.cuda.manual_seed(FLAGS.seed)
np.random.seed(FLAGS.seed)
cudnn.benchmark = True
try:
import accimage
torchvision.set_image_backend('accimage')
print('Image loader backend: accimage')
except:
print('Image loader backend: PIL')
if FLAGS.train_single:
utils.clear_folder(FLAGS.out_dir)
log_file = os.path.join(FLAGS.out_dir, 'log.txt')
print("Logging to {}\n".format(log_file))
sys.stdout = utils.StdOut(log_file)
print("PyTorch version: {}".format(torch.__version__))
print("CUDA version: {}\n".format(torch.version.cuda))
print(" " * 9 + "Args" + " " * 9 + "| " + "Type" + \
" | " + "Value")
print("-" * 50)
for arg in vars(FLAGS):
arg_str = str(arg)
var_str = str(getattr(FLAGS, arg))
type_str = str(type(getattr(FLAGS, arg)).__name__)
def train(self,
learning_rate,
reverse_token_dict,
dropout_input_keep_prob=0.8,
restore_model=False):
if not restore_model:
clear_folder(self.log_path)
clear_folder(self.model_path)
self._init_placeholders()
self._build_sequence()
self._build_optimizer(learning_rate)
init = tf.global_variables_initializer()
saver = tf.train.Saver(max_to_keep=2,
keep_checkpoint_every_n_hours=1)
else:
saver = tf.train.import_meta_graph(model_meta_file(
self.model_path))
writer = tf.summary.FileWriter(self.log_path)
merged_summary_op = tf.summary.merge_all()
with tf.Session(config=self.config) as sess:
if not restore_model:
sess.run(init)
writer.add_graph(sess.graph)
else:
print 'restore trained models from {}'.format(self.model_path)
saver.restore(sess,
tf.train.latest_checkpoint(self.model_path))
self._restore_placeholders(sess)
self._restore_operation_variables(sess)
step = 0
start_time = time.time()
while step < self.num_batches:
feed_content = self.next_feed(dropout_input_keep_prob)
_, step, summary, loss_value = sess.run([
self.train_op, self.increment_global_step_op,
merged_summary_op, self.loss
], feed_content)
if step % self.saving_steps == 0:
saver.save(sess,
os.path.join(self.model_path, 'models'),
global_step=step)
if step == 1 or step % self.display_steps == 0:
print 'step {}, minibatch loss: {}'.format(
step, loss_value)
writer.add_summary(summary, step)
if step != 1:
print 'every {} steps, it takes {:.2f} minutes...'.format(
self.display_steps,
(1. * time.time() - start_time) / 60.)
start_time = time.time()
predict_ = sess.run(self.decoder_prediction, feed_content)
for i, (inp, pred) in enumerate(
zip(feed_content[self.encoder_inputs].T,
predict_.T)):
print ' sample {}:'.format(i + 1)
print ' input > {}'.format(
map(reverse_token_dict.get, inp))
print ' predicted > {}'.format(
map(reverse_token_dict.get, pred))
if i >= 5:
break
step += 1
saver.save(sess,
os.path.join(self.model_path, 'final_model'),
global_step=step)
def main(args):
extractor = FrameFeaturesExtractor()
activity_dict = dict()
information = Information()
activity_list = os.listdir(args.input_dir)
for i, activity in enumerate(activity_list):
activity_dict[activity] = i
output_activity_dir = os.path.join(args.output_dir, activity)
if not os.path.exists(output_activity_dir):
os.makedirs(output_activity_dir)
utils.clear_folder(output_activity_dir)
curr_activ_path = os.path.join(args.input_dir, activity)
video_list = os.listdir(curr_activ_path)
for video_name in video_list:
information.size += 1
video_output_path = os.path.join(
output_activity_dir,
"{}_features.tfrecord".format(os.path.splitext(video_name)[0]))
writer = tf.python_io.TFRecordWriter(video_output_path)
current_video_path = os.path.join(curr_activ_path, video_name)
print("Extracting features from video %s of activity %s" %
(video_name, activity))
reader = tf.python_io.tf_record_iterator(current_video_path)
for data in reader:
input_example = tf.train.Example()
input_example.ParseFromString(data)
height = int(input_example.features.feature['image/height'].
int64_list.value[0])
width = int(input_example.features.feature['image/width'].
int64_list.value[0])
ids = input_example.features.feature[
'image/ids'].int64_list.value
image = np.frombuffer(
input_example.features.feature['image/array'].bytes_list.
value[0], np.uint8)
image = image.reshape((width, height, 3))
features = extractor.extract_features(image, ids)
examples = _convert_to_example(features)
for item in examples:
writer.write(item.SerializeToString())
information.embedding_sizes = len(features[0])
writer.close()
class_ids_file = os.path.join(args.output_dir, CLASS_IDS_FILENAME)
with open(class_ids_file, "w") as f:
for class_name in activity_dict.keys():
f.write("{}\t{}\n".format(activity_dict[class_name], class_name))
utils.save_obj(information,
os.path.join(args.output_dir, INFORMATION_FILENAME))
OUT_PATH = 'output'
LOG_FILE = os.path.join(OUT_PATH, 'log.txt')
BATCH_SIZE = 128 # Adjust this value according to your GPU memory
IMAGE_CHANNEL = 1
# IMAGE_CHANNEL = 3
Z_DIM = 100
G_HIDDEN = 64
X_DIM = 64
D_HIDDEN = 64
EPOCH_NUM = 25
REAL_LABEL = 1
FAKE_LABEL = 0
lr = 2e-4
seed = 1 # Change to None to get different results at each run
utils.clear_folder(OUT_PATH)
print("Logging to {}\n".format(LOG_FILE))
sys.stdout = utils.StdOut(LOG_FILE)
CUDA = CUDA and torch.cuda.is_available()
print("PyTorch version: {}".format(torch.__version__))
if CUDA:
print("CUDA version: {}\n".format(torch.version.cuda))
if seed is None:
seed = np.random.randint(1, 10000)
print("Random Seed: ", seed)
np.random.seed(seed)
torch.manual_seed(seed)
if CUDA:
torch.cuda.manual_seed(seed)
cudnn.benchmark = True # May train faster but cost more memory
def train(config,
batches,
reverse_token_dict,
dropout_input_keep_prob=0.8,
restore_model=False,
dual_outputs=False):
if not restore_model:
clear_folder(config.log_path)
clear_folder(config.model_path)
placeholders, global_step_ = _init_placeholders()
increment_global_step_op = tf.assign(global_step_,
global_step_ + config.batch_size,
name='increment_step')
decoder_prediction, decoder_logits, inference_set = _build_sequence(
placeholders, config)
mean_encoder_inputs_embedded_, mean_encoder_outputs_, final_cell_state_, final_hidden_state_ = inference_set
loss, train_op = _build_optimizer(placeholders, decoder_logits, config)
init = tf.global_variables_initializer()
saver = tf.train.Saver(max_to_keep=2, keep_checkpoint_every_n_hours=1)
else:
saver = tf.train.import_meta_graph(model_meta_file(config.model_path))
writer = tf.summary.FileWriter(config.log_path)
merged_summary_op = tf.summary.merge_all()
with tf.Session(config=config.sess_config) as sess:
if not restore_model:
sess.run(init)
writer.add_graph(sess.graph)
step = 0
else:
print 'restore trained models from {}'.format(config.model_path)
increment_global_step_op = sess.graph.get_tensor_by_name(
"increment_step:0")
saver.restore(sess, tf.train.latest_checkpoint(config.model_path))
placeholders, global_step_ = _restore_placeholders(sess)
decoder_prediction, loss, train_op = _restore_operation_variables(
sess)
step = sess.run(global_step_) ## retrieve the step from variable
start_time = time.time()
while step < config.num_batches:
if dual_outputs:
feed_content = dual_next_feed(placeholders, batches,
dropout_input_keep_prob)
else:
feed_content = next_feed(placeholders, batches,
dropout_input_keep_prob)
_ = sess.run([train_op], feed_content)
step += 1
if step == 1 or step % config.display_steps == 0:
_, summary, loss_value = sess.run(
[increment_global_step_op, merged_summary_op, loss],
feed_content)
print 'step {}, minibatch loss: {}'.format(step, loss_value)
writer.add_summary(summary, step)
if step != 1:
print 'every {} steps, it takes {:.2f} minutes...'.format(
config.display_steps,
(1. * time.time() - start_time) / 60.)
start_time = time.time()
predict_ = sess.run(decoder_prediction, feed_content)
for i, (inp, pred) in enumerate(
zip(feed_content[placeholders['encoder_inputs']].T,
predict_.T)):
print ' sample {}:'.format(i + 1)
print ' input > {}'.format(
map(reverse_token_dict.get, inp))
print ' predicted > {}'.format(
map(reverse_token_dict.get, pred))
if i >= 5:
break
if step % config.saving_steps == 0:
saver.save(sess,
os.path.join(config.model_path, 'models'),
global_step=step)
saver.save(sess,
os.path.join(config.model_path, 'final_model'),
global_step=step)
def train(self, data_generator, test_data_generator=None, dropout_input_keep_prob=0.8):
clear_folder(self.log_path)
clear_folder(self.model_path)
self._init_placeholders()
# build the model
# self.pred = self.RNN(self.x, self.meta_x, self.model_name, self.FC_layers)
self.pred = self.RNN()
print 'the predicting tensor: ', self.pred
optimizer = self._init_optimizer(self.learning_rate) # the optimizer for model building
if test_data_generator is not None:
test_data_size = test_data_generator.total_row_counts
else:
test_data_size = self.batch_size
test_eval_op = self.create_eval_op(test_data_size, 'test_eval') # eval operation using test data
train_eval_op = self.create_eval_op(self.batch_size, 'train_eval') # eval operation using train data
init = tf.global_variables_initializer()
saver = tf.train.Saver(max_to_keep=5, keep_checkpoint_every_n_hours=1)
start_time = time.time()
print 'models to be written into: ', self.model_path
print 'logs to be written into: ', self.log_path
writer = tf.summary.FileWriter(self.log_path)
with tf.Session(config=self.config) as sess:
print self.config.gpu_options
# Launch the graph
sess.run(init)
step = 1
train_MAE, test_MAE = "unavailable", "unavailable"
writer.add_graph(sess.graph)
'''
with tf.name_scope('weight_matrix'):
weight_matrix = sess.graph.get_tensor_by_name("fully_connect_layer/fully_connect_layer_2/weights:0")
self.variable_summaries(weight_matrix, 'weight_matrix')
'''
merged_summary_op = tf.summary.merge_all()
with tf.name_scope('training'):
while step * self.batch_size < self.num_epochs * data_generator.total_row_counts:
train_feed = self._generate_feed(data_generator, self.batch_size, dropout_input_keep_prob)
_, step = sess.run([optimizer, self.increment_global_step_op], feed_dict=train_feed)
if step % self.display_step == 0:
# to validate using test data
if test_data_generator is not None:
# use all the test data every time
summary, test_MAE = sess.run([merged_summary_op, test_eval_op],
feed_dict=self._generate_feed(test_data_generator,
test_data_generator.total_row_counts,
1.))
train_MAE = sess.run(train_eval_op, feed_dict=train_feed)
else:
summary, train_MAE = sess.run([merged_summary_op, train_eval_op], feed_dict=train_feed)
writer.add_summary(summary, step)
saver.save(sess, os.path.join(self.model_path, 'models'), global_step=step)
cur_time = time.time()
print "step {}, train MAE: {}, test MAE: {}, using {:.2f} seconds".format(step,
str(train_MAE),
str(test_MAE),
(cur_time - start_time))
start_time = cur_time
step += 1
saver.save(sess, os.path.join(self.model_path, 'final_model'), global_step=step)
print "Optimization Finished!"
def train(placeholders,
config,
train_op,
test_eval_op,
data_generator,
test_data_generator=None,
USE_CPU=True):
clear_folder(config.log_path)
clear_folder(config.model_path)
saver = tf.train.Saver(max_to_keep=5, keep_checkpoint_every_n_hours=1)
print 'models to be written into: ', config.model_path
print 'logs to be written into: ', config.log_path
start_time = time.time()
writer = tf.summary.FileWriter(config.log_path)
init = tf.global_variables_initializer()
merged_summary_op = tf.summary.merge_all()
generate_tensorboard_script(
config.log_path) # create the script to start a tensorboard session
if USE_CPU:
NUM_THREADS = multiprocessing.cpu_count()
sess_config = tf.ConfigProto(intra_op_parallelism_threads=NUM_THREADS)
os.environ[
'CUDA_VISIBLE_DEVICES'] = '-1' # the only way to completely not use GPU
else:
sess_config = tf.ConfigProto(log_device_placement=False)
sess_config.gpu_options.per_process_gpu_memory_fraction = 0.05
with tf.Session(config=sess_config) as sess:
sess.run(init)
step = 1
train_MAE, test_MAE = "unavailable", "unavailable"
writer.add_graph(sess.graph)
with tf.name_scope('training'):
while step * config.batch_size < config.num_epochs * data_generator.total_row_counts:
train_feed = _generate_feed(placeholders, data_generator,
config.batch_size,
config.dropout_input_keep_prob)
_ = sess.run([train_op], feed_dict=train_feed)
step += 1
if step % config.display_step == 0:
if test_data_generator is not None:
summary, test_MAE = sess.run(
[merged_summary_op, test_eval_op],
feed_dict=_generate_feed(
placeholders, test_data_generator,
test_data_generator.total_row_counts, 1.))
else:
summary, test_MAE = sess.run(
[merged_summary_op, test_eval_op],
feed_dict=_generate_feed(
placeholders, data_generator,
data_generator.total_row_counts, 1.))
train_MAE = sess.run(test_eval_op, feed_dict=train_feed)
cur_time = time.time()
writer.add_summary(summary, step)
saver.save(sess,
os.path.join(config.model_path, 'models'),
global_step=step)
print "step {}, train MAE: {}, test MAE: {}, using {:.2f} seconds".format(
step, str(train_MAE), str(test_MAE),
(cur_time - start_time))
start_time = cur_time
print "Optimization Finished!"
