def export_neglected_file_list(monitoring_dir, ROOT_DIR, LOG_DIR, backup_file_list):
today_obj = datetime.datetime.today()
today_str = str(today_obj.year)+str(today_obj.month)+str(today_obj.day)
export_name = today_str + "_neglected_files.log"
export_path = fs.join([ROOT_DIR, LOG_DIR, export_name])
if not fs.exists(fs.join([ROOT_DIR, LOG_DIR])):
try:
fs.mkdir(fs.join([ROOT_DIR, LOG_DIR]))
except:
print ("Can't create LOG_DIR in Func:", export_neglected_file_list)
try:
fs.touch(export_path)
file = fs.open(export_path, 'w')
for f in backup_file_list:
try:
file.write('================================================')
file.write('\n')
file.write(fs.filename(f))
file.write('\n')
file.write(fs.dirname(f))
file.write('\n')
except:
print("Cant' write export file in func: export_neglected_file_list")
except:
print ("cant export in func: export_neglected_file_list")
def load_meta_data(data_src, wiki_src, imdb_src):
with open(fs.join(data_src, wiki_src), 'rb') as file:
wiki_meta = pickle.load(file)
with open(fs.join(data_src, imdb_src), 'rb') as file:
imdb_meta = pickle.load(file)
return merge_meta_data(imdb_meta, wiki_meta)
def main():
if not fs.exists(DATA_DST):
fs.mkdir(DATA_DST)
meta_all = utils.shuffle_meta_data(utils.load_meta_data(DATA_SRC, WIKI_META_OBJ, IMDB_META_OBJ))
train, test = utils.split_meta_data(meta_all, TRAIN_TEST_SPLIT)
train, val = utils.split_meta_data(train, TRAIN_VAL_SPLIT)
# Free the memory
del meta_all
gc.collect()
print("Converting blocks")
print(" [train] %i Sapmles. %i Blocks required" % (len(train['path']), math.ceil(len(train['path']) / SAMPLES_PER_SPLIT)))
for i in range(math.ceil(len(train['path']) / SAMPLES_PER_SPLIT)):
X_train, y_age, y_gender = utils.get_img_array(train, DATA_SRC, age_classes, img_dim=INPUT_DIM, split=i, num_samples_per_split=SAMPLES_PER_SPLIT)
np.save(fs.add_suffix(fs.join(DATA_DST, TRAIN_DATA_OBJ), '_%02d' % i), X_train)
np.save(fs.add_suffix(fs.join(DATA_DST, TRAIN_DATA_OBJ), '_label_age_%02d' % i), y_age)
np.save(fs.add_suffix(fs.join(DATA_DST, TRAIN_DATA_OBJ), '_label_gender_%02d' % i), y_gender)
# Remove the array from memory
del X_train
del y_age
del y_gender
gc.collect()
print(" [val] %i Sapmles. 1 Block forced" % (len(val['path'])))
X_val, y_age, y_gender = utils.get_img_array(val, DATA_SRC, age_classes, img_dim=INPUT_DIM, num_samples_per_split=len(val['path']))
np.save(fs.join(DATA_DST, VAL_DATA_OBJ), X_val)
np.save(fs.add_suffix(fs.join(DATA_DST, VAL_DATA_OBJ), '_label_age'), y_age)
np.save(fs.add_suffix(fs.join(DATA_DST, VAL_DATA_OBJ), '_label_gender'), y_gender)
# Remove the array from memory
del X_val
del y_age
del y_gender
gc.collect()
print("[test] %i Sapmles. %i Blocks required" % (len(test['path']), math.ceil(len(test['path']) / SAMPLES_PER_SPLIT)))
for i in range(math.ceil(len(test['path']) / SAMPLES_PER_SPLIT)):
X_test, y_age, y_gender = utils.get_img_array(test, DATA_SRC, age_classes, img_dim=INPUT_DIM, split=i, num_samples_per_split=SAMPLES_PER_SPLIT)
np.save(fs.add_suffix(fs.join(DATA_DST, TEST_DATA_OBJ), '_%02d' % i), X_test)
np.save(fs.add_suffix(fs.join(DATA_DST, TEST_DATA_OBJ), '_label_age_%02d' % i), y_age)
np.save(fs.add_suffix(fs.join(DATA_DST, TEST_DATA_OBJ), '_label_gender_%02d' % i), y_gender)
# Remove the array from memory
del X_test
del y_age
del y_gender
gc.collect()
def main():
if not fs.exists(DATA_DST):
fs.mkdir(DATA_DST)
meta_all = utils.shuffle_meta_data(utils.load_meta_data(DATA_SRC, WIKI_META_OBJ, IMDB_META_OBJ))
train, test = utils.split_meta_data(meta_all, TRAIN_TEST_SPLIT)
train, val = utils.split_meta_data(train, TRAIN_VAL_SPLIT)
# Free the memory
del meta_all
gc.collect()
print("Converting blocks")
print(" [train] %i Sapmles" % (train_samples))
i = 0
X_train, y_age, y_gender = utils.get_img_array(train, DATA_SRC, age_classes, img_dim=INPUT_DIM, split=i, num_samples_per_split=train_samples)
np.save(fs.add_suffix(fs.join(DATA_DST, TRAIN_DATA_OBJ), '_%02d' % i), X_train)
np.save(fs.add_suffix(fs.join(DATA_DST, TRAIN_DATA_OBJ), '_label_age_%02d' % i), y_age)
np.save(fs.add_suffix(fs.join(DATA_DST, TRAIN_DATA_OBJ), '_label_gender_%02d' % i), y_gender)
# Remove the array from memory
del X_train
del y_age
del y_gender
gc.collect()
print(" [val] %i Sapmles" % (val_samples))
X_val, y_age, y_gender = utils.get_img_array(val, DATA_SRC, age_classes, img_dim=INPUT_DIM, num_samples_per_split=val_samples)
np.save(fs.join(DATA_DST, VAL_DATA_OBJ), X_val)
np.save(fs.add_suffix(fs.join(DATA_DST, VAL_DATA_OBJ), '_label_age'), y_age)
np.save(fs.add_suffix(fs.join(DATA_DST, VAL_DATA_OBJ), '_label_gender'), y_gender)
# Remove the array from memory
del X_val
del y_age
del y_gender
gc.collect()
print("[test] %i Sapmles" % (test_samples))
i = 0
X_test, y_age, y_gender = utils.get_img_array(test, DATA_SRC, age_classes, img_dim=INPUT_DIM, split=i, num_samples_per_split=test_samples)
np.save(fs.add_suffix(fs.join(DATA_DST, TEST_DATA_OBJ), '_%02d' % i), X_test)
np.save(fs.add_suffix(fs.join(DATA_DST, TEST_DATA_OBJ), '_label_age_%02d' % i), y_age)
np.save(fs.add_suffix(fs.join(DATA_DST, TEST_DATA_OBJ), '_label_gender_%02d' % i), y_gender)
# Remove the array from memory
del X_test
del y_age
del y_gender
gc.collect()
def _remove_test_dir(ROOT_DIR, TARGET_ROOT_DIR, backup_dir_name):
backup_dir_name = fs.join([ROOT_DIR, TARGET_ROOT_DIR, backup_dir_name])
try:
fs.rmdir(backup_dir_name, recursive=True)
print (backup_dir_name, "was successfully removed")
except:
print ("ERROR", backup_dir_name, "couldn't removed by func:_remove_test_dir")
def create_new_backup_root_dir(ROOT_DIR, TARGET_ROOT_DIR, backup_dir_name):
backup_dir_path = fs.join([ROOT_DIR, TARGET_ROOT_DIR, backup_dir_name])
if not fs.exists(backup_dir_path):
try:
fs.mkdir(backup_dir_path)
print (backup_dir_path, "was successfully created")
except:
print ("Cant create a backup directory in func: create_new_backup_root_dir")
def test_addpath():
TEST_MODULE = 'test_foo_bar.py'
fs.touch(fs.join(TEST_DIR, TEST_MODULE))
with pytest.raises(ImportError):
import test_foo_bar
fs.addpath(TEST_DIR)
import test_foo_bar
def make():
"""Generate the current shell scripts from the templates"""
clean()
for _file in fs.find('*.sh', SRC_DIR):
tplname = _file.replace(SRC_DIR + '/', "")
dest = fs.join(DIST_DIR, fs.filename(tplname))
tpl = env.get_template(tplname)
fs.write(dest, tpl.render())
print("Writing template %s" % tplname)
def create_archive_tree(monitoring_dir, ROOT_DIR, TARGET_ROOT_DIR, backup_dir_name, backup_dir_list):
for source_full_path in backup_dir_list:
re_pattern = re.compile(r'%s' % monitoring_dir)
source_path = re.sub(re_pattern, '', source_full_path)
archive_path = fs.join([ROOT_DIR, TARGET_ROOT_DIR, backup_dir_name, source_path])
if not fs.exists(archive_path):
try:
fs.mkdir(archive_path)
except:
print (archive_path, "Can't create archive tree in func: create_archive_tree")
def get_img_array(meta_data,
data_src,
age_classes,
img_dim=(3, 224, 224),
split=0,
num_samples_per_split=100000,
dtype=np.float32):
i_start = min(split * num_samples_per_split, len(meta_data['path']))
i_stop = min(split * num_samples_per_split + num_samples_per_split,
len(meta_data['path']))
num_samples = i_stop - i_start
X = np.zeros((num_samples, img_dim[0], img_dim[1], img_dim[2]),
dtype=dtype)
y_age = np.zeros((num_samples))
y_gender = np.zeros((num_samples))
print(' Allocating %s for dataset with shape (%i,%i,%i,%i)' % (sizeof_fmt(
X.nbytes), num_samples, img_dim[0], img_dim[1], img_dim[2]))
age_class = lambda x: age_classes.index(
next(filter(lambda e: x >= e[0] and x <= e[1], age_classes)))
for i in range(i_start, i_stop):
y_age[i - i_start] = age_class(meta_data['age'][i])
# Replace all non valid gender labels with male labels
y_gender[i - i_start] = meta_data['gender'][i] if int(
meta_data['gender'][i]) in [0, 1] else 1
abspath = fs.join(data_src, meta_data['path'][i])
# Catch errors
try:
with Image.open(abspath) as img:
img = img.resize(img_dim[1:3], RESIZE_TYPE).convert('RGB')
X[i - i_start] = np.asarray(img, dtype=dtype).transpose(
(2, 1, 0)) / 255
except OSError as e:
print("Error reading file %s" % abspath)
continue
return X, y_age, y_gender
batchsize = 64
learning_rate = 1e-2
decay_rate = 0.0
l2_reg = 1e-3
early_stopping_rounds = 10
use_class_weights = False
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
MODEL_NAME = 'VGG_16_GENDER_%i_%i_%i' % shp
DATASET_DIR = '/data/imdb-wiki-dataset'
MODEL_DIR = '/data/models'
LOGS_DIR = '/data/logs'
MODEL_PATH = fs.join(MODEL_DIR, '%s_%s.h5' % (MODEL_NAME, timestamp))
LOGS_PATH = fs.join(LOGS_DIR, '%s_%s.txt' % (MODEL_NAME, timestamp))
print("Loading Dataset ...")
# Initialize the datasets
ds_train = Dataset(DATASET_DIR, 'train')
ds_val = Dataset(DATASET_DIR, 'val')
# Get the class names for this dataset
class_names = ds_train.label_names
# Initialize the preprocessing pipeline
print("Setting up preprocessing ...")
tform = get_normalization_transform(
means=ds_train.get_mean(per_channel=True),
stds=ds_train.get_stddev(per_channel=True)
def test_join_array():
assert fs.join([os.sep, 'etc', 'var']) == os.sep + 'etc' + os.sep + 'var'
assert fs.join([os.sep, 'etc', 'var/']) == os.sep + 'etc' + os.sep + 'var/'
def test_join_arguments():
assert fs.join(os.sep, 'etc', 'var') == os.sep + 'etc' + os.sep + 'var'
assert fs.join(os.sep, 'etc', 'var/') == os.sep + 'etc' + os.sep + 'var/'
from __future__ import print_function
from fabric.api import task, local
from jinja2 import Environment, FileSystemLoader
import fs
ROOT_DIR = fs.cwd()
SRC_DIR = fs.join(ROOT_DIR, 'src')
DIST_DIR = fs.join(ROOT_DIR, 'dist')
env = Environment()
env.loader = FileSystemLoader(SRC_DIR)
@task
def clean():
"""Clean the dist/ directory"""
if (fs.isdir(DIST_DIR)):
fs.truncate(DIST_DIR)
@task
def make():
"""Generate the current shell scripts from the templates"""
clean()
for _file in fs.find('*.sh', SRC_DIR):
tplname = _file.replace(SRC_DIR + '/', "")
dest = fs.join(DIST_DIR, fs.filename(tplname))
tpl = env.get_template(tplname)
fs.write(dest, tpl.render())
print("Writing template %s" % tplname)
def get_monitoring_dir(ROOT_DIR, MONITORING_ROOT_DIR):
return fs.join([ROOT_DIR, MONITORING_ROOT_DIR])
