def extract(self, force_flag=False):
"""Checking the grid.xml file in the machine, if found use it.
Else, download from the server.
:param force_flag: force flag to extract.
"""
final_grid_xml_file = os.path.join(self.extract_dir(), 'grid.xml')
if not os.path.exists(self.extract_dir()):
mk_dir(self.extract_dir())
if force_flag or self.force_flag:
self.remove_extracted_files()
elif os.path.exists(final_grid_xml_file):
return final_grid_xml_file
# download data
local_path = self.fetch_file()
# move grid.xml to the correct directory
expected_grid_xml_file = os.path.join(local_path, 'output', 'grid.xml')
if not os.path.exists(expected_grid_xml_file):
raise FileNotFoundError('The output does not contain an '
'%s file.' % expected_grid_xml_file)
# move the file we care about to the top of the extract dir
shutil.copyfile(
os.path.join(self.extract_dir(), expected_grid_xml_file),
final_grid_xml_file)
if not os.path.exists(final_grid_xml_file):
raise CopyError('Error copying grid.xml')
return final_grid_xml_file
def fetch_file(self, retries=3):
"""Private helper to fetch a file from the sftp site.
:param retries: int - number of reattempts that should be made in
in case of network error etc.
e.g. for event 20110413170148 this file would be fetched::
20110413170148 directory
.. note:: If a cached copy of the file exits, the path to the cache
copy will simply be returned without invoking any network requests.
:return: A string for the dataset path on the local storage system.
:rtype: str
:raises: EventUndefinedError, NetworkError
"""
local_path = os.path.join(shakemap_cache_dir(), self.event_id)
local_parent_path = os.path.join(local_path, "output")
xml_file = os.path.join(local_parent_path, self.file_name())
if os.path.exists(xml_file):
return local_path
# fetch from sftp
trials = [i + 1 for i in xrange(retries)]
remote_path = os.path.join(self.sftpclient.workdir_path, self.event_id)
xml_remote_path = os.path.join(remote_path, "output", self.file_name())
for counter in trials:
last_error = None
try:
mk_dir(local_path)
mk_dir(os.path.join(local_path, "output"))
self.sftpclient.download_path(xml_remote_path, local_parent_path)
except NetworkError, e:
last_error = e
except:
def fit_model(model, X_train, y_train, X_val, y_val, nb_epochs, batch_size,
save_name, run, method):
start_decay_epoch = [30, 60]
decaylearningrate = train_callbacks.DecayLearningRate(start_decay_epoch)
mk_dir(db_name + f"_checkpoints/{method}/{run}")
callbacks = [
ModelCheckpoint(db_name + f"_checkpoints/{method}/{run}" +
"/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
monitor="val_loss",
verbose=1,
save_best_only=True,
mode="auto"), decaylearningrate
]
hist = model.fit_generator(generator=data_generator_reg(
X=X_train, Y=y_train, batch_size=batch_size),
steps_per_epoch=X_train.shape[0] // batch_size,
validation_data=(X_val, [y_val]),
epochs=nb_epochs,
verbose=1,
callbacks=callbacks)
logging.debug("Saving weights...")
model.save_weights(os.path.join(db_name + "_models/" + save_name,
save_name + '.h5'),
overwrite=True)
pd.DataFrame(hist.history).to_hdf(
os.path.join(db_name + "_models/" + save_name,
'history_' + save_name + '.h5'), "history")
def process_infolist(self, baseurl, infolist, fromdate, todate):
newdls = []
found = False
finished = False
infoWithDate = []
for info in infolist:
if info.has_key(self.DATE):
infoWithDate.append(info)
infoWithDate.sort(cmp=lambda x, y: cmp(x[self.DATE], y[self.DATE]))
for info in infoWithDate:
dateobj = info[self.DATE]
if dateobj >= fromdate and dateobj <= todate:
found = True
datestr = dateobj.date().__str__()
tmprel = os.path.join(self.name, datestr)
rawdatedir = os.path.join(self.rawdir, tmprel)
utils.mk_dir(rawdatedir)
metadatedir = os.path.join(self.metadir, tmprel)
utils.mk_dir(metadatedir)
relurl = self.download_doc(baseurl, info, tmprel)
if relurl:
newdls.append(relurl)
elif found:
finished = True
if not finished or not found or not self.INFO_ORDERED:
return False, newdls
else:
return True, newdls
def process_infolist(self, baseurl, infolist, fromdate, todate):
newdls = []
found = False
finished = False
infoWithDate = []
for info in infolist:
if info.has_key(self.DATE):
infoWithDate.append(info)
infoWithDate.sort(cmp = lambda x, y: cmp(x[self.DATE], y[self.DATE]))
for info in infoWithDate:
dateobj = info[self.DATE]
if dateobj >= fromdate and dateobj <= todate:
found = True
datestr = dateobj.date().__str__()
tmprel = os.path.join (self.name, datestr)
rawdatedir = os.path.join (self.rawdir, tmprel)
utils.mk_dir(rawdatedir)
metadatedir = os.path.join (self.metadir, tmprel)
utils.mk_dir(metadatedir)
relurl = self.download_doc(baseurl, info, tmprel)
if relurl:
newdls.append(relurl)
elif found:
finished = True
if not finished or not found or not self.INFO_ORDERED:
return False, newdls
else:
return True, newdls
def extract(self, force_flag=False):
"""Checking the grid.xml file in the machine, if found use it.
Else, download from the server.
:param force_flag: force flag to extract.
"""
final_grid_xml_file = os.path.join(self.extract_dir(), "grid.xml")
if not os.path.exists(self.extract_dir()):
mk_dir(self.extract_dir())
if force_flag or self.force_flag:
self.remove_extracted_files()
elif os.path.exists(final_grid_xml_file):
return final_grid_xml_file
# download data
local_path = self.fetch_file()
# move grid.xml to the correct directory
expected_grid_xml_file = os.path.join(local_path, "output", "grid.xml")
if not os.path.exists(expected_grid_xml_file):
raise FileNotFoundError("The output does not contain an " "%s file." % expected_grid_xml_file)
# move the file we care about to the top of the extract dir
shutil.copyfile(os.path.join(self.extract_dir(), expected_grid_xml_file), final_grid_xml_file)
if not os.path.exists(final_grid_xml_file):
raise CopyError("Error copying grid.xml")
return final_grid_xml_file
def main():
args = get_args()
input_path = args.input
batch_size = args.batch_size
nb_epochs = args.nb_epochs
depth = args.depth
k = args.width
validation_split = args.validation_split
logging.debug("Loading data...")
image, gender, age, _, image_size, _ = load_data(input_path)
X_data = image
y_data_g = np_utils.to_categorical(gender, 2)
#Quantize the age into 21 bins:
age_bins = np.linspace(0, 100, 21)
age_step = np.digitize(age, age_bins)
y_data_a = np_utils.to_categorical(age_step, 21)
#model = WideResNet(image_size, depth=depth, k=k)()
model = TYY_1stream(image_size)()
sgd = SGD(lr=0.1, momentum=0.9, nesterov=True)
model.compile(
optimizer=sgd,
loss=["categorical_crossentropy", "categorical_crossentropy"],
metrics=['accuracy'])
logging.debug("Model summary...")
model.count_params()
model.summary()
logging.debug("Saving model...")
mk_dir("models")
#with open(os.path.join("models", "WRN_{}_{}.json".format(depth, k)), "w") as f:
with open(os.path.join("models", "TYY_1stream.json"), "w") as f:
f.write(model.to_json())
mk_dir("checkpoints")
callbacks = [
LearningRateScheduler(schedule=Schedule(nb_epochs)),
ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
monitor="val_loss",
verbose=1,
save_best_only=True,
mode="auto")
]
logging.debug("Running training...")
hist = model.fit(X_data, [y_data_g, y_data_a],
batch_size=batch_size,
epochs=nb_epochs,
callbacks=callbacks,
validation_split=validation_split)
logging.debug("Saving weights...")
#model.save_weights(os.path.join("models", "WRN_{}_{}.h5".format(depth, k)), overwrite=True)
model.save_weights(os.path.join("models", "TYY_1stream.h5"),
overwrite=True)
#pd.DataFrame(hist.history).to_hdf(os.path.join("models", "history_{}_{}.h5".format(depth, k)), "history")
pd.DataFrame(hist.history).to_hdf(
os.path.join("models", "history_TYY_1steam.h5"), "history")
def download_order(self, relpath, dateobj, metainfo, onclick):
reobj = re.search('myfunViewDownLoad\s*\(\s*"(?P<ccin>\d+)"\s*,\s*"(?P<orderno>\d+)"\s*,\s*"(?P<flag>\w+)"\s*,\s*"(?P<casedetail>.+)"\s*,\s*"\w+"', onclick)
if not reobj:
self.logger.warning(u'Could not get parameters in onclick: %s' % onclick)
return None
groupdict = reobj.groupdict()
ccin = groupdict['ccin']
orderno = groupdict['orderno']
flag = groupdict['flag']
casedetail = groupdict['casedetail']
metainfo['caseno'] = casedetail
filename = self.get_filename(casedetail)
if not filename:
self.logger.warning(u'Could not get filename from %s' % casedetail)
return None
datestr = dateobj.__str__()
utils.mk_dir(os.path.join(self.rawdir, self.name, datestr))
utils.mk_dir(os.path.join(self.metadir, self.name, datestr))
relurl = os.path.join(relpath, datestr, filename)
filepath = os.path.join(self.rawdir, relurl)
metapath = os.path.join(self.metadir, relurl)
if os.path.exists(filepath):
self.logger.warning(u'Raw file already exists, skipping: %s ' % relurl)
else:
#ccin_no=001016200801769&order_no=2&flag=v&casedetail=MISC.CIVIL+APPLICATION%2F1769%2F2008&download_token_value_id=1367853726545
self.logger.info(u'Downloading %s' % relurl)
postdata = [('ccin_no', ccin), ('order_no', orderno), \
('flag', flag), ('casedetail', casedetail), \
('download_token_value_id', int(time.time())) ]
webpage = self.download_url(self.orderurl, \
referer=self.caseurl,\
loadcookies = self.cookiefile.name,\
postdata = postdata)
if webpage:
self.logger.info(u'Saving %s' % filepath)
utils.save_file(filepath, webpage)
else:
self.logger.warning(u'Could not download ccin: %s number: %s ' % (ccin, orderno))
if os.path.exists(filepath) and metainfo and \
(self.updateMeta or not os.path.exists(metapath)):
self.logger.info(u'Metainfo: %s' % metainfo)
utils.print_tag_file(metapath, metainfo)
if os.path.exists(filepath):
return relurl
return None
def attempt(request, id):
context = {}
context['queries'] = request.GET.copy()
attempt = Attempt.objects.get(id=id)
context['attempt'] = attempt
dependencies = Dependency.objects.filter(
attempt__id=id).order_by('package__name')
context['dependencies'] = dependencies
keyword_order = {'SELECT': 1, 'INSERT': 2, 'UPDATE': 3, 'DELETE': 4}
context['statistics'] = {}
for statistic in Statistic.objects.filter(attempt=attempt):
context['statistics'][statistic.description] = max(
statistic.count, context['statistics'].get(statistic.description,
0))
actions = Action.objects.filter(attempt=attempt)
context['actions'] = []
for action in actions:
fields = Field.objects.filter(action=action)
counters = Counter.objects.filter(action=action)
counters = sorted(counters,
key=lambda x: keyword_order.get(x.description, 10))
context['actions'].append({
'id': action.id,
'url': action.url,
'method': action.method,
'fields': fields,
'counters': counters
})
try:
image = Image.objects.get(attempt=attempt)
screenshot_filename = 'screenshot_{}.png'.format(attempt.id)
utils.mk_dir(
os.path.join(os.path.dirname(__file__), 'static', 'screenshots'))
screenshot = open(
os.path.join(os.path.dirname(__file__), 'static', 'screenshots',
screenshot_filename), 'wb')
screenshot.write(image.data)
screenshot.close()
context['screenshot'] = '/static/screenshots/' + screenshot_filename
except:
traceback.print_exc()
return render(request, 'attempt.html', context)
def sync(self, fromdate, todate):
dirname = os.path.join(self.rawdir, self.name)
utils.mk_dir(dirname)
dirname = os.path.join(self.metadir, self.name)
utils.mk_dir(dirname)
dls = []
for mainurl in self.mainurls:
while 1:
infolist, nexturl = self.download_info_page(mainurl)
finished, newdls = self.process_infolist(mainurl, infolist, \
fromdate, todate)
dls.extend(newdls)
if finished or nexturl == None:
break
self.logger.info(u'Going to the next page: %s' % nexturl)
mainurl = nexturl
return dls
def sync(self, fromdate, todate):
dirname = os.path.join (self.rawdir, self.name)
utils.mk_dir(dirname)
dirname = os.path.join (self.metadir, self.name)
utils.mk_dir(dirname)
dls = []
for mainurl in self.mainurls:
while 1:
infolist, nexturl = self.download_info_page(mainurl)
finished, newdls = self.process_infolist(mainurl, infolist, \
fromdate, todate)
dls.extend(newdls)
if finished or nexturl == None:
break
self.logger.info(u'Going to the next page: %s' % nexturl)
mainurl = nexturl
return dls
def download_path(self, remote_path, local_path):
""" Download remote_dir to local_dir.
:param remote_path: The remote path that will be downloaded to local,
:type remote_path: str
:param local_path: The target path on local,
:type local_path: str
EXAMPLE : remote_path = '20130111133900' will be downloaded to
local_dir/remote_path. It must be in the parent directory of remote
dir.
"""
# Check if remote_dir is exist
if not self.is_path_exist(remote_path):
print 'remote path is not exist %s' % remote_path
return False
if self.is_dir(remote_path):
# get directory name
dir_name = get_path_tail(remote_path)
# create directory in local machine
local_dir_path = os.path.join(local_path, dir_name)
mk_dir(local_dir_path)
# list all directory in remote path
list_dir = self.sftp.listdir(remote_path)
# iterate recursive
for my_dir in list_dir:
new_remote_path = os.path.join(remote_path, my_dir)
self.download_path(new_remote_path, local_dir_path)
else:
# download file to local_path
file_name = get_path_tail(remote_path)
local_file_path = os.path.join(local_path, file_name)
LOGGER.info('file %s will be downloaded to %s' %
(remote_path, local_file_path))
self.sftp.get(remote_path, local_file_path)
def get_model(method, image_size, v):
optMethod = Adam()
N_densenet = 3
depth_densenet = 3 * N_densenet + 4
if method == 'piven':
model = PI_Model(image_size, depth_densenet, method='piven')()
save_name = 'densenet_eli_reg_%d_%d' % (depth_densenet, image_size)
model.compile(optimizer=optMethod,
loss=piven_loss(eli=True),
metrics=[picp, mpiw, mae(method, v)])
elif method == 'qd':
model = PI_Model(image_size, depth_densenet, method='qd')()
save_name = 'densenet_qd_reg_%d_%d' % (depth_densenet, image_size)
model.compile(optimizer=optMethod,
loss=piven_loss(eli=False),
metrics=[picp, mpiw, mae(method, v)])
elif method == 'only_point':
model = DenseNet_reg(image_size, depth_densenet)()
save_name = 'densenet_reg_%d_%d' % (depth_densenet, image_size)
model.compile(optimizer=optMethod,
loss=["mae"],
metrics={'pred_a': 'mae'})
else:
raise ValueError(f"Invalid method name: {method}")
logging.debug("Model summary...")
model.count_params()
model.summary()
db_name = 'imdb'
logging.debug("Saving model...")
mk_dir(db_name + "_models")
mk_dir(db_name + "_models/" + save_name)
mk_dir(db_name + "_checkpoints")
mk_dir(db_name + f"_checkpoints/{method}")
with open(
os.path.join(db_name + "_models/" + save_name,
save_name + '.json'), "w") as f:
f.write(model.to_json())
return model, save_name
def fetch_file(self, retries=3):
"""Private helper to fetch a file from the sftp site.
:param retries: int - number of reattempts that should be made in
in case of network error etc.
e.g. for event 20110413170148 this file would be fetched::
20110413170148 directory
.. note:: If a cached copy of the file exits, the path to the cache
copy will simply be returned without invoking any network requests.
:return: A string for the dataset path on the local storage system.
:rtype: str
:raises: EventUndefinedError, NetworkError
"""
local_path = os.path.join(shakemap_cache_dir(), self.event_id)
local_parent_path = os.path.join(local_path, 'output')
xml_file = os.path.join(local_parent_path, self.file_name())
if os.path.exists(xml_file):
return local_path
# fetch from sftp
trials = [i + 1 for i in xrange(retries)]
remote_path = os.path.join(self.sftpclient.workdir_path, self.event_id)
xml_remote_path = os.path.join(remote_path, 'output', self.file_name())
for counter in trials:
last_error = None
try:
mk_dir(local_path)
mk_dir(os.path.join(local_path, 'output'))
self.sftpclient.download_path(xml_remote_path,
local_parent_path)
except NetworkError, e:
last_error = e
except:
def main():
args = get_args()
input_path = args.input
images_path = args.dataset_path
batch_size = args.batch_size
nb_epochs = args.nb_epochs
validation_split = args.validation_split
logging.debug("Loading data...")
dataset_name = 'imdb'
data_loader = DataManager(dataset_name, dataset_path=input_path)
ground_truth_data = data_loader.get_data()
train_keys, val_keys = split_imdb_data(ground_truth_data, validation_split)
print("Samples: Training - {}, Validation - {}".format(
len(train_keys), len(val_keys)))
input_shape = (IMG_SIZE, IMG_SIZE, 3)
# images_path = 'data/imdb_crop/'
image_generator = ImageGenerator(ground_truth_data,
batch_size,
input_shape[:2],
train_keys,
val_keys,
path_prefix=images_path,
vertical_flip_probability=0)
n_age_bins = 21
alpha = 1
model = MobileNetDeepEstimator(input_shape[0],
alpha,
n_age_bins,
weights='imagenet')()
opt = SGD(lr=0.001)
model.compile(
optimizer=opt,
loss=["binary_crossentropy", "categorical_crossentropy"],
metrics={
'gender': 'accuracy',
'age': 'accuracy'
},
)
logging.debug("Model summary...")
model.count_params()
model.summary()
logging.debug("Saving model...")
mk_dir("models")
with open(os.path.join("models", "MobileNet.json"), "w") as f:
f.write(model.to_json())
mk_dir("checkpoints")
run_id = "MobileNet - " + str(batch_size) + " " + '' \
.join(random
.SystemRandom()
.choice(string.ascii_uppercase) for _ in
range(10)
)
print(run_id)
reduce_lr = ReduceLROnPlateau(verbose=1, epsilon=0.001, patience=4)
callbacks = [
LearningRateScheduler(schedule=Schedule(nb_epochs)), reduce_lr,
ModelCheckpoint(os.path.join(
'checkpoints', 'weights.{epoch:02d}-{val_loss:.2f}.hdf5'),
monitor="val_loss",
verbose=1,
save_best_only=True,
mode="auto"),
TensorBoard(log_dir='logs/' + run_id)
]
logging.debug("Running training...")
logging.debug("steps : " + str(int((len(val_keys) / batch_size))))
hist = model.fit_generator(
image_generator.flow(mode='train'),
steps_per_epoch=int(len(train_keys) / batch_size),
epochs=nb_epochs,
callbacks=callbacks,
validation_data=image_generator.flow('val'),
validation_steps=int(len(val_keys) / batch_size))
logging.debug("Saving weights...")
model.save(os.path.join("models", "MobileNet_model.h5"))
model.save_weights(os.path.join("models", FINAL_WEIGHTS_PATH),
overwrite=True)
pd.DataFrame(hist.history).to_hdf(os.path.join("models", "history.h5"),
"history")
sys.exit(0)
leveldict = {'critical': logging.CRITICAL, 'error': logging.ERROR, \
'warning': logging.WARNING, 'info': logging.INFO, \
'debug': logging.DEBUG}
logfmt = '%(asctime)s: %(name)s: %(levelname)s %(message)s'
datefmt = '%Y-%m-%d %H:%M:%S'
if datadir == None:
print >> sys.stderr, 'No data directory specified'
print_usage(progname)
sys.exit(0)
statsdir = os.path.join(datadir, 'stats')
utils.mk_dir(statsdir)
if filename:
filename = os.path.join(statsdir, filename)
if filename:
logging.basicConfig(\
level = leveldict[debuglevel], \
format = logfmt, \
filename = filename, \
datefmt = datefmt \
)
else:
logging.basicConfig(\
level = leveldict[debuglevel], \
format = logfmt, \
if todate == None:
todate = datetime.datetime.today()
fromdate = todate - datetime.timedelta(days = 7)
else:
fromdate = todate
elif todate == None:
todate = fromdate
basedir = remlist[0]
rawdir = os.path.join(basedir, 'raw')
metadir = os.path.join(basedir, 'metatags')
statsdir = os.path.join(basedir, 'stats')
utils.mk_dir(rawdir)
utils.mk_dir(metadir)
utils.mk_dir(statsdir)
courtobjs = []
dldict = {'bombay' : bombay.Bombay, \
'kolkata' : kolkata.Kolkata, \
'kolkata_app' : kolkata_app.KolkataApp, \
'punjab' : punjab.Punjab, \
'uttaranchal' : uttaranchal.Uttaranchal, \
'delhi' : delhi.Delhi, \
'jharkhand' : jharkhand.Jharkhand, \
'gujarat' : gujarat.Gujarat, \
'rajasthan' : rajasthan.Rajasthan, \
'jodhpur' : jodhpur.Jodhpur, \
'karnataka' : karnataka.Karnataka, \
def main():
args = get_args()
input_path = args.input
batch_size = args.batch_size
nb_epochs = args.nb_epochs
depth = args.depth
k = args.width
validation_split = args.validation_split
use_augmentation = args.aug
logging.debug("Loading data...")
image, gender, age, race, _, image_size, _ = load_data(input_path)
X_data = image
y_data_g = np_utils.to_categorical(gender, 2)
y_data_a = np_utils.to_categorical(age, 101)
y_data_r = np_utils.to_categorical(race, 5)
model = WideResNet(image_size, depth=depth, k=k)()
sgd = SGD(lr=0.1, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd,
loss=[
"categorical_crossentropy", "categorical_crossentropy",
"categorical_crossentropy"
],
metrics=['accuracy'])
logging.debug("Model summary...")
model.count_params()
model.summary()
logging.debug("Saving model...")
mk_dir("models")
with open(os.path.join("models", "WRN_{}_{}.json".format(depth, k)),
"w") as f:
f.write(model.to_json())
mk_dir("checkpoints")
callbacks = [
LearningRateScheduler(schedule=Schedule(nb_epochs)),
ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
monitor="val_loss",
verbose=1,
save_best_only=True,
mode="auto")
]
logging.debug("Running training...")
data_num = len(X_data)
indexes = np.arange(data_num)
np.random.shuffle(indexes)
X_data = X_data[indexes]
y_data_g = y_data_g[indexes]
y_data_a = y_data_a[indexes]
y_data_r = y_data_r[indexes]
train_num = int(data_num * (1 - validation_split))
X_train = X_data[:train_num]
X_test = X_data[train_num:]
y_train_g = y_data_g[:train_num]
y_test_g = y_data_g[train_num:]
y_train_a = y_data_a[:train_num]
y_test_a = y_data_a[train_num:]
y_train_r = y_data_r[:train_num]
y_test_r = y_data_r[train_num:]
if use_augmentation:
datagen = ImageDataGenerator(width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
preprocessing_function=get_random_eraser(
v_l=0, v_h=255))
training_generator = MixupGenerator(X_train,
[y_train_g, y_train_a, y_train_r],
batch_size=batch_size,
alpha=0.2,
datagen=datagen)()
hist = model.fit_generator(
generator=training_generator,
steps_per_epoch=train_num // batch_size,
validation_data=(X_test, [y_test_g, y_test_a, y_test_r]),
epochs=nb_epochs,
verbose=1,
callbacks=callbacks)
else:
hist = model.fit(X_train, [y_train_g, y_train_a, y_train_r],
batch_size=batch_size,
epochs=nb_epochs,
callbacks=callbacks,
validation_data=(X_test,
[y_test_g, y_test_a, y_test_r]))
logging.debug("Saving weights...")
model.save_weights(os.path.join("models", "WRN_{}_{}.h5".format(depth, k)),
overwrite=True)
pd.DataFrame(hist.history).to_hdf(
os.path.join("models", "history_{}_{}.h5".format(depth, k)), "history")
if fromdate == None:
if todate == None:
todate = datetime.datetime.today()
fromdate = todate - datetime.timedelta(days=7)
else:
fromdate = todate
elif todate == None:
todate = fromdate
basedir = remlist[0]
rawdir = os.path.join(basedir, 'raw')
metadir = os.path.join(basedir, 'metatags')
statsdir = os.path.join(basedir, 'stats')
utils.mk_dir(rawdir)
utils.mk_dir(metadir)
utils.mk_dir(statsdir)
courtobjs = []
dldict = {'bombay' : bombay.Bombay, \
'kolkata' : kolkata.Kolkata, \
'kolkata_app' : kolkata_app.KolkataApp, \
'punjab' : punjab.Punjab, \
'uttaranchal' : uttaranchal.Uttaranchal, \
'delhi' : delhi.Delhi, \
'jharkhand' : jharkhand.Jharkhand, \
'gujarat' : gujarat.Gujarat, \
'rajasthan' : rajasthan.Rajasthan, \
'jodhpur' : jodhpur.Jodhpur, \
'karnataka' : karnataka.Karnataka, \
def main():
args = get_args()
input_path = args.input
batch_size = args.batch_size
nb_epochs = args.nb_epochs
freeze_layers = args.freeze_layers
depth = args.depth
k = args.width
validation_split = args.validation_split
logging.debug("Loading data...")
image, gender, age, _, image_size, _ = load_data(input_path)
X_data = image
y_data_g = np_utils.to_categorical(gender, 2)
y_data_a = np_utils.to_categorical(age, 101)
#Load weights
weight_file = os.path.join("pretrained_models", "weights.18-4.06.hdf5")
model = WideResNet(image_size, depth=depth, k=k)()
model.load_weights(weight_file)
# set the first 50 layers
# to non-trainable (weights will not be updated)
print(len(model.layers))
if freeze_layers > 0:
for layer in model.layers[:freeze_layers]:
layer.trainable = False
sgd = SGD(lr=0.1, momentum=0.9, nesterov=True)
model.compile(
optimizer=sgd,
loss=["categorical_crossentropy", "categorical_crossentropy"],
metrics=['accuracy'])
logging.debug("Model summary...")
model.count_params()
model.summary()
logging.debug("Saving model...")
mk_dir("models")
with open(os.path.join("models", "WRN_{}_{}.json".format(depth, k)),
"w") as f:
f.write(model.to_json())
mk_dir("checkpoints")
callbacks = [
LearningRateScheduler(schedule=Schedule(nb_epochs)),
ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
monitor="val_loss",
verbose=1,
save_best_only=True,
mode="auto")
]
logging.debug("Running training...")
# print('length of X', len(X_data))
# print('length of y_data_g', y_data_g)
# print('length of y_data_a', len(y_data_a))
hist = model.fit(X_data, [y_data_g, y_data_a],
batch_size=batch_size,
epochs=nb_epochs,
callbacks=callbacks,
validation_split=validation_split)
logging.debug("Saving weights...")
model.save_weights(os.path.join("models", "WRN_{}_{}.h5".format(depth, k)),
overwrite=True)
pd.DataFrame(hist.history).to_hdf(
os.path.join("models", "history_{}_{}.h5".format(depth, k)), "history")
def main():
args = get_args()
input_path = args.input
batch_size = args.batch_size
nb_epochs = args.nb_epochs
depth = args.depth
k = args.width
validation_split = args.validation_split
use_augmentation = args.aug
logging.debug("Loading data...")
image, gender, age, _, image_size, _ = load_data(input_path)
X_data = image
y_data_a = np_utils.to_categorical(age, 101)
#custom parameters
nb_class = 2
hidden_dim = 512
vgg_model = VGGFace(include_top=False, input_shape=(224, 224, 3))
last_layer = vgg_model.get_layer('pool5').output
x = Flatten(name='flatten')(last_layer)
x = Dense(hidden_dim, activation='relu', name='fc6')(x)
x = Dense(hidden_dim, activation='relu', name='fc7')(x)
out = Dense(nb_class, activation='softmax', name='fc8')(x)
model = Model(vgg_model.input, out)
sgd = SGD(lr=0.1, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd,
loss=["categorical_crossentropy"],
metrics=['accuracy'])
logging.debug("Model summary...")
model.count_params()
model.summary()
logging.debug("Saving model...")
mk_dir("models")
with open(os.path.join("models", "vgg_{}_{}.json".format(depth, k)),
"w") as f:
f.write(model.to_json())
mk_dir("checkpoints")
callbacks = [
LearningRateScheduler(schedule=Schedule(nb_epochs)),
ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
monitor="val_loss",
verbose=1,
save_best_only=True,
mode="auto")
]
logging.debug("Running training...")
data_num = len(X_data)
indexes = np.arange(data_num)
np.random.shuffle(indexes)
X_data = X_data[indexes]
y_data_g = y_data_g[indexes]
y_data_a = y_data_a[indexes]
train_num = int(data_num * (1 - validation_split))
X_train = X_data[:train_num]
X_test = X_data[train_num:]
y_train_g = y_data_g[:train_num]
y_test_g = y_data_g[train_num:]
y_train_a = y_data_a[:train_num]
y_test_a = y_data_a[train_num:]
if use_augmentation:
datagen = ImageDataGenerator(width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
preprocessing_function=get_random_eraser(
v_l=0, v_h=255))
training_generator = MixupGenerator(X_train, [y_train_g, y_train_a],
batch_size=batch_size,
alpha=0.2,
datagen=datagen)()
hist = model.fit_generator(generator=training_generator,
steps_per_epoch=train_num // batch_size,
validation_data=(X_test,
[y_test_g, y_test_a]),
epochs=nb_epochs,
verbose=1,
callbacks=callbacks)
else:
hist = model.fit(X_train, [y_train_g, y_train_a],
batch_size=batch_size,
epochs=nb_epochs,
callbacks=callbacks,
validation_data=(X_test, [y_test_g, y_test_a]))
logging.debug("Saving weights...")
model.save_weights(os.path.join("models", "vgg_{}_{}.h5".format(depth, k)),
overwrite=True)
pd.DataFrame(hist.history).to_hdf(
os.path.join("models", "history_{}_{}.h5".format(depth, k)), "history")
def main():
args = get_args()
validation_fold = args.validation_fold
test_set = loadmat('./fold_{}.mat'.format(validation_fold))
test_x = test_set['image']
test_y = test_set['gender']
fold_set = [0, 1, 2, 3, 4]
fold_set.remove(validation_fold)
l = []
for i in fold_set:
l.append(loadmat('./fold_{}.mat'.format(i)))
train_x = l[0]['image']
train_y = l[0]['gender']
for i in range(1, len(l)):
train_x = np.concatenate((train_x, l[i]['image']))
train_y = np.concatenate((train_y, l[i]['gender']))
#test_y = utils.to_categorical(test_y,num_classes=2)
#train_y = utils.to_categorical(test_y,num_classes=2)
print(test_y)
for i, value in np.ndenumerate(test_y):
if value == 'm':
np.put(test_y, i, 0)
elif value == 'f':
np.put(test_y, i, 1)
else:
raise Exception
print(test_y)
for i, value in np.ndenumerate(train_y):
if value == 'm':
np.put(train_y, i, 0)
elif value == 'f':
np.put(train_y, i, 1)
else:
raise Exception
cnn = CNN(input_size=IMG_SIZE)
model = cnn.get_classifier()
train_datagen = ImageDataGenerator(rescale=1. / 255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_datagen = ImageDataGenerator(rescale=1. / 255)
training_set = train_datagen.flow(train_x, train_y, batch_size=BATCH_SIZE)
test_set = test_datagen.flow(test_x, test_y, batch_size=BATCH_SIZE)
mk_dir("checkpoints")
callbacks = [
ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
monitor="val_loss",
verbose=1,
save_best_only=True,
mode="auto")
]
hist = model.fit_generator(training_set,
steps_per_epoch=len(train_x) // BATCH_SIZE,
epochs=NUM_EPOCHS,
validation_data=test_set,
validation_steps=len(test_set),
callbacks=callbacks)
model.save('my_cnn.h5')
pd.DataFrame(hist.history).to_hdf("history.h5", "history")
model = Model(input=inputs, output=predictions)
return model
if __name__ == '__main__':
model = create_model()
model.compile(optimizer=sgd, loss="categorical_crossentropy", metrics=['accuracy'])
if print_model_summary:
logging.debug("Model summary...")
model.count_params()
model.summary()
if save_model_plot:
logging.debug("Saving model plot...")
mk_dir(MODEL_PATH)
from keras.utils.visualize_util import plot
plot(model, to_file=os.path.join(MODEL_PATH, 'WRN-{0}-{1}.png'.format(depth, k)), show_shapes=True)
#Data Augmentation from page 6
logging.debug("Creating ImageDataGenerators...")
train_datagen = ImageDataGenerator(
featurewise_center=True,
featurewise_std_normalization=True,
zca_whitening=True,
horizontal_flip=True)
#NEED TO ADD (page 6): "random crops from image padded by 4 pixels on each side, filling missing pixels with reflections of original image."
#SEE: https://github.com/szagoruyko/wide-residual-networks/blob/master/augmentation.lua
train_datagen.fit(X_train, augment=True, rounds=2)
test_datagen = ImageDataGenerator(
def main():
args = get_args()
input_path = args.input
batch_size = args.batch_size
nb_epochs = args.nb_epochs
max_age = args.max_age + 1
depth = args.depth
k = args.width
transfer_learning = args.transfer_learning
validation_split = args.validation_split
use_augmentation = args.aug
initial_weights = '/home/paula/THINKSMARTER_/Model/demographics-model-prediction/pretrained_models/weights.18-4.06.hdf5'
# weight_file = '/home/paula/THINKSMARTER_/Model/age-gender-estimation-adapted/checkpoints/weights.09-4.32.hdf5'
_weight_decay = 0.0005
_use_bias = False
_weight_init = "he_normal"
logging.debug("Loading data...")
image, gender, age, _, image_size, _ = load_data(input_path)
X_data = image
y_data_g = np_utils.to_categorical(gender, 2)
y_data_a = np_utils.to_categorical(age, max_age)
if transfer_learning:
model = WideResNet(image_size, depth=depth, k=k, units_age=101)()
model.load_weights(initial_weights)
inputs = model.input
flatten = model.layers[-3].output # capa flatten
dense1 = Dense(units=2,
kernel_initializer=_weight_init,
use_bias=_use_bias,
kernel_regularizer=l2(_weight_decay),
activation="softmax")(flatten)
dense2 = Dense(units=117,
kernel_initializer=_weight_init,
use_bias=_use_bias,
kernel_regularizer=l2(_weight_decay),
activation="softmax")(flatten)
model = Model(inputs=inputs, outputs=[dense1, dense2])
# ---------------------------------
# IDEA: fine tuning (nomes entreno les dos ultimes capes)
# for layer in model.layers[:-2]:
# layer.trainable = False
else:
model = WideResNet(image_size, depth=depth, k=k, units_age=max_age)()
sgd = SGD(lr=0.1, momentum=0.9, nesterov=True)
model.compile(
optimizer=sgd,
loss=["categorical_crossentropy", "categorical_crossentropy"],
metrics=['accuracy'])
logging.debug("Model summary...")
model.count_params()
model.summary()
if args.plot_model:
plot_model(model,
to_file='experiments_pictures/model_plot.png',
show_shapes=True,
show_layer_names=True)
logging.debug("Saving model...")
mk_dir("models")
with open(os.path.join("models", "WRN_{}_{}.json".format(depth, k)),
"w") as f:
f.write(model.to_json())
mk_dir("checkpoints")
# tensorBoard = TensorBoard(log_dir='events', histogram_freq=0, batch_size=32, write_graph=True, write_grads=False, write_images=True, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None, embeddings_data=None)
callbacks = [
LearningRateScheduler(schedule=Schedule(nb_epochs)),
ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
monitor="val_loss",
verbose=1,
save_best_only=True,
mode="auto")
]
logging.debug("Running training...")
data_num = len(X_data)
indexes = np.arange(data_num)
np.random.shuffle(indexes)
X_data = X_data[indexes]
y_data_g = y_data_g[indexes]
y_data_a = y_data_a[indexes]
train_num = int(data_num * (1 - validation_split))
X_train = X_data[:train_num]
X_test = X_data[train_num:]
y_train_g = y_data_g[:train_num]
y_test_g = y_data_g[train_num:]
y_train_a = y_data_a[:train_num]
y_test_a = y_data_a[train_num:]
if use_augmentation:
datagen = ImageDataGenerator(width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
preprocessing_function=get_random_eraser(
v_l=0, v_h=255))
training_generator = MixupGenerator(X_train, [y_train_g, y_train_a],
batch_size=batch_size,
alpha=0.2,
datagen=datagen)()
hist = model.fit_generator(generator=training_generator,
steps_per_epoch=train_num // batch_size,
validation_data=(X_test,
[y_test_g, y_test_a]),
epochs=nb_epochs,
verbose=1,
callbacks=callbacks)
else:
hist = model.fit(X_train, [y_train_g, y_train_a],
batch_size=batch_size,
epochs=nb_epochs,
callbacks=callbacks,
validation_data=(X_test, [y_test_g, y_test_a]))
logging.debug("Saving weights...")
model.save_weights(os.path.join("models", "WRN_{}_{}.h5".format(depth, k)),
overwrite=True)
pd.DataFrame(hist.history).to_hdf(
os.path.join("models", "history_{}_{}.h5".format(depth, k)), "history")
with open('history_tmp.txt', 'w') as f:
for key in hist.history:
print(key, file=f)
f.write('\n')
json.dump(hist.history, f)
def main():
args = get_args()
input_path = args.input
batch_size = args.batch_size
nb_epochs = args.nb_epochs
depth = args.depth
k = args.width
validation_split = args.validation_split
use_augmentation = args.aug
image_size = 224
os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
logging.debug("Loading data and Augmentor...")
#sess = K.get_session()
#sess = tf_debug.LocalCLIDebugWrapperSession(sess)
#K.set_session(sess)
#model = WideResNet(image_size, depth=22, k=k)()
model = MyModel(image_size, trainable=False)()
adam = Adam(lr=0.01, decay=0.001)
sgd = SGD(lr=0.00001, momentum=0.9, nesterov=True, decay=0.0001)
model.compile(optimizer=sgd,
loss=["categorical_crossentropy", "MSE"],
loss_weights=[0.5, 1.0],
metrics=['accuracy'])
logging.debug("Model summary...")
model.count_params()
model.summary()
#model.load_weights(os.path.join("checkpoints", "weights.03-4.78.hdf5"))
logging.debug("Saving model...")
mk_dir("models")
with open(os.path.join("models", "WRN_{}_{}.json".format(depth, k)),
"w") as f:
f.write(model.to_json())
mk_dir("checkpoints")
train_datagen = ImageDataGenerator(rescale=1. / 255,
shear_range=0.2,
horizontal_flip=True,
validation_split=0.2)
train_generator = train_datagen.flow_from_directory(
'../../dataset/imdb_crop/new_database/',
target_size=(image_size, image_size),
batch_size=batch_size,
class_mode='categorical',
subset='training',
shuffle=True)
print(next(train_generator)[1].shape)
val_generator = train_datagen.flow_from_directory(
'../../dataset/imdb_crop/new_database/',
target_size=(image_size, image_size),
batch_size=batch_size,
class_mode='categorical',
subset='validation',
shuffle=True)
callbacks = [
LearningRateScheduler(schedule=Schedule(38138 // batch_size)),
ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
monitor="val_loss",
verbose=1,
save_best_only=True,
mode="auto")
]
#class_weight = get_class_weights(train_generator.classes)
#print(class_weight)
h = model.fit_generator(
generate_data_generator(train_generator),
use_multiprocessing=True,
epochs=10,
validation_data=generate_data_generator(val_generator),
workers=12,
steps_per_epoch=len(train_generator.classes) / batch_size,
validation_steps=len(val_generator.classes) / batch_size,
#class_weight=class_weight
)
logging.debug("Saving weights...")
model.save_weights(os.path.join("models", "WRN_{}_{}.h5".format(depth, k)),
overwrite=True)
def main():
args = get_args()
input_path = args.input
batch_size = args.batch_size
nb_epochs = args.nb_epochs
depth = args.depth
k = args.width
validation_split = args.validation_split
use_augmentation = args.aug
logging.debug("Loading data...")
image, gender, age, _, image_size, _ = load_data(input_path)
X_data = image
y_data_g = np_utils.to_categorical(gender, 2)
y_data_a = np_utils.to_categorical(age, 101)
model = WideResNet(image_size, depth=depth, k=k)()
sgd = SGD(lr=0.1, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd, loss=["categorical_crossentropy", "categorical_crossentropy"],
metrics=['accuracy'])
logging.debug("Model summary...")
model.count_params()
model.summary()
logging.debug("Saving model...")
mk_dir("models")
with open(os.path.join("models", "WRN_{}_{}.json".format(depth, k)), "w") as f:
f.write(model.to_json())
mk_dir("checkpoints")
callbacks = [LearningRateScheduler(schedule=Schedule(nb_epochs)),
ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
monitor="val_loss",
verbose=1,
save_best_only=True,
mode="auto")
]
logging.debug("Running training...")
data_num = len(X_data)
indexes = np.arange(data_num)
np.random.shuffle(indexes)
X_data = X_data[indexes]
y_data_g = y_data_g[indexes]
y_data_a = y_data_a[indexes]
train_num = int(data_num * (1 - validation_split))
X_train = X_data[:train_num]
X_test = X_data[train_num:]
y_train_g = y_data_g[:train_num]
y_test_g = y_data_g[train_num:]
y_train_a = y_data_a[:train_num]
y_test_a = y_data_a[train_num:]
if use_augmentation:
datagen = ImageDataGenerator(
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
preprocessing_function=get_random_eraser(v_l=0, v_h=255))
training_generator = MixupGenerator(X_train, [y_train_g, y_train_a], batch_size=batch_size, alpha=0.2,
datagen=datagen)()
hist = model.fit_generator(generator=training_generator,
steps_per_epoch=train_num // batch_size,
validation_data=(X_test, [y_test_g, y_test_a]),
epochs=nb_epochs, verbose=1,
callbacks=callbacks)
else:
hist = model.fit(X_train, [y_train_g, y_train_a], batch_size=batch_size, epochs=nb_epochs, callbacks=callbacks,
validation_data=(X_test, [y_test_g, y_test_a]))
logging.debug("Saving weights...")
model.save_weights(os.path.join("models", "WRN_{}_{}.h5".format(depth, k)), overwrite=True)
pd.DataFrame(hist.history).to_hdf(os.path.join("models", "history_{}_{}.h5".format(depth, k)), "history")
# else:
# path_dir = os.listdir(file_dir)
# sample = random.sample(path_dir, n)
# for name in sample:
# shutil.copyfile(os.path.join(file_dir, name),
# os.path.join(tar_dir, name))
if __name__ == '__main__':
# FaceNet
IMAGE_SIZE = 160
PADDING = 0.2
WEIGHTS = "../models/model.h5"
base_path = "test"
gender_list = ['W', 'M']
mk_dir("results")
tf.logging.set_verbosity(tf.logging.ERROR)
faceAlign = Face_aligned(keras_model=WEIGHTS)
for _, _, imgs in os.walk(base_path):
'''
Only support one face in the image
'''
for idx, im in enumerate(imgs):
img_path = os.path.join(base_path, im)
# Extract image suffix name
(_, name) = os.path.split(img_path)
image = cv2.imread(img_path)
if __name__ == '__main__':
model = create_model()
model.compile(optimizer=sgd,
loss="categorical_crossentropy",
metrics=['accuracy'])
if print_model_summary:
logging.debug("Model summary...")
model.count_params()
model.summary()
if save_model_plot:
logging.debug("Saving model plot...")
mk_dir(MODEL_PATH)
from keras.utils.visualize_util import plot
plot(model,
to_file=os.path.join(MODEL_PATH,
'WRN-{0}-{1}.png'.format(depth, k)),
show_shapes=True)
# Data Augmentation based on page 6 (see README for full details)
logging.debug("Creating ImageDataGenerators...")
train_datagen = ImageDataGenerator(featurewise_center=True,
featurewise_std_normalization=True,
zca_whitening=True,
horizontal_flip=True)
train_datagen.fit(X_train, augment=True, rounds=2)
test_datagen = ImageDataGenerator(featurewise_center=True,
def train_age():
args = get_args()
validation_fold = args.v
test_set = loadmat('./age_fold_{}.mat'.format(validation_fold))
test_x = test_set['image']
test_y = test_set['age']
test_y = [x.strip() for x in test_y] # since Matrices are of rectangular shapes in matlab
test_y = np.array(test_y)
fold_set = [0,1,2,3,4]
fold_set.remove(validation_fold)
l = []
for i in fold_set:
l.append(loadmat('./age_fold_{}.mat'.format(i)))
train_x = l[0]['image']
train_y = l[0]['age']
for i in range(1,len(l)):
train_x = np.concatenate((train_x,l[i]['image']))
train_y = np.concatenate((train_y,l[i]['age']))
train_y = [x.strip() for x in train_y]
train_y = np.array(train_y)
print(test_y[:50])
for i, value in np.ndenumerate(test_y):
if value == '(0, 2)':
np.put(test_y,i,0)
elif value == '(4, 6)':
np.put(test_y,i,1)
elif value == '(8, 12)':
np.put(test_y,i,2)
elif value == '(15, 20)':
np.put(test_y,i,3)
elif value == '(25, 32)':
np.put(test_y,i,4)
elif value == '(38, 43)':
np.put(test_y,i,5)
elif value == '(48, 53)':
np.put(test_y,i,6)
elif value == '(60, 100)':
np.put(test_y,i,7)
else:
print(value)
print(len(value))
raise Exception
print(test_y)
for i, value in np.ndenumerate(train_y):
if value == '(0, 2)':
np.put(train_y,i,0)
elif value == '(4, 6)':
np.put(train_y,i,1)
elif value == '(8, 12)':
np.put(train_y,i,2)
elif value == '(15, 20)':
np.put(train_y,i,3)
elif value == '(25, 32)':
np.put(train_y,i,4)
elif value == '(38, 43)':
np.put(train_y,i,5)
elif value == '(48, 53)':
np.put(train_y,i,6)
elif value == '(60, 100)':
np.put(train_y,i,7)
else:
print(value)
print(len(value))
raise Exception
test_y = utils.to_categorical(test_y,num_classes=8)
train_y = utils.to_categorical(train_y,num_classes=8)
#import pdb; pdb.set_trace()
cnn = AgeCNN(input_size = IMG_SIZE)
model = cnn.get_classifier()
if IMG_AUG == 1:
train_datagen = ImageDataGenerator(rescale = 1./255,
shear_range = 0.2,
zoom_range = 0.2,
horizontal_flip = True)
else:
train_datagen = ImageDataGenerator(rescale = 1./255)
test_datagen = ImageDataGenerator(rescale = 1./255)
training_set = train_datagen.flow(train_x,train_y,batch_size=BATCH_SIZE)
test_set = test_datagen.flow(test_x,test_y,batch_size=BATCH_SIZE)
mk_dir("checkpoints")
callbacks = [ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5",
monitor="val_loss",
verbose=1,
save_best_only=True,
mode="auto")
]
hist = model.fit_generator(training_set, steps_per_epoch = len(train_x)//BATCH_SIZE, epochs = NUM_EPOCHS, validation_data = test_set, validation_steps = len(test_set),callbacks=callbacks)
model.save('age_cnn.h5')
mk_dir("history")
timestr = time.strftime("%Y%m%d-%H%M%S")
pd.DataFrame(hist.history).to_hdf("./history/age_history-{}.h5".format(timestr), "history")
from utils import mk_dir, dice_coef, return_list
import torch
from torchvision import transforms
import os
DiscSeg_size = 640
# training data and label path
train_img_path = "REFUGE-Training400/Training400/Glaucoma/"
train_label_path = "Annotation-Training400/Annotation-Training400/Disc_Cup_Masks/Glaucoma/"
# validation data and label path
val_img_path = "REFUGE-Validation400/REFUGE-Validation400/"
val_label_path = "REFUGE-Validation400-GT/Disc_Cup_Masks/"
data_save_path = mk_dir("training_crop/data/")
label_save_path = mk_dir("training_crop/label/")
class MyTrainData(torch.utils.data.Dataset):
def __init__(self, data_path, label_path, data_type, train=True):
self.train = train
self.data_path = data_path
self.label_path = label_path
if self.train: # train the model
self.train_list = return_list(
data_path,
data_type) # the images name list of train and validation
else: # validate the model
self.val_list = return_list(
data_path,
def main():
args = get_args()
input_path = args.input
batch_size = args.batch_size*args.gpus
nb_epochs = args.nb_epochs
depth = args.depth
k = args.width
validation_split = args.validation_split
img_size = args.img_size
val_path = args.val_path
pretrained_fil = args.pretrained_fil
input_shape = [img_size, img_size, 3]
patience = 30
gpu_num = args.gpus
train_db = args.db
logging.debug("Loading data...")
'''
image, gender, age, _, image_size, _ = load_data(input_path)
X_data = image
y_data_g = np_utils.to_categorical(gender, 2)
y_data_a = np_utils.to_categorical(age, 101)
'''
batchdataload = BatchLoader(input_path,batch_size,img_size,train_db)
valdataload = BatchLoader(val_path,batch_size,img_size)
model = WideResNet(img_size, depth=depth, k=k)()
#model = mini_XCEPTION(input_shape,101)
with open(os.path.join("ag_models", "WRN_{}_{}.json".format(depth, k)), "w") as f:
f.write(model.to_json())
if pretrained_fil :
model.load_weights(pretrained_fil)
#sgd = SGD(lr=0.001, momentum=0.7, nesterov=True)
adam = Adam(lr=0.0001,beta_1=0.9,beta_2=0.999,epsilon=1e-5)
#model.compile(optimizer=sgd, loss=["categorical_crossentropy", "categorical_crossentropy"],
# metrics=['accuracy'])
#if gpu_num >1:
#model = multi_gpu_model(model,gpu_num)
model.compile(optimizer=adam, loss=["categorical_crossentropy"],
metrics=['accuracy'])
logging.debug("Model summary...")
#model.count_params()
model.summary()
logging.debug("Saving model...")
if not os.path.exists("./ag_models"):
mk_dir("ag_models")
reduce_lr = ReduceLROnPlateau(monitor="val_loss",factor=0.1,patience=patience*2,verbose=1,min_lr=0.0000001)
early_stop = EarlyStopping('val_loss', patience=patience)
modelcheckpoint = ModelCheckpoint("ag_models/weights.{epoch:02d}-{val_loss:.2f}.hdf5",\
monitor="val_loss",verbose=1,save_best_only=True,mode="auto",period=1000)
#mk_dir("checkpoints")
#reduce_lr = LearningRateScheduler(schedule=reduce_lr)
#callbacks = [modelcheckpoint,early_stop,reduce_lr]
callbacks = [modelcheckpoint,reduce_lr]
logging.debug("Running training...")
#whole training
error_min = 0
if whole_data :
hist = model.fit_generator(data_geneter(batchdataload), steps_per_epoch=batchdataload.batch_num,
epochs=nb_epochs, verbose=1,
callbacks=callbacks,
validation_data=data_geneter(valdataload),
nb_val_samples=valdataload.batch_num,
nb_worker=1)
logging.debug("Saving weights...")
model.save_weights(os.path.join("ag_models", "WRN_{}_{}.h5".format(depth, k)),overwrite=True)
#pd.DataFrame(hist.history).to_hdf(os.path.join("ag_models", "history_{}_{}.h5".format(depth, k)), "history")
else:
epoch_step = 0
while epoch_step < nb_epochs:
step = 0
while step < batchdataload.batch_num:
#X_data, y_data_g, y_data_a = batch_geneter(batchdataload)
X_data, y_data_g = batch_geneter(batchdataload)
#hist = model.fit(X_data, [y_data_g, y_data_a], batch_size=batch_size, epochs=1, verbose=2)
hist = model.fit(X_data, y_data_g, batch_size=batch_size, epochs=1, verbose=2)
step+=1
if step % 100 ==0:
#val_data,val_g,val_a = batch_geneter(valdataload)
val_data,val_g = batch_geneter(valdataload)
#error_t = model.evaluate(val_data,[val_g,val_a],batch_size=batch_size,verbose=1)
error_t = model.evaluate(val_data,val_g,batch_size=batch_size,verbose=1)
print ("****** Epoch {} Step {}: ***********".format(str(epoch_step),str(step)) )
print (" loss: {}".format(error_t))
if epoch_step % 5 ==0:
#logging.debug("Saving weights...")
#val_data,val_g,val_a = batch_geneter(valdataload)
#error_t = model.evaluate(val_data,[val_g,val_a],batch_size=batch_size,verbose=1)
if error_t[4] >error_min:
logging.debug("Saving weights...")
model.save_weights(os.path.join("ag_models", "WRN_{}_{}_epoch{}_step{}.h5".format(depth, k,epoch_step,step)))
error_min = error_t[4]
epoch_step+=1
if epoch_step % 5 ==0:
logging.debug("Saving weights...")
#val_data,val_g,val_a = batch_geneter(valdataload)
val_data,val_g = batch_geneter(valdataload)
#error_t = model.evaluate(val_data,[val_g,val_a],batch_size=batch_size,verbose=1)
error_t = model.evaluate(val_data,val_g,batch_size=batch_size,verbose=1)
if error_t[1] >error_min:
model.save_weights(os.path.join("ag_models", "WRN_{}_{}_epoch{}.h5".format(depth, k,epoch_step)))
error_min = error_t[4]
error_min =0
model = Model(inputs=x, outputs=[out1, out2, out3])
sgd = SGD(lr=1, momentum=0.9, nesterov=True)
model.compile( # compile model
optimizer=sgd,
loss=[
"categorical_crossentropy", "categorical_crossentropy",
"categorical_crossentropy"
],
metrics=['accuracy'])
# visualize, summarize and save model
plot_model(model, to_file='model.png')
model.summary()
print("Saving model...")
mk_dir("models")
with open(os.path.join("models", "kerasFunctionalModel.json"), "w") as f:
f.write(model.to_json())
# load training data
print("Loading data...")
image, gender, age, ethnicity, image_size = load_data("./dage.mat")
# convert labels to truth vectors
X_data = image
y_data_g = np_utils.to_categorical(gender, 2)
y_data_a = np_utils.to_categorical(age, 117)
y_data_e = np_utils.to_categorical(ethnicity, 5)
# print shapes
print("Shapes of : image, gender, age, ethnicity...")
def main():
# Set GPU memory usage
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
K.set_session(session)
tf.logging.set_verbosity(tf.logging.ERROR)
args = get_args()
IMG_SIZE = args.image_size
input_path = args.input
batch_size = args.batch_size
nb_epochs = args.nb_epochs
validation_split = args.validation_split
dataset_name = args.dataset_name
n_age_bins = args.class_num
embdding = args.embdding
lr = args.lr
logging.debug("[INFO] Loading data...")
data_loader = DataManager(dataset_name)
ground_truth_data = data_loader.get_data()
train_keys, val_keys = split_data(ground_truth_data,
validation_split=validation_split,
do_shuffle=True)
print("Samples: Training - {}, Validation - {}".format(
len(train_keys), len(val_keys)))
input_shape = (IMG_SIZE, IMG_SIZE, 3)
image_generator = ImageGenerator(ground_truth_data,
batch_size,
input_shape[:2],
train_keys,
val_keys,
path_prefix=input_path,
vertical_flip_probability=0,
eraser_probability=0,
bins=n_age_bins)
model = facenet_resnet(nb_class=n_age_bins,
embdding=embdding,
is_train=True,
weights="./models/facenet_keras_weights.h5")
model.compile(optimizer=optimizers.SGD(lr=lr,
momentum=0.9,
decay=5e-4,
nesterov=False),
loss={
'pred_g': focal_loss(alpha=.4, gamma=2),
'pred_a': mae,
"pred_e": "categorical_crossentropy"
},
loss_weights={
'pred_g': 0.2,
'pred_a': 1,
'pred_e': 0.4
},
metrics={
'pred_g': 'accuracy',
'pred_a': mae,
'pred_e': 'accuracy'
})
logging.debug("[INFO] Saving model...")
mk_dir("checkpoints")
callbacks = [
CSVLogger(os.path.join('checkpoints', 'train.csv'), append=False),
ModelCheckpoint(os.path.join(
'checkpoints',
'weights.{epoch:02d}-{val_pred_g_acc:.3f}-{val_pred_a_mae:.3f}-{val_pred_e_acc:.3f}.h5'
),
monitor="val_loss",
verbose=1,
save_best_only=True,
mode="min"),
# Use Stochastic Gradient Descent with Restart
# https://github.com/emrul/Learning-Rate
# Based on paper SGDR: STOCHASTIC GRADIENT DESCENT WITH WARM RESTARTS
# SGDRScheduler(min_lr=lr*((0.1)**3), max_lr=lr, steps_per_epoch=np.ceil(len(train_keys) /
# batch_size), lr_decay=0.9, cycle_length=5, mult_factor=1.5)
# Use Cyclical Learning Rate
# CyclicLR(mode='triangular', step_size=np.ceil(
# len(train_keys)/batch_size), base_lr=lr*((0.1)**3), max_lr=lr)
LearningRateScheduler(PolyDecay(lr, 0.9, nb_epochs).scheduler)
]
logging.debug("[INFO] Running training...")
history = model.fit_generator(
image_generator.flow(mode='train'),
steps_per_epoch=np.ceil(len(train_keys) / batch_size),
epochs=nb_epochs,
callbacks=callbacks,
validation_data=image_generator.flow('val'),
validation_steps=np.ceil(len(val_keys) / batch_size))
logging.debug("[INFO] Saving weights...")
K.clear_session()