def create_geo_alt_names(geo_type):
geo_alt_names = {}
for locale in settings.locales:
name_format = geo_type.__name__ + '{}' + locale.capitalize()
name = name_format.format('AltName')
geo_alt_names[locale] = create_model(
name = name,
fields = {
'geo': models.ForeignKey(geo_type, # Related geo type
related_name = 'alt_names_' + locale),
'name': models.CharField(max_length=200, db_index=True), # Alternate name
'is_preferred': models.BooleanField(), # True if this alternate name is an official / preferred name
'is_short': models.BooleanField(), # True if this is a short name like 'California' for 'State of California'
'objects': GeoAltNameManager(),
'__unicode__': lambda self: force_unicode(self.name),
},
app_label = 'cities',
module = 'cities.models',
options = {
'db_table': 'cities_' + un_camel(name),
'verbose_name': un_camel(name).replace('_', ' '),
'verbose_name_plural': un_camel(name_format.format('AltNames')).replace('_', ' '),
},
)
return geo_alt_names
def __init__(self, args, clients, test_loader=None):
self.args = args
self.comm_rounds = args.comm_rounds
self.num_clients = args.num_clients
self.frac = args.frac
self.clients = clients
self.client_data_num = []
self.elapsed_comm_rounds = 0
self.accuracies = np.zeros(args.comm_rounds)
self.client_accuracies = np.zeros(
(self.args.num_clients, self.args.comm_rounds))
self.selected_client_tally = np.zeros(
(self.args.comm_rounds, self.args.num_clients))
self.test_loader = test_loader
for client in self.clients:
self.client_data_num.append(len(client.train_loader))
self.client_data_num = np.array(self.client_data_num)
# The extra 1 entry in client_models and global_models are used to store
# the results after last communication round
self.client_models = np.zeros((self.comm_rounds + 1, self.num_clients),
dtype='object')
self.global_models = None # np.zeros((self.comm_rounds + 1,), dtype='object')
self.init_model = create_model(args.dataset, args.arch)
self.global_models = self.init_model
self.global_init_model = copy_model(self.init_model, args.dataset,
args.arch)
assert self.num_clients == len(
clients
), "Number of client objects does not match command line input"
def main():
args = parse_args('predict')
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
transform = transforms.Compose([
transforms.Resize(224),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
])
test_set = DataLoader(args.ext_test, transform=transform)
test_loader = torch.utils.data.DataLoader(test_set, batch_size=args.batch_size, shuffle=False)
print("==> creating model")
num_classes = args.num_classes
model = create_model(num_classes, args).to(device)
if args.average:
record = pd.read_csv(os.path.join(args.output_csv_dir, 'output.csv'), index_col=0)
sorted_r = record.sort_values('f1', ascending=False)
model_list = list(sorted_r['epoch_num'].astype(int))[:5]
for idx, epoch in enumerate(model_list):
print('\nPredict with ' f'epoch{epoch} checkpoint')
model = load_checkpoint(model, os.path.join(args.save_model_dir, f'epoch{epoch}_checkpoint.pth.tar')).to(device)
predict(test_loader, model, device, args, idx)
else:
predict(test_loader, model, device, args)
predict(test_loader, model, device, args)
def create_geo_alt_names(geo_type):
geo_alt_names = {}
for locale in settings.locales:
name_format = geo_type.__name__ + '{}' + locale.capitalize()
name = name_format.format('AltName')
geo_alt_names[locale] = create_model(
name = name,
fields = {
'geo': models.ForeignKey(geo_type, # Related geo type
related_name = 'alt_names_' + locale),
'name': models.CharField(max_length=200, db_index=True), # Alternate name
'is_preferred': models.BooleanField(), # True if this alternate name is an official / preferred name
'is_short': models.BooleanField(), # True if this is a short name like 'California' for 'State of California'
'objects': GeoAltNameManager(),
'__unicode__': lambda self: force_unicode(self.name),
},
app_label = 'cities',
module = 'cities.models',
options = {
'db_table': 'cities_' + un_camel(name),
'verbose_name': un_camel(name).replace('_', ' '),
'verbose_name_plural': un_camel(name_format.format('AltNames')).replace('_', ' '),
},
)
return geo_alt_names
def test5_test_model_list_size():
import os
from util import log_obj, create_model
models = [create_model('mnist', 'mlp') for _ in range(400)]
log_obj('./log_test/server/400_models_list.model_list', models)
filesize = os.path.getsize('./log_test/server/400_models_list.model_list')
byte_per_gb = 1E9
print(f'The log file size is {filesize/byte_per_gb}GB')
def main():
args = parse_args('test')
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
transform_test = transforms.Compose([
transforms.Resize(224),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
])
test_set = DataLoader(args.test_list, transform=transform_test)
test_loader = torch.utils.data.DataLoader(test_set,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
# Model
print("==> creating model")
num_classes = args.num_classes
model = create_model(num_classes, args).to(device)
criterion = nn.CrossEntropyLoss()
# testing
expname = args.expname.split('/')[-1]
print('\n' + expname + ': TESTING!')
train_set = os.path.basename(args.train_list).split('.')[0]
val_set = os.path.basename(args.val_list).split('.')[0]
test_set = os.path.basename(args.test_list).split('.')[0]
if args.average:
record = pd.read_csv(os.path.join(args.output_csv_dir, 'output.csv'), index_col=0)
sorted_r = record.sort_values('f1', ascending=False)
model_list = list(sorted_r['epoch_num'].astype(int))[:5]
df = pd.DataFrame(columns=['exp', 'train', 'val', 'test', 'test_loss', 'test_acc', 'f1'])
for idx, epoch in enumerate(model_list):
model = load_checkpoint(model, os.path.join(args.save_model_dir, f'epoch{epoch}_checkpoint.pth.tar'))
test_loss, test_acc, f1, _ = test(test_loader, model, criterion, device, args, epoch=idx+1)
df.loc[len(df)] = [expname, train_set, val_set, test_set, test_loss, test_acc, f1]
output_csv_file = os.path.join(args.output_csv_dir, 'test_acc.csv')
df.to_csv(output_csv_file, index=False)
else:
df = pd.DataFrame(columns=['exp', 'train', 'val', 'test', 'test_loss', 'test_acc', 'f1'])
model = load_checkpoint(model, os.path.join(save_model_dir, 'model_best_f1.pth.tar'))
test_loss, test_acc, f1, _ = test(test_loader, model, criterion, device, args)
df.loc[len(df)] = [expname, train_set, val_set, test_set, test_loss, test_acc, f1]
output_csv_file = os.path.join(output_csv_dir, 'test_f1.csv')
df.to_csv(output_csv_file, index=False)
def main(args):
config = create_config()
model = create_model(args.mode)
dm = create_data_module(config)
print(f'Training for {args.max_epochs} epochs')
# Output training parameters
config.display()
trainer = Trainer.from_argparse_args(args)
trainer.fit(model, dm)
def __init__(self, args, train_loader, test_loader, client_id=None):
self.args = args
print("Creating model for client " + str(client_id))
self.model = create_model(self.args.dataset, self.args.arch)
print("Copying model for client " + str(client_id))
self.init_model = copy_model(self.model, self.args.dataset,
self.args.arch)
print("Done Copying model for client " + str(client_id))
self.test_loader = test_loader
self.train_loader = train_loader
self.client_id = client_id
self.elapsed_comm_rounds = 0
self.accuracies = np.zeros((args.comm_rounds, self.args.client_epoch))
self.losses = np.zeros((args.comm_rounds, self.args.client_epoch))
self.prune_rates = np.zeros(args.comm_rounds)
assert self.model, "Something went wrong and the model cannot be initialized"
def main(args):
with open('classes.json', 'r') as f:
classes = json.load(f)
# Convert key values to ints
classes = {int(k): v for k, v in classes.items()}
model = create_model(args.mode, path_to_checkpoint=args.checkpoint_path)
model.eval()
image = Image.open(args.image_path)
tensor = ToTensor()(image).unsqueeze(0)
prediction = torch.argmax(model(tensor), dim=1).item()
print(f'Predicted class: {classes[prediction]}')
def main(**kwargs):
config_name = kwargs.get("config_name", "config.json")
path = osp.join(osp.dirname(osp.realpath(__file__)), config_name)
config_file = open(path)
config = json.load(config_file,
object_hook=lambda d: namedtuple('x', d.keys())
(*d.values()))
config_file.seek(0)
logger.info(str(json.load(config_file)))
config_file.close()
num_unrolls = config.num_steps // config.unroll_length
with tf.Session() as sess:
# tf.get_default_graph().finalize()
model = util.create_model(sess, config, logger)
step, loss, reset, fx_array, x_array = model.step()
best_cost = [float('inf')] * 3
epoch_cost = 0
total_cost = 0
for e in range(config.num_epochs):
cost, _ = util.run_epoch(sess, loss, [step], reset, num_unrolls)
epoch_cost += cost
total_cost += cost
if (e + 1) % config.log_period == 0:
lm_e = epoch_cost / config.log_period
logger.info('Epoch {}, Mean Error: {:.3f}'.format(e, lm_e))
epoch_cost = 0
if (e + 1) % config.evaluation_period == 0:
elm_e = total_cost / config.evaluation_period
logger.info('Current {} epochs, Mean Error: {:.3f}'.format(
config.evaluation_period, elm_e))
mbc = max(best_cost)
if config.save_path is not None and total_cost < mbc:
save_path = osp.join(osp.dirname(osp.realpath(__file__)),
config.save_path)
best_cost.remove(mbc)
best_cost.append(total_cost)
logger.info('Save current model ...')
model.saver.save(sess, save_path, global_step=e)
total_cost = 0
def main():
config_file = open('./config.json')
config = json.load(config_file,
object_hook=lambda d: namedtuple('x', d.keys())
(*d.values()))
config_file.seek(0)
logger.info(str(json.load(config_file)))
config_file.close()
num_unrolls = config.num_steps // config.unroll_length
with tf.Session() as sess:
# tf.get_default_graph().finalize()
model = util.create_model(sess, config, logger)
step, loss, reset, fx_array, x_array = model.step()
best_evaluation = float('inf')
total_cost = 0
for e in range(config.num_epochs):
cost, _ = util.run_epoch(sess, loss, [step], reset, num_unrolls)
total_cost += cost
if (e + 1) % config.log_period == 0:
lm_e = total_cost / config.log_period
logger.info('Epoch {}, Mean Error: {:.3f}'.format(e, lm_e))
total_cost = 0
if (e + 1) % config.evaluation_period == 0:
eval_cost = 0
for _ in range(config.evaluation_epochs):
cost, _ = util.run_epoch(sess, loss, [
step,
], reset, num_unrolls)
eval_cost += cost
elm_e = eval_cost / config.evaluation_epochs
logger.info('EVALUATION, Mean Error: {:.3f}'.format(elm_e))
if config.save_path is not None and eval_cost < best_evaluation:
logger.info('Save current model ...')
model.saver.save(sess, config.save_path, global_step=e)
best_evaluation = eval_cost
def AFCNN(x_train, y_train, x_test, y_test):
class_num = 4 # 分类的种类
x_train = x_train.reshape(x_train.shape[0], 1, x_train.shape[1], 1)
x_test = x_test.reshape(x_test.shape[0], 1, x_test.shape[1], 1)
# one-hot
Y_train_one_hot = to_categorical(y_train, int(class_num)) # four labels
Y_test_one_hot = to_categorical(y_test, int(class_num)) # four labels
model = create_model() # 创建CNN模型
# model.fit(x_train, y_train,batch_size = 32, epochs = 10,verbose = 1,validation_data=(test_data,test_target),callbacks=callbacks_list)
model.fit(x_train, Y_train_one_hot, batch_size=32, epochs=5, verbose=1, validation_data=(x_test, Y_test_one_hot))
# test_pred,train_pred = stacking(train_data=x_train,train_target=Y_train_one_hot,test_data=x_test,test_target=Y_test_one_hot, n_fold=5)
test_prob = model.predict(x_test)
test_p = [np.argmax(item) for item in test_prob] # 将矩阵按列合并
score = model.evaluate(x_test, Y_test_one_hot, verbose=1)
# print('acc==', score)
# tmp_target = [np.argmax(item) for item in Y_test_one_hot]
# print(confusion_matrix(tmp_target, test_p))
# return test_p, test_prob
return test_p
def main():
print("==> loading model")
num_classes = args.num_classes
model = create_model(num_classes, args).to(device)
model = load_checkpoint(model, checkpoint).to(device)
transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
])
input_data = args.input
if os.path.isdir(input_data):
input_folder_lst = [
os.path.join(input_data, i) for i in os.listdir(input_data)
]
for input_folder in input_folder_lst:
print(os.path.basename(input_folder))
input_data = os.path.join(input_folder, 'patch.json')
filter_patches(model, input_data, transform)
else:
filter_patches(model, input_data, transform)
def create_geo_alt_names(geo_type):
geo_alt_names = {}
for locale in settings.locales:
name_format = geo_type.__name__ + "{}" + locale.capitalize()
name = name_format.format("AltName")
geo_alt_names[locale] = create_model(
name=name,
fields={
"geo": models.ForeignKey(geo_type, related_name="alt_names_" + locale), # Related geo type
"name": models.CharField(max_length=200, db_index=True), # Alternate name
"is_preferred": models.BooleanField(), # True if this alternate name is an official / preferred name
"is_short": models.BooleanField(), # True if this is a short name like 'California' for 'State of California'
"objects": GeoAltNameManager(),
"__unicode__": lambda self: force_unicode(self.name),
},
app_label="cities",
module="cities.models",
options={
"db_table": "cities_" + un_camel(name),
"verbose_name": un_camel(name).replace("_", " "),
"verbose_name_plural": un_camel(name_format.format("AltNames")).replace("_", " "),
},
)
return geo_alt_names
default='gru',
choices=['gru', 'lstm'],
help='type of recurrent layer to use (gru or lstm)')
parser.add_argument('--model_weights', help='pickle file of trained weights')
args = parser.parse_args(gen_be=False)
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
be.bsz = 1
# load the bAbI dataset
babi = babi_handler(args.data_dir, args.task)
valid_set = QA(*babi.test)
# create model
model_inference = create_model(babi.vocab_size, args.rlayer_type)
model_inference.load_params(args.model_weights)
model_inference.initialize(dataset=valid_set)
ex_story, ex_question, ex_answer = babi.test_parsed[0]
stitch_sentence = lambda words: \
" ".join(words).replace(" ?", "?").replace(" .", ".\n").replace("\n ", "\n")
print "\nThe vocabulary set from this task has {} words:".format(
babi.vocab_size)
print stitch_sentence(babi.vocab)
print "\nExample from test set:"
print "\nStory"
print stitch_sentence(ex_story)
print "Question"
print stitch_sentence(ex_question)
print "\nAnswer"
def create_postal_codes():
@property
def parent(self):
for parent_name in reversed(['country'] + RegionBase.levels):
parent_obj = getattr(self, parent_name)
if parent_obj: return parent_obj
return None
@property
def hierarchy(self):
"""Get hierarchy, root first"""
list = self.parent.hierarchy
list.append(self)
return list
@property
def names(self):
"""Get a hierarchy of non-null names, root first"""
return [e for e in [
force_unicode(self.country),
force_unicode(self.region_name),
force_unicode(self.subregion_name),
force_unicode(self.district_name),
force_unicode(self.name),
] if e]
@property
def name_full(self):
"""Get full name including hierarchy"""
return u', '.join(reversed(self.names))
postal_codes = {}
for country in settings.postal_codes:
name_format = "{}" + country
name = name_format.format('PostalCode')
postal_codes[country] = create_model(
name = name,
fields = {
'country': models.ForeignKey(Country,
related_name = 'postal_codes_' + country),
'code': models.CharField(max_length=20, primary_key=True),
'name': models.CharField(max_length=200, db_index=True),
# Region names for each admin level, region may not exist in DB
'region_name': models.CharField(max_length=100, db_index=True),
'subregion_name': models.CharField(max_length=100, db_index=True),
'district_name': models.CharField(max_length=100, db_index=True),
'region': models.ForeignKey(Region, null=True, blank=True, related_name = 'postal_codes_' + country),
'subregion': models.ForeignKey(Subregion, null=True, blank=True, related_name = 'postal_codes_' + country),
'location': models.PointField(),
'objects': models.GeoManager(),
'parent': parent,
'hierarchy': hierarchy,
'names': names,
'name_full': name_full,
'__unicode__': lambda self: force_unicode(self.code),
},
app_label = 'cities',
module = 'cities.models',
options = {
'db_table': 'cities_' + un_camel(name),
'verbose_name': un_camel(name).replace('_', ' '),
'verbose_name_plural': un_camel(name_format.format('PostalCodes')).replace('_', ' '),
},
)
return postal_codes
#
# Model Running
#
######################################################
logger.info("building model")
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
initializer = tf.contrib.layers.xavier_initializer()
with tf.variable_scope("model", reuse=None, initializer=initializer):
model = util.create_model(session=sess,
Model=BILSTM_CRF,
ckpt_file=save_path,
labelKey=labelKey,
label2Idx=label2Idx,
word2Idx=word2Idx,
num_steps=num_steps,
num_epochs=num_epochs,
embedding_matrix=embeddings,
logger=logger)
logger.info("training model")
logger.info("start training")
model.train(sess, save_path, X_train, y_train, X_dev, y_dev,
save_model_name, logger)
print("final best f1 is: %f" % (model.max_f1))
end_time = time.time()
print("time used %f(hour)" % ((end_time - start_time) / 3600))
parser.add_argument('--rlayer_type', default='gru', choices=['gru', 'lstm'],
help='type of recurrent layer to use (gru or lstm)')
parser.add_argument('--model_weights',
help='pickle file of trained weights')
args = parser.parse_args(gen_be=False)
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
be.bsz = 1
# load the bAbI dataset
babi = babi_handler(args.data_dir, args.task)
valid_set = QA(*babi.test)
# create model
model_inference = create_model(babi.vocab_size, args.rlayer_type)
model_inference.load_params(args.model_weights)
model_inference.initialize(dataset=valid_set)
ex_story, ex_question, ex_answer = babi.test_parsed[0]
stitch_sentence = lambda words: \
" ".join(words).replace(" ?", "?").replace(" .", ".\n").replace("\n ", "\n")
print "\nThe vocabulary set from this task has {} words:".format(babi.vocab_size)
print stitch_sentence(babi.vocab)
print "\nExample from test set:"
print "\nStory"
print stitch_sentence(ex_story)
print "Question"
print stitch_sentence(ex_question)
print "\nAnswer"
print ex_answer
# Set this variable to your MLflow server's DNS name
mlflow_server = '178.128.106.223'
# Tracking URI
mlflow_tracking_uri = 'http://' + mlflow_server + ':5000'
print("MLflow Tracking URI: %s" % (mlflow_tracking_uri))
mlflow.set_tracking_uri(mlflow_tracking_uri)
if __name__ == "__main__":
#init_mlflow()
h2o.init()
# VARIABLE INITIATION
target = str(sys.argv[1]) if len(sys.argv) > 1 else target
csv_path = str(sys.argv[2]) if len(sys.argv) > 1 else csv_path
delimiter = str(sys.argv[3]) if len(sys.argv) > 1 else delimiter
run_time = int(sys.argv[4]) if len(sys.argv) > 1 else run_time
export_path = str(sys.argv[5]) if len(sys.argv) > 1 else export_path
# LOAD DATA
data = util.load_csv_to_pandas(csv_path, delimiter)
# MODEL EXPERIMENT
model = util.create_model(h2o, mlflow, data, target, run_time)
# MODEL EXPORT
util.save_model(model, save_directory=export_path, mlflow=mlflow.h2o)
import numpy as np
from util import create_model
x = np.random.random((3, 7228))
model = create_model(x)
model.load_weights('weights.best.hdf5')
print(model.predict(x))
# print('x-max-seq-unit: {}'.format(X_max_seq_len))
# print('y-max-seq-unit: {}'.format(y_max_seq_len))
if not os.path.exists(SAVED_DIR):
os.mkdir(SAVED_DIR)
while EPOCHS > 0:
epochs = min(EPOCHS, SAVE_MODEL_AFTER_EPOCH)
# load model
if os.path.isfile(SAVED_MODEL_PATH): # retrieving model
model = load_model(SAVED_MODEL_PATH)
with open(SAVED_EPOCH_PATH, 'r') as f:
prev_epoch = int(f.read())
else: # creating model
model = create_model(X_num_of_unit, X_max_seq_len, y_num_of_unit,
y_max_seq_len, NUM_OF_HIDDEN_UNIT)
prev_epoch = 0
# printing model summary
# model.summary()
# shuffle data
# to avoid local minima
indices = np.arange(len(X))
np.random.shuffle(indices)
X = X[indices]
y = y[indices]
# checkpoint = ModelCheckpoint(filepath=SAVED_MODEL_PATH, monitor='acc',
# verbose=1, save_best_only='true', mode='max')
# callback_list = [checkpoint]
def create_postal_codes():
@property
def parent(self):
"""Returns region if available, otherwise country"""
return self.region if self.region else self.country
@property
def hierarchy(self):
"""Get hierarchy, root first"""
list = self.parent.hierarchy
list.append(self)
return list
@property
def names(self):
"""Get a hierarchy of non-null names, root first"""
return [
e for e in [
force_unicode(self.country),
force_unicode(self.region_0_name),
force_unicode(self.region_1_name),
force_unicode(self.region_2_name),
force_unicode(self.name),
] if e
]
@property
def name_full(self):
"""Get full name including hierarchy"""
return u', '.join(reversed(self.names))
postal_codes = {}
for country in settings.postal_codes:
name_format = "{}" + country
name = name_format.format('PostalCode')
postal_codes[country] = create_model(
name=name,
fields={
'country':
models.ForeignKey(Country,
related_name='postal_codes_' + country),
'code':
models.CharField(max_length=20, db_index=True),
'name':
models.CharField(max_length=200, db_index=True),
'region_0_name':
models.CharField(max_length=100,
db_index=True,
verbose_name="region 0 name (state)"),
'region_1_name':
models.CharField(max_length=100,
db_index=True,
verbose_name="region 1 name (county)"),
'region_2_name':
models.CharField(max_length=100,
db_index=True,
verbose_name="region 2 name (community)"),
'region':
models.ForeignKey(Region,
null=True,
blank=True,
related_name='postal_codes_' + country),
'location':
models.PointField(),
'objects':
models.GeoManager(),
'parent':
parent,
'hierarchy':
hierarchy,
'names':
names,
'name_full':
name_full,
'__unicode__':
lambda self: force_unicode(self.code),
},
app_label='cities',
module='cities.models',
options={
'db_table':
'cities_' + un_camel(name),
'verbose_name':
un_camel(name).replace('_', ' '),
'verbose_name_plural':
un_camel(name_format.format('PostalCodes')).replace('_', ' '),
},
)
return postal_codes
def create_postal_codes():
@property
def parent(self):
"""Returns region if available, otherwise country"""
return self.region if self.region else self.country
@property
def hierarchy(self):
"""Get hierarchy, root first"""
list = self.parent.hierarchy
list.append(self)
return list
@property
def names(self):
"""Get a hierarchy of non-null names, root first"""
return [e for e in [
force_unicode(self.country),
force_unicode(self.region_0_name),
force_unicode(self.region_1_name),
force_unicode(self.region_2_name),
force_unicode(self.name),
] if e]
@property
def name_full(self):
"""Get full name including hierarchy"""
return u', '.join(reversed(self.names))
postal_codes = {}
for country in settings.postal_codes:
name_format = "{}" + country
name = name_format.format('PostalCode')
postal_codes[country] = create_model(
name = name,
fields = {
'country': models.ForeignKey(Country,
related_name = 'postal_codes_' + country),
'code': models.CharField(max_length=20, db_index=True),
'name': models.CharField(max_length=200, db_index=True),
'region_0_name': models.CharField(max_length=100, db_index=True, verbose_name="region 0 name (state)"),
'region_1_name': models.CharField(max_length=100, db_index=True, verbose_name="region 1 name (county)"),
'region_2_name': models.CharField(max_length=100, db_index=True, verbose_name="region 2 name (community)"),
'region': models.ForeignKey(Region, null=True, blank=True,
related_name = 'postal_codes_' + country),
'location': models.PointField(),
'objects': models.GeoManager(),
'parent': parent,
'hierarchy': hierarchy,
'names': names,
'name_full': name_full,
'__unicode__': lambda self: force_unicode(self.code),
},
app_label = 'cities',
module = 'cities.models',
options = {
'db_table': 'cities_' + un_camel(name),
'verbose_name': un_camel(name).replace('_', ' '),
'verbose_name_plural': un_camel(name_format.format('PostalCodes')).replace('_', ' '),
},
)
return postal_codes
def train(model,
version,
train_generator,
validation_generator,
steps_per_epoch,
save=True,
csv=True,
plot=True,
connected=True,
dropout=0.0,
dense=4096,
BN=False,
AveragePooling=True,
MutiFC=False,
report=True,
load_model=False):
"""
training code for pet(cat and dog) emotion. by now only support 'cat' or 'dog'
:param report: if show classification report
:param test_face: will use pure face test data
:param MutiFC: add multiple fully connected layers
:param AveragePooling: add global average pooling layer
:param BN: if add batch normalization before fully-connected layer
:param load_model: if load existed model
:param dense: set number of cells in fully-connected layer
:param dropout: set dropout ratio (float)
:param connected: if add a fully connected layer
:param classify: cat or dog
:param model: the name of pre-trained model
:param version: the current version
:param plot: decide if need to plot & save images of acc and loss
:param csv: if save the history to csv
:param save: if save the model
"""
# load pre-trained model from keras.application
# create model
model, modelName = util.create_model(model, connected, dropout, dense, BN,
AveragePooling, MutiFC)
MODEL_PATH = '../../../../Data for project/New_Data/cat/'
# file name for model
modelPath = os.sep + modelName + version
# compile model
model.compile(optimizer=keras.optimizers.Adam(),
loss=keras.losses.categorical_crossentropy,
metrics=['accuracy'])
model.summary()
print('Now is running: ', ' Cat Final ', version)
if not load_model:
# add tensorboard log save_path
log_dir = const.LOG_PATH + modelPath
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1)
# fit the model
history = model.fit(
train_generator,
steps_per_epoch=steps_per_epoch,
epochs=const.EPOCH,
validation_data=validation_generator,
workers=6,
# callbacks=[tensorboard_callback]
)
else:
# show classification report on predicting
util.show_predict_report(validation_generator)
# save model history to csv for further analyse
if csv:
pd.DataFrame(history.history).to_csv(MODEL_PATH + modelPath + '.csv')
# summarize history for accuracy & loss
if plot:
util.plotInfo(MODEL_PATH, modelPath, history)
# save model
if save:
try:
model.save(MODEL_PATH + modelPath + '.h5')
except OSError:
# I found a bug in saving model with tf. it shows os error but it did save model
print("OSError, is it saved? ")
pass
# show each class report
if report:
y_actual = validation_generator.classes
y_predicted = model.predict(validation_generator)
class_ = util.get_confusion_matrix(y_actual=y_actual,
y_predicted=y_predicted)
print(class_)
#tokenize the input
t,reverse_word_map = word_mapping(df)
t.fit_on_texts(df.texts + df.target)
print('done tokenizing')
#pickle t
pickle.dump(t,open('tokenizer.pickle', 'wb'))
print('done saving tokenizer')
#prepare the data set
PREDICTORS,TARGET = prepare_sets(df,t)
#split the data set
predictors_train,target_train = generate_sets(PREDICTORS,TARGET,1)
#experiment parameters:
max_list_predictors = len(max(PREDICTORS,key=len))
max_sequence_len=max_list_predictors + 1
total_words = len(t.word_index) + 1
output_dim = np.shape(target_train)[1]
neurons = 300
epochs = 1
#create the model
model = create_model(max_sequence_len,total_words,neurons,output_dim)
#train it and run the experiment
model = train_and_run_experiment(model,epochs,predictors_train,target_train)
args.save_path = 'babi.p'
if args.callback_args['save_path'] is None:
args.callback_args['save_path'] = args.save_path
# setup backend
args.batch_size = 32
be = gen_backend(**extract_valid_args(args, gen_backend))
# load the bAbI dataset
babi = babi_handler(args.data_dir, args.task)
train_set = QA(*babi.train)
valid_set = QA(*babi.test)
# create model
model = create_model(babi.vocab_size, args.rlayer_type)
# setup callbacks
callbacks = Callbacks(model, train_set, eval_set=valid_set, **args.callback_args)
# train model
model.fit(train_set,
optimizer=Adam(),
num_epochs=args.epochs,
cost=GeneralizedCost(costfunc=CrossEntropyMulti()),
callbacks=callbacks)
# output accuracies
print('Train Accuracy = %.1f%%' % (model.eval(train_set, metric=Accuracy())*100))
print('Test Accuracy = %.1f%%' % (model.eval(valid_set, metric=Accuracy())*100))
type=str,
default='none',
help='none|std_dev|init_weights')
parser.add_argument(
'--prune_method',
type=str,
default='l1',
help='l1|old_super_mask|new_super_mask|mix_l1_super_mask')
parser.add_argument('--server_prune_threshold', type=float, default=0.8)
parser.add_argument('--proj_name', type=str, default='CELL')
args = parser.parse_args()
seed_everything(seed=args.seed, workers=True)
model = create_model(cls=models[args.dataset][args.arch],
device=args.device)
train_loaders, test_loaders, class_idxs = DataLoaders(
num_users=args.n_clients,
dataset_name=args.dataset,
n_class=args.n_classes,
nsamples=args.n_samples,
mode=args.dataset_mode,
batch_size=args.batch_size,
rate_unbalance=args.rate_unbalance,
num_workers=args.n_workers)
clients = []
for i in range(args.n_clients):
client = Client(i, args, train_loaders[i], test_loaders[i],
class_idxs[i])
clients.append(client)
def main(config, checkpoint_dir=None):
args = parse_args('train')
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
torch.manual_seed(10)
if tune_hyperparam:
batch_size = config['batch_size']
lr = config['lr']
else:
batch_size = args.batch_size
lr = args.lr
gamma = config['gamma']
best_f1 = 0
transform_train = transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomRotation(30),
transforms.RandomHorizontalFlip(p=0.5),
transforms.RandomVerticalFlip(p=0.5),
transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4),
GaussianBlur(),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
RandomErasing()
])
transform_val = transforms.Compose([
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
])
train_set = DataLoader(args.train_list, transform=transform_train)
if args.resample:
train_loader = torch.utils.data.DataLoader(
train_set,
batch_size=batch_size,
sampler=ImbalancedDatasetSampler(
train_set,
num_samples=len(train_set),
callback_get_label=train_set.data),
num_workers=args.num_workers)
else:
train_loader = torch.utils.data.DataLoader(
train_set,
batch_size=batch_size,
shuffle=True,
num_workers=args.num_workers)
val_set = DataLoader(args.val_list, transform=transform_val)
val_loader = torch.utils.data.DataLoader(val_set,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
# Load model
print("==> Creating model")
num_classes = args.num_classes
model = create_model(num_classes, args).to(device)
# choose loss function
if args.weighted_loss:
targets = [i['target'] for i in train_set.data]
weights = compute_class_weight('balanced',
classes=np.unique(targets),
y=np.array(targets))
criterion = select_loss_func(choice=args.loss,
weights=torch.tensor(weights,
dtype=torch.float),
gamma=gamma)
else:
criterion = select_loss_func(choice=args.loss, gamma=gamma)
# choose optimizer
print('==> {} optimizer'.format(args.optimizer))
if args.optimizer == 'SAM':
base_optimizer = torch.optim.SGD
optimizer = SAM(model.parameters(),
base_optimizer,
lr=lr,
momentum=0.9)
elif args.optimizer == 'ADAM':
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
else:
optimizer = torch.optim.SGD(model.parameters(),
lr,
momentum=0.9,
weight_decay=1e-3,
nesterov=True)
# set up logger
writer = SummaryWriter(log_dir=args.log_dir)
start_epoch = args.start_epoch
if args.dataset == 'renal':
df = pd.DataFrame(columns=[
'model', 'lr', 'epoch_num', 'train_loss', 'val_loss', 'train_acc',
'val_acc', 'normal', 'obsolescent', 'solidified', 'disappearing',
'non_glom', 'f1'
])
elif args.dataset == 'ham':
df = pd.DataFrame(columns=[
'model', 'lr', 'epoch_num', 'train_loss', 'val_loss', 'train_acc',
'val_acc', 'MEL', 'NV', 'BCC', 'AKIEC', 'BKL', 'DF', 'VASC', 'f1'
])
else:
raise ValueError('no such dataset exists!')
# start training
for epoch in range(start_epoch, args.epochs):
epoch += 1
if args.optimizer != 'ADAM':
cur_lr = adjust_learning_rate(lr, optimizer, epoch)
else:
cur_lr = lr
print('\nEpoch: [%d | %d] LR: %f' % (epoch, args.epochs, cur_lr))
train_loss, train_acc, train_f1, train_f1s = train(
train_loader, model, optimizer, criterion, device, args)
val_loss, val_acc, val_f1, val_f1s = validate(val_loader, model,
criterion, epoch, device,
args)
if tune_hyperparam:
tune.report(loss=val_loss, accuracy=val_f1)
writer.add_scalars("loss/", {
'train': train_loss,
'val': val_loss
}, epoch)
writer.add_scalars("f1/", {'train': train_f1, 'val': val_f1}, epoch)
# write to csv
df.loc[epoch] = [
args.network, cur_lr, epoch, train_loss, val_loss, train_acc,
val_acc
] + val_f1s + [val_f1]
output_csv_file = os.path.join(args.output_csv_dir, 'output.csv')
df.to_csv(output_csv_file, index=False)
# save model
is_best_f1 = val_f1 > best_f1
best_f1 = max(val_f1, best_f1)
save_checkpoint(
{
'epoch': epoch,
'state_dict': model.state_dict(),
'acc': val_acc,
'best_f1': best_f1,
'optimizer': optimizer.state_dict(),
}, is_best_f1, epoch, args.save_model_dir)
print('Best f1:')
print(best_f1)
i = i + 1
except (IOError, OSError):
print("failed to process {}".format(image_path))
X = np.array(images, dtype='float32')
y = np.eye(num_classes, dtype='uint8')[labels]
return X, y
NUM_CLASSES = 43
IMG_SIZE = 48
TRAINING_DATA_PATH = "./GTSRB/Final_Training/Images/"
model = create_model(NUM_CLASSES, IMG_SIZE)
X, y = get_training_data(TRAINING_DATA_PATH, NUM_CLASSES, IMG_SIZE)
learning_rate = 0.01
sgd = SGD(lr=learning_rate, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
batch_size = 32
epochs = 30
history = model.fit(X,
y,
batch_size=batch_size,
args.save_path = 'babi.p'
if args.callback_args['save_path'] is None:
args.callback_args['save_path'] = args.save_path
# setup backend
args.batch_size = 32
be = gen_backend(**extract_valid_args(args, gen_backend))
# load the bAbI dataset
babi = babi_handler(args.data_dir, args.task)
train_set = QA(*babi.train)
valid_set = QA(*babi.test)
# create model
model = create_model(babi.vocab_size, args.rlayer_type)
# setup callbacks
callbacks = Callbacks(model, eval_set=valid_set, **args.callback_args)
# train model
if args.test_only:
model.fit(train_set,
optimizer=Adam(),
num_epochs=args.epochs,
cost=GeneralizedCost(costfunc=CrossEntropyMulti()),
callbacks=callbacks)
# output accuracies
print('Train Accuracy = %.1f%%' %
(model.eval(train_set, metric=Accuracy()) * 100))
def __init__(self, docs, labels):
threading.Thread.__init__(self)
self.docs = docs
self.labels = labels
self.model = util.create_model()
