def log_in(client):
"""Authorizes ImgurClient to use user account"""
config = ConfigParser()
config.read('auth.ini')
access_token = config.get('credentials', 'access_token')
refresh_token = config.get('credentials', 'refresh_token')
if len(access_token) > 0 and len(refresh_token) > 0:
client.set_user_auth(access_token, refresh_token)
return client
authorization_url = client.get_auth_url('pin')
webbrowser.open(authorization_url)
pin = input('Please input your pin\n>\t')
credentials = client.authorize(pin) # grant_type default is 'pin'
access_token = credentials['access_token']
refresh_token = credentials['refresh_token']
config.set('credentials', 'access_token', access_token)
config.set('credentials', 'refresh_token', refresh_token)
save_config(config)
client.set_user_auth(access_token, refresh_token)
return client
def on_remove_current_blog(self, action, parameter):
"""
Removes current blog
"""
dialog = Gtk.MessageDialog(
parent=self.main_window,
text="Confirm removing blog",
buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL,
Gtk.STOCK_REMOVE, Gtk.ResponseType.OK)
)
dialog.get_action_area().get_children()[1].get_style_context().add_class("destructive-action")
response = dialog.run()
if response == Gtk.ResponseType.OK:
dialog.destroy()
blog = utils.get_blog_by_id(self.config, self.config["active_blog"])
if blog:
index = self.config["blogs"].index(blog)
self.config["blogs"].pop(index)
self.config["active_blog"] = None
header_bar = self.main_window.get_children()[1]
subtitle = header_bar.get_custom_title().get_children()[1]
subtitle.set_text("")
utils.save_config(self.config)
# Build menu items
self.main_window.select_blog_menu.remove_all()
for item in self.config["blogs"]:
# Creat actions and add new item in the menu
self.create_select_blog_action(item["id"])
self.main_window.select_blog_menu.append(item["name"], "app.select_blog_%s" % item["id"])
else:
self.main_window.infobar.get_content_area().get_children()[0].set_text("No blog to remove")
self.main_window.infobar.get_action_area().get_children()[1].props.visible = False
self.main_window.infobar.show()
else:
dialog.destroy()
def main(_):
prepare_dirs_and_logger(config)
if not config.task.lower().startswith('tsp'):
raise Exception("[!] Task should starts with TSP")
if config.max_enc_length is None:
config.max_enc_length = config.max_data_length
if config.max_dec_length is None:
config.max_dec_length = config.max_data_length
rng = np.random.RandomState(config.random_seed)
tf.set_random_seed(config.random_seed)
trainer = Trainer(config, rng)
save_config(config.model_dir, config)
if config.is_train:
trainer.train()
else:
if not config.load_path:
raise Exception("[!] You should specify `load_path` to load a pretrained model")
trainer.test()
tf.logging.info("Run finished.")
def on_post_button_clicked(self, target):
"""
Send post to remote server
"""
blog = utils.get_blog_by_id(self.app.config, self.app.config["active_blog"])
if blog:
if blog["provider"] == "blogger":
service = BloggerProvider(blog["username"], blog["password"])
tags = self.tag_entry.get_text().split(",")
tags = [x.strip() for x in tags]
tags = [unicode(x, "utf-8") for x in tags]
# Add new tags to the blog
new_tags = []
for tag in tags:
if tag not in blog["tags"]:
new_tags.append(tag)
if new_tags:
index = self.app.config["blogs"].index(blog)
self.app.config["blogs"][index]["tags"].extend(new_tags)
utils.save_config(self.app.config)
result = service.send_post(blog["id"], self.title_entry.get_text(), self.sourceview.get_buffer().props.text, tags)
self.infobar.get_content_area().get_children()[0].set_text(result)
self.infobar.get_action_area().get_children()[1].props.visible = True
self.infobar.show()
else:
self.infobar.get_content_area().get_children()[0].set_text("No blog is selected")
# Hide New button
self.infobar.get_action_area().get_children()[1].props.visible = False
self.infobar.show()
def main(config):
prepare_dirs_and_logger(config)
save_config(config)
if config.is_train:
from trainer import Trainer
if config.dataset == 'line':
from data_line import BatchManager
elif config.dataset == 'ch':
from data_ch import BatchManager
elif config.dataset == 'kanji':
from data_kanji import BatchManager
elif config.dataset == 'baseball' or\
config.dataset == 'cat':
from data_qdraw import BatchManager
batch_manager = BatchManager(config)
trainer = Trainer(config, batch_manager)
trainer.train()
else:
from tester import Tester
if config.dataset == 'line':
from data_line import BatchManager
elif config.dataset == 'ch':
from data_ch import BatchManager
elif config.dataset == 'kanji':
from data_kanji import BatchManager
elif config.dataset == 'baseball' or\
config.dataset == 'cat':
from data_qdraw import BatchManager
batch_manager = BatchManager(config)
tester = Tester(config, batch_manager)
tester.test()
def run(self, container):
if container.get('template'):
self.join_subtasks(self.run_subtask('container.template.fetch'))
template = self.dispatcher.call_sync(
'container.template.query',
[('template.name', '=', container['template'].get('name'))],
{'single': True}
)
template['template'].pop('readme')
if template is None:
raise TaskException(errno.ENOENT, 'Template {0} not found'.format(container['template'].get('name')))
result = {}
for key in container:
if container[key]:
result[key] = container[key]
deep_update(template, result)
container = template
self.join_subtasks(self.run_subtask('container.cache.update', container['template']['name']))
else:
normalize(container, {
'config': {},
'devices': []
})
normalize(container, {
'enabled': True,
'immutable': False
})
normalize(container['config'], {
'memsize': 512,
'ncpus': 1
})
self.init_dataset(container)
for res in container['devices']:
self.create_device(container, res)
self.init_files(container)
id = self.datastore.insert('containers', container)
self.dispatcher.dispatch_event('container.changed', {
'operation': 'create',
'ids': [id]
})
container = self.datastore.get_by_id('containers', id)
save_config(
self.dispatcher.call_sync(
'volume.resolve_path',
container['target'],
os.path.join('vm', container['name'])
),
'vm-{0}'.format(container['name']),
container
)
return id
def run(self, id, updated_params):
container = self.datastore.get_by_id('containers', id)
if container['immutable']:
raise TaskException(errno.EACCES, 'Cannot modify immutable container {0}.'.format(id))
try:
delete_config(
self.dispatcher.call_sync(
'volume.resolve_path',
container['target'],
os.path.join('vm', container['name'])
),
'vm-{0}'.format(container['name'])
)
except (RpcException, OSError):
pass
if 'template' in updated_params:
readme = updated_params['template'].pop('readme')
if readme:
root = self.dispatcher.call_sync('container.get_container_root', container['id'])
with open(os.path.join(root, 'README.md'), 'w') as readme_file:
readme_file.write(readme)
if 'devices' in updated_params:
self.join_subtasks(self.run_subtask('container.cache.update', container['template']['name']))
for res in updated_params['devices']:
existing = first_or_default(lambda i: i['name'] == res['name'], container['devices'])
if existing:
self.update_device(container, existing, res)
else:
self.create_device(container, res)
if not updated_params.get('enabled', True):
self.join_subtasks(self.run_subtask('container.stop', id))
container.update(updated_params)
self.datastore.update('containers', id, container)
self.dispatcher.dispatch_event('container.changed', {
'operation': 'update',
'ids': [id]
})
container = self.datastore.get_by_id('containers', id)
save_config(
self.dispatcher.call_sync(
'volume.resolve_path',
container['target'],
os.path.join('vm', container['name'])
),
'vm-{0}'.format(container['name']),
container
)
def run(self, id, updated_fields):
share = self.datastore.get_by_id('shares', id)
remove_unchanged(updated_fields, share)
path = self.dispatcher.call_sync('share.get_directory_path', share['id'])
try:
delete_config(
path,
'{0}-{1}'.format(share['type'], share['name'])
)
except OSError:
pass
if 'type' in updated_fields:
old_share_type = share['type']
new_share_type = self.dispatcher.call_sync('share.supported_types').get(updated_fields['type'])
if share['target_type'] == 'DATASET':
pool, dataset = split_dataset(share['target_path'])
self.join_subtasks(
self.run_subtask('volume.dataset.update', dataset, {
'permissions_type': new_share_type['perm_type']
})
)
share.update(updated_fields)
self.join_subtasks(self.run_subtask('share.{0}.delete'.format(old_share_type), id))
self.join_subtasks(self.run_subtask('share.{0}.create'.format(updated_fields['type']), share))
else:
self.join_subtasks(self.run_subtask('share.{0}.update'.format(share['type']), id, updated_fields))
if 'permissions' in updated_fields:
path = self.dispatcher.call_sync('share.translate_path', id)
self.join_subtasks(self.run_subtask('file.set_permissions', path, updated_fields['permissions']))
self.dispatcher.dispatch_event('share.changed', {
'operation': 'update',
'ids': [share['id']]
})
updated_share = self.datastore.get_by_id('shares', id)
path = self.dispatcher.call_sync('share.get_directory_path', updated_share['id'])
try:
save_config(
path,
'{0}-{1}'.format(updated_share['type'], updated_share['name']),
updated_share
)
except OSError as err:
self.add_warning(TaskWarning(errno.ENXIO, 'Cannot save backup config file: {0}'.format(str(err))))
def activate_blog(self, blog_id):
"""
Makes blog active
"""
self.config["active_blog"] = blog_id
utils.save_config(self.config)
blog = utils.get_blog_by_id(self.config, blog_id)
if blog:
window = self.get_windows()[0]
header_bar = window.get_children()[1]
text = u'<a href="{href}">{name}</a>'.format(href=blog["link"], name=blog["name"])
subtitle = header_bar.get_custom_title().get_children()[1]
subtitle.props.use_markup = True
subtitle.props.track_visited_links = False
subtitle.set_markup(text)
def unload(self):
"""
Closes the data handler and closes the engine cleanly.
This should be called when closing the client application, or when you're
sure you're not going to use the engine anymore. This does all the necessary
procedures to close the data handler cleanly and then itself.
"""
self.msg.info(self.name, "Unloading...")
self.data_handler.unload()
# Save config file
utils.save_config(self.userconfig, self.userconfigfile)
self.loaded = False
def run(self, id, updated_params):
container = self.datastore.get_by_id('containers', id)
try:
delete_config(
self.dispatcher.call_sync(
'volume.resolve_path',
container['target'],
os.path.join('vm', container['name'])
),
'vm-{0}'.format(container['name'])
)
except (RpcException, OSError):
pass
if 'devices' in updated_params:
for res in updated_params['devices']:
existing = first_or_default(lambda i: i['name'] == res['name'], container['devices'])
if existing:
self.update_device(container, existing, res)
else:
self.create_device(container, res)
if not updated_params.get('enabled', True):
self.join_subtasks(self.run_subtask('container.stop', id))
container.update(updated_params)
self.datastore.update('containers', id, container)
self.dispatcher.dispatch_event('container.changed', {
'operation': 'update',
'ids': [id]
})
container = self.datastore.get_by_id('containers', id)
save_config(
self.dispatcher.call_sync(
'volume.resolve_path',
container['target'],
os.path.join('vm', container['name'])
),
'vm-{0}'.format(container['name']),
container
)
def run_dialog(dialog):
"""
Recreates dialog loop
"""
response = dialog.run()
if response == Gtk.ResponseType.OK:
content_widgets = dialog.get_content_area().get_children()
username = content_widgets[0].get_text()
password = content_widgets[1].get_text()
provider = content_widgets[2].get_active_id()
if provider == "blogger":
service = BloggerProvider(username, password)
data = service.get_blogs()
if data["status"] == "ok":
message = "<b>Following blogs will be added:</b> \n"
for item in data["blogs"]:
message = message + item["name"] + "\n"
ok_dialog = Gtk.MessageDialog(parent=dialog, text=message, buttons=(Gtk.STOCK_APPLY, Gtk.ResponseType.OK),
use_markup=True)
ok_dialog.run()
ok_dialog.destroy()
dialog.destroy()
for item in data["blogs"]:
if not utils.get_blog_by_id(self.config, item["id"]):
self.config["blogs"].append(item)
# Creat actions and add new item in the menu
self.create_select_blog_action(item["id"])
main_window = self.get_windows()[0]
main_window.select_blog_menu.append(item["name"], "app.select_blog_%s" % item["id"])
utils.save_config(self.config)
if data["status"] == "error":
error_dialog = Gtk.MessageDialog(parent=dialog, text=data["error"], buttons=(Gtk.STOCK_APPLY, Gtk.ResponseType.OK))
error_dialog.run()
error_dialog.destroy()
run_dialog(dialog)
else:
dialog.destroy()
def admin_config_view(request):
"""
Configure default UI parameter settings
"""
check_login(request)
session = request.session
subjects = ["Arts", "Business", "Humanities", "Mathematics and Statistics",
"Science and Technology", "Social Sciences"]
form = Form(request, schema=ConfigSchema)
config = load_config(request)
# Check for successful form completion
if 'form.submitted' in request.POST:
form.validate()
for key in ['service_document_url', 'workspace_url']:
config[key] = form.data[key]
for key in ['title', 'summary', 'subject', 'keywords', 'language', 'google_code']:
config['metadata'][key] = form.data[key]
for key in ['authors', 'maintainers', 'copyright', 'editors', 'translators']:
config['metadata'][key] = [x.strip() for x in form.data[key].split(',')]
save_config(config, request)
response = {
'form': FormRenderer(form),
'subjects': subjects,
'languages': languages,
'roles': [('authors', 'Authors'),
('maintainers', 'Maintainers'),
('copyright', 'Copyright holders'),
('editors', 'Editors'),
('translators',
'Translators')
],
'request': request,
'config': config,
}
return response
def main(config):
"""
config参数：Namespace(D_arch='DCGAN', batch_size=1, beta1=0.5, beta2=0.999, ckpt_path=None, conv_hidden_num=128,
d_lr=2e-05, data_dir='data', data_format='NCHW', dataset='DF_train_data', g_lr=2e-05, gamma=0.5, gpu=0,
grayscale=False, img_H=256, img_W=256, is_train=True, lambda_k=0.001, load_path='', log_dir='logs',
log_level='INFO', log_step=200, lr_update_step=50000, max_step=80, model=11, model_dir='path_to_directory_of_model',
num_log_samples=3, num_worker=4, optimizer='adam', pretrained_path=None, random_seed=123, sample_per_image=64,
save_model_secs=1000, split='train', start_step=0, test_data_path=None, test_one_by_one=False, use_gpu=True,
z_num=2)
"""
prepare_dirs_and_logger(config)
if config.gpu > -1:
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"] = str(config.gpu)
config.data_format = 'NHWC'
trainer = None
if 1 == config.model:
print("使用PG2()，即Market-1501 数据库，并初始化")
trainer = PG2(config)
trainer.init_net()
elif 11 == config.model:
print("使用PG2_256()，即DeepFashion数据库，并初始化")
trainer = PG2_256(config)
trainer.init_net()
if config.is_train:
print("开始训练")
save_config(config) # 存储参数到json文件
trainer.train() # 开始训练
else:
print("开始测试")
if not config.load_path:
raise Exception("[!] 没有指定 `load_path` 用于读取预训练的模型")
trainer.test()
def unload(self):
"""
Closes the data handler and closes the engine
This should be called when closing the client application, or when you're
sure you're not going to use the engine anymore. This does all the necessary
procedures to close the data handler and then itself.
"""
#if not self.loaded:
# raise utils.wmalError("Engine is not loaded.")
if self.last_show:
self.data_handler.set_show_attr(self.last_show, 'playing', False)
self._emit_signal('playing', self.last_show)
self.msg.info(self.name, "Unloading...")
self.data_handler.unload()
# Save config file
#utils.save_config(self.config, self.configfile)
utils.save_config(self.userconfig, self.userconfigfile)
self.loaded = False
def train():
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower,
FLAGS.zeros)
# "sentences[0]:[['我', 'O'], ['要', 'O'], ['看', 'O'], ['乌', 'B-SLOC'], ['鲁', 'I-SLOC'], ['木', 'I-SLOC'], ['齐', 'I-SLOC'], ['市', 'I-SLOC'], ['第', 'I-SLOC'], ['四', 'I-SLOC'], ['十', 'I-SLOC'], ['九', 'I-SLOC'], ['中', 'I-SLOC'], ['学', 'I-SLOC'], ['东', 'I-SLOC'], ['门', 'I-SLOC'], ['去', 'O'], ['乌', 'B-ELOC'], ['鲁', 'I-ELOC'], ['木', 'I-ELOC'], ['齐', 'I-ELOC'], ['推', 'I-ELOC'], ['拿', 'I-ELOC'], ['职', 'I-ELOC'], ['业', 'I-ELOC'], ['学', 'I-ELOC'], ['校', 'I-ELOC'], ['南', 'I-ELOC'], ['门', 'I-ELOC'], ['沿', 'O'], ['西', 'B-ROAD'], ['虹', 'I-ROAD'], ['东', 'I-ROAD'], ['路', 'I-ROAD'], ['的', 'O'], ['监', 'B-TYPE'], ['控', 'I-TYPE']]"
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
# print("train_sentences[0]:{}".format(train_sentences[0]))
# "train_sentences[0]:[['我', 'O'], ['要', 'O'], ['看', 'O'], ['乌', 'B-SLOC'], ['鲁', 'I-SLOC'], ['木', 'I-SLOC'], ['齐', 'I-SLOC'], ['市', 'I-SLOC'], ['第', 'I-SLOC'], ['四', 'I-SLOC'], ['十', 'I-SLOC'], ['九', 'I-SLOC'], ['中', 'I-SLOC'], ['学', 'I-SLOC'], ['东', 'I-SLOC'], ['门', 'E-SLOC'], ['去', 'O'], ['乌', 'B-ELOC'], ['鲁', 'I-ELOC'], ['木', 'I-ELOC'], ['齐', 'I-ELOC'], ['推', 'I-ELOC'], ['拿', 'I-ELOC'], ['职', 'I-ELOC'], ['业', 'I-ELOC'], ['学', 'I-ELOC'], ['校', 'I-ELOC'], ['南', 'I-ELOC'], ['门', 'E-ELOC'], ['沿', 'O'], ['西', 'B-ROAD'], ['虹', 'I-ROAD'], ['东', 'I-ROAD'], ['路', 'E-ROAD'], ['的', 'O'], ['监', 'B-TYPE'], ['控', 'E-TYPE']]"
update_tag_scheme(dev_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist
# print("map_file:{}".format(FLAGS.map_file))
# print("pre_emb:{}".format(FLAGS.pre_emb))
# map_file: maps.pkl
# pre_emb: False
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
dico_chars_train = char_mapping(
train_sentences, FLAGS.lower)[0] # character -> count dict
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences,
FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
FLAGS.lower)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), 0, len(test_data)))
# '3027 / 0 / 361 sentences in train / dev / test.'
# print("batch_size:{}".format(FLAGS.batch_size))
# batch_size: 20
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
make_path(FLAGS)
# print("config_file:{}".format(FLAGS.config_file))
# config_file: config_file
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
log_path = os.path.join("log", FLAGS.log_file)
# log_path:log/train.log
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
# print("steps_per_epoch:{}".format(steps_per_epoch))
# steps_per_epoch: 152
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
# print("steps_check:{}".format(FLAGS.steps_check))
# steps_check: 100
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
export(model, sess, "ner", "export_model")
def save_configuration(self):
self.config['debug'] = False
if (self.debug):
self.config['debug'] = True
save_config(self.config, CONFIG_FILENAME)
print(bcolors.WARNING + "Configuration saved" + bcolors.INFO)
def main():
args = parse_args()
if args.seed is not None:
print("Using seed = {}".format(args.seed))
# torch.manual_seed(args.seed)
mx.random.seed(seed_state=args.seed)
np.random.seed(seed=args.seed)
# Save configuration to file
config = {k: v for k, v in args.__dict__.items()}
config['timestamp'] = "{:.0f}".format(datetime.utcnow().timestamp())
config['local_timestamp'] = str(datetime.now())
run_dir = "./run/neumf_" + args.data + "/{}".format(config['timestamp'])
print("Saving config and results to {}".format(run_dir))
if not os.path.exists(run_dir) and run_dir != '':
os.makedirs(run_dir)
utils.save_config(config, run_dir) #defined in utils.py
# Check that GPUs are actually available
use_cuda = not args.no_cuda and mx.test_utils.list_gpus()
t1 = time.time()
# Load Data
print('Loading data')
train_dataset = CFTrainDataset(
os.path.join(args.data, TRAIN_RATINGS_FILENAME), args.negative_samples)
#in original file, use 8 core as defaul
# the parameter：shuffle means random the samples
train_dataloader = mx.gluon.data.DataLoader(dataset=train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers)
test_ratings = load_test_ratings(
os.path.join(args.data, TEST_RATINGS_FILENAME)) # noqa: E501
test_negs = load_test_negs(os.path.join(args.data, TEST_NEG_FILENAME))
nb_users, nb_items = train_dataset.nb_users, train_dataset.nb_items
print('Load data done [%.1f s]. #user=%d, #item=%d, #train=%d, #test=%d' %
(time.time() - t1, nb_users, nb_items, train_dataset.mat.nnz,
len(test_ratings)))
if (use_cuda):
ctx = mx.gpu(0)
# default to use NO.1 gpu can use docker to select a nvidia
else:
ctx = mx.cpu(0)
# Create model
model = NeuMF(nb_users,
nb_items,
mf_dim=args.factors,
mf_reg=0.,
mlp_layer_sizes=args.layers,
mlp_layer_regs=[0. for i in args.layers],
ctx=ctx)
model.initialize(ctx=ctx)
model.hybridize()
print(model)
# todo 9: to change the function in utils
# print("{} parameters".format(utils.count_parameters(model)))
# model.collect_params()
# Save model text description
with open(os.path.join(run_dir, 'model.txt'), 'w') as file:
file.write(str(model))
# model.save_parameters(os.path.join("/home/net.params", 'net.params'))
# Create files for tracking training
valid_results_file = os.path.join(run_dir, 'valid_results.csv')
# Calculate initial Hit Ratio and NDCG
hits, ndcgs = val_epoch(model,
test_ratings,
test_negs,
args.topk,
processes=args.processes,
ctx=ctx)
print('Initial HR@{K} = {hit_rate:.4f}, NDCG@{K} = {ndcg:.4f}'.format(
K=args.topk, hit_rate=np.mean(hits), ndcg=np.mean(ndcgs)))
############# hyperparameters
# Add optimizer and loss to graph
lr = args.learning_rate
bs = args.batch_size
trainer = mx.gluon.Trainer(model.collect_params(), 'adam',
{'learning_rate': lr})
mxnet_criterion = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss(
) # equivalent to lossfunction
# training
for epoch in range(args.epochs):
begin = time.time()
# tqdm shows the percentage of the process
loader = tqdm.tqdm(train_dataloader)
for batch_index, (user, item, label) in enumerate(loader):
# TODO 7: search the autograd in mxnet
# todo : let user act in gpu
user = nd.array(user, ctx=ctx)
item = nd.array(item, ctx=ctx)
label = nd.array(label, ctx=ctx)
# compute the gradient automatically
with autograd.record():
outputs = model(user, item)
loss = mxnet_criterion(outputs, label.T)
loss.backward()
trainer.step(bs)
for x in loss.mean().asnumpy().tolist():
loss_number = x
description = ('Epoch {} Loss {:.4f}'.format(epoch, loss_number))
loader.set_description(description)
train_time = time.time() - begin
begin = time.time()
hits, ndcgs = val_epoch(model,
test_ratings,
test_negs,
args.topk,
output=valid_results_file,
epoch=epoch,
processes=args.processes,
ctx=ctx)
val_time = time.time() - begin
print(
'Epoch {epoch}: HR@{K} = {hit_rate:.4f}, NDCG@{K} = {ndcg:.4f},'
' train_time = {train_time:.2f}, val_time = {val_time:.2f}'.format(
epoch=epoch,
K=args.topk,
hit_rate=np.mean(hits),
ndcg=np.mean(ndcgs),
train_time=train_time,
val_time=val_time))
if args.threshold is not None:
if np.mean(hits) >= args.threshold:
print("Hit threshold of {}".format(args.threshold))
# Save model text description after modelling
with open(os.path.join(run_dir, 'model.txt'), 'w') as file:
file.write(str(model))
# model.save_parameters(os.path.join("/home/net.params",'net.params'))
return 0
def train():
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower,
FLAGS.zeros) #训练集 101218 句子
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower,
FLAGS.zeros) #验证集 7827句子
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower,
FLAGS.zeros) #测试集 16804句子
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema) #更新标注iob转换成iobes
update_tag_scheme(test_sentences, FLAGS.tag_schema) #更新标注iob转换成iobes
update_tag_scheme(dev_sentences, FLAGS.tag_schema) #更新标注iob转换成iobes
# create maps if not exist
if not os.path.isfile(FLAGS.map_file): #判断maps.pkl是否存在
# create dictionary for word
if FLAGS.pre_emb: #是否使用预先训练的模型(训练好的字向量) 测试集的数据不在训练集中
dico_chars_train = char_mapping(train_sentences,
FLAGS.lower)[0] #字频统计下来 dico_chars
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(),
FLAGS.emb_file,
list(
itertools.chain.from_iterable( #拉平，变成一个list
[[w[0] for w in s] for s in test_sentences]) #w[0] 是个字
)) #每个字建个字典，每个词建个字典
else:
#每个字的id,标记的id
_c, char_to_id, id_to_char = char_mapping(train_sentences,
FLAGS.lower)
# Create a dictionary and a mapping for tags 每个标记的id
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences) #字频，排序，写入文件
#with open('maps.txt','w',encoding='utf8') as f1:
#f1.writelines(str(char_to_id)+" "+id_to_char+" "+str(tag_to_id)+" "+id_to_tag+'\n')
with open(FLAGS.map_file, "wb") as f: #持久化下来
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset( #字词 数字特征化
train_sentences, char_to_id, tag_to_id, FLAGS.lower)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), 0, len(test_data)))
train_manager = BatchManager(train_data, FLAGS.batch_size) #训练集每次60个句子进行迭代
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
make_path(FLAGS) #创建文件log,result,ckpt
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id) #字符对应的id,标签对应的id
save_config(config, FLAGS.config_file) #每次的数据不一样都要生成一个config_file，
make_path(FLAGS) #创建文件log,result,ckpt 模型中的文件
log_path = os.path.join("log", FLAGS.log_file) #读取log路径
logger = get_logger(log_path) #定义log日志的写入格式
print_config(config, logger) #写入log日志
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True #设置GPU自适应，用多少使用多少
#tf_config.gpu_options.per_process_gpu_memory_fraction=True 设置GPU的使用率，占比
steps_per_epoch = train_manager.len_data #总共分多少批，取多少次
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
#模型初始化结束
logger.info("start training")
loss = []
# with tf.device("/gpu:0"):没有Gpu注释掉 卷积神经网络要求句子的长度一样，
for i in range(100): #迭代多少次，每次把数据拿过来
for batch in train_manager.iter_batch(shuffle=True): #随机的拿
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
# best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)比上次模型好的话，就保存
if i % 7 == 0:
save_model(sess, model, FLAGS.ckpt_path, logger)
with open(FLAGS.map_file, "rb") as f:
if pyversion == 'three':
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
else:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f,
protocol=2)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
app = Flask(__name__)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
tf_config = tf.ConfigProto()
sess = tf.Session(config=tf_config)
sess.run(tf.global_variables_initializer())
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config,
id_to_char, logger)
@app.route('/', methods=['POST', 'GET'])
def get_text_input():
text = request.args.get('inputStr')
def run(self, id, updated_fields, enable_service=False):
share = self.datastore.get_by_id('shares', id)
if not share:
raise TaskException(errno.ENOENT, 'Share not found')
if share['immutable']:
raise TaskException(errno.EACCES, 'Cannot modify immutable share {0}.'.format(id))
if 'name' in updated_fields or 'type' in updated_fields:
share.update(updated_fields)
if self.datastore.exists(
'shares',
('id', '!=', id),
('type', '=', share['type']),
('name', '=', share['name'])
):
raise TaskException(errno.EEXIST, 'Share {0} of type {1} already exists'.format(
share['name'],
share['type']
))
path_after_update = updated_fields.get('target_path', share['target_path'])
type_after_update = updated_fields.get('target_type', share['target_type'])
permissions = updated_fields.pop('permissions', None)
share_path = self.dispatcher.call_sync('share.expand_path', path_after_update, type_after_update)
if not os.path.exists(share_path):
raise TaskException(
errno.ENOENT,
'Selected share target {0} does not exist'.format(path_after_update)
)
share = self.datastore.get_by_id('shares', id)
remove_unchanged(updated_fields, share)
path = self.dispatcher.call_sync('share.get_directory_path', share['id'])
try:
delete_config(
path,
'{0}-{1}'.format(share['type'], share['name'])
)
except (OSError, ValueError):
pass
if 'type' in updated_fields:
old_share_type = share['type']
new_share_type = self.dispatcher.call_sync('share.supported_types').get(updated_fields['type'])
if share['target_type'] == 'DATASET':
pool, dataset = split_dataset(share['target_path'])
self.join_subtasks(
self.run_subtask('volume.dataset.update', dataset, {
'permissions_type': new_share_type['perm_type']
})
)
share.update(updated_fields)
self.run_subtask_sync('share.{0}.delete'.format(old_share_type), id)
self.run_subtask_sync('share.{0}.create'.format(updated_fields['type']), share)
else:
self.run_subtask_sync('share.{0}.update'.format(share['type']), id, updated_fields)
if permissions:
path = self.dispatcher.call_sync('share.translate_path', id)
self.run_subtask_sync('file.set_permissions', path, permissions)
self.dispatcher.dispatch_event('share.changed', {
'operation': 'update',
'ids': [share['id']]
})
updated_share = self.datastore.get_by_id('shares', id)
path = self.dispatcher.call_sync('share.get_directory_path', updated_share['id'])
try:
save_config(
path,
'{0}-{1}'.format(updated_share['type'], updated_share['name']),
updated_share
)
except OSError as err:
self.add_warning(TaskWarning(errno.ENXIO, 'Cannot save backup config file: {0}'.format(str(err))))
service_state = self.dispatcher.call_sync('service.query', [('name', '=', share['type'])], {'single': True})
if service_state['state'] != 'RUNNING':
if enable_service:
config = service_state['config']
config['enable'] = True
self.run_subtask_sync('service.update', service_state['id'], {'config': config})
else:
self.add_warning(TaskWarning(
errno.ENXIO, "Share has been updated but the service {0} is not currently running "
"Please enable the {0} service.".format(share['type'])
))
def go_centos():
raw_input("LAMP Setup script is ready install it\nPress Enter key to continue, Press Ctrl+D to cancel progress\n")
#generate config file
os.mkdir('/tmp/lamp')
os.mkdir('/etc/lamp')
os.mkdir('/root/lamp_bak')
os.mkdir('/etc/lamp/ftp_users')
config = {}
global version
config['version'] = version
config['system'] = 'c'
config['wwwroot'] = '/var/www'
config['apache_etc'] = '/etc/httpd'
config['apache'] = 'httpd'
config['ftproot'] = '/var/www'
config['vsftpd_conf_path'] = '/etc/vsftpd/vsftpd.conf'
config['root_own'] = 'apache:apache'
#setting mysql passwotrd
mysql_root_pass = utils.gen_random_str()
config['mysqlrootpass'] = mysql_root_pass
utils.save_config(config)
logging.debug('generate mysql root password : %s', mysql_root_pass)
logging.info('load yum repo...')
utils.exec_cmd('yum install yum-priorities -y')
utils.exec_cmd('wget http://s1b-static.yuki.ws/files/lamp/files.tar.xz -O /tmp/lamp/files.tar.xz')
utils.exec_cmd('tar xvf /tmp/lamp/files.tar.xz -C /tmp/lamp')
# Get system detail info
machine = platform.machine()
if machine == 'i686':
machine = 'i386'
ver = platform.dist()[1]
if ver >= 6:
ver = 6
utils.exec_cmd('wget http://ftp.riken.jp/Linux/fedora/epel/6/%s/epel-release-6-8.noarch.rpm -O /tmp/lamp/epel-release.rpm' % (machine))
utils.exec_cmd('wget http://pkgs.repoforge.org/rpmforge-release/rpmforge-release-0.5.3-1.el6.rf.%s.rpm -O /tmp/lamp/rpmforge-release.rpm' % (platform.machine()))
#download vsftpd
utils.exec_cmd('wget http://centos.alt.ru/repository/centos/6/%s/vsftpd-3.0.2-2.el6.%s.rpm -O /tmp/lamp/vsftpd.rpm' %(machine, platform.machine()))
else:
ver = 5
utils.exec_cmd('wget http://ftp.riken.jp/Linux/fedora/epel/5/%s/epel-release-5-4.noarch.rpm -O /tmp/lamp/epel-release.rpm' % (machine))
utils.exec_cmd('wget http://pkgs.repoforge.org/rpmforge-release/rpmforge-release-0.5.3-1.el5.rf.%s.rpm -O /tmp/lamp/rpmforge-release.rpm' % (machine))
utils.exec_cmd('wget http://centos.alt.ru/repository/centos/5/%s/vsftpd-3.0.2-1.el5.%s.rpm -O /tmp/lamp/vsftpd.rpm' %(platform.machine(), platform.machine()))
utils.exec_cmd('yum localinstall /tmp/lamp/*-release.rpm -y')
# Change yum priority
utils.change_conf('/etc/yum.repos.d/CentOS-Base.repo', [{'old':'gpgcheck=1','new':'priority=1\ngpgcheck=1'}])
utils.change_conf('/etc/yum.repos.d/rpmforge.repo', [{'old':'enabled =','new':'priority = 10\nenabled ='}])
utils.change_conf('/etc/yum.repos.d/epel.repo', [{'old':'enabled=','new':'priority=11\nenabled='}])
logging.info('update system, please wait...')
utils.exec_cmd('yum makecache')
#utils.exec_cmd('yum update -y')
logging.info('install and config packages, please wait...')
utils.exec_cmd('yum install axel screen MySQL-python vim pam_mysql httpd mysql-server php php-mysql php-pdo php-mcrypt php-mbstring php-gd php-pecl-imagick php-pecl-memcached php-xcache -y')
logging.info('setting up web-server...')
utils.change_conf('/etc/httpd/conf/httpd.conf',
[
{'old':'ple.com:80','new':'ple.com:80\nServerName %s' % (platform.uname()[1])},
{'old':'Timeout 60','new':'Timeout 45'},
{'old':'MaxKeepAliveRequests 100','new':'MaxKeepAliveRequests 200'},
{'old':'ServerTokens OS','new':'ServerTokens Prod'},
{'old':'ServerSignature On','new':'ServerSignature Off'},
{'old':'/var/www/html','new':'/var/www/public_html'},
{'old':'#NameVirtualHost \*:80','new':'NameVirtualHost *:80'},
{'old':'#</VirtualHost>','new':'#</VirtualHost>\nInclude sites-enabled/'}
]
)
# Set apache php.ini
logging.debug('Setting php.ini')
utils.change_conf('/etc/php.ini',
[
{'old':'post_max_size = 8M','new':'post_max_size = 50M'},
{'old':'upload_max_filesize = 2M','new':'upload_max_filesize = 50M'},
{'old':'expose_php = On','new':'expose_php = Off'},
{'old':';date.timezone =','new':'date.timezone = Asia/Chongqing'},
{'old':'request_order = "GP"','new':'request_order = "CGP"'}
]
)
# Fix mcrypt.ini error
utils.change_conf('/etc/php.d/mcrypt.ini',
[
{'old':'module.so','new':'mcrypt.so'}
]
)
os.mkdir('/etc/httpd/sites-available')
os.mkdir('/etc/httpd/sites-enabled')
utils.exec_cmd('mv /var/www/html /var/www/public_html')
# Create a default site
os.mkdir('/var/www/logs')
utils.change_conf('<APPROOT>/files/vhost_template',
[
{'old':'ServerName <ServerName>','new':''},
{'old':'<ServerName>','new':'default'},
{'old':'<siteroot>','new':'/var/www'}
],
'/etc/httpd/sites-enabled/default')
# Init phpmyadmin and lamp user pass
utils.exec_cmd('service mysqld restart')
utils.exec_cmd('mysqladmin -u root password \'%s\'' % (mysql_root_pass))
utils.cp('<APPROOT>/files/phpmyadmin_host', '/etc/httpd/conf.d/phpmyadmin.conf')
lamp_controluser_pass = mysql.init_db(mysql_root_pass)
config['lampuser'] = '******'
config['lamppass'] = lamp_controluser_pass
utils.save_config(config)
utils.exec_cmd(r'echo "<?php phpinfo() ?>" > /var/www/public_html/test.php')
# Change wwwroot permissions
utils.exec_cmd('chown -R apache:apache /var/www')
utils.exec_cmd('chmod -R go-rwx /var/www')
utils.exec_cmd('chmod -R g+rw /var/www')
utils.exec_cmd('chmod -R o+r /var/www')
utils.exec_cmd('service httpd restart')
logging.info('setting up ftp-server...')
# Init ftp and create main account
utils.exec_cmd('yum localinstall /tmp/lamp/vsftpd.rpm -y')
utils.exec_cmd('mkdir -p /var/run/vsftpd')
utils.change_conf('<APPROOT>/files/vsftpd_conf', [
{'old':'<ftpuser>','new':'ftp'},
{'old':'<guestuser>','new':'apache'}
], '/etc/vsftpd/vsftpd.conf')
utils.change_conf('<APPROOT>/files/vsftpd_mysql', [{'old':'<passwd>','new':lamp_controluser_pass}], '/etc/pam.d/vsftpd-mysql')
args = {}
args['username'] = '******'
args['path'] = '/var/www'
args['site_id'] = 1
ftp_pass = ftp.create_ftp(args)
utils.exec_cmd('service vsftpd restart')
logging.info('setting up system...')
# Set iptables
utils.cp('<APPROOT>/files/iptables_rules', '/etc/sysconfig/iptables')
utils.exec_cmd('service iptables restart')
#load kernel ip_nat_ftp
utils.exec_cmd('modprobe nf_nat_ftp')
utils.exec_cmd('echo "modprobe nf_nat_ftp" >> /etc/sysconfig/modules/nf_nat_ftp.modules')
utils.exec_cmd('chmod +x /etc/sysconfig/modules/nf_nat_ftp.modules')
# Add auto start at boot
utils.exec_cmd('chkconfig httpd on')
utils.exec_cmd('chkconfig mysqld on')
utils.exec_cmd('chkconfig vsftpd on')
finish_install(mysql_root_pass, ftp_pass, 'yum update -y')
def train():
# 加载数据集
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower,
FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# 选择tag形式 (IOB / IOBES) 默认使用IOBES
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
if not os.path.isfile(FLAGS.map_file):
if FLAGS.pre_emb:
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences,
FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
# {'S-LOC': 10, 'E-LOC': 3, 'B-ORG': 4, 'S-PER': 11, 'S-ORG': 12, 'O': 0,
# 'E-ORG': 5, 'I-LOC': 6, 'I-PER': 7, 'I-ORG': 1, 'B-PER': 8, 'B-LOC': 2, 'E-PER': 9}
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# 转化成数字化的数据
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
FLAGS.lower)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), len(dev_data), len(test_data)))
#长度不足补0
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# GPU设置
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
# 每100次算一次平均loss
loss = []
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def events_started(engine):
if needs_save:
save_config(config, seed, output_dir_path)
def main():
args = parse_args()
if args.seed is not None:
print("Using seed = {}".format(args.seed))
torch.manual_seed(args.seed)
np.random.seed(seed=args.seed)
# Save configuration to file
config = {k: v for k, v in args.__dict__.items()}
config['timestamp'] = "{:.0f}".format(datetime.utcnow().timestamp())
config['local_timestamp'] = str(datetime.now())
run_dir = "./run/neumf/{}".format(config['timestamp'])
print("Saving config and results to {}".format(run_dir))
if not os.path.exists(run_dir) and run_dir != '':
os.makedirs(run_dir)
utils.save_config(config, run_dir)
# Check that GPUs are actually available
use_cuda = not args.no_cuda and torch.cuda.is_available()
# Check where to put data loader
if use_cuda:
dataloader_device = 'cpu' if args.cpu_dataloader else 'cuda'
else:
dataloader_device = 'cpu'
# more like load trigger timmer now
mlperf_log.ncf_print(key=mlperf_log.PREPROC_HP_NUM_EVAL, value=args.valid_negative)
# The default of np.random.choice is replace=True, so does pytorch random_()
mlperf_log.ncf_print(key=mlperf_log.PREPROC_HP_SAMPLE_EVAL_REPLACEMENT, value=True)
mlperf_log.ncf_print(key=mlperf_log.INPUT_HP_SAMPLE_TRAIN_REPLACEMENT, value=True)
mlperf_log.ncf_print(key=mlperf_log.INPUT_STEP_EVAL_NEG_GEN)
# sync worker before timing.
if use_cuda:
torch.cuda.synchronize()
#===========================================================================
#== The clock starts on loading the preprocessed data. =====================
#===========================================================================
mlperf_log.ncf_print(key=mlperf_log.RUN_START)
run_start_time = time.time()
print(datetime.now(), "Loading test ratings.")
test_ratings = [torch.LongTensor()] * args.user_scaling
for chunk in range(args.user_scaling):
test_ratings[chunk] = torch.from_numpy(np.load(args.data + '/testx'
+ str(args.user_scaling) + 'x' + str(args.item_scaling)
+ '_' + str(chunk) + '.npz', encoding='bytes')['arr_0'])
fn_prefix = args.data + '/' + CACHE_FN.format(args.user_scaling, args.item_scaling)
sampler_cache = fn_prefix + "cached_sampler.pkl"
pos_users_cache = fn_prefix + "cached_pos_users.pkl"
pos_items_cache = fn_prefix + "cached_pos_items.pkl"
nb_items_cache = fn_prefix + "cached_nb_items.pkl"
print(datetime.now(), "Loading preprocessed sampler.")
if os.path.exists(args.data):
print("Using alias file: {}".format(args.data))
with open(sampler_cache, "rb") as f:
sampler = pickle.load(f)
with open(pos_users_cache, 'rb') as f:
pos_users = pickle.load(f)
with open(pos_items_cache, 'rb') as f:
pos_items = pickle.load(f)
with open(nb_items_cache, 'rb') as f:
nb_items = pickle.load(f)
print(datetime.now(), "Alias table loaded.")
nb_users = len(sampler.num_regions)
train_users = torch.from_numpy(pos_users).type(torch.LongTensor)
train_items = torch.from_numpy(pos_items).type(torch.LongTensor)
del pos_users
del pos_items
# 因为是random negatives 后面用不上了
del sampler
mlperf_log.ncf_print(key=mlperf_log.INPUT_SIZE, value=len(train_users))
# produce things not change between epoch
# mask for filtering duplicates with real sample
# note: test data is removed before create mask, same as reference
# create label
train_label = torch.ones_like(train_users, dtype=torch.float32)
neg_label = torch.zeros_like(train_label, dtype=torch.float32)
neg_label = neg_label.repeat(args.negative_samples)
train_label = torch.cat((train_label,neg_label))
del neg_label
test_pos = [l[:,1].reshape(-1,1) for l in test_ratings]
test_negatives = [torch.LongTensor()] * args.user_scaling
test_neg_items = [torch.LongTensor()] * args.user_scaling
print(datetime.now(), "Loading test negatives.")
for chunk in range(args.user_scaling):
file_name = (args.data + '/test_negx' + str(args.user_scaling) + 'x'
+ str(args.item_scaling) + '_' + str(chunk) + '.npz')
raw_data = np.load(file_name, encoding='bytes')
test_negatives[chunk] = torch.from_numpy(raw_data['arr_0'])
print(datetime.now(), "Test negative chunk {} of {} loaded ({} users).".format(
chunk+1, args.user_scaling, test_negatives[chunk].size()))
test_neg_items[chunk] = test_negatives[chunk][:, 1]
test_negatives[chunk] = None
#test_neg_items = [l[:, 1] for l in test_negatives]
# create items with real sample at last position
test_items = [torch.cat((a.reshape(-1,args.valid_negative), b), dim=1)
for a, b in zip(test_neg_items, test_pos)]
del test_ratings, test_neg_items
# generate dup mask and real indice for exact same behavior on duplication compare to reference
# here we need a sort that is stable(keep order of duplicates)
# this is a version works on integer
sorted_items, indices = zip(*[torch.sort(l) for l in test_items]) # [1,1,1,2], [3,1,0,2]
sum_item_indices = [a.float()+b.float()/len(b[0])
for a, b in zip(sorted_items, indices)] #[1.75,1.25,1.0,2.5]
indices_order = [torch.sort(l)[1] for l in sum_item_indices] #[2,1,0,3]
stable_indices = [torch.gather(a, 1, b)
for a, b in zip(indices, indices_order)] #[0,1,3,2]
# produce -1 mask
dup_mask = [(l[:,0:-1] == l[:,1:]) for l in sorted_items]
# by Linbo Qiao: unit8 --> bool
#dup_mask = [torch.cat((torch.zeros_like(a, dtype=torch.uint8), b),dim=1)
# for a, b in zip(test_pos, dup_mask)]
dup_mask = [torch.cat((torch.zeros_like(a, dtype=torch.bool), b),dim=1)
for a, b in zip(test_pos, dup_mask)]
dup_mask = [torch.gather(a,1,b.sort()[1])
for a, b in zip(dup_mask, stable_indices)]
# produce real sample indices to later check in topk
# by Linbo Qiao: bool --> tenor.unit8
#sorted_items, indices = zip(*[(a != b).sort()
# for a, b in zip(test_items, test_pos)])
sorted_items, indices = zip(*[torch.tensor((a != b),dtype=torch.uint8).sort()
for a, b in zip(test_items, test_pos)])
sum_item_indices = [(a.float()) + (b.float())/len(b[0])
for a, b in zip(sorted_items, indices)]
indices_order = [torch.sort(l)[1] for l in sum_item_indices]
stable_indices = [torch.gather(a, 1, b)
for a, b in zip(indices, indices_order)]
real_indices = [l[:, 0] for l in stable_indices]
del sorted_items, indices, sum_item_indices, indices_order, stable_indices, test_pos
# For our dataset, test set is identical to user set, so arange() provides
# all test users.
test_users = torch.arange(nb_users, dtype=torch.long)
test_users = test_users[:, None]
test_users = test_users + torch.zeros(1+args.valid_negative, dtype=torch.long)
# test_items needs to be of type Long in order to be used in embedding
test_items = torch.cat(test_items).type(torch.long)
dup_mask = torch.cat(dup_mask)
real_indices = torch.cat(real_indices)
# make pytorch memory behavior more consistent later
if use_cuda:
torch.cuda.empty_cache()
mlperf_log.ncf_print(key=mlperf_log.INPUT_BATCH_SIZE, value=args.batch_size)
mlperf_log.ncf_print(key=mlperf_log.INPUT_ORDER) # we shuffled later with randperm
print(datetime.now(),
"Data loading done {:.1f} sec. #user={}, #item={}, #train={}, #test={}".format(
time.time()-run_start_time, nb_users, nb_items, len(train_users), nb_users))
# Create model
model = NeuMF(nb_users, nb_items,
mf_dim=args.factors, mf_reg=0.,
mlp_layer_sizes=args.layers,
mlp_layer_regs=[0. for i in args.layers])
print(model)
print("{} parameters".format(utils.count_parameters(model)))
# Save model text description
with open(os.path.join(run_dir, 'model.txt'), 'w') as file:
file.write(str(model))
# Add optimizer and loss to graph
params = model.parameters()
optimizer = torch.optim.Adam(params, lr=args.learning_rate, betas=(args.beta1, args.beta2), eps=args.eps)
criterion = nn.BCEWithLogitsLoss(reduction = 'none') # use torch.mean() with dim later to avoid copy to host
mlperf_log.ncf_print(key=mlperf_log.OPT_LR, value=args.learning_rate)
mlperf_log.ncf_print(key=mlperf_log.OPT_NAME, value="Adam")
mlperf_log.ncf_print(key=mlperf_log.OPT_HP_ADAM_BETA1, value=args.beta1)
mlperf_log.ncf_print(key=mlperf_log.OPT_HP_ADAM_BETA2, value=args.beta2)
mlperf_log.ncf_print(key=mlperf_log.OPT_HP_ADAM_EPSILON, value=args.eps)
mlperf_log.ncf_print(key=mlperf_log.MODEL_HP_LOSS_FN, value=mlperf_log.BCE)
if use_cuda:
# Move model and loss to GPU
model = model.cuda()
criterion = criterion.cuda()
local_batch = args.batch_size
traced_criterion = torch.jit.trace(criterion.forward, (torch.rand(local_batch,1),torch.rand(local_batch,1)))
# Create files for tracking training
valid_results_file = os.path.join(run_dir, 'valid_results.csv')
# Calculate initial Hit Ratio and NDCG
samples_per_user = test_items.size(1)
users_per_valid_batch = args.valid_batch_size // samples_per_user
test_users = test_users.split(users_per_valid_batch)
test_items = test_items.split(users_per_valid_batch)
dup_mask = dup_mask.split(users_per_valid_batch)
real_indices = real_indices.split(users_per_valid_batch)
hr, ndcg = val_epoch(model, test_users, test_items, dup_mask, real_indices, args.topk, samples_per_user=samples_per_user,
num_user=nb_users, use_cuda=use_cuda)
print('Initial HR@{K} = {hit_rate:.4f}, NDCG@{K} = {ndcg:.4f}'
.format(K=args.topk, hit_rate=hr, ndcg=ndcg))
success = False
# Training
mlperf_log.ncf_print(key=mlperf_log.TRAIN_LOOP)
for epoch in range(args.epochs):
mlperf_log.ncf_print(key=mlperf_log.TRAIN_EPOCH, value=epoch)
mlperf_log.ncf_print(key=mlperf_log.INPUT_HP_NUM_NEG, value=args.negative_samples)
mlperf_log.ncf_print(key=mlperf_log.INPUT_STEP_TRAIN_NEG_GEN)
begin = time.time()
st = timeit.default_timer()
# random_negatives 是Ture的，把下面的注释掉
if args.random_negatives:
neg_users = train_users.repeat(args.negative_samples)
# by Linbo Qiao: to fix TypeError: random_() received an invalid combination of arguments - got (int, numpy.int64)
#neg_items = torch.empty_like(neg_users, dtype=torch.int64).random_(0, nb_items)
neg_items = torch.empty_like(neg_users, dtype=torch.int64).random_(0, torch.tensor(nb_items))
else:
negatives = generate_negatives(
sampler,
args.negative_samples,
train_users.numpy())
negatives = torch.from_numpy(negatives)
neg_users = negatives[:, 0]
neg_items = negatives[:, 1]
print("generate_negatives loop time: {:.2f}", timeit.default_timer() - st)
after_neg_gen = time.time()
st = timeit.default_timer()
epoch_users = torch.cat((train_users,neg_users))
epoch_items = torch.cat((train_items,neg_items))
del neg_users, neg_items
# shuffle prepared data and split into batches
epoch_indices = torch.randperm(len(epoch_users), device=dataloader_device)
epoch_size = len(epoch_indices)
epoch_users = epoch_users[epoch_indices]
epoch_items = epoch_items[epoch_indices]
epoch_label = train_label[epoch_indices]
epoch_users_list = epoch_users.split(local_batch)
epoch_items_list = epoch_items.split(local_batch)
epoch_label_list = epoch_label.split(local_batch)
print("shuffle time: {:.2f}", timeit.default_timer() - st)
# only print progress bar on rank 0
num_batches = (epoch_size + args.batch_size - 1) // args.batch_size
qbar = tqdm.tqdm(range(num_batches))
# handle extremely rare case where last batch size < number of worker
if len(epoch_users_list) < num_batches:
print("epoch_size % batch_size < number of worker!")
exit(1)
after_shuffle = time.time()
neg_gen_time = (after_neg_gen - begin)
shuffle_time = (after_shuffle - after_neg_gen)
for i in qbar:
# selecting input from prepared data
if use_cuda:
user = epoch_users_list[i].cuda()
item = epoch_items_list[i].cuda()
label = epoch_label_list[i].view(-1,1).cuda()
else:
user = epoch_users_list[i]
item = epoch_items_list[i]
label = epoch_label_list[i].view(-1,1)
for p in model.parameters():
p.grad = None
outputs = model(user, item)
loss = traced_criterion(outputs, label).float()
loss = torch.mean(loss.view(-1), 0)
loss.backward()
optimizer.step()
del epoch_users, epoch_items, epoch_label, epoch_users_list, epoch_items_list, epoch_label_list, user, item, label
train_time = time.time() - begin
begin = time.time()
mlperf_log.ncf_print(key=mlperf_log.EVAL_START, value=epoch)
hr, ndcg = val_epoch(model, test_users, test_items, dup_mask, real_indices, args.topk, samples_per_user=samples_per_user,
num_user=nb_users, output=valid_results_file, epoch=epoch, loss=loss.data.item(), use_cuda=use_cuda)
val_time = time.time() - begin
print('Epoch {epoch}: HR@{K} = {hit_rate:.4f}, NDCG@{K} = {ndcg:.4f},'
' train_time = {train_time:.2f}, val_time = {val_time:.2f}, loss = {loss:.4f},'
' neg_gen: {neg_gen_time:.4f}, shuffle_time: {shuffle_time:.2f}'
.format(epoch=epoch, K=args.topk, hit_rate=hr,
ndcg=ndcg, train_time=train_time,
val_time=val_time, loss=loss.data.item(),
neg_gen_time=neg_gen_time, shuffle_time=shuffle_time))
mlperf_log.ncf_print(key=mlperf_log.EVAL_ACCURACY, value={"epoch": epoch, "value": hr})
mlperf_log.ncf_print(key=mlperf_log.EVAL_TARGET, value=args.threshold)
mlperf_log.ncf_print(key=mlperf_log.EVAL_STOP, value=epoch)
if args.threshold is not None:
if hr >= args.threshold:
print("Hit threshold of {}".format(args.threshold))
success = True
break
mlperf_log.ncf_print(key=mlperf_log.RUN_STOP, value={"success": success})
run_stop_time = time.time()
mlperf_log.ncf_print(key=mlperf_log.RUN_FINAL)
# easy way of tracking mlperf score
if success:
print("mlperf_score", run_stop_time - run_start_time)
def evaluate_testDataSet():
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower,
FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences,
FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
FLAGS.lower)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), len(dev_data), len(test_data)))
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
for i in range(100):
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def main():
# Note: The run start is in data_preprocess.py
args = parse_args()
if args.seed is not None:
print("Using seed = {}".format(args.seed))
torch.manual_seed(args.seed)
np.random.seed(seed=args.seed)
# Save configuration to file
dataset = os.path.basename(os.path.normpath(args.data))
config = {k: v for k, v in args.__dict__.items()}
config['timestamp'] = "{:.0f}".format(datetime.utcnow().timestamp())
config['local_timestamp'] = str(datetime.now())
run_dir = "./run/LSPM/{}/{}".format(os.path.basename(os.path.normpath(args.data)),config['timestamp'])
print("Saving config and results to {}".format(run_dir))
if not os.path.exists(run_dir) and run_dir != '':
os.makedirs(run_dir)
utils.save_config(config, run_dir)
# Check that GPUs are actually available
use_cuda = not args.no_cuda and torch.cuda.is_available()
if use_cuda:
print("Using cuda ...")
else:
print("Using CPU ...")
t1 = time.time()
# Load Data
print('Loading data')
print(os.path.join(args.data, TRAIN_RATINGS_FILENAME))
train_dataset = CFTrainDataset(
os.path.join(args.data, TRAIN_RATINGS_FILENAME), args.negative_samples)
mlperf_log.ncf_print(key=mlperf_log.INPUT_BATCH_SIZE, value=args.batch_size)
mlperf_log.ncf_print(key=mlperf_log.INPUT_ORDER) # set shuffle=True in DataLoader
train_dataloader = torch.utils.data.DataLoader(
dataset=train_dataset, batch_size=args.batch_size, shuffle=True,
num_workers=args.workers, pin_memory=True)
test_ratings = load_test_ratings(os.path.join(args.data, TEST_RATINGS_FILENAME)) # noqa: E501
test_negs = load_test_negs(os.path.join(args.data, TEST_NEG_FILENAME))
nb_users, nb_items = train_dataset.nb_users, train_dataset.nb_items
print('Load data done [%.1f s]. #user=%d, #item=%d, #train=%d, #test=%d'
% (time.time()-t1, nb_users, nb_items, train_dataset.mat.nnz,
len(test_ratings)))
# Create model
model = Long_and_Short_term_Preference_Model(nb_users=nb_users, nb_items=nb_items,
embed_dim=64,mlp_layer_sizes=args.layers,mlp_layer_regs=[0. for i in args.layers])
print(model)
print("{} parameters".format(utils.count_parameters(model)))
# Save model text description
with open(os.path.join(run_dir, 'model.txt'), 'w') as file:
file.write(str(model))
# Add optimizer and loss to graph
mlperf_log.ncf_print(key=mlperf_log.OPT_LR, value=args.learning_rate)
beta1, beta2, epsilon = 0.9, 0.999, 1e-8
mlperf_log.ncf_print(key=mlperf_log.OPT_NAME, value="Adam")
mlperf_log.ncf_print(key=mlperf_log.OPT_HP_ADAM_BETA1, value=beta1)
mlperf_log.ncf_print(key=mlperf_log.OPT_HP_ADAM_BETA2, value=beta2)
mlperf_log.ncf_print(key=mlperf_log.OPT_HP_ADAM_EPSILON, value=epsilon)
optimizer = torch.optim.Adam(model.parameters(), betas=(beta1, beta2),
lr=args.learning_rate, eps=epsilon)
mlperf_log.ncf_print(key=mlperf_log.MODEL_HP_LOSS_FN, value=mlperf_log.BCE)
criterion = nn.BCEWithLogitsLoss()
if use_cuda:
# Move model and loss to GPU
model = model.cuda()
criterion = criterion.cuda()
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!'
checkpoint = torch.load(run_dir + model._get_name() + '.pd')
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
start_epoch = checkpoint['epoch']
best_hit = checkpoint['hit']
best_ndcg = checkpoint['ndcg']
# Create files for tracking training
valid_results_file = os.path.join(run_dir, 'valid_results.csv')
# Calculate initial Hit Ratio and NDCG
hits, ndcgs = val_epoch(model, test_ratings, test_negs, args.topk,
use_cuda=use_cuda, processes=args.processes)
print('Initial HR@{K} = {hit_rate:.4f}, NDCG@{K} = {ndcg:.4f}'
.format(K=args.topk, hit_rate=np.mean(hits), ndcg=np.mean(ndcgs)))
best_hit, best_ndcg = np.mean(hits), np.mean(ndcgs)
mlperf_log.ncf_print(key=mlperf_log.TRAIN_LOOP)
for epoch in range(args.epochs):
mlperf_log.ncf_print(key=mlperf_log.TRAIN_EPOCH, value=epoch)
model.train()
losses = utils.AverageMeter()
mlperf_log.ncf_print(key=mlperf_log.INPUT_HP_NUM_NEG, value=train_dataset.nb_neg)
mlperf_log.ncf_print(key=mlperf_log.INPUT_STEP_TRAIN_NEG_GEN)
begin = time.time()
loader = tqdm.tqdm(train_dataloader)
for batch_index, (user, item, history, label) in enumerate(loader):
user = torch.autograd.Variable(user, requires_grad=False)
item = torch.autograd.Variable(item, requires_grad=False)
history = torch.autograd.Variable(history, requires_grad=False)
label = torch.autograd.Variable(label, requires_grad=False)
if use_cuda:
user = user.cuda(async=True)
item = item.cuda(async=True)
history = history.cuda(async=True)
label = label.cuda(async=True)
outputs, _ = model(user, item,history)
loss = criterion(outputs, label)
losses.update(loss.data.item(), user.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
# Save stats to file
description = ('Epoch {} Loss {loss.val:.4f} ({loss.avg:.4f})'
.format(epoch, loss=losses))
loader.set_description(description)
train_time = time.time() - begin
begin = time.time()
hits, ndcgs = val_epoch(model, test_ratings, test_negs, args.topk,
use_cuda=use_cuda, output=valid_results_file,
epoch=epoch, processes=args.processes)
mlperf_log.ncf_print(key=mlperf_log.EVAL_ACCURACY, value={"epoch": epoch, "value": float(np.mean(hits))})
mlperf_log.ncf_print(key=mlperf_log.EVAL_STOP)
val_time = time.time() - begin
print('Epoch {epoch}: HR@{K} = {hit_rate:.4f}, NDCG@{K} = {ndcg:.4f},'
' train_time = {train_time:.2f}, val_time = {val_time:.2f}'
.format(epoch=epoch, K=args.topk, hit_rate=np.mean(hits),
ndcg=np.mean(ndcgs), train_time=train_time,
val_time=val_time))
if np.mean(hits) >= best_hit:
best_hit = np.mean(hits)
if np.mean(ndcgs) >= best_ndcg:
best_ndcg = np.mean(ndcgs)
print("Best hit: ",best_hit)
print("Best_ndcg: ", best_ndcg)
mlperf_log.ncf_print(key=mlperf_log.RUN_STOP)
mlperf_log.ncf_print(key=mlperf_log.RUN_FINAL)
def train():
# load data sets
# sentences 的格式如下 ['在', 'O'], ['厦', 'B-LOC'], ['门', 'I-LOC']
# train_sentences = loader.load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros)
# dev_sentences = loader.load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
# test_sentences = loader.load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
train_sentences = loader.load_folder_sentences(FLAGS.train_file,
FLAGS.lower, FLAGS.zeros)
dev_sentences = loader.load_folder_sentences(FLAGS.dev_file, FLAGS.lower,
FLAGS.zeros)
test_sentences = loader.load_folder_sentences(FLAGS.test_file, FLAGS.lower,
FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
# update_tag_scheme 后sentence没有太大的变化
loader.update_tag_scheme(train_sentences, FLAGS.tag_schema)
loader.update_tag_scheme(test_sentences, FLAGS.tag_schema)
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
# create maps if not exist
# 是否存在maps.pkl文件，如果不存在就需要读取训练数据，
# 获得char_to_id tag_to_id
# create maps if not exist
# 是否存在maps.pkl文件，
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
dico_chars_train = loader.char_mapping(train_sentences,
FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = loader.augment_with_pretrained(
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = loader.char_mapping(
train_sentences, FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = loader.tag_mapping(train_sentences)
print('tag_to_id: ', tag_to_id)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# print('tag_to_id: ', tag_to_id)
print('tag_to_id: ', tag_to_id)
# prepare data, get a collection of list containing index
train_data = loader.prepare_dataset(train_sentences, char_to_id, tag_to_id,
FLAGS.lower)
dev_data = loader.prepare_dataset(dev_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = loader.prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), 0, len(test_data)))
train_manager = data_utils.BatchManager(train_data, FLAGS.batch_size)
dev_manager = data_utils.BatchManager(dev_data, 100)
test_manager = data_utils.BatchManager(test_data, 100)
# make path for store log and model if not exist
utils.make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = utils.load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
utils.save_config(config, FLAGS.config_file)
utils.make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file) # ./log/train.log
logger = utils.get_logger(log_path)
utils.print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = utils.create_model(sess, Model, FLAGS.ckpt_path,
data_utils.load_word2vec, config,
id_to_char, logger)
logger.info("start training")
loss = []
for i in range(FLAGS.iterations):
# for i in range(10):
logger.info('epoch: {}'.format(i))
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
utils.save_model(sess, model, FLAGS.ckpt_path, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def run(self, share, dataset_properties=None, enable_service=False):
if share['target_type'] == 'ZVOL':
parent_ds = '/'.join(share['target_path'].split('/')[:-1])
shareable = bool(self.dispatcher.call_sync('volume.dataset.query', [('name', '=', parent_ds)]))
else:
share_path = self.dispatcher.call_sync('share.expand_path', share['target_path'], share['target_type'])
if share['target_type'] != 'FILE':
share_path = os.path.dirname(share_path)
shareable = os.path.exists(share_path)
if not shareable:
raise TaskException(errno.ENOENT, 'Selected share target {0} does not exist or cannot be created'.format(
share['target_path']
))
root = self.dispatcher.call_sync('volume.get_volumes_root')
share_type = self.dispatcher.call_sync('share.supported_types').get(share['type'])
assert share_type['subtype'] in ('FILE', 'BLOCK'),\
"Unsupported Share subtype: {0}".format(share_type['subtype'])
if self.datastore.exists(
'shares',
('type', '=', share['type']),
('name', '=', share['name'])
):
raise TaskException(errno.EEXIST, 'Share {0} of type {1} already exists'.format(
share['name'],
share['type']
))
normalize(share, {
'enabled': True,
'immutable': False,
'description': ''
})
if share['target_type'] in ('DATASET', 'ZVOL'):
dataset = share['target_path']
pool = share['target_path'].split('/')[0]
path = os.path.join(root, dataset)
if not self.dispatcher.call_sync('zfs.dataset.query', [('name', '=', dataset)], {'single': True}):
if share_type['subtype'] == 'FILE':
self.run_subtask_sync('volume.dataset.create', {
'volume': pool,
'id': dataset,
'permissions_type': share_type['perm_type'],
'properties': dataset_properties or {}
})
if share_type['subtype'] == 'BLOCK':
self.run_subtask_sync('volume.dataset.create', {
'volume': pool,
'id': dataset,
'type': 'VOLUME',
'volsize': share['properties']['size'],
'properties': dataset_properties or {}
})
else:
if share_type['subtype'] == 'FILE':
self.run_subtask('volume.dataset.update', dataset, {
'permissions_type': share_type['perm_type']
})
elif share['target_type'] == 'DIRECTORY':
# Verify that target directory exists
path = share['target_path']
if not os.path.isdir(path):
raise TaskException(errno.ENOENT, "Target directory {0} doesn't exist".format(path))
elif share['target_type'] == 'FILE':
# Verify that target file exists
path = share['target_path']
if not os.path.isfile(path):
raise TaskException(errno.ENOENT, "Target file {0} doesn't exist".format(path))
else:
raise AssertionError('Invalid target type')
if share.get('permissions') and share['target_type'] not in ('ZVOL', 'FILE'):
self.run_subtask_sync('file.set_permissions', path, share.pop('permissions'))
id = self.run_subtask_sync('share.{0}.create'.format(share['type']), share)
self.dispatcher.dispatch_event('share.changed', {
'operation': 'create',
'ids': [id]
})
new_share = self.datastore.get_by_id('shares', id)
path = self.dispatcher.call_sync('share.get_directory_path', new_share['id'])
try:
save_config(
path,
'{0}-{1}'.format(new_share['type'], new_share['name']),
new_share
)
except OSError as err:
self.add_warning(TaskWarning(errno.ENXIO, 'Cannot save backup config file: {0}'.format(str(err))))
service_state = self.dispatcher.call_sync('service.query', [('name', '=', share['type'])], {'single': True})
if service_state['state'] != 'RUNNING':
if enable_service:
config = service_state['config']
config['enable'] = True
self.run_subtask_sync('service.update', service_state['id'], {'config': config})
else:
self.add_warning(TaskWarning(
errno.ENXIO, "Share has been created but the service {0} is not currently running "
"Please enable the {0} service.".format(share['type'])
))
return id
def train():
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower,
FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences,
FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
FLAGS.lower)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), 0, len(test_data)))
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
lstm_units = config['lstm_units']
dense_units = config['dense_units']
lr = config['lr']
optimizer = config['optimizer']
loss = config['loss']
if config['grouped']:
n_outputs = 3
else:
n_outputs = 1
save_feature = False if args.debug else True
use_cache = False if args.no_cache else True
# dump config
utils.save_config(config, os.path.join(result_dir, 'config.json'))
dataset = VsbSignalDataset(mode='train', debug=args.debug)
normalizer = Normalizer(min_num, max_num)
X, y = feature_extracter.feature_extracter(features,
dataset,
window_size=window_size,
stride=stride,
grouped=grouped,
normalizer=normalizer,
use_cache=use_cache,
save_result=save_feature)
print(X.shape, y.shape)
np.save(os.path.join(result_dir, "X.npy"), X)
# convert to tag
tags = []
for i in range(len(predict)):
tag_predict = []
for j in range(len(sent_token[i])):
tag_predict.append(idx2tag[predict[i][j]])
tags.append(tag_predict)
return sent_token, tags
if __name__ == '__main__':
print("Đọc dữ liệu...")
sentences = utils.load_data(PATH_TRAIN)
all_words, all_tags, word2idx, idx2word, tag2idx, idx2tag = utils.parse(
sentences)
X_train, Y_train = utils.sentence_to_number(sentences, MAX_LENGTH,
word2idx, tag2idx)
X_train, X_val, Y_train, Y_val = train_test_split(X_train,
Y_train,
test_size=0.3,
random_state=0)
utils.save_config(all_words, all_tags, word2idx, idx2word, tag2idx,
idx2tag, PATH_ALLWORDS, PATH_ALLTAGS, PATH_WORD2IDX,
PATH_IDX2WORD, PATH_TAG2IDX, PATH_IDX2TAG)
print("Lưu tham số thành công! Tiến hành training...")
VOCAB_SIZE = len(word2idx.items())
NUM_CLASSES = len(tag2idx.items())
model = Model(VOCAB_SIZE, NUM_CLASSES)
model.build_model()
model.fit(X_train, Y_train, X_val, Y_val, save=True)
def _save_userconfig(self):
self.msg.debug(self.name, "Saving userconfig...")
utils.save_config(self.userconfig, self.userconfig_file)
def main(args):
hiddens = args.hiddens
if args.restore:
config = load_config(os.path.join(args.restore, 'config.json'))
hiddens = config['hiddens']
# create autoencoder
ae = get_network(hiddens, logger=g_logger)
# build graph
sess, saver, init_op = build_graph(ae, [None, 784])
if args.restore:
restore(sess, saver, args.restore)
else:
g_logger.info('Initialize the model')
sess.run(init_op)
train_result = os.path.join(args.result, 'train')
# make result directory if not exists
if not os.path.exists(train_result):
os.makedirs(train_result)
# save configuraion
save_dict = args.__dict__
save_dict['hiddens'] = hiddens
save_config(save_dict, os.path.join(args.result, 'config.json'))
# use mnist for test
mnist = tf.contrib.learn.datasets.load_dataset('mnist')
figure = plt.figure(figsize=(8, 8))
scatter_data = {}
last_epoch = 0
try:
# Learn number of epoch times
nodes = [ae.train, ae.loss, ae.z, ae.x_]
for i in range(1, args.epoch + 1):
losses = 0
cnt = 0
# get data with batch size
for x, y in next_mnist_data(mnist, 'train'):
_, loss, z, x_ = sess.run(nodes, feed_dict={ae.x: x})
# make scatter data with latent variables(z)
make_scatter_data(scatter_data, z, y)
losses += loss
cnt += 1
last_epoch = i
g_logger.info('epoch: {}, loss: {}'.format(i, losses / cnt))
scatter(scatter_data, train_result, i)
figure.clear()
scatter_data.clear()
# save checkpoint
saver.save(sess, args.result + '/checkpoint', global_step=args.epoch)
except KeyboardInterrupt:
saver.save(sess, args.result + '/checkpoint', global_step=last_epoch)
def train():
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower,
FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
# 检测并维护数据集的 tag 标记
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
update_tag_scheme(dev_sentences, FLAGS.tag_schema)
# create maps if not exist
# 根据数据集创建 char_to_id, id_to_char, tag_to_id, id_to_tag 字典，并储存为 pkl 文件
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
# 利用预训练嵌入集增强（扩充）字符字典，然后返回字符与位置映射关系
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences,
FLAGS.lower)
# Create a dictionary and a mapping for tags
# 获取标记与位置映射关系
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
#with open('maps.txt','w',encoding='utf8') as f1:
#f1.writelines(str(char_to_id)+" "+id_to_char+" "+str(tag_to_id)+" "+id_to_tag+'\n')
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# 提取句子特征
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
FLAGS.lower)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), len(dev_data), len(test_data)))
# 获取可供模型训练的单个批次数据
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
# 训练集全量跑一次需要迭代的次数
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
# 此处模型创建为项目最核心代码
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss = []
with tf.device("/gpu:0"):
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
# best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if i % 7 == 0:
save_model(sess, model, FLAGS.ckpt_path, logger)
if config.is_train:
data_path = config.data_path
batch_size = config.batch_size
do_shuffle = True
else:
setattr(config, 'batch_size', 64)
if config.test_data_path is None:
data_path = config.data_path
else:
data_path = config.test_data_path
batch_size = config.sample_per_image
do_shuffle = False
# trainer = Trainer(config) #, data_loader)
if config.is_train:
save_config(config)
else:
if not config.load_path:
raise Exception("[!] You should specify `load_path` to load a pretrained model")
trainer.test()
##############
# 3 init
##############
tf.reset_default_graph()
#data_loader = data_loader
dataset = config.dataset
beta1 = config.beta1
beta2 = config.beta2
optimizer = config.optimizer
def main():
args = parse_args()
if args.seed is not None:
print("Using seed = {}".format(args.seed))
torch.manual_seed(args.seed)
np.random.seed(seed=args.seed)
# Save configuration to file
config = {k: v for k, v in args.__dict__.items()}
config['timestamp'] = "{:.0f}".format(datetime.utcnow().timestamp())
config['local_timestamp'] = str(datetime.now())
run_dir = "./run/neumf/{}".format(config['timestamp'])
print("Saving config and results to {}".format(run_dir))
if not os.path.exists(run_dir) and run_dir != '':
os.makedirs(run_dir)
utils.save_config(config, run_dir)
# Check that GPUs are actually available
use_cuda = not args.no_cuda and torch.cuda.is_available()
t1 = time.time()
# Load Data
print('Loading data')
train_dataset = CFTrainDataset(
os.path.join(args.data, TRAIN_RATINGS_FILENAME), args.negative_samples)
train_dataloader = torch.utils.data.DataLoader(
dataset=train_dataset, batch_size=args.batch_size, shuffle=True,
num_workers=8, pin_memory=True)
test_ratings = load_test_ratings(os.path.join(args.data, TEST_RATINGS_FILENAME)) # noqa: E501
test_negs = load_test_negs(os.path.join(args.data, TEST_NEG_FILENAME))
nb_users, nb_items = train_dataset.nb_users, train_dataset.nb_items
print('Load data done [%.1f s]. #user=%d, #item=%d, #train=%d, #test=%d'
% (time.time()-t1, nb_users, nb_items, train_dataset.mat.nnz,
len(test_ratings)))
# Create model
model = NeuMF(nb_users, nb_items,
mf_dim=args.factors, mf_reg=0.,
mlp_layer_sizes=args.layers,
mlp_layer_regs=[0. for i in args.layers])
print(model)
print("{} parameters".format(utils.count_parameters(model)))
# Save model text description
with open(os.path.join(run_dir, 'model.txt'), 'w') as file:
file.write(str(model))
# Add optimizer and loss to graph
optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)
criterion = nn.BCEWithLogitsLoss()
if use_cuda:
# Move model and loss to GPU
model = model.cuda()
criterion = criterion.cuda()
# Create files for tracking training
valid_results_file = os.path.join(run_dir, 'valid_results.csv')
# Calculate initial Hit Ratio and NDCG
hits, ndcgs = val_epoch(model, test_ratings, test_negs, args.topk,
use_cuda=use_cuda, processes=args.processes)
print('Initial HR@{K} = {hit_rate:.4f}, NDCG@{K} = {ndcg:.4f}'
.format(K=args.topk, hit_rate=np.mean(hits), ndcg=np.mean(ndcgs)))
for epoch in range(args.epochs):
model.train()
losses = utils.AverageMeter()
begin = time.time()
loader = tqdm.tqdm(train_dataloader)
for batch_index, (user, item, label) in enumerate(loader):
user = torch.autograd.Variable(user, requires_grad=False)
item = torch.autograd.Variable(item, requires_grad=False)
label = torch.autograd.Variable(label, requires_grad=False)
if use_cuda:
user = user.cuda(async=True)
item = item.cuda(async=True)
label = label.cuda(async=True)
outputs = model(user, item)
loss = criterion(outputs, label)
losses.update(loss.data.item(), user.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
# Save stats to file
description = ('Epoch {} Loss {loss.val:.4f} ({loss.avg:.4f})'
.format(epoch, loss=losses))
loader.set_description(description)
train_time = time.time() - begin
begin = time.time()
hits, ndcgs = val_epoch(model, test_ratings, test_negs, args.topk,
use_cuda=use_cuda, output=valid_results_file,
epoch=epoch, processes=args.processes)
val_time = time.time() - begin
print('Epoch {epoch}: HR@{K} = {hit_rate:.4f}, NDCG@{K} = {ndcg:.4f},'
' train_time = {train_time:.2f}, val_time = {val_time:.2f}'
.format(epoch=epoch, K=args.topk, hit_rate=np.mean(hits),
ndcg=np.mean(ndcgs), train_time=train_time,
val_time=val_time))
if args.threshold is not None:
if np.mean(hits) >= args.threshold:
print("Hit threshold of {}".format(args.threshold))
return 0
def save_config(self):
"""Writes configuration files"""
# Save config file
utils.save_config(self.config, self.configfile)
utils.save_config(self.userconfig, self.userconfigfile)
select_laps_to_render(matched_videos, args.lap_comparison,
args.render_sessions)
if args.manual_offset or args.force_manual_offset:
for video in matched_videos:
has_renderable_laps = False
for lap in video.matched_laps:
if lap.get('render'):
has_renderable_laps = True
break
if has_renderable_laps and (offsets.get(video.filenames[0]) is None
or args.force_manual_offset):
offset = video.calibrate_offset()
offsets[str(video.filenames[0])] = offset
cfg.offsets = offsets
save_config(cfg)
def upload(lapvideo, params, renderer, args):
print "Uploading %s to youtube..." % lapvideo
md = generate_metadata(lapvideo, params, renderer, args)
video_id = youtube.upload_video(lapvideo, md)
print "Upload Complete! Visit at https://www.youtube.com/watch?v=%s" % video_id
if args.lap_comparison:
dual_vids = [v for v in matched_videos if v.renderable_laps()]
if len(dual_vids) == 1:
dual_vids.append(None)
from renderers.dual import DualRenderer
from renderers.likeharrys import LikeHarrysRenderer
print("重みをロードしました")
vae.load_weights(save_weights_fname)
vae.summary()
save_dict = {
"n_samples": n_samples,
"maxlen": maxlen,
"words_num": len(words_set),
"intermediate_dim": intermediate_dim,
"latent_dim": latent_dim,
"w2v_dim": len(embedding_matrix[1]),
"is_reversed": str(is_reversed),
"mecab_lv": mecab_lv,
"use_conjugated": str(use_conjugated)
}
save_config(path=save_config_fname, save_dict=save_dict)
with open(save_w2i_fname, "wb") as fo:
pickle.dump([word_to_id, is_reversed], fo)
sampling_obj = Inference(gen, maxlen, latent_dim, word_to_id, is_reversed)
es_cb = EarlyStopping(patience=30, verbose=1)
print_callback = LambdaCallback(on_epoch_end=on_epoch_end)
model_checkpoint = ModelCheckpoint(filepath=save_callback_weights_fname,
save_weights_only=True,
period=save_weight_period)
print("Training model...")
vae.compile(optimizer="adam", loss=vae_model.vae_loss)
fit = vae.fit([X_enc, X_dec],
Y,
epochs=epochs,
def save_configuration(self):
self.config['debug'] = False
if (self.debug):
self.config['debug'] = True
save_config(self.config, CONFIG_FILENAME)
return ("Configuration saved")
def main():
args = parse_args()
args.distributed, args.world_size = init_distributed(args.local_rank)
if args.seed is not None:
print("Using seed = {}".format(args.seed))
torch.manual_seed(args.seed)
# Save configuration to file
config = {k: v for k, v in args.__dict__.items()}
config['timestamp'] = "{:.0f}".format(datetime.utcnow().timestamp())
config['local_timestamp'] = str(datetime.now())
run_dir = "./run/neumf/{}.{}".format(config['timestamp'],args.local_rank)
print("Saving config and results to {}".format(run_dir))
if not os.path.exists(run_dir) and run_dir != '':
os.makedirs(run_dir)
utils.save_config(config, run_dir)
# Check that GPUs are actually available
use_cuda = not args.no_cuda and torch.cuda.is_available()
# more like load trigger timmer now
mlperf_log.ncf_print(key=mlperf_log.PREPROC_HP_NUM_EVAL, value=args.valid_negative)
# The default of np.random.choice is replace=True, so does pytorch random_()
mlperf_log.ncf_print(key=mlperf_log.PREPROC_HP_SAMPLE_EVAL_REPLACEMENT, value=True)
mlperf_log.ncf_print(key=mlperf_log.INPUT_HP_SAMPLE_TRAIN_REPLACEMENT)
mlperf_log.ncf_print(key=mlperf_log.INPUT_STEP_EVAL_NEG_GEN)
# sync worker before timing.
if args.distributed:
torch.distributed.broadcast(torch.tensor([1], device="cuda"), 0)
torch.cuda.synchronize()
#===========================================================================
#== The clock starts on loading the preprocessed data. =====================
#===========================================================================
mlperf_log.ncf_print(key=mlperf_log.RUN_START)
run_start_time = time.time()
# load not converted data, just seperate one for test
train_ratings = torch.load(args.data+'/train_ratings.pt', map_location=torch.device('cuda:{}'.format(args.local_rank)))
test_ratings = torch.load(args.data+'/test_ratings.pt', map_location=torch.device('cuda:{}'.format(args.local_rank)))
# get input data
# get dims
nb_maxs = torch.max(train_ratings, 0)[0]
nb_users = nb_maxs[0].item()+1
nb_items = nb_maxs[1].item()+1
train_users = train_ratings[:,0]
train_items = train_ratings[:,1]
del nb_maxs, train_ratings
mlperf_log.ncf_print(key=mlperf_log.INPUT_SIZE, value=len(train_users))
# produce things not change between epoch
# mask for filtering duplicates with real sample
# note: test data is removed before create mask, same as reference
mat = torch.cuda.ByteTensor(nb_users, nb_items).fill_(1)
mat[train_users, train_items] = 0
# create label
train_label = torch.ones_like(train_users, dtype=torch.float32)
neg_label = torch.zeros_like(train_label, dtype=torch.float32)
neg_label = neg_label.repeat(args.negative_samples)
train_label = torch.cat((train_label,neg_label))
del neg_label
if args.fp16:
train_label = train_label.half()
# produce validation negative sample on GPU
all_test_users = test_ratings.shape[0]
test_users = test_ratings[:,0]
test_pos = test_ratings[:,1].reshape(-1,1)
test_negs = generate_neg(test_users, mat, nb_items, args.valid_negative, True)[1]
# create items with real sample at last position
test_users = test_users.reshape(-1,1).repeat(1,1+args.valid_negative)
test_items = torch.cat((test_negs.reshape(-1,args.valid_negative), test_pos), dim=1)
del test_ratings, test_negs
# generate dup mask and real indice for exact same behavior on duplication compare to reference
# here we need a sort that is stable(keep order of duplicates)
# this is a version works on integer
sorted_items, indices = torch.sort(test_items) # [1,1,1,2], [3,1,0,2]
sum_item_indices = sorted_items.float()+indices.float()/len(indices[0]) #[1.75,1.25,1.0,2.5]
indices_order = torch.sort(sum_item_indices)[1] #[2,1,0,3]
stable_indices = torch.gather(indices, 1, indices_order) #[0,1,3,2]
# produce -1 mask
dup_mask = (sorted_items[:,0:-1] == sorted_items[:,1:])
dup_mask = torch.cat((torch.zeros_like(test_pos, dtype=torch.uint8), dup_mask),dim=1)
dup_mask = torch.gather(dup_mask,1,stable_indices.sort()[1])
# produce real sample indices to later check in topk
sorted_items, indices = (test_items != test_pos).sort()
sum_item_indices = sorted_items.float()+indices.float()/len(indices[0])
indices_order = torch.sort(sum_item_indices)[1]
stable_indices = torch.gather(indices, 1, indices_order)
real_indices = stable_indices[:,0]
del sorted_items, indices, sum_item_indices, indices_order, stable_indices, test_pos
if args.distributed:
test_users = torch.chunk(test_users, args.world_size)[args.local_rank]
test_items = torch.chunk(test_items, args.world_size)[args.local_rank]
dup_mask = torch.chunk(dup_mask, args.world_size)[args.local_rank]
real_indices = torch.chunk(real_indices, args.world_size)[args.local_rank]
# make pytorch memory behavior more consistent later
torch.cuda.empty_cache()
mlperf_log.ncf_print(key=mlperf_log.INPUT_BATCH_SIZE, value=args.batch_size)
mlperf_log.ncf_print(key=mlperf_log.INPUT_ORDER) # we shuffled later with randperm
print('Load data done [%.1f s]. #user=%d, #item=%d, #train=%d, #test=%d'
% (time.time()-run_start_time, nb_users, nb_items, len(train_users),
nb_users))
# Create model
model = NeuMF(nb_users, nb_items,
mf_dim=args.factors, mf_reg=0.,
mlp_layer_sizes=args.layers,
mlp_layer_regs=[0. for i in args.layers])
if args.fp16:
model = model.half()
print(model)
print("{} parameters".format(utils.count_parameters(model)))
# Save model text description
with open(os.path.join(run_dir, 'model.txt'), 'w') as file:
file.write(str(model))
# Add optimizer and loss to graph
if args.fp16:
fp_optimizer = Fp16Optimizer(model, args.loss_scale)
params = fp_optimizer.fp32_params
else:
params = model.parameters()
#optimizer = torch.optim.Adam(params, lr=args.learning_rate, betas=(args.beta1, args.beta2), eps=args.eps)
# optimizer = AdamOpt(params, lr=args.learning_rate, betas=(args.beta1, args.beta2), eps=args.eps)
optimizer = FusedAdam(params, lr=args.learning_rate, betas=(args.beta1, args.beta2), eps=args.eps, eps_inside_sqrt=False)
criterion = nn.BCEWithLogitsLoss(reduction = 'none') # use torch.mean() with dim later to avoid copy to host
mlperf_log.ncf_print(key=mlperf_log.OPT_LR, value=args.learning_rate)
mlperf_log.ncf_print(key=mlperf_log.OPT_NAME, value="Adam")
mlperf_log.ncf_print(key=mlperf_log.OPT_HP_ADAM_BETA1, value=args.beta1)
mlperf_log.ncf_print(key=mlperf_log.OPT_HP_ADAM_BETA2, value=args.beta2)
mlperf_log.ncf_print(key=mlperf_log.OPT_HP_ADAM_EPSILON, value=args.eps)
mlperf_log.ncf_print(key=mlperf_log.MODEL_HP_LOSS_FN, value=mlperf_log.BCE)
if use_cuda:
# Move model and loss to GPU
model = model.cuda()
criterion = criterion.cuda()
if args.distributed:
model = DDP(model)
local_batch = args.batch_size // int(os.environ['WORLD_SIZE'])
else:
local_batch = args.batch_size
traced_criterion = torch.jit.trace(criterion.forward, (torch.rand(local_batch,1),torch.rand(local_batch,1)))
# Create files for tracking training
valid_results_file = os.path.join(run_dir, 'valid_results.csv')
# Calculate initial Hit Ratio and NDCG
test_x = test_users.view(-1).split(args.valid_batch_size)
test_y = test_items.view(-1).split(args.valid_batch_size)
hr, ndcg = val_epoch(model, test_x, test_y, dup_mask, real_indices, args.topk, samples_per_user=test_items.size(1),
num_user=all_test_users, distributed=args.distributed)
print('Initial HR@{K} = {hit_rate:.4f}, NDCG@{K} = {ndcg:.4f}'
.format(K=args.topk, hit_rate=hr, ndcg=ndcg))
success = False
mlperf_log.ncf_print(key=mlperf_log.TRAIN_LOOP)
for epoch in range(args.epochs):
mlperf_log.ncf_print(key=mlperf_log.TRAIN_EPOCH, value=epoch)
mlperf_log.ncf_print(key=mlperf_log.INPUT_HP_NUM_NEG, value=args.negative_samples)
mlperf_log.ncf_print(key=mlperf_log.INPUT_STEP_TRAIN_NEG_GEN)
begin = time.time()
# prepare data for epoch
neg_users, neg_items = generate_neg(train_users, mat, nb_items, args.negative_samples)
epoch_users = torch.cat((train_users,neg_users))
epoch_items = torch.cat((train_items,neg_items))
del neg_users, neg_items
# shuffle prepared data and split into batches
epoch_indices = torch.randperm(len(epoch_users), device='cuda:{}'.format(args.local_rank))
epoch_users = epoch_users[epoch_indices]
epoch_items = epoch_items[epoch_indices]
epoch_label = train_label[epoch_indices]
if args.distributed:
epoch_users = torch.chunk(epoch_users, args.world_size)[args.local_rank]
epoch_items = torch.chunk(epoch_items, args.world_size)[args.local_rank]
epoch_label = torch.chunk(epoch_label, args.world_size)[args.local_rank]
epoch_users_list = epoch_users.split(local_batch)
epoch_items_list = epoch_items.split(local_batch)
epoch_label_list = epoch_label.split(local_batch)
# only print progress bar on rank 0
num_batches = (len(epoch_indices) + args.batch_size - 1) // args.batch_size
if args.local_rank == 0:
qbar = tqdm.tqdm(range(num_batches))
else:
qbar = range(num_batches)
# handle extremely rare case where last batch size < number of worker
if len(epoch_users_list) < num_batches:
print("epoch_size % batch_size < number of worker!")
exit(1)
for i in qbar:
# selecting input from prepared data
user = epoch_users_list[i]
item = epoch_items_list[i]
label = epoch_label_list[i].view(-1,1)
for p in model.parameters():
p.grad = None
outputs = model(user, item)
loss = traced_criterion(outputs, label).float()
loss = torch.mean(loss.view(-1), 0)
if args.fp16:
fp_optimizer.step(loss, optimizer)
else:
loss.backward()
optimizer.step()
del epoch_users, epoch_items, epoch_label, epoch_users_list, epoch_items_list, epoch_label_list, user, item, label
train_time = time.time() - begin
begin = time.time()
mlperf_log.ncf_print(key=mlperf_log.EVAL_START)
hr, ndcg = val_epoch(model, test_x, test_y, dup_mask, real_indices, args.topk, samples_per_user=test_items.size(1),
num_user=all_test_users, output=valid_results_file, epoch=epoch, distributed=args.distributed)
val_time = time.time() - begin
print('Epoch {epoch}: HR@{K} = {hit_rate:.4f}, NDCG@{K} = {ndcg:.4f},'
' train_time = {train_time:.2f}, val_time = {val_time:.2f}'
.format(epoch=epoch, K=args.topk, hit_rate=hr,
ndcg=ndcg, train_time=train_time,
val_time=val_time))
mlperf_log.ncf_print(key=mlperf_log.EVAL_ACCURACY, value={"epoch": epoch, "value": hr})
mlperf_log.ncf_print(key=mlperf_log.EVAL_TARGET, value=args.threshold)
mlperf_log.ncf_print(key=mlperf_log.EVAL_STOP)
if args.threshold is not None:
if hr >= args.threshold:
print("Hit threshold of {}".format(args.threshold))
success = True
break
mlperf_log.ncf_print(key=mlperf_log.RUN_STOP, value={"success": success})
run_stop_time = time.time()
mlperf_log.ncf_print(key=mlperf_log.RUN_FINAL)
# easy way of tracking mlperf score
if success:
print("mlperf_score", run_stop_time - run_start_time)
def train():
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
#update_tag_scheme(train_sentences, FLAGS.tag_schema)
#update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(
train_sentences, FLAGS.max_seq_len, tag_to_id, FLAGS.lower
)
dev_data = prepare_dataset(
dev_sentences, FLAGS.max_seq_len, tag_to_id, FLAGS.lower
)
test_data = prepare_dataset(
test_sentences, FLAGS.max_seq_len, tag_to_id, FLAGS.lower
)
print("%i / %i / %i sentences in train / dev / test." % (
len(train_data), 0, len(test_data)))
train_len=len(train_data)
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, FLAGS.batch_size)
test_manager = BatchManager(test_data, FLAGS.batch_size)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, config, logger)
logger.info("start training")
loss = []
for i in range(FLAGS.max_epoch):
from tqdm import tqdm
for batch in tqdm(train_manager.iter_batch(shuffle=True)):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step%steps_per_epoch, steps_per_epoch, np.mean(loss)))
loss = []
print("save result epoch:",i," ***************************************************")
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger,i)
if i>=8:
save_model(sess, model, FLAGS.ckpt_path, logger, global_steps=step)
evaluate(sess, model, "test", test_manager, id_to_tag, logger,i)
def train():
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(),
FLAGS.emb_file,
list(itertools.chain.from_iterable(
[[w[0] for w in s] for s in test_sentences])
)
)
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(
train_sentences, char_to_id, tag_to_id, FLAGS.lower
)
dev_data = prepare_dataset(
dev_sentences, char_to_id, tag_to_id, FLAGS.lower
)
test_data = prepare_dataset(
test_sentences, char_to_id, tag_to_id, FLAGS.lower
)
print("%i / %i / %i sentences in train / dev / test." % (
len(train_data), 0, len(test_data)))
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
#print batch
step, batch_loss = model.run_step(sess, True, batch)
#print step
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step%steps_per_epoch, steps_per_epoch, np.mean(loss)))
loss = []
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def train():
train_sentences = load_sentences(FLAGS.train_file)
dev_sentences = load_sentences(FLAGS.dev_file)
test_sentences = load_sentences(FLAGS.test_file)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
update_tag_scheme(dev_sentences, FLAGS.tag_schema)
if not os.path.isfile(FLAGS.map_file):
if FLAGS.pre_emb:
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences,
FLAGS.lower)
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, 'wb') as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, 'rb') as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id)
train_manager = BatchManager(train_data, FLAGS.batch_size, FLAGS.num_steps)
dev_manager = BatchManager(dev_data, 100, FLAGS.num_steps)
test_manager = BatchManager(test_data, 100, FLAGS.num_steps)
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
os.environ["CUDA_VISIBLE_DEVICES"] = "3"
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
tf_config = tf.ConfigProto(gpu_options=gpu_options)
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(75):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{},".format(
iteration, step % steps_per_epoch, steps_per_epoch))
loss = []
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def main(args):
# Setup logging
logger = setup_logging(args)
# Read params of model
params = fetch_model_params(args.model)
# Fetch appropriate input functions
input_fn = generic_text
pred_input_fn = pred_input
handle_pred_output_fn = handle_pred_output
if params["mlm_training"]:
mlm_sample_text_fn = partial(mlm_sample_text, params)
input_fn = partial(generic_text, sample_text_fn=mlm_sample_text_fn)
# Fetch encoder per params
encoder = fetch_encoder(params)
pred_input_fn = partial(pred_input_fn,
path_to_prompt=args.prompt,
logger=logger,
enc=encoder)
# Sample from Dataset if check dataset flag is on
if args.check_dataset:
check_dataset(input_fn, params)
# Confirm deletion of checkpoint files if --new flag is set
if args.new:
if yes_or_no(
f"Are you sure you want to remove '{params['model_path']}' to start afresh?"
):
remove_gs_or_filepath(params["model_path"])
else:
exit()
# Save config to logdir for experiment management
save_config(params, params["model_path"])
# Add to params: auto_layout, auto_layout_and_mesh_shape, use_tpu, num_cores
mesh_shape = mtf.convert_to_shape(params["mesh_shape"])
params["num_cores"] = mesh_shape.size
params["auto_layout"] = args.auto_layout
params["auto_layout_and_mesh_shape"] = args.auto_layout_and_mesh_shape
params["use_tpu"] = True if not args.tpu is None else False
params["gpu_ids"] = args.gpu_ids
params["steps_per_checkpoint"] = args.steps_per_checkpoint
# Expand attention types param
params["attention_types"] = expand_attention_types_params(
params["attention_types"])
assert len(params["attention_types"]) == params[
"n_layer"] # Assert that the length of expanded list = num layers
params["predict_batch_size"] = params.get("predict_batch_size",
1) # Default to 1
params["predict"] = args.predict
params['model'] = params.get(
"model", "GPT"
) # Default model selection to GPT since it's the only option for now
# Sample quality of MoE models suffers when using the faster sampling method, so default to slow_sampling if
# moe layers are present
params[
"slow_sampling"] = True if params["moe_layers"] is not None else False
logger.info(f"params = {params}")
# Get eval tasks from params
eval_tasks = params.get("eval_tasks", [])
has_predict_or_eval_steps_or_eval_tasks = params[
"predict_steps"] > 0 or params["eval_steps"] > 0 or len(eval_tasks) > 0
for t in eval_tasks:
assert t in task_descriptors, f"Eval task '{t}' is not known"
task_descriptors[t]["init_fn"](params)
# Set up TPUs and Estimator
if args.tpu == "colab":
tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(
) if params["use_tpu"] else None
else:
tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(
args.tpu) if params["use_tpu"] else None
config = tpu_config.RunConfig(
cluster=tpu_cluster_resolver,
model_dir=params["model_path"],
save_checkpoints_steps=None, # Disable the default saver
save_checkpoints_secs=None, # Disable the default saver
log_step_count_steps=params["iterations"],
save_summary_steps=params["iterations"],
tpu_config=tpu_config.TPUConfig(
num_shards=mesh_shape.size,
iterations_per_loop=params["iterations"],
num_cores_per_replica=1,
per_host_input_for_training=tpu_config.InputPipelineConfig.
BROADCAST))
estimator = tpu_estimator.TPUEstimator(
use_tpu=params["use_tpu"],
model_fn=model_fn,
config=config,
train_batch_size=params["train_batch_size"],
eval_batch_size=params["train_batch_size"],
predict_batch_size=params["predict_batch_size"],
params=params)
def _make_task_estimator(task):
task_params = params.copy()
task_params["eval_task"] = task
return tpu_estimator.TPUEstimator(
use_tpu=params["use_tpu"],
model_fn=model_fn,
config=config,
train_batch_size=params["train_batch_size"],
eval_batch_size=params["eval_batch_size"],
predict_batch_size=params["predict_batch_size"],
params=task_params)
eval_task_estimators = {
task: _make_task_estimator(task)
for task in eval_tasks
}
current_step = int(
estimator_lib._load_global_step_from_checkpoint_dir(
params["model_path"]))
logger.info(f"Current step {current_step}")
if args.predict:
# Predict
predictions = estimator.predict(input_fn=pred_input_fn)
logger.info("Predictions generated")
enc = fetch_encoder(params)
handle_pred_output_fn(
predictions,
logger,
enc,
params,
out_name=f"predictions_{args.sacred_id}_{current_step}")
return
if args.eval:
for task in eval_tasks:
logger.info(f"Starting evaluation task '{task}'")
task_info = task_descriptors[task]["get_task_info_fn"](params)
task_estimator = eval_task_estimators[task]
task_input_fn = task_descriptors[task]["input_fn"]
eval_results = task_estimator.evaluate(input_fn=task_input_fn,
steps=task_info["n_steps"],
name=task)
logger.info(f"Eval task '{task}' results: {eval_results}")
return
elif has_predict_or_eval_steps_or_eval_tasks:
# Eval and train - stop and predict and/or eval every checkpoint
while current_step < params["train_steps"]:
next_checkpoint = min(current_step + args.steps_per_checkpoint,
params["train_steps"])
estimator.train(input_fn=partial(input_fn, eval=False),
max_steps=next_checkpoint)
current_step = next_checkpoint
if params["predict_steps"] > 0:
logger.info("Running prediction...")
predictions = estimator.predict(input_fn=pred_input_fn)
enc = fetch_encoder(params)
handle_pred_output_fn(
predictions,
logger,
enc,
params,
out_name=f"predictions_{args.sacred_id}_{current_step}")
if params["eval_steps"] > 0:
logger.info("Running evaluation...")
eval_results = estimator.evaluate(input_fn=partial(input_fn,
eval=True),
steps=params["eval_steps"])
logger.info(f"Eval results: {eval_results}")
for task in eval_tasks:
logger.info(f"Starting evaluation task '{task}'")
task_info = task_descriptors[task]["get_task_info_fn"](params)
task_estimator = eval_task_estimators[task]
task_input_fn = task_descriptors[task]["input_fn"]
eval_results = task_estimator.evaluate(
input_fn=task_input_fn,
steps=task_info["n_steps"],
name=task)
logger.info(f"Eval task '{task}' results: {eval_results}")
return
else:
# Else, just train
while current_step < params["train_steps"]:
# Else, don't stop and restart
estimator.train(input_fn=partial(input_fn, eval=False),
max_steps=params["train_steps"])
def main(config):
# NOTE: Run this in shell first.
if tf.__version__[:3] != '1.1':
sys.exit(
'***NOTE!***: FIRST RUN:\n"source ~/began/BEGAN-tensorflow/tf1.1/bin/activate"'
)
# NOTE: Other setup requirements.
print(
'\nREQUIREMENTS:\n 1. The file "target_num_user_weights.npy" should '
'contain the user-provided labels for the desired target number.\n '
'2. Use the flag --mnist_class to set the class of MNIST digit that'
'matches that of the user-provided labels.\n 3. In trainer_iwgn.py, '
'"num_user_labeled" must correspond to the number of labels in '
'"target_num_user_weights.npy".')
print('Press "c" to continue.')
pdb.set_trace()
prepare_dirs_and_logger(config)
rng = np.random.RandomState(config.random_seed)
tf.set_random_seed(config.random_seed)
if config.is_train:
data_path = config.data_path
batch_size = config.batch_size
do_shuffle = True
else:
setattr(config, 'batch_size', 64)
if config.test_data_path is None:
data_path = config.data_path
else:
data_path = config.test_data_path
batch_size = config.sample_per_image
do_shuffle = False
#dir_loader = 'train8020'
#dir_loader = 'train6040'
#dir_loader = 'train4060'
dir_loader = 'train2080'
#dir_loader = 'train1090'
#dir_loader = 'train0510'
#dir_loader = 'trainBig0510'
#dir_loader = 'train_all_1090'
#dir_loader = 'train_small_5050'
config.pct = [int(dir_loader[-4:][:2]), int(dir_loader[-4:][2:])]
dir_target = 'train5050'
data_loader = get_loader(data_path,
config.batch_size,
config.scale_size,
config.data_format,
config.split,
target=dir_loader)
data_loader_target = get_loader(data_path,
config.batch_size,
config.scale_size,
config.data_format,
config.split,
target=dir_target)
trainer = Trainer(config, data_loader, data_loader_target)
if config.is_train:
save_config(config)
trainer.train()
else:
if not config.load_path:
raise Exception(
"[!] You should specify `load_path` to load a pretrained model"
)
trainer.test()
def train():
# load data sets：返回的是语料集的[['字'，'标'],...]元组
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower,
FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# 由loader.py负责处理数据
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb: # 判断是否用之前训练好的词向量
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
# dico_chars_train应该只接收了dico <注意后面的[0]> ，即训练数据的不重复统计的字集
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences]))
# chain.from_iterable(iterables): 一个备用链构造函数，其中的iterables是一个迭代变量，生成迭代序列
# 所以这里的list生成的就是test_sentences里的字集
)
# 这里dico_chars是在train_set字典基础上添加wiki_100中包含的test_set里的字构成的字典
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences,
FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
# 通过pickle模块的序列化操作我们能够将程序中运行的对象信息保存到文件中去，永久存储。
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
FLAGS.lower)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
# xxx_data 以句子为单位存储[字符，字符id，标签id/chars长度的全是“0”对应标签id的list <train = True/False>，标签]
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), 0, len(test_data)))
train_manager = BatchManager(train_data,
FLAGS.batch_size) # 默认的batch_size为20
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# 定义了3个BatchManager类：这个类中包含batch_data和len_data
# batch_data 是按句子长短顺序排序后一个batch大小的data列表数据，而且每个batch中的数据都padding到统一长短
# len_data 是所分batch的数量
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto(
) # tf.ConfigProto一般用在创建session的时候。用来对session进行参数配置
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def go_debian():
raw_input("LAMP Setup script is ready install it\nPress Enter key to continue, Press Ctrl+D to cancel the installation\n")
#generate config file
os.mkdir('/tmp/lamp')
os.mkdir('/etc/lamp')
os.mkdir('/root/lamp_bak')
os.mkdir('/etc/lamp/ftp_users')
config = {}
global version
config['version'] = version
config['system'] = 'u'
config['wwwroot'] = '/var/www'
config['apache_etc'] = '/etc/apache2'
config['apache'] = 'apache2'
config['ftproot'] = '/var/www'
config['vsftpd_conf_path'] = '/etc/vsftpd.conf'
config['root_own'] = 'www-data:www-data'
'''
logging.info('download files ...')
utils.exec_cmd('wget http://s1b-static.yuki.ws/files/lamp/files.tar.xz -O /tmp/lamp/files.tar.xz')
utils.exec_cmd('tar xvf /tmp/lamp/files.tar.xz -C /tmp/lamp')
'''
#setting mysql passwotrd
mysql_root_pass = utils.gen_random_str()
config['mysqlrootpass'] = mysql_root_pass
utils.save_config(config)
logging.debug('generate mysql root password : %s', mysql_root_pass)
debconf_tmp = open('/tmp/lamp/debconf.tmp', 'w+')
debconf_tmp.write('mysql-server mysql-server/root_password password %s\nmysql-server mysql-server/root_password_again password %s\n' % (mysql_root_pass, mysql_root_pass))
debconf_tmp.write('iptables-persistent iptables-persistent/autosave_v4 boolean true\niptables-persistent iptables-persistent/autosave_v6 boolean true\n')
debconf_tmp.close()
utils.exec_cmd('debconf-set-selections /tmp/lamp/debconf.tmp')
os.remove('/tmp/lamp/debconf.tmp')
del debconf_tmp
logging.info('update data packages info...')
utils.exec_cmd('apt-get update')
logging.info('update system, please wait...')
#utils.exec_cmd('apt-get upgrade -y')
logging.info('install and config packages, please wait...')
utils.exec_cmd('apt-get install -y vim axel curl unzip build-essential python-mysqldb python-software-properties php5 apache2 libapache2-mod-php5 mysql-server php5-mysql php5-curl php5-gd php5-mcrypt php5-imagick php5-memcached php5-sqlite php5-xcache iptables-persistent libpam-mysql')
logging.info('setting up web-server...')
logging.debug('setting apache conf')
httpd_conf = open('/etc/apache2/httpd.conf', 'w')
httpd_conf.write('ServerName %s\n' % (platform.uname()[1]))
httpd_conf.close()
del httpd_conf
utils.change_conf('/etc/apache2/apache2.conf',
[
{'old':'Timeout 300','new':'Timeout 45'},
{'old':'MaxKeepAliveRequests 100','new':'MaxKeepAliveRequests 200'}
]
)
utils.change_conf('/etc/apache2/conf.d/security',
[
{'old':'ServerTokens OS','new':'ServerTokens Prod'},
{'old':'ServerSignature On','new':'ServerSignature Off'}
]
)
# Enable mod-rewrite
utils.exec_cmd('a2enmod rewrite')
# Set apache php.ini
logging.debug('Setting php.ini')
utils.change_conf('/etc/php5/apache2/php.ini',
[
{'old':'post_max_size = 8M','new':'post_max_size = 50M'},
{'old':'upload_max_filesize = 2M','new':'upload_max_filesize = 50M'},
{'old':'expose_php = On','new':'expose_php = Off'},
{'old':'display_errors = Off','new':'display_errors = On'},
{'old':';date.timezone =','new':'date.timezone = Asia/Chongqing'},
{'old':'request_order = "GP"','new':'request_order = "CGP"'}
]
)
# Change default www dir
utils.exec_cmd('mkdir -p /var/www/public_html')
utils.exec_cmd('mv /var/www/index.html /var/www/public_html/')
utils.change_conf('/etc/apache2/sites-enabled/000-default', [{'old':'/var/www','new':'/var/www/public_html'}])
# Init phpmyadmin and lamp user pass
utils.exec_cmd('service mysql restart')
lamp_controluser_pass = mysql.init_db(mysql_root_pass)
utils.cp('<APPROOT>/files/phpmyadmin_host', '/etc/apache2/mods-available/phpmyadmin.conf')
utils.exec_cmd('ln -s /etc/apache2/mods-available/phpmyadmin.conf /etc/apache2/mods-enabled/phpmyadmin.conf')
config['lampuser'] = '******'
config['lamppass'] = lamp_controluser_pass
utils.save_config(config)
# create test php script
utils.exec_cmd(r'echo "<?php phpinfo() ?>" > /var/www/public_html/test.php')
# Change wwwroot permissions
utils.exec_cmd('chown -R www-data:www-data /var/www')
utils.exec_cmd('chmod -R go-rwx /var/www')
utils.exec_cmd('chmod -R g+rw /var/www')
utils.exec_cmd('chmod -R o+r /var/www')
utils.exec_cmd('service apache2 restart')
logging.info('setting up ftp-server...')
# Init ftp and create main account
#ftp_pass = ftp.init_ftp()
if platform.machine() == 'x86_64':
utils.exec_cmd('axel -q -n 3 -o /tmp/lamp/vsftpd.deb http://ftp.jaist.ac.jp/pub/Linux/ubuntu/pool/main/v/vsftpd/vsftpd_3.0.2-1ubuntu1_amd64.deb')
#add fix 12.04 pam.d-mysql bugs "libgcc_s.so.1 must be installed for pthread_cancel to work"
utils.exec_cmd('DEBIAN_FRONTEND=noninteractive apt-get install -qq libpam-ldap')
else:
utils.exec_cmd('axel -q -n 3 -o /tmp/lamp/vsftpd.deb http://ftp.jaist.ac.jp/pub/Linux/ubuntu/pool/main/v/vsftpd/vsftpd_3.0.2-1ubuntu1_i386.deb')
returncode = utils.exec_cmd('dpkg -i /tmp/lamp/vsftpd.deb')
if returncode == 2:
logging.debug('install vsftpd failed!')
utils.change_conf('<APPROOT>/files/vsftpd_conf', [
{'old':'<ftpuser>','new':'ftp'},
{'old':'<guestuser>','new':'www-data'}
], '/etc/vsftpd.conf')
#add fix 500 OOPS: priv_sock_get_cmd
if platform.machine() == 'x86_64':
utils.change_conf('/etc/vsftpd.conf', [{'old':'ftp_users','new':'ftp_users\nseccomp_sandbox=NO'}])
utils.change_conf('<APPROOT>/files/vsftpd_mysql', [{'old':'<passwd>','new':lamp_controluser_pass}], '/etc/pam.d/vsftpd-mysql')
#set master ftp account
args = {}
args['username'] = '******'
args['path'] = '/var/www'
args['site_id'] = 1
ftp_pass = ftp.create_ftp(args)
utils.exec_cmd('service vsftpd restart')
logging.info('setting up system...')
# Set iptables
utils.cp('<APPROOT>/files/iptables_rules', '/etc/iptables/rules.v4')
utils.exec_cmd('service iptables-persistent restart')
#load kernel ip_nat_ftp
utils.exec_cmd('modprobe nf_nat_ftp')
utils.exec_cmd('echo "nf_nat_ftp" >> /etc/modules')
# Add auto start at boot
utils.exec_cmd('update-rc.d apache2 defaults')
utils.exec_cmd('update-rc.d mysql defaults')
utils.exec_cmd('update-rc.d vsftpd defaults')
finish_install(mysql_root_pass, ftp_pass, 'apt-get upgrade -y')
"user-read-playback-state user-modify-playback-state user-read-currently-playing",
"spotify_redirect_uri_playlist":
"https://spoofy.baka.tokyo/callback_playlist/",
"spotify_scopes_playlist":
"playlist-modify-public playlist-read-collaborative",
"spotify_connect_name": "Spoofy Bot",
"http_host": "127.0.0.1",
"http_port": 5000
}
if __name__ == '__main__':
try:
config = utils.load_config()
except FileNotFoundError:
config = DEFAULT_CONFIG.copy()
utils.save_config(config)
# If discord bot token is not present, error out and quit as we cannot do anything useful.
if 'bot_token' not in config.keys() or config['bot_token'] in [
"", DEFAULT_CONFIG['bot_token']
]:
config['bot_token'] = DEFAULT_CONFIG['bot_token']
utils.save_config(config)
print(
"Please configure the bot_token by modifying the 'config.json' file!",
flush=True)
sys.exit(1)
bot_token = config['bot_token']
# If the encryption key is missing, generate it and save it to the config.
if 'encryption_key_passphrase' not in config.keys(
def save_config(self):
"""Writes all configuration files to disk."""
# Save config file
utils.save_config(self.config, self.configfile)
def main():
global msglogger
script_dir = os.path.dirname(__file__)
args = parse_args()
# Distiller loggers
msglogger = apputils.config_pylogger('logging.conf',
args.name,
output_dir=args.output_dir)
tflogger = TensorBoardLogger(msglogger.logdir)
# tflogger.log_gradients = True
# pylogger = PythonLogger(msglogger)
if args.seed is not None:
msglogger.info("Using seed = {}".format(args.seed))
torch.manual_seed(args.seed)
np.random.seed(seed=args.seed)
args.qe_mode = str(args.qe_mode).split('.')[1]
args.qe_clip_acts = str(args.qe_clip_acts).split('.')[1]
apputils.log_execution_env_state(sys.argv)
if args.gpus is not None:
try:
args.gpus = [int(s) for s in args.gpus.split(',')]
except ValueError:
msglogger.error(
'ERROR: Argument --gpus must be a comma-separated list of integers only'
)
exit(1)
if len(args.gpus) > 1:
msglogger.error('ERROR: Only single GPU supported for NCF')
exit(1)
available_gpus = torch.cuda.device_count()
for dev_id in args.gpus:
if dev_id >= available_gpus:
msglogger.error(
'ERROR: GPU device ID {0} requested, but only {1} devices available'
.format(dev_id, available_gpus))
exit(1)
# Set default device in case the first one on the list != 0
torch.cuda.set_device(args.gpus[0])
# Save configuration to file
config = {k: v for k, v in args.__dict__.items()}
config['timestamp'] = "{:.0f}".format(datetime.utcnow().timestamp())
config['local_timestamp'] = str(datetime.now())
run_dir = msglogger.logdir
msglogger.info("Saving config and results to {}".format(run_dir))
if not os.path.exists(run_dir) and run_dir != '':
os.makedirs(run_dir)
utils.save_config(config, run_dir)
# Check that GPUs are actually available
use_cuda = not args.no_cuda and torch.cuda.is_available()
t1 = time.time()
# Load Data
training = not (args.eval or args.qe_calibration
or args.activation_histograms)
msglogger.info('Loading data')
if training:
train_dataset = CFTrainDataset(
os.path.join(args.data, TRAIN_RATINGS_FILENAME),
args.negative_samples)
train_dataloader = torch.utils.data.DataLoader(
dataset=train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
nb_users, nb_items = train_dataset.nb_users, train_dataset.nb_items
else:
train_dataset = None
train_dataloader = None
nb_users, nb_items = (138493, 26744)
test_ratings = load_test_ratings(
os.path.join(args.data, TEST_RATINGS_FILENAME)) # noqa: E501
test_negs = load_test_negs(os.path.join(args.data, TEST_NEG_FILENAME))
msglogger.info(
'Load data done [%.1f s]. #user=%d, #item=%d, #train=%s, #test=%d' %
(time.time() - t1, nb_users, nb_items,
str(train_dataset.mat.nnz) if training else 'N/A', len(test_ratings)))
# Create model
model = NeuMF(nb_users,
nb_items,
mf_dim=args.factors,
mf_reg=0.,
mlp_layer_sizes=args.layers,
mlp_layer_regs=[0. for i in args.layers],
split_final=args.split_final)
if use_cuda:
model = model.cuda()
msglogger.info(model)
msglogger.info("{} parameters".format(utils.count_parameters(model)))
# Save model text description
with open(os.path.join(run_dir, 'model.txt'), 'w') as file:
file.write(str(model))
compression_scheduler = None
start_epoch = 0
optimizer = None
if args.load:
if training:
model, compression_scheduler, optimizer, start_epoch = apputils.load_checkpoint(
model, args.load)
if args.reset_optimizer:
start_epoch = 0
optimizer = None
else:
model = apputils.load_lean_checkpoint(model, args.load)
# Add loss to graph
criterion = nn.BCEWithLogitsLoss()
if use_cuda:
criterion = criterion.cuda()
if training and optimizer is None:
optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)
msglogger.info('Optimizer Type: %s', type(optimizer))
msglogger.info('Optimizer Args: %s', optimizer.defaults)
if args.compress:
compression_scheduler = distiller.file_config(model, optimizer,
args.compress)
model.cuda()
# Create files for tracking training
valid_results_file = os.path.join(run_dir, 'valid_results.csv')
if args.qe_calibration or args.activation_histograms:
calib = {
'portion':
args.qe_calibration,
'desc_str':
'quantization calibration stats',
'collect_func':
partial(distiller.data_loggers.collect_quant_stats,
inplace_runtime_check=True,
disable_inplace_attrs=True)
}
hists = {
'portion':
args.activation_histograms,
'desc_str':
'activation histograms',
'collect_func':
partial(distiller.data_loggers.collect_histograms,
activation_stats=None,
nbins=2048,
save_hist_imgs=True)
}
d = calib if args.qe_calibration else hists
distiller.utils.assign_layer_fq_names(model)
num_users = int(np.floor(len(test_ratings) * d['portion']))
msglogger.info(
"Generating {} based on {:.1%} of the test-set ({} users)".format(
d['desc_str'], d['portion'], num_users))
test_fn = partial(val_epoch,
ratings=test_ratings,
negs=test_negs,
K=args.topk,
use_cuda=use_cuda,
processes=args.processes,
num_users=num_users)
d['collect_func'](model=model,
test_fn=test_fn,
save_dir=run_dir,
classes=None)
return 0
if args.eval:
if args.quantize_eval and args.qe_calibration is None:
model.cpu()
quantizer = quantization.PostTrainLinearQuantizer.from_args(
model, args)
dummy_input = (torch.tensor([1]), torch.tensor([1]),
torch.tensor([True], dtype=torch.bool))
quantizer.prepare_model(dummy_input)
model.cuda()
distiller.utils.assign_layer_fq_names(model)
if args.eval_fp16:
model = model.half()
# Calculate initial Hit Ratio and NDCG
begin = time.time()
hits, ndcgs = val_epoch(model,
test_ratings,
test_negs,
args.topk,
use_cuda=use_cuda,
processes=args.processes)
val_time = time.time() - begin
hit_rate = np.mean(hits)
msglogger.info(
'Initial HR@{K} = {hit_rate:.4f}, NDCG@{K} = {ndcg:.4f}, val_time = {val_time:.2f}'
.format(K=args.topk,
hit_rate=hit_rate,
ndcg=np.mean(ndcgs),
val_time=val_time))
hit_rate = 0
if args.quantize_eval:
checkpoint_name = 'quantized'
apputils.save_checkpoint(0,
'NCF',
model,
optimizer=None,
extras={'quantized_hr@10': hit_rate},
name='_'.join([args.name, 'quantized'])
if args.name else checkpoint_name,
dir=msglogger.logdir)
return 0
total_samples = len(train_dataloader.sampler)
steps_per_epoch = math.ceil(total_samples / args.batch_size)
best_hit_rate = 0
best_epoch = 0
for epoch in range(start_epoch, args.epochs):
msglogger.info('')
model.train()
losses = utils.AverageMeter()
begin = time.time()
if compression_scheduler:
compression_scheduler.on_epoch_begin(epoch, optimizer)
loader = tqdm.tqdm(train_dataloader)
for batch_index, (user, item, label) in enumerate(loader):
user = torch.autograd.Variable(user, requires_grad=False)
item = torch.autograd.Variable(item, requires_grad=False)
label = torch.autograd.Variable(label, requires_grad=False)
if use_cuda:
user = user.cuda(async=True)
item = item.cuda(async=True)
label = label.cuda(async=True)
if compression_scheduler:
compression_scheduler.on_minibatch_begin(
epoch, batch_index, steps_per_epoch, optimizer)
outputs = model(user, item, torch.tensor([False],
dtype=torch.bool))
loss = criterion(outputs, label)
if compression_scheduler:
compression_scheduler.before_backward_pass(
epoch,
batch_index,
steps_per_epoch,
loss,
optimizer,
return_loss_components=False)
losses.update(loss.data.item(), user.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
if compression_scheduler:
compression_scheduler.on_minibatch_end(epoch, batch_index,
steps_per_epoch,
optimizer)
# Save stats to file
description = (
'Epoch {} Loss {loss.val:.4f} ({loss.avg:.4f})'.format(
epoch, loss=losses))
loader.set_description(description)
steps_completed = batch_index + 1
if steps_completed % args.log_freq == 0:
stats_dict = OrderedDict()
stats_dict['Loss'] = losses.avg
stats = ('Performance/Training/', stats_dict)
params = model.named_parameters(
) if args.log_params_histograms else None
distiller.log_training_progress(stats, params, epoch,
steps_completed,
steps_per_epoch, args.log_freq,
[tflogger])
tflogger.log_model_buffers(model,
['tracked_min', 'tracked_max'],
'Quant/Train/Acts/TrackedMinMax',
epoch, steps_completed,
steps_per_epoch, args.log_freq)
train_time = time.time() - begin
begin = time.time()
hits, ndcgs = val_epoch(model,
test_ratings,
test_negs,
args.topk,
use_cuda=use_cuda,
output=valid_results_file,
epoch=epoch,
processes=args.processes)
val_time = time.time() - begin
if compression_scheduler:
compression_scheduler.on_epoch_end(epoch, optimizer)
hit_rate = np.mean(hits)
mean_ndcgs = np.mean(ndcgs)
stats_dict = OrderedDict()
stats_dict['HR@{0}'.format(args.topk)] = hit_rate
stats_dict['NDCG@{0}'.format(args.topk)] = mean_ndcgs
stats = ('Performance/Validation/', stats_dict)
distiller.log_training_progress(stats,
None,
epoch,
steps_completed=0,
total_steps=1,
log_freq=1,
loggers=[tflogger])
msglogger.info(
'Epoch {epoch}: HR@{K} = {hit_rate:.4f}, NDCG@{K} = {ndcg:.4f}, AvgTrainLoss = {loss.avg:.4f}, '
'train_time = {train_time:.2f}, val_time = {val_time:.2f}'.format(
epoch=epoch,
K=args.topk,
hit_rate=hit_rate,
ndcg=mean_ndcgs,
loss=losses,
train_time=train_time,
val_time=val_time))
is_best = False
if hit_rate > best_hit_rate:
best_hit_rate = hit_rate
is_best = True
best_epoch = epoch
extras = {
'current_hr@10': hit_rate,
'best_hr@10': best_hit_rate,
'best_epoch': best_epoch
}
apputils.save_checkpoint(epoch,
'NCF',
model,
optimizer,
compression_scheduler,
extras,
is_best,
dir=run_dir)
if args.threshold is not None:
if np.mean(hits) >= args.threshold:
msglogger.info("Hit threshold of {}".format(args.threshold))
break
def main(config):
save_config(config)
data_loader = get_loader(config.batch_size, config.project_root)
trainer = Trainer(config, data_loader)
trainer.train()
return
def run(self, share):
root = self.dispatcher.call_sync('volume.get_volumes_root')
share_type = self.dispatcher.call_sync('share.supported_types').get(share['type'])
assert share_type['subtype'] in ('FILE', 'BLOCK'),\
"Unsupported Share subtype: {0}".format(share_type['subtype'])
if self.datastore.exists(
'shares',
('type', '=', share['type']),
('name', '=', share['name'])
):
raise TaskException(errno.EEXIST, 'Share {0} of type {1} already exists'.format(
share['name'],
share['type']
))
normalize(share, {
'enabled': True,
'immutable': False,
'description': ''
})
if share['target_type'] in ('DATASET', 'ZVOL'):
dataset = share['target_path']
pool = share['target_path'].split('/')[0]
path = os.path.join(root, dataset)
if not self.dispatcher.call_sync('zfs.dataset.query', [('name', '=', dataset)], {'single': True}):
if share_type['subtype'] == 'FILE':
self.join_subtasks(self.run_subtask('volume.dataset.create', {
'volume': pool,
'id': dataset,
'permissions_type': share_type['perm_type'],
}))
if share_type['subtype'] == 'BLOCK':
self.join_subtasks(self.run_subtask('volume.dataset.create', {
'volume': pool,
'id': dataset,
'type': 'VOLUME',
'volsize': share['properties']['size'],
}))
else:
if share_type['subtype'] == 'FILE':
self.run_subtask('volume.dataset.update', dataset, {
'permissions_type': share_type['perm_type']
})
elif share['target_type'] == 'DIRECTORY':
# Verify that target directory exists
path = share['target_path']
if not os.path.isdir(path):
raise TaskException(errno.ENOENT, "Target directory {0} doesn't exist".format(path))
elif share['target_type'] == 'FILE':
# Verify that target file exists
path = share['target_path']
if not os.path.isfile(path):
raise TaskException(errno.ENOENT, "Target file {0} doesn't exist".format(path))
else:
raise AssertionError('Invalid target type')
if share.get('permissions'):
self.join_subtasks(self.run_subtask('file.set_permissions', path, share['permissions']))
ids = self.join_subtasks(self.run_subtask('share.{0}.create'.format(share['type']), share))
self.dispatcher.dispatch_event('share.changed', {
'operation': 'create',
'ids': ids
})
new_share = self.datastore.get_by_id('shares', ids[0])
path = self.dispatcher.call_sync('share.get_directory_path', new_share['id'])
try:
save_config(
path,
'{0}-{1}'.format(new_share['type'], new_share['name']),
new_share
)
except OSError as err:
self.add_warning(TaskWarning(errno.ENXIO, 'Cannot save backup config file: {0}'.format(str(err))))
return ids[0]
