def save_to_file(self, algorithm_name, number_of_elements,
number_of_operations, algorithm_time):
output_filename = self.output_file_name()
output_path = self.output_path()
my_writer = Writer(output_path, output_filename)
my_writer.save_to_file(algorithm_name, self.name, number_of_elements,
number_of_operations, algorithm_time)
def convertRam2Xml(self, schema, xml_path):
"""
Create ram representation of incoming Schema object
:param schema:
:param xml_path:
:return:
"""
if schema is None:
raise ParseError("Schema not found", self)
node = self.create_schema(schema)
node.appendChild(self.xml.createElement("custom"))
if schema.domains:
domains = self.xml.createElement("domains")
for domain in self.create_domain(schema.domains):
domains.appendChild(domain)
node.appendChild(domains)
tables = self.xml.createElement("tables")
for table in self.create_table(schema.tables):
tables.appendChild(table)
node.appendChild(tables)
try:
self.xml.appendChild(node)
if xml_path:
Writer.write_xml(xml_path, self.xml)
except Exception:
raise ItemNotFoundException("domains, tables", schema.name)
def pack(self):
packet_data = self.buffer
Writer.__init__(self)
self.writeShort(self.id)
self.writeUInt(len(packet_data), 3)
self.writeShort(0) # Version
self.buffer += packet_data + b'\xff\xff\x00\x00\x00\x00\x00'
def label(self, start: int = 0, end: int = None):
if end is None:
end = len(self.dataframe)
dataframe = self.dataframe.iloc[start:end]
for index, row in dataframe.iterrows():
title = row["title"]
text = row["original_text"]
persons = list(filter(lambda person: len(person) > 2, row["persons"]))
if len(persons) > 0:
personPattern = re.compile("|".join(persons), re.IGNORECASE)
title = personPattern.sub("<Person>", title)
text = personPattern.sub("<Person>", text)
title = self.partyPattern.sub("<Partei>", title)
text = self.partyPattern.sub("<Partei>", text)
mediaPattern = re.compile("|".join(["bild", "bildplus", "taz", "tagesschau"]), re.IGNORECASE)
title = mediaPattern.sub("<Zeitung>", title)
text = mediaPattern.sub("<Zeitung>", text)
print("================================================")
print(title)
print("++++++++++++++++++++++++++++++++++++++++++++++++")
print(text)
print("================================================")
self.get_polarity_input(dataframe, index)
self.get_subjectivity_input(dataframe, index)
Writer.write_dataframe(dataframe, "labeled_paragraphs")
def __init__(self):
super(Simulator, self).__init__()
self.model = cfg.model(cfg)
self.optim = cfg.optim(cfg.learning_rate)
self.loss = cfg.loss
self.epoch = tf.Variable(0)
self.writer = Writer(cfg)
# Restore if save exists
if Path('./simulator_saves/best').is_dir():
self.model, self.optim, self.epoch = self.writer.restore(
model=self.model, optim=self.optim, epoch=self.epoch)
self.preprocessing()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-c",
"--config",
type=str,
required=True,
help="yaml file for config.")
parser.add_argument(
"-n",
"--name",
type=str,
default=None,
help="Name of the model. Used for both logging and saving chkpt.",
)
args = parser.parse_args()
hp = load_hparam(args.config)
if args.name is not None:
hp.log.name = args.name
# random seed
if hp.train.random_seed is None:
hp.train.random_seed = random.randint(1, 10000)
set_random_seed(hp.train.random_seed)
# set log/checkpoint dir
hp.log.chkpt_dir = os.path.join(hp.log.chkpt_dir, hp.log.name)
hp.log.log_dir = os.path.join(hp.log.log_dir, hp.log.name)
os.makedirs(hp.log.chkpt_dir, exist_ok=True)
os.makedirs(hp.log.log_dir, exist_ok=True)
# set logger
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(levelname)s - %(message)s",
handlers=[
logging.FileHandler(
os.path.join(hp.log.log_dir,
"%s-%d.log" % (hp.log.name, time.time()))),
logging.StreamHandler(),
],
)
logger = logging.getLogger()
# set writer (tensorboard / wandb)
writer = Writer(hp, hp.log.log_dir)
hp_str = yaml.dump(hp.to_dict())
logger.info("Config:")
logger.info(hp_str)
if hp.data.train_dir == "" or hp.data.test_dir == "":
logger.error("train or test data directory cannot be empty.")
raise Exception("Please specify directories of data in %s" %
args.config)
train_loop(hp, logger, writer)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-c',
'--config',
type=str,
required=True,
help="yaml file for config.")
parser.add_argument('-p',
'--checkpoint_path',
type=str,
default=None,
help="path of checkpoint pt file for resuming")
parser.add_argument(
'-n',
'--name',
type=str,
required=True,
help="Name of the model. Used for both logging and saving chkpt.")
args = parser.parse_args()
hp = HParam(args.config)
hp_str = yaml.dump(hp)
args_str = yaml.dump(vars(args))
pt_dir = os.path.join(hp.log.chkpt_dir, args.name)
log_dir = os.path.join(hp.log.log_dir, args.name)
os.makedirs(pt_dir, exist_ok=True)
os.makedirs(log_dir, exist_ok=True)
logging.basicConfig(level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler(
os.path.join(
log_dir,
'%s-%d.log' % (args.name, time.time()))),
logging.StreamHandler()
])
logger = logging.getLogger()
logger.info('Config by yaml file')
logger.info(hp_str)
logger.info('Command Line Config')
logger.info(args_str)
if hp.data.train == '' or hp.data.test == '':
logger.error("train or test data directory cannot be empty.")
raise Exception("Please specify directories of data in %s" %
args.config)
writer = Writer(hp, log_dir)
train_loader = create_dataloader(hp, args, DataloaderMode.train)
test_loader = create_dataloader(hp, args, DataloaderMode.test)
train(args, pt_dir, train_loader, test_loader, writer, logger, hp, hp_str)
def main(DEVICE):
# define model, optimizer, scheduler
model = VQVC().to(DEVICE)
recon_loss = nn.L1Loss().to(DEVICE)
vocoder = get_vocgan(
ckpt_path=args.vocoder_pretrained_model_path).to(DEVICE)
mel_stat = torch.tensor(np.load(args.mel_stat_path)).to(DEVICE)
optimizer = Adam(model.parameters(), lr=args.init_lr)
scheduler = WarmupScheduler(optimizer,
warmup_epochs=args.warmup_steps,
initial_lr=args.init_lr,
max_lr=args.max_lr,
milestones=args.milestones,
gamma=args.gamma)
global_step = load_checkpoint(checkpoint_path=args.model_checkpoint_path,
model=model,
optimizer=optimizer,
scheduler=scheduler)
# load dataset & dataloader
train_dataset = SpeechDataset(mem_mode=args.mem_mode,
meta_dir=args.prepro_meta_train,
dataset_name=args.dataset_name,
mel_stat_path=args.mel_stat_path,
max_frame_length=args.max_frame_length)
eval_dataset = SpeechDataset(mem_mode=args.mem_mode,
meta_dir=args.prepro_meta_eval,
dataset_name=args.dataset_name,
mel_stat_path=args.mel_stat_path,
max_frame_length=args.max_frame_length)
train_data_loader = DataLoader(dataset=train_dataset,
batch_size=args.train_batch_size,
shuffle=True,
drop_last=True,
pin_memory=True,
num_workers=args.n_workers)
eval_data_loader = DataLoader(dataset=eval_dataset,
batch_size=args.train_batch_size,
shuffle=False,
pin_memory=True,
drop_last=True)
# tensorboard
writer = Writer(args.model_log_path) if args.log_tensorboard else None
# train the model!
train(train_data_loader, eval_data_loader, model, recon_loss, vocoder,
mel_stat, optimizer, scheduler, global_step, writer, DEVICE)
def encode(self):
Writer.__init__(self)
self.writeVint(0)
self.writeVint(self.dataDB["LowID"])
self.writeVint(0)
self.writeVint(0) # Count brawlers
self.writeVint(14) # Count array
for x in range(14):
self.writeVint(x + 1)
self.writeVint(x)
self.writeString(self.dataDB["name"]) # Name
self.writeVint(100)
self.writeVint(28000000)
self.writeVint(43000000)
self.writeBool(False) # Is club
self.writeVint(0)
def train_threshold(self) -> float:
"""
Train the threshold with labeled data.
:return The best threshold.
"""
threshold: float = 0
best_threshold: float = 0
best_score: float = 0
thresholds: List[float] = []
f1_scores: List[Tuple[float, float, float, float]] = []
self.tfidf_sentiment.get_context_polarity(8)
self.tfidf_sentiment.calculate_sentiment_score(overwrite=True)
# Iterate over different thresholds, increase with every loop
while threshold <= 0.005:
self.tfidf_sentiment.map_sentiment(threshold=threshold, overwrite=True)
# Optimize over the sum of all f1 scores for SentiWs
f1_sentiws, _, _ = self.f1_score(training=True)
f1_sum = f1_sentiws[0] + f1_sentiws[1] + f1_sentiws[2]
thresholds.append(threshold)
f1_scores.append((f1_sum, f1_sentiws[0], f1_sentiws[1], f1_sentiws[2]))
# Replace best threshold if current one is better
if f1_sum > best_score:
best_score = f1_sum
best_threshold = threshold
threshold += 0.0000001
# Visualize the training
self.visualize_threshold(thresholds, f1_scores, best_threshold, 0.005)
# Adjust the sentiment with best threshold
self.tfidf_sentiment.map_sentiment(threshold=best_threshold, overwrite=True)
Writer.write_dataframe(self.dataframe, "labeled_paragraphs")
return best_threshold
def __init__(self, opt, tensorboard=True):
# opt is the global options shared in the coding env
self.opt = opt
self.tensorboard = tensorboard
if tensorboard:
import threading
# self.t = threading.Thread(target=launchTensorBoard, args=([os.path.join(opt.checkpoints_dir, opt.name), opt.port]))
# self.t.daemon = True
launchTensorBoard(os.path.join(opt.checkpoints_dir, opt.name),
opt.port)
self.t.start()
# self.t.join(20)
self.writer = SummaryWriter(
os.path.join(opt.checkpoints_dir, opt.name))
else:
self.writer = Writer(os.path.join(opt.checkpoints_dir, opt.name))
log_filename = os.path.join(opt.checkpoints_dir,
'%s_log.txt' % opt.name)
self.log = open(log_filename, 'w')
self.write_opt(opt)
self.save_image_root = os.path.join(opt.checkpoints_dir, opt.name)
def __init__(self, dataset, net, device, args):
self.dataset = dataset
self.net = net
self.device = torch.device(
device if torch.cuda.is_available() else 'cpu')
self.args = args
self.forward = self.net.to(self.device)
self.loss_fn = self._setup_loss_fn()
# logs save in text and visualize in tensorboard
self.recoder = Writer(self.args.log_dir)
def _execute_transform(self):
execution_info = ETLExecutionInfo("TRANSFORM")
loaded_jobs = JobsLoader.Instance().loaded_jobs
job_manager = JobsManager(loaded_jobs)
results = job_manager.run()
results = PosProcessor.Instance().run(results)
Writer.Instance().run(results)
execution_info.end()
Log.Instance().appendFinalReport(
"[TRANSFORM executed in: " +
str(execution_info.execution_data['value']) + " minutes ]")
return execution_info.execution_data
def __init__(self):
super(AlphaDoom, self).__init__()
self.mcts = MCTS(cfg)
self.replay = Replay(cfg)
self.autoencoder = Simulator()
self.autoencoder.train()
#self.autoencoder = AutoEncoder()
#tf.train.Checkpoint(model=self.autoencoder).restore(tf.train.latest_checkpoint('./simulator_saves/best'))
# Load selected model
self.model = cfg.model(cfg)
self.loss1 = cfg.loss1
self.loss2 = cfg.loss2
self.optim = cfg.optim(cfg.learning_rate)
self.epoch = tf.Variable(0)
self.writer = Writer(cfg)
# Restore if save exists
if Path('./alphadoom_saves/best').is_dir():
self.model, self.optim, self.epoch = self.writer.restore(
self.model, self.optim, self.epoch)
self.vizdoom = VizDoom(cfg)
def run_test(epoch=-1, name="evaluator_pretrained"):
print('Running Test')
opt = TestOptions().parse()
opt.serial_batches = True # no shuffle
opt.name = name # checkpoint dir name, evaluator/gan/vae_pretrained
dataset = DataLoader(opt)
model = create_model(opt)
writer = Writer(opt)
# test
writer.reset_counter()
for i, data in enumerate(dataset):
model.set_input(data)
ncorrect, nexamples = model.test()
writer.update_counter(ncorrect, nexamples)
writer.print_acc(epoch, writer.acc)
return writer.acc
class ClientHelloMessage():
def __init__(self):
self.w = Writer()
def add_header(self, data, id: int):
header = b''
header += self.w.writeShort(id)
header += len(data).to_bytes(3, 'big')
header += self.w.writeShort(0)
header += data
return header
def send_client_hello(self, major: int, minor: int):
message = b''
message += self.w.writeInt(2)
message += self.w.writeInt(11)
message += self.w.writeInt(major)
message += self.w.writeInt(0)
message += self.w.writeInt(minor)
message += self.w.writeInt(0)
message += self.w.writeInt(2)
message += self.w.writeInt(2)
return self.add_header(message, 10100)
def train(args):
os.makedirs(args.checkpoint_dir, exist_ok=True)
logging = GetLogging(args.logfile)
train_dataset = CustomerDataset(args.input,
upsample_factor=hop_length,
local_condition=True,
global_condition=False)
device = torch.device("cuda" if args.use_cuda else "cpu")
generator, discriminator = create_model(args)
print(generator)
print(discriminator)
num_gpu = torch.cuda.device_count() if args.use_cuda else 1
global_step = 0
g_parameters = list(generator.parameters())
g_optimizer = optim.Adam(g_parameters, lr=args.g_learning_rate)
d_parameters = list(discriminator.parameters())
d_optimizer = optim.Adam(d_parameters, lr=args.d_learning_rate)
writer = Writer(args.checkpoint_dir, sample_rate=sample_rate)
generator.to(device)
discriminator.to(device)
if args.resume is not None:
restore_step = attempt_to_restore(generator, discriminator,
g_optimizer, d_optimizer,
args.resume, args.use_cuda, logging)
global_step = restore_step
customer_g_optimizer = Optimizer(g_optimizer, args.g_learning_rate,
global_step, args.warmup_steps,
args.decay_learning_rate)
customer_d_optimizer = Optimizer(d_optimizer, args.d_learning_rate,
global_step, args.warmup_steps,
args.decay_learning_rate)
criterion = nn.MSELoss().to(device)
stft_criterion = MultiResolutionSTFTLoss()
for epoch in range(args.epochs):
collate = CustomerCollate(upsample_factor=hop_length,
condition_window=args.condition_window,
local_condition=True,
global_condition=False)
train_data_loader = DataLoader(train_dataset,
collate_fn=collate,
batch_size=args.batch_size,
num_workers=args.num_workers,
shuffle=True,
pin_memory=True)
#train one epoch
for batch, (samples, conditions) in enumerate(train_data_loader):
start = time.time()
batch_size = int(conditions.shape[0] // num_gpu * num_gpu)
samples = samples[:batch_size, :].to(device)
conditions = conditions[:batch_size, :, :].to(device)
losses = {}
if num_gpu > 1:
g_outputs = parallel(generator, (conditions, ))
else:
g_outputs = generator(conditions)
sc_loss, mag_loss = stft_criterion(g_outputs.squeeze(1),
samples.squeeze(1))
g_loss = sc_loss + mag_loss
losses['sc_loss'] = sc_loss.item()
losses['mag_loss'] = mag_loss.item()
losses['g_loss'] = g_loss.item()
customer_g_optimizer.zero_grad()
g_loss.backward()
nn.utils.clip_grad_norm_(g_parameters, max_norm=0.5)
customer_g_optimizer.step_and_update_lr()
time_used = time.time() - start
logging.info(
"Step: {} --sc_loss: {:.3f} --mag_loss: {:.3f} --Time: {:.2f} seconds"
.format(global_step, sc_loss, mag_loss, time_used))
if global_step % args.checkpoint_step == 0:
save_checkpoint(args, generator, discriminator, g_optimizer,
d_optimizer, global_step, logging)
if global_step % args.summary_step == 0:
writer.logging_loss(losses, global_step)
target = samples.cpu().detach()[0, 0].numpy()
predict = g_outputs.cpu().detach()[0, 0].numpy()
writer.logging_audio(target, predict, global_step)
writer.logging_histogram(generator, global_step)
writer.logging_histogram(discriminator, global_step)
global_step += 1
def train_loop(rank, hp, world_size=1):
# reload hp
hp = DotDict(hp)
if hp.model.device.lower() == "cuda" and world_size != 0:
setup(hp, rank, world_size)
if rank != 0:
logger = None
writer = None
else:
# set logger
logger = make_logger(hp)
# set writer (tensorboard / wandb)
writer = Writer(hp, hp.log.log_dir)
hp_str = yaml.dump(hp.to_dict())
logger.info("Config:")
logger.info(hp_str)
if hp.data.train_dir == "" or hp.data.test_dir == "":
logger.error("train or test data directory cannot be empty.")
raise Exception("Please specify directories of data")
logger.info("Set up train process")
if hp.model.device.lower() == "cuda" and world_size != 0:
hp.model.device = rank
torch.cuda.set_device(rank)
else:
hp.model.device = hp.model.device.lower()
# make dataloader
if logger is not None:
logger.info("Making train dataloader...")
train_loader = create_dataloader(hp, DataloaderMode.train, rank,
world_size)
if logger is not None:
logger.info("Making test dataloader...")
test_loader = create_dataloader(hp, DataloaderMode.test, rank, world_size)
# init Model
net_arch = Net_arch(hp)
loss_f = torch.nn.MSELoss()
model = Model(hp, net_arch, loss_f, rank, world_size)
# load training state
if hp.load.resume_state_path is not None:
model.load_training_state(logger)
else:
if logger is not None:
logger.info("Starting new training run.")
try:
epoch_step = 1 if hp.data.divide_dataset_per_gpu else world_size
for model.epoch in itertools.count(model.epoch + 1, epoch_step):
if model.epoch > hp.train.num_iter:
break
train_model(hp, model, train_loader, writer, logger)
if model.epoch % hp.log.chkpt_interval == 0:
model.save_network(logger)
model.save_training_state(logger)
test_model(hp, model, test_loader, writer)
cleanup()
if logger is not None:
logger.info("End of Train")
except Exception as e:
if logger is not None:
logger.info("Exiting due to exception: %s" % e)
traceback.print_exc()
cleanup()
def encode(self):
Writer.__init__(self)
best_val_loss = 1e100
best_do = -1
l2 = 0.0001
do_layers = 1
bn = True
for lr in [0.0005, 0.005]:
for l2 in [0.00001, 0.0001, 0.0005]:
for do_layers in [1, 2]:
for bn in [True, False]:
args.checkpoints_dir = out_dir_ii = out_dir_i + '/lr_{}_l2_{}_do_{}_bn_{}/checkpoints'.format(
lr, l2, do_layers, bn)
utils.makedirs(args.checkpoints_dir)
out_dir_ii = out_dir_i + '/lr_{}_l2_{}_do_{}_bn_{}'.format(
lr, l2, do_layers, bn)
writer = Writer(args, outdir=out_dir_ii)
# Defining model
model = MLPModelWPathways(in_channels,
n_classes,
args.num_layers,
args.hidden_gcn,
args.hidden_fc,
pathway_edge_index.to(device),
n_cmt,
mode=args.feature_agg_mode,
batchnorm=bn,
do_layers=do_layers).to(device)
args.model_parameters = utils.count_parameters(model)
# writer.save_args()
hit10 = np.sum(ranks <= 10, axis=0) / len(ranks)
mrr_sum = (1. / ranks).sum(axis=0)
mrr = np.tile(
np.array([mrr_sum[0] + mrr_sum[2], mrr_sum[1] + mrr_sum[3]]) /
(2 * len(ranks)), 2)
result = pd.DataFrame({
"mrr": mrr,
"mean rank": mean_rank,
"hit10": hit10
},
index=[
"tail: raw ranking", "tail: filtered ranking",
"head: raw ranking", "head: filtered ranking"
])
result["hit10"] = result["hit10"].apply(lambda x: "%.2f%%" % (x * 100))
ranks = pd.DataFrame(
ranks,
columns=["tail:raw", "tail:filtered", "head:raw", "head:filtered"])
return ranks, result
dis.eval()
ranks, result = evaluate()
writer = Writer(configs)
logger = Logger(configs)
writer.write(result)
if configs.log:
logger.write(ranks)
)
sys.exit()
comparison = Comparison(labeled_file)
# Train the score threshold
optimal_threshold = comparison.train_threshold()
print("Optimal threshold: {}\n".format(optimal_threshold))
# Train the window and the score threshold
optimal_context_thresholds = comparison.train_context_thresholds()
print("Optimal context thresholds: {} (window), {} (score)\n".format(
optimal_context_thresholds[0], optimal_context_thresholds[1]))
# Save paragraphs to disk
Writer.write_dataframe(df_paragraphs, "paragraphs")
# Show GUI
if args.show_gui:
gui = SentimentGUI(df_paragraphs)
gui.show_gui()
# Compare labeled data with results
if args.compare:
labeled_file = Path("src/output/labeled_paragraphs.json")
if not labeled_file.exists():
print(
'You have to provide a labeled file "labeled_paragraphs.json" for comparison in the output folder'
)
sys.exit()
db_name = args.db_name
ddl_path = args.ddl_path
db_path = args.db_path
url_mssql = args.mssql_url
pg_url = args.pg_url
downloader = MSSQLDownloader(mssql_queries, url_mssql)
print("downloading db from " + url_mssql)
schema = downloader.load('dbo')
converter = Converter()
converter.convertRam2Xml(schema, result_path + 'dbo.xml')
ddl_generator = DBInitialisator()
ddl = ddl_generator.generate_ddl(schema)
Writer.write(ddl_path + schema.name + ".ddl", '\n'.join(ddl))
print("ddl saved to {}".format(ddl_path + schema.name + ".ddl"))
pg_init = DBInitialisator()
conn = postgresql.open(pg_url)
conn.execute(pg_init.drop_database(db_name))
print("creating database {}".format(db_name))
conn.execute(pg_init.create_database_ddl(db_name))
conn.close()
print("connecting to database {}".format(db_name))
conn = postgresql.open(pg_url + '/' + db_name.lower())
conn.execute('\n '.join(ddl))
data_transfer = DataTransfering(db_name, mssql_url,
class Simulator(object):
def __init__(self):
super(Simulator, self).__init__()
self.model = cfg.model(cfg)
self.optim = cfg.optim(cfg.learning_rate)
self.loss = cfg.loss
self.epoch = tf.Variable(0)
self.writer = Writer(cfg)
# Restore if save exists
if Path('./simulator_saves/best').is_dir():
self.model, self.optim, self.epoch = self.writer.restore(
model=self.model, optim=self.optim, epoch=self.epoch)
self.preprocessing()
def preprocessing(self):
if cfg.package_data or not Path('./data.pkl').is_file():
vizdoom = VizDoom(cfg)
memory = []
for episode in trange(cfg.gather_epochs):
vizdoom.new_episode()
s0 = vizdoom.get_preprocessed_state()
while not vizdoom.is_episode_finished():
action = random.choice(cfg.actions)
vizdoom.make_action(action)
s1 = vizdoom.get_preprocessed_state()
action = np.reshape(
action, [1, 1, len(cfg.actions)]).astype(np.float32)
memory.append([s0, action, s1])
s0 = s1
with open('data.pkl', 'wb') as f:
pickle.dump(memory, f)
# Load data
with open(cfg.data_dir, 'rb') as f:
s0, action, s1 = zip(*pickle.load(f))
self.size = len(s0)
self.data = tf.data.Dataset.from_tensor_slices(
(np.array(s0), np.array(action), np.array(s1)))
def update(self, s0, action, s1):
# Normalize
s0_n = tf.image.per_image_standardization(s0)
truth = tf.image.per_image_standardization(s1) - s0_n
# Construct graph
with tf.GradientTape() as tape:
# Approximate next frame
logits = self.model(s0_n, action)
# Compare generated transformation matrix with truth
loss = tf.reduce_mean(self.loss(truth, logits))
# Log stats, images
self.writer.log(self.optim, tape, loss)
self.writer.log_state("logits", logits)
self.writer.log_state("truth_logits", truth)
# Compute/apply gradients
grads = tape.gradient(loss, self.model.trainable_weights)
grads_and_vars = zip(grads, self.model.trainable_weights)
self.optim.apply_gradients(grads_and_vars)
self.writer.global_step.assign_add(1)
def train(self):
for epoch in trange(self.epoch.numpy(), cfg.epochs):
# Uniform shuffle
batch = self.data.shuffle(self.size).batch(cfg.batch_size)
for s0, action, s1 in batch:
self.update(s0, action, s1)
self.epoch.assign_add(1)
self.writer.save(self.model, self.optim, self.epoch)
def predict(self, s0, action):
s0_n = tf.image.per_image_standardization(s0)
logits = self.model(s0_n, action[None])
return logits + s0_n
DATA_DIR = 'data/books_processed/'
LR = 0.0025
model_param = dict(max_n=MAX_SEQ_LEN,
nb_tokens=50256,
d_model=768,
d_ff=3072,
activation=gelu,
h=12,
dropout_value=0.1,
weight_init=0,
bias_init=0.02,
mask=True,
n_block=1)
writer = Writer(experiment_name='GPT-1')
# load data
transformer = Transformer(**model_param).to(device)
transformer.apply(GPT1_weight_init)
#transformer.load_state_dict(torch.load('model_weights/GPT1_3_9000'))
print('Model loaded')
book_dataset = BookCorpusDataset(DATA_DIR, nb_tokens=MAX_SEQ_LEN + 1)
print('Dataset loaded')
dataset_loader = torch.utils.data.DataLoader(book_dataset,
batch_size=BATCH,
shuffle=SHUFFLE,
num_workers=NUM_WORKERS)
elif db:
print("reading database {} ".format(db))
db_downloader = DBDownloader(sqlite_queries, db, None)
schemas_from_db = db_downloader.load()
schemas = schemas_from_db
arr = db.replace('.db', '').split('\\')
db_name = arr[arr.__len__() - 1]
pg_init = DBInitialisator()
print("establishing connection...")
url = postgressql_url
conn = postgresql.open(url)
conn.execute(pg_init.drop_database(db_name))
print("creating database {}".format(db_name))
conn.execute(pg_init.create_database_ddl(db_name))
conn.close()
print("connecting to database {}".format(db_name))
conn = postgresql.open(url + '/' + db_name.lower())
for schema in schemas.values():
print("generating ddl for schema {}".format(schema.name))
ddl = pg_init.generate_ddl(schema) # generate ddl instructions
Writer.write(ddl_path + db_name + ".ddl", '\n '.join(ddl))
print("ddl saved to {}".format(ddl_path + db_name + ".ddl"))
conn.execute('\n '.join(ddl))
print("schema '{}' was created".format(schema.name))
conn.close()
def train_loop(rank, cfg):
logger = get_logger(cfg, os.path.basename(__file__))
if cfg.device == "cuda" and cfg.dist.gpus != 0:
cfg.device = rank
# turn off background generator when distributed run is on
cfg.data.use_background_generator = False
setup(cfg, rank)
torch.cuda.set_device(cfg.device)
# setup writer
if is_logging_process():
# set log/checkpoint dir
os.makedirs(cfg.log.chkpt_dir, exist_ok=True)
# set writer (tensorboard / wandb)
writer = Writer(cfg, "tensorboard")
cfg_str = OmegaConf.to_yaml(cfg)
logger.info("Config:\n" + cfg_str)
if cfg.data.train_dir == "" or cfg.data.test_dir == "":
logger.error("train or test data directory cannot be empty.")
raise Exception("Please specify directories of data")
logger.info("Set up train process")
logger.info("BackgroundGenerator is turned off when Distributed running is on")
# download MNIST dataset before making dataloader
# TODO: This is example code. You should change this part as you need
_ = torchvision.datasets.MNIST(
root=hydra.utils.to_absolute_path("dataset/meta"),
train=True,
transform=torchvision.transforms.ToTensor(),
download=True,
)
_ = torchvision.datasets.MNIST(
root=hydra.utils.to_absolute_path("dataset/meta"),
train=False,
transform=torchvision.transforms.ToTensor(),
download=True,
)
# Sync dist processes (because of download MNIST Dataset)
if cfg.dist.gpus != 0:
dist.barrier()
# make dataloader
if is_logging_process():
logger.info("Making train dataloader...")
train_loader = create_dataloader(cfg, DataloaderMode.train, rank)
if is_logging_process():
logger.info("Making test dataloader...")
test_loader = create_dataloader(cfg, DataloaderMode.test, rank)
# init Model
net_arch = Net_arch(cfg)
loss_f = torch.nn.CrossEntropyLoss()
model = Model(cfg, net_arch, loss_f, rank)
# load training state / network checkpoint
if cfg.load.resume_state_path is not None:
model.load_training_state()
elif cfg.load.network_chkpt_path is not None:
model.load_network()
else:
if is_logging_process():
logger.info("Starting new training run.")
try:
if cfg.dist.gpus == 0 or cfg.data.divide_dataset_per_gpu:
epoch_step = 1
else:
epoch_step = cfg.dist.gpus
for model.epoch in itertools.count(model.epoch + 1, epoch_step):
if model.epoch > cfg.num_epoch:
break
train_model(cfg, model, train_loader, writer)
if model.epoch % cfg.log.chkpt_interval == 0:
model.save_network()
model.save_training_state()
test_model(cfg, model, test_loader, writer)
if is_logging_process():
logger.info("End of Train")
except Exception as e:
if is_logging_process():
logger.error(traceback.format_exc())
else:
traceback.print_exc()
finally:
if cfg.dist.gpus != 0:
cleanup()
def main():
opt = TrainOptions().parse()
if opt == None:
return
dataset = DataLoader(opt)
dataset_size = len(dataset) * opt.num_grasps_per_object
model = create_model(opt)
writer = Writer(opt)
total_steps = 0
for epoch in range(opt.epoch_count, opt.niter + opt.niter_decay + 1):
epoch_start_time = time.time()
iter_data_time = time.time()
epoch_iter = 0
for i, data in enumerate(dataset):
iter_start_time = time.time()
if total_steps % opt.print_freq == 0:
t_data = iter_start_time - iter_data_time
total_steps += opt.batch_size
epoch_iter += opt.batch_size
model.set_input(data)
model.optimize_parameters()
if total_steps % opt.print_freq == 0:
loss_types = []
if opt.arch == "vae":
loss = [
model.loss, model.kl_loss, model.reconstruction_loss,
model.confidence_loss, model.l2_loss
]
loss_types = [
"total_loss", "kl_loss", "reconstruction_loss",
"confidence loss", "l2_loss"
]
elif opt.arch == "gan":
loss = [
model.loss, model.reconstruction_loss,
model.confidence_loss
]
loss_types = [
"total_loss", "reconstruction_loss", "confidence_loss"
]
else:
loss = [
model.loss, model.classification_loss,
model.confidence_loss
]
loss_types = [
"total_loss", "classification_loss", "confidence_loss"
]
t = (time.time() - iter_start_time) / opt.batch_size
writer.print_current_losses(epoch, epoch_iter, loss, t, t_data,
loss_types)
writer.plot_loss(loss, epoch, epoch_iter, dataset_size,
loss_types)
if i % opt.save_latest_freq == 0:
print('saving the latest model (epoch %d, total_steps %d)' %
(epoch, total_steps))
model.save_network('latest', epoch)
iter_data_time = time.time()
if epoch % opt.save_epoch_freq == 0:
print('saving the model at the end of epoch %d, iters %d' %
(epoch, total_steps))
model.save_network('latest', epoch)
model.save_network(str(epoch), epoch)
print('End of epoch %d / %d \t Time Taken: %d sec' %
(epoch, opt.niter + opt.niter_decay,
time.time() - epoch_start_time))
model.update_learning_rate()
if opt.verbose_plot:
writer.plot_model_wts(model, epoch)
if epoch % opt.run_test_freq == 0:
acc = run_test(epoch, name=opt.name)
writer.plot_acc(acc, epoch)
writer.close()
class Visualizer:
def __init__(self, opt, tensorboard=True):
# opt is the global options shared in the coding env
self.opt = opt
self.tensorboard = tensorboard
if tensorboard:
import threading
# self.t = threading.Thread(target=launchTensorBoard, args=([os.path.join(opt.checkpoints_dir, opt.name), opt.port]))
# self.t.daemon = True
launchTensorBoard(os.path.join(opt.checkpoints_dir, opt.name),
opt.port)
self.t.start()
# self.t.join(20)
self.writer = SummaryWriter(
os.path.join(opt.checkpoints_dir, opt.name))
else:
self.writer = Writer(os.path.join(opt.checkpoints_dir, opt.name))
log_filename = os.path.join(opt.checkpoints_dir,
'%s_log.txt' % opt.name)
self.log = open(log_filename, 'w')
self.write_opt(opt)
self.save_image_root = os.path.join(opt.checkpoints_dir, opt.name)
def add_text(self, tag, text):
self.writer.add_text(tag, text)
def write_opt(self, opt):
args = vars(self.opt)
self.write_log_rightnow('------------ Options -------------\n')
for k, v in sorted(args.items()):
self.write_log_rightnow('%s: %s\n' % (str(k), str(v)))
self.write_log_rightnow('-------------- End ----------------\n')
def write_acc(self, acc):
self.write_log_rightnow('Acc: %f\n' % acc)
def write_log_rightnow(self, str):
self.log.write(str)
self.log.flush()
os.fsync(self.log)
def write_confmat(self, conf_mat):
self.write_log_rightnow('Conf Mat:\n')
for line in conf_mat:
for e in line:
self.write_log_rightnow('%f ' % e)
self.write_log_rightnow('\n')
def add_log(self, str):
# print str
self.write_log_rightnow('%s\n' % str)
def write_epoch(self, epoch):
self.write_log_rightnow('Epoch %d\n' % epoch)
def plot_errors(self, errors, main_fig_title='errors', time_step=None):
# example: main_fig_title = 'GAN_depth_to_image', sub_fig_title = 'train_loss'
# scalars_dict = {'item': value, 'item2':value},
# when time_step is None, it uses internal global_time_step
# for k, v in errors.items():
tag = main_fig_title + '/'
self.writer.add_scalars(tag, errors, time_step)
def kill(self):
# print self.t.getName()
self.log.close()
def display_image(self, images, time_step=None):
# images : {'real': real_im,'fake':fake_im}
for item in images.keys():
self.writer.add_image(item, images[item], time_step)
def save_images(self, images, epoch):
for item in images.keys():
cur_image_fullpath = os.path.join(
self.save_image_root, 'Epoch_%06d_%s.png' % (epoch, item))
simple_util.save_image(images[item], cur_image_fullpath)
def print_errors(self, epoch, batch, batch_all, errors):
message = '(epoch: %d, (%d/%d)) ' % (epoch, batch, batch_all)
for k, v in errors.items():
message += '%s: %.3f ' % (k, v)
print(message)
def train_loop(rank, hp, world_size=0):
if hp.model.device == "cuda" and world_size != 0:
hp.model.device = rank
# turn off background generator when distributed run is on
hp.data.use_background_generator = False
setup(hp, rank, world_size)
torch.cuda.set_device(hp.model.device)
# setup logger / writer
if rank != 0:
logger = None
writer = None
else:
# set logger
logger = make_logger(hp)
# set writer (tensorboard / wandb)
writer = Writer(hp, os.path.join(hp.log.log_dir, "tensorboard"))
hp_str = yaml.dump(hp.to_dict())
logger.info("Config:")
logger.info(hp_str)
if hp.data.train_dir == "" or hp.data.test_dir == "":
logger.error("train or test data directory cannot be empty.")
raise Exception("Please specify directories of data")
logger.info("Set up train process")
logger.info(
"BackgroundGenerator is turned off when Distributed running is on")
# download MNIST dataset before making dataloader
# TODO: This is example code. You should change this part as you need
_ = torchvision.datasets.MNIST(
root="dataset/meta",
train=True,
transform=torchvision.transforms.ToTensor(),
download=True,
)
_ = torchvision.datasets.MNIST(
root="dataset/meta",
train=False,
transform=torchvision.transforms.ToTensor(),
download=True,
)
# Sync dist processes (because of download MNIST Dataset)
if world_size != 0:
dist.barrier()
# make dataloader
if logger is not None:
logger.info("Making train dataloader...")
train_loader = create_dataloader(hp, DataloaderMode.train, rank,
world_size)
if logger is not None:
logger.info("Making test dataloader...")
test_loader = create_dataloader(hp, DataloaderMode.test, rank, world_size)
# init Model
net_arch = Net_arch(hp)
loss_f = torch.nn.CrossEntropyLoss()
model = Model(hp, net_arch, loss_f, rank, world_size)
# load training state / network checkpoint
if hp.load.resume_state_path is not None:
model.load_training_state(logger)
elif hp.load.network_chkpt_path is not None:
model.load_network(logger=logger)
else:
if logger is not None:
logger.info("Starting new training run.")
try:
if world_size == 0 or hp.data.divide_dataset_per_gpu:
epoch_step = 1
else:
epoch_step = world_size
for model.epoch in itertools.count(model.epoch + 1, epoch_step):
if model.epoch > hp.train.num_epoch:
break
train_model(hp, model, train_loader, writer, logger)
if model.epoch % hp.log.chkpt_interval == 0:
model.save_network(logger)
model.save_training_state(logger)
test_model(hp, model, test_loader, writer, logger)
if logger is not None:
logger.info("End of Train")
except Exception as e:
if logger is not None:
logger.error(traceback.format_exc())
else:
traceback.print_exc()
finally:
if world_size != 0:
cleanup()
