def split(
self
): # process if the dataset split into two parts, which contains lots of missing value
self.train = loader('train.csv')
self.test = loader('test.csv')
self.test, self.id = dp.Pandas_dataProcess().label_extract(self.test,
id=True,
istest=True)
self.train, feature_name = dp.Pandas_dataProcess().trans_categorical(
self.train)
self.test = dp.Pandas_dataProcess().trans_categorical(
self.test, feature_name=feature_name, istest=True)
self.train = dp.Pandas_dataProcess().id_droper(self.train)
self.train = self.train.values.tolist()
self.test = self.test.values.tolist()
self.misstrain, self.normaltrain = dp.List_dataProcess(
).missingset_split(self.train)
self.misstrain, self.misstrain_y = dp.List_dataProcess().label_extract(
self.misstrain)
self.normaltrain, self.normaltrain_y = dp.List_dataProcess(
).label_extract(self.normaltrain)
self.misstest, self.normaltest = dp.List_dataProcess(
).missingset_split(self.test)
self.normaltrain = dp.Pandas_dataProcess().trans_topanda(
self.normaltrain)
self.normaltrain = dp.Pandas_dataProcess().fill_missvalue(
self.normaltrain, value=self.missing)
self.normaltest = dp.Pandas_dataProcess().trans_topanda(
self.normaltest)
self.normaltest = dp.Pandas_dataProcess().fill_missvalue(
self.normaltest, value=self.missing)
def normal(self): # process if the dataset not split into two parts
self.train = loader('train.csv')
self.train_x, self.train_y = dp.Pandas_dataProcess().label_extract(self.train, real_extract=False)
self.test = loader('test.csv')
self.test, self.id = dp.Pandas_dataProcess().label_extract(self.test, id=True, istest=True)
self.train_x, feature_name = dp.Pandas_dataProcess().trans_categorical(self.train_x)
self.test = dp.Pandas_dataProcess().trans_categorical(self.test, feature_name=feature_name, istest=True)
self.train_x = dp.Pandas_dataProcess().fill_missvalue(self.train_x, value=0)
self.test = dp.Pandas_dataProcess().fill_missvalue(self.test, value=0)
self.train_x = self.train_x.iloc[random.permutation(len(self.train_x))]
self.train_x, self.train_y = dp.Pandas_dataProcess().label_extract(self.train_x)
def split(self): # process if the dataset split into two parts, which contains lots of missing value
self.train = loader('train.csv')
self.test = loader('test.csv')
self.test, self.id = dp.Pandas_dataProcess().label_extract(self.test, id=True, istest=True)
self.train, feature_name = dp.Pandas_dataProcess().trans_categorical(self.train)
self.test = dp.Pandas_dataProcess().trans_categorical(self.test, feature_name=feature_name, istest=True)
self.train = dp.Pandas_dataProcess().id_droper(self.train)
self.train = self.train.values.tolist()
self.test = self.test.values.tolist()
self.misstrain, self.normaltrain = dp.List_dataProcess().missingset_split(self.train)
self.misstrain, self.misstrain_y = dp.List_dataProcess().label_extract(self.misstrain)
self.normaltrain, self.normaltrain_y = dp.List_dataProcess().label_extract(self.normaltrain)
self.misstest, self.normaltest = dp.List_dataProcess().missingset_split(self.test)
self.normaltrain = dp.Pandas_dataProcess().trans_topanda(self.normaltrain)
self.normaltrain = dp.Pandas_dataProcess().fill_missvalue(self.normaltrain, value=self.missing)
self.normaltest = dp.Pandas_dataProcess().trans_topanda(self.normaltest)
self.normaltest = dp.Pandas_dataProcess().fill_missvalue(self.normaltest, value=self.missing)
def doclassify(self):
train = loader("train.csv")
target = train['target'].values
test = loader("test.csv")
id_test = test['ID'].values
train['v22-1'] = train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))
test['v22-1'] = test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[0]))
train['v22-2'] = train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1]))
test['v22-2'] = test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[1])) # to process v22 features which is an important categorical feature
train['v22-3'] = train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))
test['v22-3'] = test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[2]))
train['v22-4'] = train['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))
test['v22-4'] = test['v22'].fillna('@@@@').apply(lambda x:'@'*(4-len(str(x)))+str(x)).apply(lambda x:ord(x[3]))
drop_list = ['v91', 'v1', 'v8', 'v25', 'v29', 'v34', 'v41', 'v46', 'v54', 'v67', 'v97', 'v105', 'v122', 'v38', 'v72','v24', 'v52']
train = train.drop(['ID', 'target'] + drop_list, axis=1).fillna(self.missing)
test = test.drop(['ID'] + drop_list, axis=1).fillna(self.missing)
# train, train_y = dp.Pandas_dataProcess().label_extract(train, real_extract=True)
# test, id = dp.Pandas_dataProcess().label_extract(test, id=True, istest=True)
# train, feature_name = dp.Pandas_dataProcess().trans_categorical(train)
# test = dp.Pandas_dataProcess().trans_categorical(test, feature_name=feature_name, istest=True)
# train = dp.Pandas_dataProcess().fill_missvalue(train, value='mean')
# test = dp.Pandas_dataProcess().fill_missvalue(test, value='mean')
refcols=list(train.columns)
print refcols
for elt in refcols:
if train[elt].dtype == 'O':
train[elt], temp = pd.factorize(train[elt])
test[elt] = temp.get_indexer(test[elt])
else:
train[elt] = train[elt].round(5)
test[elt] = test[elt].round(5)
a = dp.AddNearestNeighbourLinearFeatures(n_neighbours=self.n_ft, max_elts=self.max_elts, verbose=True, random_state=self.rnd)
a.fit(train, target)
train = a.transform(train)
test = a.transform(test)
clf = ensemble.ExtraTreesClassifier(n_estimators=1200, max_features=30, criterion='entropy', min_samples_split=2,
max_depth=35, min_samples_leaf=2, n_jobs=-1, random_state=self.rnd)
clf.fit(train,target)
pred_et = clf.predict_proba(test)
submission = pd.read_csv('sample_submission.csv')
submission.index = submission.ID
submission.PredictedProb = pred_et[:, 1]
submission.to_csv('./addNNLinearFt.csv', index=False)
submission.PredictedProb.hist(bins=30)
def normal(self): # process if the dataset not split into two parts
self.train = loader('train.csv')
self.train_x, self.train_y = dp.Pandas_dataProcess().label_extract(
self.train, real_extract=False)
self.test = loader('test.csv')
self.test, self.id = dp.Pandas_dataProcess().label_extract(self.test,
id=True,
istest=True)
self.train_x, feature_name = dp.Pandas_dataProcess().trans_categorical(
self.train_x)
self.test = dp.Pandas_dataProcess().trans_categorical(
self.test, feature_name=feature_name, istest=True)
self.train_x = dp.Pandas_dataProcess().fill_missvalue(self.train_x,
value=0)
self.test = dp.Pandas_dataProcess().fill_missvalue(self.test, value=0)
self.train_x = self.train_x.iloc[random.permutation(len(self.train_x))]
self.train_x, self.train_y = dp.Pandas_dataProcess().label_extract(
self.train_x)
def plot2d():
ld = dl.loader(dimensions=2)
tsi = ld.getTestInp()
tso = ld.getTestOut()
tri = ld.getTrainInp()
tro = ld.getTrainOut()
mlp = MLP(tri, tro, (20,25), epoches=2500)
plt.plot(tri, tro, "r+")
out = mlp.calc(tri)
plt.plot(tri, out, "b-")
out = mlp.calc(tsi)
plt.plot(tsi, out, "go")
plt.show()
def plot3d():
ld = dl.loader(dimensions=3)
ai = ld.getAllInp()
ao = ld.getAllOut()
x, y, z = mesh_data(ai.T[0], ai.T[1], ao.T[0])
fig = plt.figure()
axes = fig.gca(projection='3d')
axes.scatter(x, y, z, color="#bbff8016")
mlp = MLP(ai, ao, (25,), epoches=1000)
out = mlp.calc(ai)
x, y, z = mesh_data(ai.T[0], ai.T[1], out)
axes.plot_surface(x, y, z, cmap=cm.get_cmap("coolwarm"),
linewidth=0, antialiased=True)
plt.show()
def painting():
Not_number = [
'v3', 'v22', 'v24', 'v30', 'v31', 'v47', 'v52', 'v56', 'v66', 'v71',
'v74', 'v75', 'v79', 'v91', 'v107', 'v110', 'v112', 'v113', 'v125'
]
start = time.time()
print 'painting...'
dataset = dataloader.loader('train.csv')
p = Paint()
# p.missingVL_abs(dataset)
# p.correlation_digonal(dataset)
# p.hexbin(dataset, 'v83', 'v130')
# p.hexbin(dataset, 'v1', 'v37')
p.label_prop(dataset, 'v125')
# p.label_prop(dataset, 'v24')
# p.hexbin(dataset, 'v10', 'v131')
# p.distribution(dataset, 'v1')
# p.distribution(dataset, 'v131')
# p.distribution(dataset, 'target')
end = time.time()
print '-' * 100
print 'total run time is:', end - start, 's'
def sta():
dataset = loader('train.csv')
stat.Categorical_stat().co_relation(dataset)
return support_index
def observer(self,step = 1):
color = array([[0,0,1], [1,0,0],[0,1,0] ] )
#color for classifier
datacolor = [color[lbl] for lbl in self.label]
X= zeros(( len(self.label),len(self.points[0]) ))
#init a the data matrix
for i in xrange(0,len(self.label)):
X[i][0],X[i][1] = self.points[i][0],self.points[i][1]
#clear the previous figure
pyplot.plot(hold = False)
pyplot.hold = True
pyplot.scatter(X[:,0],X[:,1],c = datacolor)
pyplot.savefig("data/twofeature.png",format = "png")
def get_label(self):
return label
def get_points(self):
return points
if __name__ == "__main__":
label,points = dataloader.loader("twofeature.txt")
svm = SVM(label = label,points = points)
svm.observer()
import cv2
import numpy as np
from dataloader import loader
from unet import Models
from sklearn.model_selection import train_test_split
from keras.preprocessing.image import ImageDataGenerator
from keras.optimizers import Adam
import matplotlib.pyplot as plt
w = 256
h = 256
c = 3
mod = Models(w, h, c)
auto_encoder = mod.arch3()
load_img = loader()
auto_encoder.summary()
x_data, y_data = load_img.load()
x_data = np.array(x_data, dtype='float') / 255.0
y_data = np.array(y_data, dtype='float') / 255.0
opt = Adam(lr=0.001, decay=0.001 / 50)
train_x, test_x, train_y, test_y = train_test_split(x_data,
y_data,
test_size=0.1,
random_state=30)
auto_encoder.compile(optimizer=Adam(lr=1e-4),
loss='binary_crossentropy',
metrics=['accuracy'])
history = auto_encoder.fit(train_x,
train_y,
batch_size=1,
def sta():
dataset = loader('train.csv')
stat.Categorical_stat().co_relation(dataset)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
np.random.seed(seed)
random.seed(seed)
setup_seed(44)
rel2id, id2rel = map_id_rel()
print(id2rel)
USE_CUDA = torch.cuda.is_available()
#USE_CUDA=False
path = "data/questions.json"
train, dev = loader(path)
data = train
train_text = data['text']
train_mask = data['mask']
train_label = data['label']
train_text = [t.numpy() for t in train_text]
train_mask = [t.numpy() for t in train_mask]
train_text = torch.tensor(train_text)
train_mask = torch.tensor(train_mask)
train_label = torch.tensor(train_label)
data = dev
dev_text = data['text']
dev_mask = data['mask']
def doclassify(self):
train = loader("train.csv")
target = train['target'].values
test = loader("test.csv")
id_test = test['ID'].values
train['v22-1'] = train['v22'].fillna('@@@@').apply(lambda x: '@' * (
4 - len(str(x))) + str(x)).apply(lambda x: ord(x[0]))
test['v22-1'] = test['v22'].fillna('@@@@').apply(lambda x: '@' * (
4 - len(str(x))) + str(x)).apply(lambda x: ord(x[0]))
train['v22-2'] = train['v22'].fillna('@@@@').apply(lambda x: '@' * (
4 - len(str(x))) + str(x)).apply(lambda x: ord(x[1]))
test['v22-2'] = test['v22'].fillna(
'@@@@'
).apply(lambda x: '@' * (4 - len(str(x))) + str(x)).apply(
lambda x: ord(x[1])
) # to process v22 features which is an important categorical feature
train['v22-3'] = train['v22'].fillna('@@@@').apply(lambda x: '@' * (
4 - len(str(x))) + str(x)).apply(lambda x: ord(x[2]))
test['v22-3'] = test['v22'].fillna('@@@@').apply(lambda x: '@' * (
4 - len(str(x))) + str(x)).apply(lambda x: ord(x[2]))
train['v22-4'] = train['v22'].fillna('@@@@').apply(lambda x: '@' * (
4 - len(str(x))) + str(x)).apply(lambda x: ord(x[3]))
test['v22-4'] = test['v22'].fillna('@@@@').apply(lambda x: '@' * (
4 - len(str(x))) + str(x)).apply(lambda x: ord(x[3]))
drop_list = [
'v91', 'v1', 'v8', 'v25', 'v29', 'v34', 'v41', 'v46', 'v54', 'v67',
'v97', 'v105', 'v122', 'v38', 'v72', 'v24', 'v52'
]
train = train.drop(['ID', 'target'] + drop_list,
axis=1).fillna(self.missing)
test = test.drop(['ID'] + drop_list, axis=1).fillna(self.missing)
# train, train_y = dp.Pandas_dataProcess().label_extract(train, real_extract=True)
# test, id = dp.Pandas_dataProcess().label_extract(test, id=True, istest=True)
# train, feature_name = dp.Pandas_dataProcess().trans_categorical(train)
# test = dp.Pandas_dataProcess().trans_categorical(test, feature_name=feature_name, istest=True)
# train = dp.Pandas_dataProcess().fill_missvalue(train, value='mean')
# test = dp.Pandas_dataProcess().fill_missvalue(test, value='mean')
refcols = list(train.columns)
print refcols
for elt in refcols:
if train[elt].dtype == 'O':
train[elt], temp = pd.factorize(train[elt])
test[elt] = temp.get_indexer(test[elt])
else:
train[elt] = train[elt].round(5)
test[elt] = test[elt].round(5)
a = dp.AddNearestNeighbourLinearFeatures(n_neighbours=self.n_ft,
max_elts=self.max_elts,
verbose=True,
random_state=self.rnd)
a.fit(train, target)
train = a.transform(train)
test = a.transform(test)
clf = ensemble.ExtraTreesClassifier(n_estimators=1200,
max_features=30,
criterion='entropy',
min_samples_split=2,
max_depth=35,
min_samples_leaf=2,
n_jobs=-1,
random_state=self.rnd)
clf.fit(train, target)
pred_et = clf.predict_proba(test)
submission = pd.read_csv('sample_submission.csv')
submission.index = submission.ID
submission.PredictedProb = pred_et[:, 1]
submission.to_csv('./addNNLinearFt.csv', index=False)
submission.PredictedProb.hist(bins=30)
return model
elif type == "dns201":
model = torch.hub.load('pytorch/vision:v0.9.0',
'densenet201',
pretrained=False)
return model
elif type == "rsnxt-50":
model = pretrainedmodels.__dict__["se_resnext50_32x4d"](
num_classes=2, pretrained=None)
return model
if __name__ == '__main__':
dataloaders, dataset_sizes = loader(
"/content/drive/MyDrive/competitions/recog-r2/train.csv", 0.2)
model_ft = mdl("res50")
model_ft = model_ft.to(DEVICE)
criterion = nn.CrossEntropyLoss()
# Observe that all parameters are being optimized
# optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)
optimizer_ft = optim.Adam(model_ft.parameters(), lr=0.001)
# Decay LR by a factor of 0.1 every 7 epochs
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft,
step_size=7,
gamma=0.1)
import torch
import torchvision
import torch.nn as nn
import datetime as dt
import Levenshtein as L
import torch.optim as optim
from model import ZLNet
from utils import details
from constant import CHARSET
from dataloader import loader
DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
test_data_file = '../data/test.npy'
testloader = loader(test_data_file)
model = ZLNet().to(DEVICE)
model.load_state_dict(torch.load("model.pt"))
model.eval()
result = []
for inputs, lens in testloader:
lens = lens.to(DEVICE)
inputs = inputs.to(DEVICE)
charindice = model(inputs, lens)
charindice = charindice[0]
pred = ''.join([CHARSET[idx] for idx in charindice])
result.append(pred[1:-1])
save_ckp(checkpoint, checkpoint_path)
print()
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))
# load best model weights
model.load_state_dict(best_model_wts)
plot(loss_p,acc_p,num_epochs)
return model, best_acc
if __name__ == '__main__':
dataloaders,dataset_sizes = loader(use_pretrained=True)
model_ft = mdl("res34")
# model_ft = drklrd()
model_ft = model_ft.to(config.DEVICE)
criterion = nn.CrossEntropyLoss()
optimizer_ft = optim.Adam(model_ft.parameters(), lr=0.0001)
# Decay LR by a factor of 0.1 every 7 epochs
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)
checkpoint_path = "/content/drive/MyDrive/competitions/mosaic-r2/weights/res34_albu_seg.pt"
model_ft, best_acc = train_model(model_ft, dataloaders, criterion, optimizer_ft, exp_lr_scheduler, dataset_sizes, checkpoint_path, num_epochs=config.NUM_EPOCHS)
def __init__(self, config_path):
with open(os.path.join(config_path, "config.yml")) as cf:
config = yaml.load(cf, Loader=yaml.FullLoader)
self.num_layers = config["num_layers"]
self.d_model = config["d_model"]
self.dff = config["dff"]
self.num_heads = config["num_heads"]
self.dropout_rate = config["dropout_rate"]
self.max_length = config["max_length"]
self.epochs = config["epochs"]
self.batch_size = config["batch_size"]
self.target_vocab_size = config["target_vocab_size"]
self.checkpoint = config["checkpoint"]
self.max_checkpoint = config["max_checkpoint"]
self.custom_checkpoint = config["custom_checkpoint"]
self.eval_limit = config["eval_limit"]
self.exit_phrase = config["exit_phrase"]
if config["storage_path"] != None:
self.storage_path = config["storage_path"]
else:
self.storage_path = "./"
if config["ckpt_path"] != None:
self.ckpt_path = config["ckpt_path"]
else:
self.ckpt_path = "./"
if not self.storage_path.endswith("/"):
self.storage_path += "/"
if not self.ckpt_path.endswith("/"):
self.ckpt_path += "/"
self.data_path = f"{self.storage_path}data"
self.checkpoint_path = f"{self.ckpt_path}checkpoints/train"
self.tokenizer_path = f"{self.storage_path}tokenizers"
self.inputs_savepath = f"{self.tokenizer_path}/inputs_token"
self.outputs_savepath = f"{self.tokenizer_path}/outputs_token"
if not os.path.exists(f"{self.ckpt_path}checkpoints"):
os.mkdir(f"{self.ckpt_path}checkpoints")
if not os.path.exists(f"{self.ckpt_path}checkpoints/train"):
os.mkdir(f"{self.ckpt_path}checkpoints/train")
if not os.path.exists(f"{self.storage_path}tokenizers"):
os.mkdir(f"{self.storage_path}tokenizers")
if not os.path.exists(f"{self.storage_path}models"):
os.mkdir(f"{self.storage_path}models")
if config["mode"] in ["train", "eval"]:
if os.path.exists(os.path.join(
config_path, "data/train.from")) and os.path.exists(
os.path.join(config_path, "data/train.to")):
pass
else:
if config["reddit_data"]:
print("Starting to generate train data from Subreddits.")
get_data(config_path)
loader(config_path)
self.inputs, self.outputs = load_data(
f"{self.data_path}/training_data.txt")
try:
self.inputs_tokenizer, self.outputs_tokenizer = load_tokenizers(
inputs_outputs_savepaths=[
self.inputs_savepath, self.outputs_savepath
])
except:
print(
"No tokenizers has been created yet, creating new tokenizers..."
)
self.inputs_tokenizer, self.outputs_tokenizer = create_tokenizers(
inputs_outputs=[self.inputs, self.outputs],
inputs_outputs_savepaths=[
self.inputs_savepath, self.outputs_savepath
],
target_vocab_size=self.target_vocab_size)
self.input_vocab_size = self.inputs_tokenizer.vocab_size + 2
self.target_vocab_size = self.outputs_tokenizer.vocab_size + 2
self.learning_rate = CustomSchedule(self.d_model)
self.optimizer = tf.keras.optimizers.Adam(self.learning_rate,
beta_1=0.9,
beta_2=0.98,
epsilon=1e-9)
self.train_loss = tf.keras.metrics.Mean(name='train_loss')
self.train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
name='train_accuracy')
self.transformer = Transformer(self.num_layers,
self.d_model,
self.num_heads,
self.dff,
self.input_vocab_size,
self.target_vocab_size,
pe_input=self.input_vocab_size,
pe_target=self.target_vocab_size,
rate=self.dropout_rate)
self.ckpt = tf.train.Checkpoint(transformer=self.transformer,
optimizer=self.optimizer)
self.ckpt_manager = tf.train.CheckpointManager(
self.ckpt, self.checkpoint_path, max_to_keep=self.max_checkpoint)
if self.custom_checkpoint:
self.ckpt.restore(self.custom_checkpoint)
print(f"Custom checkpoint restored: {self.custom_checkpoint}")
# if a checkpoint exists, restore the latest checkpoint.
elif self.ckpt_manager.latest_checkpoint:
self.ckpt.restore(self.ckpt_manager.latest_checkpoint)
print(
f"Latest checkpoint restored: {self.ckpt_manager.latest_checkpoint}"
)
if config["mode"] == "train":
print("\nMODE: train\n===========\n")
self.train_dataset = prepare_data(
self.batch_size, [self.inputs, self.outputs],
[self.inputs_tokenizer, self.outputs_tokenizer],
self.max_length)
self.train()
eval_indexes = random.choices(range(len(self.inputs)),
k=int(len(self.inputs) * 0.01))
for i in eval_indexes:
predicted_sentence, attention_weights, sentence, result = self.reply(
self.inputs[i])
print(f"\nInput: {self.inputs[i]}")
print(f"Predicted: {predicted_sentence}")
print(f"Sample output: {self.outputs[i]}")
elif config["mode"] == "eval":
print("\nMODE: eval\n==========\n")
self.inputs = self.inputs[:self.eval_limit]
self.outputs = self.outputs[:self.eval_limit]
for (ins, outs) in zip(self.inputs, self.outputs):
predicted_sentence, attention_weights, sentence, result = self.reply(
ins)
print(f"\nInput: {ins}")
print(f"Predicted: {predicted_sentence}")
print(f"Sample output: {outs}")
elif config["mode"] == "test":
print("\nMODE: test\n==========\n")
while True:
usr_input = input("[USER]: ")
if usr_input == self.exit_phrase:
print("Exiting test mode...")
break
else:
predicted_sentence, _, _, _ = self.reply(usr_input)
print(f"[BOT]: {predicted_sentence}")
elif config["mode"] == "script":
print("\nMODE: script\n==========\n")
import argparse
from analog.load import ExperimentLog
from dataloader import loader
from plots import plot_learning_curves
parser = argparse.ArgumentParser()
parser.add_argument('--logdir', type=str, required=True)
parser.add_argument('--exp_names', nargs='+', type=str, required=True)
parser.add_argument('--std_type', type=str, choices=['time', 'run'])
parser.add_argument('--min_t', type=float, default=0.)
parser.add_argument('--max_t', type=float, default=100.)
args = parser.parse_args()
start_date = 'last'
stop_date = None
expdata: ExperimentLog = ExperimentLog()
for exp_name in args.exp_names:
expdata.extend(loader(args.logdir, exp_name, start_date=start_date, stop_date=stop_date))
# def filter(args) -> bool:
# return ('noscale' in args and args['noscale'] and 'value' in args['algo']) \
# or 'advantage' in args['algo']
# expdata = filter(expdata, filter)
plot_learning_curves(expdata, ['Return'], args.exp_names[0], mint=args.min_t, maxt=args.max_t, gtype=args.std_type + "_std")
expdata.repr_rawlogs("Return", 5)