def main():
parser = argparse.ArgumentParser(
description="rfretrieval: Atmospheric retrieval with random forests.")
parser.set_defaults(func=show_usage, parser=parser)
parser.add_argument("--quiet", action='store_true')
subparsers = parser.add_subparsers()
parser_train = subparsers.add_parser('train', help="train a model")
parser_train.add_argument(
"training_dataset",
type=str,
help="JSON file with the training dataset description")
parser_train.add_argument(
"model_path",
type=str,
help="path where the trained model will be saved")
parser_train.add_argument("--num-trees",
type=int,
default=1000,
help="number of trees in the forest")
parser_train.add_argument(
"--num-jobs",
type=int,
default=5,
help="number of parallel jobs for fitting the random forest")
parser_train.add_argument(
"--feature-importance",
action='store_true',
help="compute feature importances after training")
parser_train.set_defaults(func=main_train)
parser_test = subparsers.add_parser(
'predict', help="use a trained model to perform a prediction")
parser_test.set_defaults(func=main_predict)
parser_test.add_argument("model_path",
type=str,
help="path to the trained model")
parser_test.add_argument("data_file",
type=str,
help="NPY file with the data for the prediction")
parser_test.add_argument(
"output_path",
type=str,
help="path to write the results of the prediction")
parser_test.add_argument(
"--plot-posterior",
action='store_true',
help="plot and save the scatter matrix of the posterior distribution")
args = parser.parse_args()
config_logger(level=logging.WARNING if args.quiet else logging.INFO)
args.func(**vars(args))
def __init__(self, envs, args, net):
self.envs = envs
self.args = args
# define the newtork...
self.net = net
self.old_net = copy.deepcopy(self.net)
# if use the cuda...
if self.args.cuda:
self.net.cuda()
self.old_net.cuda()
# define the optimizer...
self.optimizer = optim.Adam(self.net.parameters(), self.args.lr, eps=self.args.eps)
# check saving folder..
if not os.path.exists(self.args.save_dir):
os.mkdir(self.args.save_dir)
# env folder..
self.model_path = os.path.join(self.args.save_dir, self.args.env_name)
if not os.path.exists(self.model_path):
os.mkdir(self.model_path)
# logger folder
if not os.path.exists(self.args.log_dir):
os.mkdir(self.args.log_dir)
self.log_path = self.args.log_dir + self.args.env_name + '.log'
# get the observation
self.batch_ob_shape = (self.args.num_workers * self.args.nsteps, ) + self.envs.observation_space.shape
self.obs = np.zeros((self.args.num_workers, ) + self.envs.observation_space.shape, dtype=self.envs.observation_space.dtype.name)
self.obs[:] = self.envs.reset()
self.dones = [False for _ in range(self.args.num_workers)]
self.logger = config_logger(self.log_path)
def main():
cmd_parser = argparse.ArgumentParser(
description='Simcoin a cryptocurrency simulator.',
usage='''<command> [<args>]
The commands are:
nodes creates the {} for a simulation
network creates the {} for a simulation
ticks creates the {} for a simulation
simulate executes a simulation based on the {}, {} and {}
run runs all above commands
multi-run run the simulation multiple times
'''.format(
config.nodes_csv_file_name,
config.network_csv_file_name,
config.ticks_csv_file_name,
config.nodes_csv_file_name,
config.network_csv_file_name,
config.ticks_csv_file_name,
))
cmd_parser.add_argument('command', help='Subcommand to run')
# parse_args defaults to [1:] for args, but you need to
# exclude the rest of the args too, or validation will fail
args = cmd_parser.parse_args(sys.argv[1:2])
command = args.command
if command not in commands:
print('Unrecognized command')
cmd_parser.print_help()
exit(1)
# use dispatch pattern to invoke method with same name
if not os.path.exists(config.data_dir):
os.makedirs(config.data_dir)
bitcoin.SelectParams('regtest')
args = _parse_args()
utils.config_logger(args.verbose)
logging.info("Arguments called with: {}".format(sys.argv))
logging.info("Parsed arguments in simcoin.py: {}".format(args))
logging.info('Executing command={}'.format(command))
commands[command]()
def main():
cmd_parser = argparse.ArgumentParser(description='BlockPerf',
usage='''<command> [<args>]
The commands are:
nodes creates the {} for a run
network creates peer {} for a run
simulate executes a simulation based on the {} nd {}
run runs all above commands
'''.format(
config.nodes_csv_file_name,
config.network_csv_file_name,
config.nodes_csv_file_name,
config.network_csv_file_name,
))
cmd_parser.add_argument('command', help='Subcommand to run')
args = cmd_parser.parse_args(sys.argv[1:2])
command = args.command
if command not in commands:
print('Unrecognized command')
cmd_parser.print_help()
exit(1)
# use dispatch pattern to invoke method with same name
if not os.path.exists(config.data_dir):
os.makedirs(config.data_dir)
bitcoin.SelectParams('regtest')
args = _parse_args()
utils.config_logger(args.verbose)
logging.info("Arguments called with: {}".format(sys.argv))
logging.info('Executing command={}'.format(command))
commands[command]()
"'multi2', 'multi2_ext'. Use multichar labels to reduce sequence size")
parser.add_argument('--remove-splits',
action="store_true",
default=False,
help="Clean splits and chords during kern cleaning")
args = parser.parse_args()
data_prep_job = f'{args.id}_{args.label_encoder}{"_splits" if not args.remove_splits else ""}' # noqa E501
outdir = Path(args.out_dir) / data_prep_job
outdir.mkdir(parents=True, exist_ok=True)
args.out_dir = str(outdir)
# Logging config.
log_file = outdir / f'{datetime.now().strftime("%Y%m%d-%H%M%S")}_{data_prep_job}.log' # noqa E501
logger = config_logger('data_prep',
console_level='ERROR',
log_file=log_file)
# Main execution.
logger.info("Preprocessing humdrum data...")
root = Path(args.data_dir)
scores = sorted([x.relative_to(root) for x in root.rglob('*.krn')])
logger.info("Spliting train/test samples...")
scores_train, scores_test = train_test_split(scores,
test_size=args.test_split,
random_state=45)
middle = round(len(scores_test) /
2) # Favor validation if number of samples is odd
scores_val = scores_test[:middle]
def main():
config_logger()
log.info('Start main')
boo()
goo()
log.info('Finish main')
import pika
import sys
import logging
from utils import config_logger
log = logging.getLogger(__name__)
def send_tasks(channel, queue_name, count):
for i in range(count):
msg = 'Task {}'.format(i)
channel.basic_publish(exchange='',
routing_key=queue_name,
body=msg,
properties=pika.BasicProperties(delivery_mode=2))
log.info('{} published'.format(msg))
def main(queue_name='tasks', tasks_count=5):
logging.getLogger('pika').setLevel(logging.ERROR)
connection = pika.BlockingConnection(
pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
# channel.queue_declare(queue=queue_name, durable=True)
send_tasks(channel, queue_name, tasks_count)
connection.close()
if __name__ == '__main__':
config_logger(log_file='logs/tasks_producer.log')
main()
def main(args):
# init some setting
# config logging
log_path = os.path.join(args.log_root, '{}.log'.format(args.model_name))
logger = config_logger(log_path)
gpu_idx = args.gpu
if not gpu_idx:
ctx = mx.cpu()
else:
ctx = mx.gpu(gpu_idx - 1)
logger.info("Using ctx: {}".format(ctx))
# Loading vocab and model
ch_bert, ch_vocab = gluonnlp.model.get_model(
args.bert_model,
dataset_name=args.ch_bert_dataset,
pretrained=True,
ctx=ctx,
use_pooler=False,
use_decoder=False,
use_classifier=False)
model = BertClass(bert=ch_bert, max_seq_len=args.max_seq_len, ctx=ctx)
logger.info("Model Creating Completed.")
# init or load params for model
if args.istrain:
model.output_dense.initialize(init.Xavier(), ctx)
else:
model.load_parameters(args.model_params_path, ctx=ctx)
logger.info("Parameters Initing and Loading Completed")
model.hybridize()
if args.istrain:
# Loading dataloader
assiant = DatasetAssiantTransformer(ch_vocab=ch_vocab,
max_seq_len=args.max_seq_len)
dataset = ClassDataset(args.train_file_path)
train_dataset, dev_dataset = train_valid_split(dataset,
valid_ratio=0.1)
train_dataiter = ClassDataLoader(train_dataset,
batch_size=args.batch_size,
assiant=assiant,
shuffle=True).dataiter
dev_dataiter = ClassDataLoader(dev_dataset,
batch_size=args.batch_size,
assiant=assiant,
shuffle=True).dataiter
logger.info("Data Loading Completed")
else:
assiant = DatasetAssiantTransformer(ch_vocab=ch_vocab,
max_seq_len=args.max_seq_len,
istrain=args.istrain)
test_dataset = ClassTestDataset(args.test_file_path)
test_dataiter = ClassDataLoader(test_dataset,
batch_size=args.batch_size,
assiant=assiant,
shuffle=True).dataiter
# build trainer
finetune_trainer = gluon.Trainer(ch_bert.collect_params(), args.optimizer,
{"learning_rate": args.finetune_lr})
trainer = gluon.Trainer(model.collect_params("dense*"), args.optimizer,
{"learning_rate": args.train_lr})
loss_func = gloss.SoftmaxCELoss()
if args.istrain:
logger.info("## Trainning Start ##")
train_and_valid(ch_bert=ch_bert,
model=model,
ch_vocab=ch_vocab,
train_dataiter=train_dataiter,
dev_dataiter=dev_dataiter,
trainer=trainer,
finetune_trainer=finetune_trainer,
epochs=args.epochs,
loss_func=loss_func,
ctx=ctx,
lr=args.train_lr,
batch_size=args.batch_size,
params_save_step=args.params_save_step,
params_save_path_root=args.params_save_path_root,
eval_step=args.eval_step,
log_step=args.log_step,
check_step=args.check_step,
logger=logger,
num_train_examples=len(train_dataset),
warmup_ratio=args.warmup_ratio)
else:
predict(ch_bert=ch_bert,
model=model,
ch_vocab=ch_vocab,
test_dataiter=test_dataiter,
logger=logger,
ctx=ctx)
'SMOD':
'GHS_SMOD_POP_GLOBE_R2019A/GHS_SMOD_POP2015_GLOBE_R2019A_54009_1K/V2-0/tiles/'
'GHS_SMOD_POP2015_GLOBE_R2019A_54009_1K_V2_0_{column}_{row}.zip',
'POP':
'GHS_POP_MT_GLOBE_R2019A/GHS_POP_E2015_GLOBE_R2019A_54009_1K/V1-0/tiles/'
'GHS_POP_E2015_GLOBE_R2019A_54009_1K_V1_0_{column}_{row}.zip'
}
OUTPUT_GHS = {
'SMOD': '{country_iso3}_SMOD_2015_1km_mosaic.tif',
'POP': '{country_iso3}_POP_2015_1km_mosaic.tif'
}
GHS_CRS = 'ESRI:54009'
URBAN_MIN_MAX = (21, 30)
RURAL_MIN_MAX = (11, 13)
utils.config_logger()
logger = logging.getLogger(__name__)
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('country_iso3', help='Country ISO3')
parser.add_argument('-d',
'--download-ghs',
action='store_true',
help='Download the GHS data')
return parser.parse_args()
def main(country_iso3, download_ghs=False):
parser.add_argument('embeddings',
help='Text or numpy file with word embeddings')
parser.add_argument('--vocab',
help='Vocabulary file (only needed if numpy'
'embedding file is given)')
parser.add_argument('-a',
help='Plot attention values graph',
dest='attention',
action='store_true')
parser.add_argument('-i',
help='Run inference classifier',
dest='inference',
action='store_true')
args = parser.parse_args()
utils.config_logger(verbose=False)
logger = utils.get_logger()
params = ioutils.load_params(args.load)
if args.inference:
label_dict = ioutils.load_label_dict(args.load)
number_to_label = {v: k for (k, v) in label_dict.items()}
logger.info('Reading model')
sess = tf.InteractiveSession()
model_class = utils.get_model_class(params)
model = model_class.load(args.load, sess)
word_dict, embeddings = ioutils.load_embeddings(args.embeddings,
args.vocab,
generate=False,
load_extra_from=args.load,
normalize=True)
parser.add_argument('--seed', default=45, type=int, help='Seed to generators')
parser.add_argument('--mixed-precision',
action='store_true',
help='Uses mixed precision to train a model '
'(suggested with volta and above)')
if __name__ == '__main__':
args = parser.parse_args()
config = configparser.ConfigParser()
save_folder = os.path.dirname(args.model_path)
os.makedirs(save_folder, exist_ok=True) # Ensure save folder exists
# Logging config.
train_job = f"train_{Path(args.model_path).with_suffix('.log').name}"
log_file = f'{save_folder}/{datetime.now().strftime("%Y%m%d-%H%M%S")}_{train_job}'
logger = config_logger('train', log_file=log_file)
# Main execution.
# Get train config.
config.read(args.config_path)
model_name = config['train']['model']
model_conf = config[model_name]
audio_conf = config['audio']
if model_name == "deepspeech":
from deepspeech.loss import Loss
from deepspeech.model import DeepSpeech as Model
else:
raise NotImplementedError
train_conf = config['train']
def combine_datasets(data_dir, dataset_config, sample_rate=22050):
logger = config_logger('combine_datasets', console_level='INFO')
if not isinstance(data_dir, Path):
data_dir = Path(data_dir)
group = data_dir.name
# Label file must be the same. Just make a copy and rename.
labels_path = [
i for i in data_dir.rglob(f'*{dataset_config}/labels*.json')
][0]
outpath = data_dir / f'labels_{group}_{dataset_config}.json'
logger.info(f'Saving encoder labels to {outpath}')
outpath.write_text(labels_path.read_text())
dataset_partitions = ['test', 'train', 'val']
extension = 'csv'
for dataset_partition in dataset_partitions:
logger.info(
f'Looking for partition {dataset_partition} from configuration {dataset_config}.'
)
all_filenames = [
i for i in data_dir.rglob(
f'*{dataset_config}/{dataset_partition}*.{extension}')
]
logger.info(f'Found {len(all_filenames)} files.')
# combine all files in the list
combined_csv = pd.concat([
pd.read_csv(f, header=None).assign(filename=f.name)
for f in all_filenames
]).rename(columns={
0: "audio",
1: "seq"
})
logger.info(combined_csv.groupby('filename')['audio'].count())
logger.info(
f"Combined partition {dataset_partition}: {combined_csv['audio'].count()} samples."
)
durations = []
total_duration = 0
audio_errors = 0
for audio_file in combined_csv['audio'].tolist():
try:
y = load_audio(str(audio_file))
duration = len(y) / sample_rate
except Exception as e:
logger.exception(
f"Exception while loading {audio_file} audio. Reason: {e}")
audio_errors += 1
duration = 0
continue
durations.append(duration)
combined_csv = combined_csv.assign(duration=durations)
# SortaGrad
combined_csv = combined_csv.sort_values(by='duration')
total_duration = combined_csv['duration'].sum()
logger.info(f'Total duration: {total_duration/60/60} hours.')
logger.info(f'Found {audio_errors} errors during loading.')
# export to csv
outpath = data_dir / f'{dataset_partition}_{group}_{dataset_config}.{extension}'
logger.info(f'Saving to {outpath}')
combined_csv.drop(['filename', 'duration'],
axis=1).to_csv(outpath,
index=False,
header=False,
encoding='iso-8859-1')
def run(self):
utils.config_logger(self.args)
tracker.submit(
self.num_workers, fun_submit=self.submit(), pscmd=self.cmd)
parser.add_argument(
'--verbose',
action="store_true",
help="logger.info out decoded output and error of each sample")
parser.add_argument('--output-path',
default=None,
type=str,
help="Where to save raw acoustic output")
args = parser.parse_args()
if __name__ == '__main__':
save_folder = os.path.dirname(args.model_path)
manifest_name = '_'.join([*Path(args.test_manifest).parts[-2:]])
test_job = f"test_{manifest_name}_{Path(args.model_path).with_suffix('.log').name}"
log_file = f'{save_folder}/{datetime.now().strftime("%Y%m%d-%H%M%S")}_{test_job}'
logger = config_logger('test', log_file=log_file, console_level='ERROR')
torch.set_grad_enabled(False)
model, _ = load_model(args.model_path)
device = torch.device("cuda" if args.cuda else "cpu")
label_decoder = LabelDecoder(model.labels)
model.eval()
model = model.to(device)
test_dataset = SpectrogramDataset(audio_conf=model.audio_conf,
manifest_filepath=args.test_manifest,
labels=model.labels)
test_sampler = BucketingSampler(test_dataset, batch_size=args.batch_size)
test_loader = AudioDataLoader(test_dataset,
batch_sampler=test_sampler,
num_workers=args.num_workers)
parser.add_argument('--cnn')
parser.add_argument('--pred')
parser.add_argument('--test-set')
parser.add_argument('--output', default=None)
parser.add_argument('--gpu', default='0')
parser.add_argument('--keep-ratio', type=float, default=0.1)
parser.add_argument('--consider-trains', action='store_true')
parser.add_argument('--test-train', action='store_true')
parser.add_argument('-v', '--verbose', action='store_true')
args = parser.parse_args()
config_logger(args.pred + '.' + args.test_set + '.log',
verbose=args.verbose)
set_gpu(args.gpu)
test_sets = json.load(open('materials/imagenet-testsets.json', 'r'))
train_wnids = test_sets['train']
test_wnids = test_sets[args.test_set]
logging.info('pred: {}'.format(args.pred))
logging.info('cnn: {}'.format(args.cnn))
logging.info('test set: {}, {} classes, ratio={}'.format(
args.test_set, len(test_wnids), args.keep_ratio))
logging.info('consider train classifiers: {}'.format(args.consider_trains))
pred_file = torch.load(args.pred)
pred_wnids = pred_file['wnids']
pred_vectors = pred_file['pred']
parser.add_argument("--batch_size", "--b", type=int)
parser.add_argument("--mean", type=tuple)
parser.add_argument("--std", type=tuple)
parser.add_argument("--epochs", type=int)
parser.add_argument("--pretrain_path", type=str)
parser.add_argument("--logs", type=str, default="./logs/")
parser.add_argument("--desc", type=str, default="no_description_specified")
parser.add_argument("--kd", action="store_true")
parser.add_argument("--save", action="store_true")
parser.add_argument("--test", action="store_true")
parser.add_argument("--logfile", type=str, required=True)
parser.set_defaults(save=False)
parser.set_defaults(kd=False)
parser.set_defaults(test=False)
args = parser.parse_args()
loggername = datetime.now().strftime(f"D%d_%H:%M:%S_{args.desc}.log")
loggername = os.path.join(args.logs, loggername)
logger = config_logger(args.logfile, args.test)
assert os.environ.get("CONFIG_PATH") is not None
config = json.load(open(os.environ.get("CONFIG_PATH"), "r"))
for arg in vars(args):
if not getattr(args, arg) and arg in config.keys():
setattr(args, arg, config[arg])
logger.info("{} : {}".format(arg, getattr(args, arg)))
main(args)
print('Received: {}'.format(body))
ch.basic_ack(delivery_tag=method.delivery_tag)
log.info('Message {} received and processed'.format(body))
def main():
logging.getLogger('pika').setLevel(logging.ERROR)
if len(sys.argv) < 2:
worker_id = 1
else:
worker_id = sys.argv[1]
worker_name = 'Worker_{}'.format(worker_id)
connection = pika.BlockingConnection(
pika.ConnectionParameters('localhost'))
channel = connection.channel()
channel.queue_declare(queue='tasks')
log.info('{} started'.format(worker_name))
print("[*] {}: waiting for messages. To exit press CTRL+C".format(
worker_name))
channel.basic_qos(prefetch_count=1)
channel.basic_consume(callback, queue='tasks')
channel.start_consuming()
log.info('{} finished'.format(worker_name))
if __name__ == '__main__':
config_logger(log_file='logs/worker.log')
main()
#!/usr/bin/env python
import logging
import re
from utils import config_logger, add_logger
logger = logging.getLogger(__name__)
config_logger(logger)
class Token(object):
"""A token"""
name = None
value = None
def __init__(self, name, value):
self.name = name
self.value = value
def __str__(self):
return "{cls}({name},{value})".format(cls=self.__class__.__name__,
name=self.name,
value=repr(self.value))
class TokenizerException(Exception):
pass
class Lexer(object):
def main(args):
# init some setting
# config logging
log_path = os.path.join(args.log_root, '{}.log'.format(args.model_name))
if not os.path.exists(args.log_root):
os.makedirs(args.log_root)
logger = config_logger(log_path)
gpu_idx = args.gpu
if not gpu_idx:
ctx = mx.cpu()
else:
ctx = mx.gpu(gpu_idx - 1)
logger.info("Using ctx: {}".format(ctx))
# Loading vocab and model
src_bert, src_vocab = gluonnlp.model.get_model(
args.bert_model,
dataset_name=args.src_bert_dataset,
pretrained=True,
ctx=ctx,
use_pooler=False,
use_decoder=False,
use_classifier=False)
_, tgt_vocab = gluonnlp.model.get_model(args.bert_model,
dataset_name=args.tgt_bert_dataset,
pretrained=True,
ctx=ctx,
use_pooler=False,
use_decoder=False,
use_classifier=False)
mt_model = MTModel_Hybird(src_vocab=src_vocab,
tgt_vocab=tgt_vocab,
embedding_dim=args.mt_emb_dim,
model_dim=args.mt_model_dim,
head_num=args.mt_head_num,
layer_num=args.mt_layer_num,
ffn_dim=args.mt_ffn_dim,
dropout=args.mt_dropout,
att_dropout=args.mt_att_dropout,
ffn_dropout=args.mt_ffn_dropout,
ctx=ctx)
logger.info("Model Creating Completed.")
# init or load params for model
mt_model.initialize(init.Xavier(), ctx)
if args.src_bert_load_path:
src_bert.load_parameters(args.src_bert_load_path, ctx=ctx)
if args.mt_model_load_path:
mt_model.load_parameters(args.mt_model_load_path, ctx=ctx)
logger.info("Parameters Initing and Loading Completed")
src_bert.hybridize()
mt_model.hybridize()
# Loading dataloader
assiant = DatasetAssiantTransformer(src_vocab=src_vocab,
tgt_vocab=tgt_vocab,
max_src_len=args.max_src_len,
max_tgt_len=args.max_tgt_len)
train_dataset = MTDataset(args.train_data_path)
eval_dataset = MTDataset(args.eval_data_path)
train_dataiter = MTDataLoader(train_dataset,
batch_size=args.batch_size,
assiant=assiant,
shuffle=True).dataiter
dev_dataiter = MTDataLoader(eval_dataset,
batch_size=args.batch_size,
assiant=assiant,
shuffle=True).dataiter
logger.info("Data Loading Completed")
# build trainer
finetune_trainer = gluon.Trainer(src_bert.collect_params(), args.optimizer,
{"learning_rate": args.finetune_lr})
trainer = gluon.Trainer(mt_model.collect_params(), args.optimizer,
{"learning_rate": args.train_lr})
# loss function
if args.label_smooth:
loss_func = MaskedCELoss(sparse_label=False)
else:
loss_func = MaskedCELoss()
logger.info("## Trainning Start ##")
train_and_valid(src_bert=src_bert,
mt_model=mt_model,
src_vocab=src_vocab,
tgt_vocab=tgt_vocab,
train_dataiter=train_dataiter,
dev_dataiter=dev_dataiter,
trainer=trainer,
finetune_trainer=finetune_trainer,
epochs=args.epochs,
loss_func=loss_func,
ctx=ctx,
lr=args.train_lr,
batch_size=args.batch_size,
params_save_path_root=args.params_save_path_root,
eval_step=args.eval_step,
log_step=args.log_step,
check_step=args.check_step,
label_smooth=args.label_smooth,
logger=logger,
num_train_examples=len(train_dataset),
warmup_ratio=args.warmup_ratio)
parser.add_argument('--save-epoch', type=int, default=300)
parser.add_argument('--save-path', default='gcn-dense-att')
parser.add_argument('--seed', type=int, default=0)
parser.add_argument('--gpu', default='0')
parser.add_argument('--no-pred', action='store_true')
args = parser.parse_args()
random.seed(args.seed)
set_gpu(args.gpu)
save_path = args.save_path
save_path = osp.join('save', args.trainval, save_path)
ensure_path(save_path)
config_logger(save_path+'/train.log')
logging.info(args)
graph = json.load(open('materials/imagenet-dense-grouped-graph.json', 'r'))
wnids = graph['wnids']
n = len(wnids)
edges_set = graph['edges_set']
logging.info('edges_set{}'.format([len(l) for l in edges_set]))
lim = 4
for i in range(lim + 1, len(edges_set)):
edges_set[lim].extend(edges_set[i])
edges_set = edges_set[:lim + 1]
logging.info('edges_set{}'.format( [len(l) for l in edges_set]))
from app import create_flask_app
from common_constants.constants import FLASK_CONFIG_MODULE
from database import session
from restful_apis import create_restful_api
from utils import config_logger
from flask_jwt_extended import JWTManager
app = create_flask_app('Flask Jwt')
app.config.from_object(FLASK_CONFIG_MODULE)
app.config['JWT_SECRET_KEY'] = 'demo_app'
jwt = JWTManager(app)
app.config['JWT_HEADER_TYPE'] = 'JWT'
config_logger(app)
create_restful_api(app)
def close_session(resp):
session.close() # used to remove actual session
# session.remove() https://groups.google.com/forum/#!msg/sqlalchemy/twoHzgXcR60/nZqMKkCz9UwJ
return resp
app.teardown_request(close_session)
app.teardown_appcontext(close_session)
if __name__ == '__main__':
app.run()
parser.add_argument('--k', type=int, default=2)
parser.add_argument('--seed', type=int, default=0)
parser.add_argument('--gpu', default='0')
parser.add_argument('--no-pred', action='store_true')
parser.add_argument('-v', '--verbose', action='store_true')
args = parser.parse_args()
random.seed(args.seed)
set_gpu(args.gpu)
save_path = args.save_path
save_path += '-k={}'.format(args.k)
save_path = osp.join('save', args.trainval, save_path)
ensure_path(save_path)
config_logger(save_path + '/train.log', verbose=args.verbose)
logging.info(args)
graph = json.load(open('materials/imagenet-induced-graph.json', 'r'))
wnids = graph['wnids']
n = len(wnids)
edges = graph['edges']
edges = edges + [(v, u) for (u, v) in edges]
edges = edges + [(u, u) for u in range(n)]
word_vectors = torch.tensor(graph['vectors']).cuda()
word_vectors = F.normalize(word_vectors)
fcfile = json.load(open('materials/fc-weights.json', 'r'))
train_wnids = [x[0] for x in fcfile]
return hit, tot
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--cnn')
parser.add_argument('--pred')
parser.add_argument('--gpu', default='0')
parser.add_argument('--consider-trains', action='store_true')
parser.add_argument('--output', default=None)
args = parser.parse_args()
config_logger(args.pred + '.awa2.log')
set_gpu(args.gpu)
awa2_split = json.load(open('materials/awa2-split.json', 'r'))
train_wnids = awa2_split['train']
test_wnids = awa2_split['test']
logging.info('pred: {}'.format(args.pred))
logging.info('cnn: {}'.format(args.cnn))
logging.info('train: {}, test: {}'.format(len(train_wnids),
len(test_wnids)))
logging.info('consider train classifiers: {}'.format(args.consider_trains))
accs = []
for epoch in range(900, 3100, 300):
#!/usr/bin/env python import logging from utils import add_logger, config_logger config_logger(logging.getLogger())
help='Number of batches between '
'performance reports',
default=100,
type=int)
parser.add_argument('-v',
help='Verbose',
action='store_true',
dest='verbose')
parser.add_argument('--optim',
help='Optimizer algorithm',
default='adagrad',
choices=['adagrad', 'adadelta', 'adam'])
args = parser.parse_args()
utils.config_logger(args.verbose)
logger = utils.get_logger('train')
logger.info('Training with following options: %s' % ' '.join(sys.argv))
#train_pairs = ioutils.read_corpus(args.train, args.lower, args.lang)
#valid_pairs = ioutils.read_corpus(args.validation, args.lower, args.lang)
# whether to generate embeddings for unknown, padding, null
word_dict, embeddings = ioutils.load_embeddings(args.embeddings,
args.vocab,
True,
normalize=True)
# test
#print(word_dict)
print(embeddings)
logger.info('Converting words to indices')
# find out which labels are there in the data
# (more flexible to different datasets)
def main(management, hps):
# Setup Experiment
task = hps.task
config_logger(management.log_file, saving=management.save_logs)
logger = logging.getLogger('Exp')
train_logger = logging.getLogger('Exp.Train')
eval_logger = logging.getLogger('Exp.Eval')
try:
state = State(hps.seed, management)
except SingletonError:
State.instance = None
state = State(hps.seed, management)
logger.info(
f"Initializing experiment `{management.exp_name}` with hyperparameters:\n%s",
repr(hps))
stats = accumulator()
# Setup Data
if task == 'dirac':
def train_data():
dirac = State().convert(torch.Tensor(1, 1).fill_(hps.dirac_target))
while True:
yield dirac
train_iter = train_data()
else:
dataset_cfg = dict(
type=task,
root=management.data_path,
download=True,
preload_to_gpu=management.preload_to_gpu,
num_threads=management.num_workers,
batch_size=hps.batch_size,
)
train_data = Dataset(**dataset_cfg, mode='train')
eval_data = Dataset(**dataset_cfg, mode='test')
train_iter = train_data.sampler(infinite=True, project=0)
# Setup Generator
if task == 'dirac':
generator = dirac.DiracGenerator()
stats.g_params.append(generator.param.clone().detach().cpu())
else:
generator = model.Generator(dimz=hps.generator_dimz,
dimh=hps.generator_dimh,
default_batch_size=hps.batch_size)
test_generator = generator
if hps.generator_alpha_ema is not None:
test_generator = deepcopy(generator)
test_generator.to(device=State().device)
test_generator.train()
generator.to(device=State().device)
generator.train()
generator_optim = optim.init_optimizer(generator.parameters(),
type=hps.optimizer,
lr=hps.generator_lr,
betas=hps.generator_betas,
wd=hps.generator_wd)
logger.info("Generator:\n%s", generator)
# Setup Critic
if task == 'dirac':
critic = dirac.DiracCritic()
stats.c_params.append(critic.param.clone().detach().cpu())
else:
critic = model.Critic(dimh=hps.critic_dimh, sn=hps.critic_use_sn)
critic.to(device=State().device)
critic.train()
critic_optim = optim.init_optimizer(critic.parameters(),
type=hps.optimizer,
lr=hps.critic_lr,
betas=hps.critic_betas,
wd=hps.critic_wd)
logger.info("Critic:\n%s", critic)
# Train
step = 0
train_loss_meter = running_average_meter()
train_step = gan.make_train_step(hps.loss_type,
critic_inner_iters=hps.critic_inner_iters,
reg_type=hps.critic_reg_type,
reg_cf=hps.critic_reg_cf,
alpha_ema=hps.generator_alpha_ema)
if task != 'dirac':
eval_step = gan.make_eval_step(os.path.join(
management.exp_path, task + '_inception_stats.npz'),
eval_data.sampler(infinite=False,
project=0),
hps.generator_dimz,
persisting_Z=100,
device=State().device)
logger.info("Training")
while True:
if step >= hps.max_iters: break
step += 1
train_loss = train_step(train_iter, critic, critic_optim, generator,
test_generator, generator_optim)
train_loss_meter.update(train_loss.clone().detach())
if step % management.log_every == 0 and task != 'dirac':
train_logger.info("step %d | loss(%s) %.3f (%.3f)", step,
hps.loss_type, train_loss_meter.avg.item(),
train_loss_meter.val.item())
if task == 'dirac':
stats.g_params.append(test_generator.param.clone().detach().cpu())
stats.c_params.append(critic.param.clone().detach().cpu())
if step % management.eval_every == 0 and task != 'dirac':
eval_iter = eval_data.sampler(infinite=False, project=0)
samples, results = eval_step(eval_iter, critic, test_generator)
if management.viz:
from IPython.display import clear_output, display, update_display
grid_img = torchvision.utils.make_grid(samples,
nrow=10,
normalize=True,
value_range=(-1., 1.),
padding=0)
plt.imshow(grid_img.permute(1, 2, 0).cpu())
display(plt.gcf())
eval_logger.info(
"step %d | " +
' | '.join([f'{k} {v:.3f}' for k, v in results.items()]), step)
torchvision.utils.save_image(samples.cpu(),
os.path.join(management.log_path,
f'samples-{step}.png'),
nrow=10,
normalize=True,
value_range=(-1., 1.),
padding=0)
logger.info("Final Evaluation")
if task == 'dirac':
g_params = torch.stack(stats.g_params)
c_params = torch.stack(stats.c_params)
trajectory = torch.cat([c_params, g_params], dim=-1).numpy()
logger.info(f"Final point in parameter space: {trajectory[-1]}")
anima = dirac.animate(trajectory, hps)
if management.viz:
from IPython.display import HTML, display
display(HTML(anima.to_html5_video()))
anima.save(os.path.join(management.log_path, 'evolution.mp4'))
else:
eval_iter = eval_data.sampler(infinite=False, project=0)
samples, results = eval_step(eval_iter, critic, test_generator)
logger.info(
"step %d | " +
' | '.join([f'{k} {v:.3f}' for k, v in results.items()]), step)
torchvision.utils.save_image(samples,
os.path.join(management.log_path,
f'samples-final.png'),
nrow=10,
normalize=True,
value_range=(-1., 1.),
padding=0)
