def run_test(cfile):
"""
Runs a configuration file
"""
config = load_config_file(f"../TestConfigs/{cfile}.json", id=True)
# print(config)
dispatch = TrainDispatch()
runconfig = RunConfig(impl=1,
verbose=True,
tboard=False,
best=True,
early=True,
multi=1,
proxy=False,
save=False,
remote=False,
info=True,
log=False)
train_process, architecture = dispatch.dispatch(config['arch']['mode'])
lresults = train_process(architecture, config, runconfig)
print(lresults)
parser.add_argument('--lower',
type=int,
help='Select lower worst',
default=100)
parser.add_argument('--exp',
help='Experiment type',
default="eastwest9597")
parser.add_argument('--mode', help='Architecture mode', default='seq2seq')
parser.add_argument('--suff', help='Datafile suffix', default='12')
parser.add_argument('--testdb',
action='store_true',
default=False,
help='Use test database')
args = parser.parse_args()
config = load_config_file(args.config)
if args.testdb:
mongoconnection = mongolocaltest
client = MongoClient(mongoconnection.server)
db = client[mongoconnection.db]
if mongoconnection.passwd is not None:
db.authenticate(mongoconnection.user, password=mongoconnection.passwd)
col = db[mongoconnection.col]
exps = col.find({'experiment': args.exp, 'arch.mode': args.mode})
lexps = []
for e in exps:
lexps.append((np.sum(np.array(e['result'])[:, 1]), e['site']))
import time
__author__ = 'bejar'
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--testdb',
action='store_true',
default=False,
help='Use test database')
args = parser.parse_args()
lfiles = glob.glob('measure*.json')
lfiles = sorted(lfiles)
if args.testdb:
mongoconnection = mongolocaltest
client = MongoClient(mongoconnection.server)
db = client[mongoconnection.db]
if mongoconnection.user is not None:
db.authenticate(mongoconnection.user, password=mongoconnection.passwd)
col = db[mongoconnection.col]
count = 0
for file in tqdm(lfiles):
config = load_config_file(file, upload=True)
col.insert_one(config)
os.rename(file, f'done_{file}')
count += 1
print(f'{count} Processed {time.ctime()}')
'Equalize the number of experiments for the top npar configurations')
parser.add_argument(
'--nbatches',
type=int,
default=1,
help=
'Number of batches of sites for new configurations or intensification')
parser.add_argument('--ext',
type=int,
default=12,
help='Extension of the datafiles')
args = parser.parse_args()
# Template configuration file
configB = load_config_file(args.config, upload=True)
# Parameters space configuration file (all parameters and values to explore)
configP = load_config_file(args.pconfig, upload=True)
# Variables used from the experiments
arch = list(configP['arch'].keys()) if 'arch' in configP else []
data = list(configP['data'].keys()) if 'data' in configP else []
train = list(configP['train'].keys()) if 'train' in configP else []
# DB stuff
if args.testdb:
mongoconnection = mongolocaltest
client = MongoClient(mongoconnection.server)
db = client[mongoconnection.db]
if mongoconnection.user is not None:
db.authenticate(mongoconnection.user, password=mongoconnection.passwd)
col = db[mongoconnection.col]
:Description: Test
:Authors: bejar
:Version:
:Created on: 09/11/2018 11:52
"""
from Wind.Architectures.RNNS2SArchitecture import RNNS2SArchitecture
from Wind.Misc import load_config_file
from Wind.Train import RunConfig
__author__ = 'bejar'
if __name__ == '__main__':
config = load_config_file("../examples/config_RNN_ED_s2s.json", id=True)
config['idimensions'] = (12, 6)
config['odimensions'] = 12
runconfig = RunConfig(1, 0, False, False, False, False, False, False,
False)
print(config)
net = RNNS2SArchitecture(config, runconfig)
net.generate_model()
net.summary()
__author__ = 'HPAI-BSC'
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--sec', help='Sites Section')
parser.add_argument('--site', help='Initial Site')
parser.add_argument('--config', help='Config file')
parser.add_argument('--irange', type=int, help='Initial example')
parser.add_argument('--erange', type=int, help='Final example')
parser.add_argument('--byinstance', action='store_true', default=False, help='Visualize by instance')
parser.add_argument('--meandev', action='store_true', default=False, help='Visualize mean and deviation')
args = parser.parse_args()
print(f"Config file: /{args.sec}/{args.site}/{args.config}")
config = load_config_file(wind_models_path + f"/{args.sec}/{args.site}/{args.config}", id=True,
abspath=True)
if config['arch']['mode'] == 'regdir':
datamode = False
arch = 'RNNDir'
elif config['arch']['mode'] == 'seq2seq':
datamode = 's2s'
arch = 'RNNS2S'
elif config['arch']['mode'] == 'mlps2s':
datamode = 'mlp'
arch = 'MLPRegS2S'
ahead = config['data']['ahead']
lag = config['data']['lag']
train_x, train_y, val_x, val_y, test_x, test_y = generate_dataset(config['data'], ahead=ahead, mode=datamode,
args = parser.parse_args()
if not args.gpulog:
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
if args.dev is not None:
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
# The GPU id to use
os.environ["CUDA_VISIBLE_DEVICES"] = args.dev
verbose = 1 if args.verbose else 0
impl = 2 if args.gpu else 1
if args.config is None:
config = getconfig(proxy=args.proxy, mode=args.exp, secpat=args.secpat)
elif args.mino:
config = load_config_file(args.config, id=False, mino=True)
else:
config = load_config_file(args.config, id=True)
run_config = RunConfig(impl, verbose, args.tboard, args.best, args.early,
args.multi, args.proxy, args.save, args.remote,
args.info, args.log)
dispatch = TrainDispatch()
if config is not None:
############################################
# Data
if not 'site' in config:
