def get_config(node_id, config, env):
init_nexus(config)
logging.info("Fetching the nodes list...")
if node_id not in nexus.list_node_ids():
logging.error("Node ID not found.")
return 1
logging.info("Node found, fetching the classes info...")
classes_log = nexus.private.fetch_system_logs("classes", node_id=node_id)
if not classes_log:
logging.error("Missing classes data.")
return 1
remote_node_id = classes_log.split(',')[0]
if not remote_node_id == node_id:
logging.error("Mismatching node_id?!")
return 1
classes = classes_log.split(',')[1:]
logging.info("Generating config...")
CONFIG_DIR = os.path.join(config['paths']['config_repo'], 'endpoint')
node_config = c.make_config(node_id, classes, config_dir=CONFIG_DIR)
print json.dumps(node_config, indent=4)
return 0
def main():
args = parse_arg()
configs = [make_config(k,
embd=args.embd,
tsne=args.tsne,
small=args.small,
shuffled_class=args.shuffled_class) for k in args.datasets]
if args.make_graph:
visualize(configs[0], k_nearest=args.k_nearest, M_sample=args.M)
valss = [test_job(config=k, k_nearest=args.k_nearest, M_sample=args.M) for k in tqdm(configs)]
plot_with_err(*valss)
def get_config(self, token):
parts = token.split(',')
if len(parts) < 2: abort(400)
UUID = parts[0]
sig = parts[-1].encode()
classes = parts[1:-1]
if not check_sig(','.join(parts[:-1]), sig): abort(403)
result = config.make_config(UUID, classes)
result = json.dumps(result, indent=4)
return Response(result, mimetype='application/json')
def create_configuration(tests):
""" Creates and saves configuration file based on chosen system calls.
@param tests: List of system calls to be executed during testing.
"""
# create configuration file and save it
configuration = config.make_config(tests)
if not os.path.exists(commons.TESTING_PATH + '/restricted'):
os.makedirs(commons.TESTING_PATH + '/restricted')
# TODO Check if you can write into it
print('Creating configuration')
config_file = open(commons.TESTING_PATH + '/medusa.conf', 'w')
config_file.write(configuration)
config_file.close()
# create constable configuration file that refers to medusa.conf
config_file = open(commons.TESTING_PATH + '/constable.conf', 'w')
config_file.write(config.constable_config)
config_file.close()
def create_configuration(tests):
""" Creates and saves configuration file based on chosen system calls.
@param tests: List of system calls to be executed during testing.
"""
# create configuration file and save it
configuration = config.make_config(tests)
if not os.path.exists(commons.TESTING_PATH + '/restricted'):
os.makedirs(commons.TESTING_PATH + '/restricted')
# TODO Check if you can write into it
print('Creating configuration')
config_file = open(commons.TESTING_PATH + '/medusa.conf', 'w')
config_file.write(configuration)
config_file.close()
# create constable configuration file that refers to medusa.conf
config_file = open(commons.TESTING_PATH + '/constable.conf', 'w')
config_file.write(config.constable_config)
config_file.close()
def fetch_data(node_id, c):
nexus.private.set_node_id(node_id)
if node_id not in list_node_ids():
return NodeData(node_id=node_id, error="Node ID not found.")
classes_log = nexus.private.fetch_system_logs("classes")
if not classes_log:
return NodeData(node_id=node_id, error="Missing classes data.")
remote_node_id = classes_log.split(',')[0]
if not remote_node_id == node_id:
return NodeData(node_id=node_id, error="Mismatching node_id?!")
classes = classes_log.split(',')[1:]
node_config = config.make_config(node_id, classes)
timestamp = nexus.get_timestamp()
if timestamp is None:
return NodeData(node_id=node_id, error="Missing timesync data.")
timestamp = arrow.get(timestamp)
iwconfig_log = nexus.private.fetch_system_logs("iwconfig")
if not iwconfig_log:
return NodeData(node_id=node_id, error="Missing iwconfig data.")
wifi_quality = iwconfig_log.split(',')[1]
if wifi_quality == 'N/A': wifi_quality = None
else: wifi_quality = int(wifi_quality)
versions_log = nexus.private.fetch_system_logs("versions")
if not versions_log:
return NodeData(node_id=node_id, error="Missing versions data.")
versions = versions_log.split(',')
versions = dict(zip((
"timestamp",
"ansible",
"piloteur-code",
"piloteur-blobs",
), versions))
return NodeData(node_id=node_id,
classes=classes,
config=node_config,
timestamp=timestamp,
versions=versions,
wifi_quality=wifi_quality)
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate
from config import make_config
from flask_cors import CORS
from flask_babel import Babel
import os
import eventlet
eventlet.monkey_patch()
try:
os.remove("storage.db")
except Exception:
pass
app = Flask(__name__)
make_config(app)
db = SQLAlchemy(app)
migrate = Migrate(app, db)
socketio = SocketIO(app, async_mode='eventlet')
babel = Babel(app)
CORS(app)
from .util import (
body_message,
config_broker,
connection_broker,
construct_scenario,
)
from .controller import main_controller
from .model import baby_monitor, smartphone, smart_tv
from .solution import observer_model
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
#
""" This script evaluates full F2F models on the val/test sets."""
import cv2
import os
import sys
import pprint
import torch
from mypython.logger import create_logger
#-------------------------------------------------------------------------------
from config import make_config
opt, configs, _ = make_config()
assert opt['train_single_level'] is None
assert opt['n_target_frames'] == 1 # >1 means batch prediction of >1 time steps
assert opt['n_target_frames_ar'] >= 1
#-------------------------------------------------------------------------------
# Start logging
logger = create_logger(os.path.join(opt['logs'], 'eval.log'))
logger.info('============ Initialized logger ============')
logger.info(pprint.pformat(opt))
#-------------------------------------------------------------------------------
# Create dataloader
valsetConfig = configs['valset']
from data_multiscale import load_cityscapes_val
logger.info(pprint.pprint(valsetConfig))
valsetConfig['loaded_model'] = None
"""
@author: cdtang
@file: preprocess.py
@time: 19-11-11 下午10:22
"""
import numpy as np
import json
from collections import defaultdict
import pickle
from config import make_config
from random import choice
import time
config = make_config()
def clear_entity(text):
# ban = {
# '，': ',',
# '·': '•',
# '：': ':',
# '！': '!',
# }
# for i in ban:
# if i in text:
# text = text.replace(i, ban[i])
return text.lower()
# This file is for ensemble and k-best prediction
from model_task import SRL_LSTM
from config import make_config
from io_utils import *
import argparse
import numpy as np
import random
import json
parser = argparse.ArgumentParser()
parser.add_argument('model_dir', type=str, help='Current model tag to save')
args = parser.parse_args()
_config = make_config(os.path.join(args.model_dir, "run.json"))
_config.load_all_data_file()
dp_list = [_config.dev_dp] + _config.test_dp
pred_dp_list = [_config.dev_pred_dp] + _config.test_pred_dp
parse_list = [_config.dev_parse] + _config.test_parse
pred_parse_list = [_config.dev_pred_parse] + _config.test_pred_parse
srl_list = [_config.dev_srl] + _config.test_srl
gold_list = [_config.dev_eval] + _config.test_eval
is_dev = True
is_debug = False
use_nbest = 1
model = SRL_LSTM(_config, use_nbest)
model.build_graph()
performance_dict = {'pred': {}, 'gold': {}}
with tf.Session() as sess:
import time
from train import train_model, save_model
from dataset import make_dataset, save_dataset, load_dataset
from config import make_config
import sys
config = make_config("""{
"target":"example",
"activation":"relu",
"solver":"adam",
"limit": 1000
}""")
try:
dataset = load_dataset(config)
except:
dataset = make_dataset(config)
save_dataset(config, dataset)
print("loaded dataset")
def features():
start_time = time.time()
make_dataset(config)
end_time = time.time()
return end_time - start_time
def train(dataset):
X, Y = dataset
# obs_space need to be flattened before passed to PPOTFPolicy
flat_obs_space = self._prep.observation_space
self._policy_mapping[name] = PolicyWrapper(
LoadPolicy,
params=(flat_obs_space, self._action_space, {}))
self._policy_mapping[name].set_preprocessor(self._prep)
weights = state[name]
self._policy_mapping[name].set_weights(weights)
def policies(self):
return self._policy_mapping
config = make_config(
use_stacked_observation=True,
use_rgb=False,
action_type=1,
)
load_path = "checkpoint_1/checkpoint-1"
# load saved model
# NOTE: the saved model includes two agent policy model with name AGENT-0 and AGENT-1 respectively
policy_handler = RLlibTFCheckpointPolicy(
Path(__file__).parent / load_path,
"DQN",
[f"AGENT-{i}" for i in range(2)],
config.spec["obs"],
config.spec["act"],
)
# Agent specs in your submission must be correlated to each scenario type, in other words, one agent spec for one scenario type.
# LICENSE file in the root directory of this source tree.
#
"""This script is useful to assemble F2Fi models into a F2F model.
NB: assumes multiscale architecture_f2fi and parallel architecture,
etc. see modelsConfig below"""
import os
import torch
import sys
from mypython.logger import create_logger
from torch.nn.parameter import Parameter
from mytorch.implementation_utils import get_nb_parameters
#-------------------------------------------------------------------------------
from config import make_config
opt, configs, checkpoint = make_config()
assert opt['architecture'] == 'parallel'
assert opt['architecture_f2fi'] == 'multiscale'
#-------------------------------------------------------------------------------
# Start logging
logger = create_logger(os.path.join(opt['logs'], 'assemble_models.log'))
logger.info('============ Initialized logger ============')
f2f_paths = {
'f2f5': os.environ['F2F5_WEIGHTS'],
'f2f4': os.environ['F2F4_WEIGHTS'],
'f2f3': os.environ['F2F3_WEIGHTS'],
'f2f2': os.environ['F2F2_WEIGHTS']
}
else:
config.seed = seed
config.num_actors = num_actors
config.lr_init = lr_init
config.discount = discount
config.batch_size = batch_size
config.num_simulations = num_simulations
config.num_unroll_steps = num_unroll_steps
config.window_size = window_size
config.window_step = window_step
config.td_steps = td_steps
date = set_tags(
meta_config, config)
yield config, date
if __name__ == '__main__':
meta_config = make_config()
if meta_config.load_state:
tz = pytz.timezone(meta_config.time_zone)
date = datetime.datetime.now(tz=tz).strftime("%d-%b-%Y_%H-%M-%S")
state = torch.load(meta_config.load_state, map_location='cpu')
launch(state['config'], date, state=state)
else:
for config, date in config_generator(meta_config):
launch(config, date)
