def get_visdom(visdom_path=None, **conf_updates):
visdom_conf = dict(raise_exceptions=True)
if visdom_path is not None:
visdom_conf.update(load_conf(visdom_path))
visdom_conf.update(conf_updates)
return Visdom(**visdom_conf)
def get_mongo_obs(mongo_conf=None, mongo_path=None):
if mongo_conf is None:
mongo_conf = load_conf(mongo_path)
db_url = get_mongo_connection_url(mongo_conf)
return MongoObserver.create(url=db_url,
db_name=mongo_conf['db'],
collection=mongo_conf['collection'])
def get_mongo_collection(mongo_conf=None, mongo_path=None, collection=None):
if mongo_conf is None:
mongo_conf = load_conf(mongo_path)
db = get_mongo_db(mongo_conf)
if collection is None:
collection = mongo_conf['collection']
return db[collection]
def set_observers(experiment):
if os.path.isfile(VISDOM_CONF_PATH):
visdom_conf = load_conf(VISDOM_CONF_PATH)
experiment.observers.append(LogObserver.create(visdom_conf))
if os.path.isfile(MONGO_CONF_PATH):
experiment.observers.append(get_mongo_obs(mongo_path=MONGO_CONF_PATH))
else:
experiment.observers.append(
FileStorageObserver.create(LOCAL_SAVE_PATH))
def get_mongo_connection_url(mongo_conf=None, mongo_path=None):
if mongo_conf is None:
mongo_conf = load_conf(mongo_path)
if 'user' in mongo_conf and 'passwd' in mongo_conf:
db_user = '******'.format(mongo_conf['user'], mongo_conf['passwd'])
else:
db_user = None
db_host = '{}:{}'.format(mongo_conf['host'], mongo_conf['port'])
auth_db = mongo_conf.get('auth_db', mongo_conf['db'])
return f'mongodb://{db_user}@{db_host}/{auth_db}' if db_user else f'mongodb://{db_host}'
def __init__(self, task_gen, ll_models, cuda, n_it_max, n_tasks, patience,
grace_period, num_hp_samplings, visdom_traces_folder,
batch_sizes, plot_tasks, log_steps, log_epoch, name,
task_save_folder, use_ray, use_ray_logging, redis_address,
use_threads, local_mode, smoke_test, sacred_run, log_dir,
norm_models, resources, seed):
self.task_gen = task_gen
self.sims = None
self.sims_comp = None
self.name = name
self.smoke_test = smoke_test
assert isinstance(ll_models, dict)
if 'finetune-mt-head' in ll_models:
assert 'multitask-head' in ll_models and \
isinstance(ll_models['multitask-head'],
MultitaskHeadLLModel), \
'Fine tune should be associated with multitak LLModel'
ll_models['finetune-mt-head'].set_source_model(
ll_models['multitask-head'])
if 'finetune-mt-leg' in ll_models:
assert 'multitask-leg' in ll_models and \
isinstance(ll_models['multitask-leg'], MultitaskLegLLModel), \
'Fine tune leg should be associated with multitak Leg LLModel'
ll_models['finetune-mt-leg'].set_source_model(
ll_models['multitask-leg'])
self.ll_models = ll_models
self.norm_models = norm_models
keys = list(self.ll_models.keys())
self.norm_models_idx = [keys.index(nm) for nm in self.norm_models]
if cuda and torch.cuda.is_available():
self.device = 'cuda'
else:
self.device = 'cpu'
self.use_ray = use_ray
self.redis_address = redis_address
self.use_threads = use_threads
self.local_mode = local_mode
self.use_ray_logging = use_ray_logging
self.n_it_max = n_it_max
self.n_tasks = n_tasks
self.patience = patience
self.grace_period = grace_period
self.num_hp_samplings = num_hp_samplings
self.resources = resources
self.plot_tasks = plot_tasks
self.batch_sizes = batch_sizes
if os.path.isfile(VISDOM_CONF_PATH):
self.visdom_conf = load_conf(VISDOM_CONF_PATH)
else:
self.visdom_conf = None
self.log_steps = log_steps
self.log_epoch = log_epoch
self.sacred_run = sacred_run
self.seed = seed
self.exp_name = get_env_name(sacred_run.config, sacred_run._id)
self.exp_dir = os.path.join(log_dir, self.exp_name)
init_model_path = os.path.join(self.exp_dir, 'model_initializations')
if not os.path.isdir(init_model_path):
os.makedirs(init_model_path)
self.visdom_traces_folder = os.path.join(visdom_traces_folder,
self.exp_name)
self.task_save_folder = os.path.join(task_save_folder, self.exp_name)
main_env = get_env_name(sacred_run.config, sacred_run._id, main=True)
trace_file = os.path.join(self.visdom_traces_folder, main_env)
self.main_viz = visdom.Visdom(env=main_env,
log_to_filename=trace_file,
**self.visdom_conf)
task_env = '{}_tasks'.format(self.exp_name)
trace_file = '{}/{}'.format(self.visdom_traces_folder,
task_env)
self.task_env = visdom.Visdom(env=task_env,
log_to_filename=trace_file,
**self.visdom_conf)
self.summary = {'model': list(self.ll_models.keys()),
'speed': [],
'accuracy': []}
update_summary(self.summary, self.main_viz)
self.sacred_run.info['transfers'] = defaultdict(dict)
self.task_envs_str = defaultdict(list)
self.plot_labels = defaultdict()
self.plot_labels.default_factory = self.plot_labels.__len__
self.tune_register_lock = Lock()
self.eval_lock = Lock()
# Init metrics
self.metrics = defaultdict(lambda: [[] for _ in self.ll_models])
self.training_times_it = [[] for _ in self.ll_models]
self.training_times_s = [[] for _ in self.ll_models]
self.all_perfs = [[] for _ in self.ll_models]
self.all_perfs_normalized = [[] for _ in self.ll_models]
self.ideal_potentials = [[] for _ in self.ll_models]
self.current_potentials = [[] for _ in self.ll_models]
from src.edge_device import edge_devices
from src.learners import mp
from src.ml import get_dataset
from src.ml import initialize_models
from src.network import random_graph, network_graph
from src.plots import plot_train_history
from src.utils import exp_details, load_conf, fixed_seed, save
if __name__ == '__main__':
# load experiment configuration from CLI arguments
args = load_conf()
# =================================
args.mp = 0
args.epochs = 10
args.iid = 1
args.unequal = 0
args.num_users = 100
args.rounds = 400
# =================================
fixed_seed(False)
# print experiment details
exp_details(args)
# load dataset and initialize user groups
train_ds, test_ds, user_groups = get_dataset(args)
# build users models
models = initialize_models(args, same=True)
# set up the network topology
topology = random_graph(models, rho=0.3)
# include physical edge devices (count < 1 to only use simulated nodes)
edge = edge_devices(args, count=-1)
# build the network graph
def __init__(self, task_gen, ll_models, cuda, n_it_max, n_ep_max,
augment_data, normalize, single_pass, n_tasks, patience,
grace_period, num_hp_samplings, visdom_traces_folder,
plot_all, batch_sizes, plot_tasks, lca, log_steps, log_epoch,
name, task_save_folder, load_tasks_from, use_ray,
use_ray_logging, redis_address, use_processes, local_mode,
smoke_test, stream_setting, sacred_run, log_dir, norm_models,
val_per_task, schedule_mode, split_optims, ref_params_id,
seed):
self.task_gen = task_gen
self.sims = None
self.sims_comp = None
self.name = name
assert isinstance(ll_models, dict)
self.ll_models = ll_models
self.learner_names = list(self.ll_models.keys())
self.norm_models = norm_models
keys = list(self.ll_models.keys())
self.norm_models_idx = [keys.index(nm) for nm in self.norm_models]
if cuda and torch.cuda.is_available():
self.device = 'cuda'
else:
self.device = 'cpu'
self.use_ray = use_ray
self.redis_address = redis_address
self.use_processes = use_processes
self.local_mode = local_mode
self.use_ray_logging = use_ray_logging
self.single_pass = single_pass
self.n_it_max = n_it_max
self.n_ep_max = n_ep_max
self.augment_data = augment_data
self.normalize = normalize
self.schedule_mode = schedule_mode
self.n_tasks = n_tasks
self.patience = patience
self.grace_period = grace_period
self.num_hp_samplings = num_hp_samplings
self.stream_setting = stream_setting
self.val_per_task = val_per_task
self.split_optims = split_optims
self.plot_tasks = plot_tasks
self.batch_sizes = batch_sizes
if os.path.isfile(VISDOM_CONF_PATH):
self.visdom_conf = load_conf(VISDOM_CONF_PATH)
else:
self.visdom_conf = None
self.lca = lca
self.log_steps = log_steps
self.log_epoch = log_epoch
self.sacred_run = sacred_run
self.seed = seed
self.exp_name = get_env_name(sacred_run.config, sacred_run._id)
self.exp_dir = os.path.join(log_dir, self.exp_name)
self.init_model_path = os.path.join(self.exp_dir,
'model_initializations')
if not os.path.isdir(self.init_model_path):
os.makedirs(self.init_model_path)
if ref_params_id is None:
self.ref_params_path = None
else:
assert isinstance(ref_params_id, int)
self.ref_params_path = os.path.join(log_dir, str(ref_params_id),
'model_initializations',
'ref.pth')
self.visdom_traces_folder = os.path.join(visdom_traces_folder,
self.exp_name)
self.load_tasks = load_tasks_from is not None
if self.load_tasks:
self.data_path = os.path.join(task_save_folder,
str(load_tasks_from))
assert os.path.isdir(self.data_path), \
'{} doesn\'t exists'.format(self.data_path)
else:
self.data_path = os.path.join(task_save_folder, self.exp_name)
main_env = get_env_name(sacred_run.config, sacred_run._id, main=True)
trace_file = os.path.join(self.visdom_traces_folder, main_env)
self.main_viz_params = {
'env': main_env,
'log_to_filename': trace_file,
**self.visdom_conf
}
self.main_viz = visdom.Visdom(**self.main_viz_params)
task_env = '{}_tasks'.format(self.exp_name)
trace_file = '{}/{}'.format(self.visdom_traces_folder, task_env)
self.task_env = visdom.Visdom(env=task_env,
log_to_filename=trace_file,
**self.visdom_conf)
self.plot_all = plot_all
self.summary = {
'model': list(self.ll_models.keys()),
'speed': [float('nan')] * len(self.ll_models),
'accuracy_t': [float('nan')] * len(self.ll_models),
'accuracy_now': [float('nan')] * len(self.ll_models)
}
update_summary(self.summary, self.main_viz)
self.param_summary = defaultdict(list)
self.param_summary['Task id'] = list(range(self.n_tasks))
self.sacred_run.info['transfers'] = defaultdict(dict)
self.task_envs_str = defaultdict(list)
self.best_task_envs_str = defaultdict(list)
# List of dicts. Each dict contains the parameters of a Visdom env for
# the corresponding task per learner. In the current version this envs
# are never used directly but modified for each training to contain
# the actual parameters used.
self.training_envs = []
self.task_envs = []
self.plot_labels = defaultdict()
self.plot_labels.default_factory = self.plot_labels.__len__
self.tune_register_lock = threading.Lock()
self.eval_lock = threading.Lock()
# Init metrics
self.metrics = defaultdict(lambda: [[] for _ in self.ll_models])
self.training_times_it = [[] for _ in self.ll_models]
self.training_times_s = [[] for _ in self.ll_models]
self.all_perfs = [[] for _ in self.ll_models]
self.all_perfs_normalized = [[] for _ in self.ll_models]
self.ideal_potentials = [[] for _ in self.ll_models]
self.current_potentials = [[] for _ in self.ll_models]
self.n_params = [[] for _ in self.ll_models]
perc=split_c["dev_perc"])
if __name__ == '__main__':
parser = ArgumentParser()
parser.add_argument(
"-c",
"--conf_name",
type=str,
help="path to configuration file for slicing",
)
logging.getLogger().setLevel(logging.INFO)
parsed_opts = parser.parse_args()
config_values = load_conf(parsed_opts.conf_name)
data_c = config_values["data"]
valid_c = config_values["validity_values"]
split_c = config_values["spliting_values"]
base_dir = Path(data_c["data_path"])
basin_path = base_dir / "vector_data/basin/Dudh_Koshi_Glacier.shp"
countries = data_c["country"]
years = data_c["year"]
if not years:
years = [
year_path.name
for year_path in Path(base_dir, 'img_data').iterdir()
def get_gridfs(mongo_conf=None, mongo_path=None):
if mongo_conf is None:
mongo_conf = load_conf(mongo_path)
return gridfs.GridFS(get_mongo_db(mongo_conf))
def get_mongo_db(mongo_conf=None, mongo_path=None):
if mongo_conf is None:
mongo_conf = load_conf(mongo_path)
connection_url = get_mongo_connection_url(mongo_conf)
return MongoClient(connection_url)[mongo_conf['db']]