def check_data_integrity(bot):
'''Checks and fixes the file integrity of Data.\n
A.K.A Replaces files which are currupt, missing, ETC.\n
Also replaces values that have been set incorrectly.'''
for server in bot.servers:
server_path = "Data/" + server.id
#This is for the main server folder.
if not os.path.exists(server_path):
os.makedirs(server_path)
#And this is for the json files contained within.
check_json(server_path + "/RolesConfig.json", {})
check_json(server_path + "/CommandConfig.json", {})
check_json(server_path + "/EventConfig.json", {})
check_json(server_path + "/FunctionConfig.json", {})
check_json(server_path + "/UserConfig.json", {})
if check_json(server_path + "/ServerConfig.json", server_config):
set_default_config_values(server)
update_config_keys(server)
replace_invalid_values(server)
remove_absent_servers(bot)
for member in server.members:
if member.server.get_channel(configs.get_config(configs.server_config, server.id)["MainChannel"]).permissions_for(member).administrator:
if member.id not in configs.get_config(configs.user_config, server.id):
configs.set_config(server.id, "Users", member.id)
configs.set_config(server.id, "Users", member.id + " / GodMode / true")
def __call__(self, cls):
# establish clsfile, pkg_root for cls:
clsfile = inspect.getfile(cls)
mod = inspect.getmodule(cls)
pkg = mod.__name__.split('.')[0]
pkg_root = os.path.abspath(pr.resource_filename(pkg, '.'))
# try to get default values from default config and section:
defaults = {
'__class_file': clsfile,
'__module': cls.__module__,
'__pkg_root': pkg_root,
}
if self.defaults_fn is not None and self.def_sections is not None:
if not os.path.isabs(self.defaults_fn):
# locate defaults_fn in same directory as class file
self.defaults_fn = os.path.join(os.path.dirname(clsfile),
self.defaults_fn)
else:
# locate defaults_fn in package root, unless it already points to a real file:
if not os.path.exists(self.defaults_fn):
self.defaults_fn = os.path.join(pkg_root,
self.defaults_fn[1:])
def_config = get_config(self.defaults_fn)
for dsect in self.def_sections:
defaults.update(to_dict(def_config, dsect))
# get class config and inject values:
config_fn = os.path.join(os.path.dirname(clsfile),
replace_ext(os.path.basename(clsfile), 'ini'))
cls_config = get_config(config_fn, defaults=defaults)
load_attributes_from_config(cls, cls_config, cls.__name__)
return cls
def main():
config = get_config(mode='train')
val_config = get_config(mode='valid')
with open(os.path.join(config.save_path, 'config.json'), 'w') as json_f:
config.to_json(json_f)
raw_data = load_json(config.all_path)
train_data_loader = get_loader(raw_data=raw_data,
max_len=config.max_len,
batch_size=config.batch_size,
shuffle=True,
user_map_dict=config.user_map_dict,
max_users=config.max_users)
raw_data = load_json(val_config.all_path)
eval_data_loader = get_loader(raw_data=raw_data,
max_len=val_config.max_len,
batch_size=val_config.eval_batch_size,
shuffle=False,
user_map_dict=config.user_map_dict,
max_users=config.max_users)
model_solver = getattr(solvers, "Solver{}".format(config.model))
solver = model_solver(config,
train_data_loader,
eval_data_loader,
is_train=True)
solver.build()
solver.train()
solver.writer.close()
return config
def test_register_and_retrieve_config(self):
form_builder = self.instance.get_form_builder()
lazydictionary_post = get_config('test')
form = form_builder({'setting1':'wooot', 'setting2':'2', 'site':'1'}, {})
self.assertTrue(form.is_valid(), form.errors)
form.save()
self.assertNotEqual(0, len(get_config('test').items()))
self.assertNotEqual(0, len(lazydictionary_post.items()))
def get_dataset(dataset_name, params):
if dataset_name == "quickdraw":
ds_config = get_config(dataset_name)().parse(
"split={}".format("c300_msl65_scaled_48"))
dataset_proto = get_dataset_sketch(dataset_name)(SKETCH_DATA_DIR,
ds_config)
dataset = dataset_proto.load(repeat=True)[0]
dataset = dataset.map(lambda im, lab: tf.py_func(
lambda x, y: (x[0], quickdraw_lookup[y[0].decode('utf-8').rstrip(
'\x00')]), (im, lab), (tf.float32, tf.int64)))
dataset = dataset.map(lambda im, lab:
(im, tf.one_hot(lab, len(quickdraw_classes))))
train, height, width, colors = input_fn_dataset(
dataset, params['batch_size'])
return DataSet(dataset_name, train, None, None, height, width, colors,
len(quickdraw_classes))
elif dataset_name == "fs_omniglot":
ds_config = get_config(params['fso_config'])().parse("mode=batch")
dataset_proto = get_dataset_sketch(dataset_name)(SKETCH_DATA_DIR,
ds_config)
(dataset, _), class_list = dataset_proto.load(repeat=True)
fso_lookup = {
class_string: i
for i, class_string in enumerate(class_list)
}
dataset = dataset.map(lambda im, lab: tf.py_func(
lambda x, y: (x[0], fso_lookup[y[0].decode('utf-8').rstrip('\x00')]
), (im, lab), (tf.float32, tf.int64)))
dataset = dataset.map(lambda im, lab:
(im, tf.one_hot(lab, len(class_list))))
train, height, width, colors = input_fn_dataset(
dataset, params['batch_size'])
return DataSet(dataset_name, train, None, None, height, width, colors,
len(class_list))
else:
train, height, width, colors = _DATASETS[dataset_name + '_train'](
batch_size=params['batch_size'])
test = _DATASETS[dataset_name + '_test'](batch_size=1)[0]
train = train.map(lambda v: dict(
x=v['x'], label=tf.one_hot(v['label'], _NCLASS[dataset_name])))
test = test.map(lambda v: dict(
x=v['x'], label=tf.one_hot(v['label'], _NCLASS[dataset_name])))
if dataset_name + '_train_once' in _DATASETS:
train_once = _DATASETS[dataset_name +
'_train_once'](batch_size=1)[0]
train_once = train_once.map(lambda v: dict(
x=v['x'], label=tf.one_hot(v['label'], _NCLASS[dataset_name])))
else:
train_once = None
return DataSet(dataset_name, train, test, train_once, height, width,
colors, _NCLASS[dataset_name])
def test_is_not_encrypted(self):
form_builder = self.complex_instance.get_form_builder()
lazydictionary_post = get_config('testcomplex')
test_user = User.objects.get_or_create(username='******')[0]
form = form_builder({'amount': '5.00', 'user': test_user.pk, 'site': '1'}, {})
self.assertTrue(form.is_valid(), form.errors)
form.save()
self.assertTrue(form.is_valid(), form.errors)
self.assertNotEqual(0, len(get_config('testcomplex').items()))
self.assertNotEqual(0, len(lazydictionary_post.items()))
conf = Configuration.objects.get(key='testcomplex')
self.assertTrue('5.00' in conf.data)
def sms_request(mob_number, vcode):
data = urllib.parse.urlencode({
'appid': get_config('appid'),
'to': mob_number,
'content': get_config(key='content') + vcode,
'signature': get_config('app_key')
})
data = data.encode('utf-8')
request = urllib.request.Request(
"https://api.submail.cn/message/send.json", method='POST')
request.add_header("Content-Type",
"application/x-www-form-urlencoded;charset=utf-8")
f = urllib.request.urlopen(request, data)
return f.read().decode('utf-8')
def test_complex_config(self):
form_builder = self.complex_instance.get_form_builder()
lazydictionary_post = get_config('testcomplex')
test_user = User.objects.get_or_create(username='******')[0]
form = form_builder({'amount':'5.00', 'user':test_user.pk, 'site':'1'}, {})
self.assertTrue(form.is_valid(), form.errors)
form.save()
self.assertNotEqual(0, len(get_config('testcomplex').items()))
self.assertNotEqual(0, len(lazydictionary_post.items()))
config = get_config('testcomplex')
self.assertTrue(isinstance(config['amount'], Decimal))
self.assertEqual(Decimal('5.00'), config['amount'])
self.assertTrue(isinstance(config['user'], User))
self.assertEqual(test_user.pk, config['user'].pk)
def test_complex_config(self):
form_builder = self.complex_instance.get_form_builder()
lazydictionary_post = get_config('testcomplex')
test_user = User.objects.get_or_create(username='******')[0]
form = form_builder({'amount':'5.00', 'user':test_user.pk, 'site':'1'}, {})
self.assertTrue(form.is_valid(), form.errors)
form.save()
self.assertNotEqual(0, len(get_config('testcomplex').items()))
self.assertNotEqual(0, len(lazydictionary_post.items()))
nuke_cache()
config = get_config('testcomplex')
self.assertTrue(isinstance(config['amount'], Decimal))
self.assertEqual(Decimal('5.00'), config['amount'])
self.assertTrue(isinstance(config['user'], User))
self.assertEqual(test_user.pk, config['user'].pk)
def main():
config = get_config(mode='test')
with open(os.path.join(config.save_path, 'config.json'), 'r') as json_f:
temp_config_str = json_f.read()
config.max_users = int(
re.findall(r"'max_users': ([0-9]+?),", temp_config_str)[0])
config.max_len = int(
re.findall(r"'max_len': ([0-9]+?),", temp_config_str)[0])
config.rnn_hidden_size = int(
re.findall(r"'rnn_hidden_size': ([0-9]+?),", temp_config_str)[0])
raw_data = load_json(config.all_path)
test_data_loader = get_loader(raw_data=raw_data,
max_len=config.max_len,
batch_size=config.batch_size,
shuffle=False,
user_map_dict=config.user_map_dict,
max_users=config.max_users)
model_solver = getattr(solvers, "Solver{}".format(config.model))
solver = model_solver(config, None, test_data_loader, is_train=False)
solver.build()
solver.test()
return config
def get_racy_default(opt, config,
default=NotUsed, prj=NotUsed, option_value=Undefined):
if config is None:
config = get_option('CONFIG',config=configs.DEFAULT_CONFIG)
defaults_source = configs.get_config(config, raise_on_not_found = False)
if not defaults_source:
defaults_source = configs.get_config(configs.DEFAULT_CONFIG)
res = defaults_source.get(opt)
allowedvalues.check_value_with_msg(opt, res, config, exceptionnal_case)
return res
def get_user_options(opt, config, default, prj=NotUsed, option_value=Undefined):
"""Lookup for a user.options file in RACY_CONFIG_DIR. Get the user
option 'opt' if exists. A user option is overrided by a user config option
"""
import racy.renv as renv
files = [
( renv.dirs.config, 'user.options'),
]
file_opts = get_file_options(None, files)
res = file_opts.get(opt, default)
options_file = rutils.get_first_existing_file(files)
ckconfig = config
if options_file is not None:
ckconfig = ":".join([options_file, config])
allowedvalues.check_value_with_msg(opt, res, ckconfig, exceptionnal_case)
old_res = res
loc = renv.dirs.user_configs
source = configs.get_config(config, path=loc,
include_defaults = False, raise_on_not_found = False)
if source:
res = source.get(opt, res)
if old_res != res:
ckconfig = ":".join([loc,config])
allowedvalues.check_value_with_msg(opt, res, ckconfig,
exceptionnal_case)
return res
def make(self, key):
# Find path with sorted data
config_type = (Experiment() & key).fetch1('config_type')
cfg = configs.get_config(config_type)
sorted_path = os.path.join(data_path, cfg['sorted_path'].format(**key))
if not os.path.isdir(sorted_path):
print('Neural recordings for {session_id} in {experiment_id} are not found'.format(**key))
return
possible_paths = [sorted_path]
possible_paths.extend([os.path.join(sorted_path, path) for path in os.listdir(sorted_path) if os.path.isdir(os.path.join(sorted_path, path))])
print(possible_paths)
sorted_path = [path for path in possible_paths if os.path.isfile(os.path.join(path, 'phy.log'))]
print(sorted_path)
sorted_path = sorted_path[0]
try:
key['spike_times'] = np.load(os.path.join(sorted_path, 'spike_times.npy'))
key['spike_clusters'] = np.load(os.path.join(sorted_path, 'spike_clusters.npy'))
key['spike_templates'] = np.load(os.path.join(sorted_path, 'spike_templates.npy'))
key['amplitudes'] = np.load(os.path.join(sorted_path, 'amplitudes.npy'))
key['channel_positions'] = np.load(os.path.join(sorted_path, 'channel_positions.npy'))
key['channel_map'] = np.load(os.path.join(sorted_path, 'channel_map.npy'))
key['cluster_info'] = pd.read_csv(os.path.join(sorted_path, 'cluster_info.tsv'), sep='\t', header=0, index_col=0).to_dict()
self.insert1(key)
print('Populated sorted recordings for {session_id} in {experiment_id}'.format(**key))
except Exception as e:
print("Error populating sorted recording for {session_id} in {experiment_id}: ".format(**key), e)
def make(self, key):
import h5py
config_type = (Experiment() & key).fetch1('config_type')
cfg = configs.get_config(config_type)
wavesurfer_path = os.path.join(data_path, cfg['wavesurfer_path'].format(**key))
print("Wavesurfer path", wavesurfer_path)
if not os.path.isdir(wavesurfer_path):
print('Ball recordings for {session_id} in {experiment_id} are not found'.format(**key))
return
wavesurfer_files = [f for f in os.listdir(wavesurfer_path) if f.endswith('.h5') and key['subsession_id'] in f] # Assumes subsession_ids are not overlapping!!!
print(wavesurfer_files)
# speedV0 = 16800 # analog signal when the animal is not moving
# sweep = '0001'
with h5py.File(os.path.join(wavesurfer_path, wavesurfer_files[0]), "r") as ws_file:
sweep = [sub for sub in ws_file.keys() if 'sweep' in sub][0]
trigger_traces = np.array(ws_file.get(sweep + '/analogScans'))
# ball_speed = (trigger_traces[2, :]-speedV0)/speedV0 # recorded with 20kHz
key['ball_readout'] = trigger_traces # ball_speed
sync_trace = (EphysRaw() & key).fetch1('sync_trace')
try:
key['pxi_scaling_factor'], key['pxi_offset'] = self._get_pxi_matching(trigger_traces, sync_trace)
self.insert1(key)
except Exception as e:
print(e)
def test_nuke_cache(self):
my_config = get_config('test')
my_config._load()
nuke_cache()
self.assertFalse(hasattr(my_config.data, 'config'))
my_config._load()
self.assertTrue(hasattr(my_config.data, 'config'))
def send_email(data):
NOREPLY_EMAIL = '*****@*****.**'
mailgun_request_data = {
"from":
"%s <%s>" % (data.get('sender_name', 'Team Trotto'),
data.get('reply_to', NOREPLY_EMAIL)),
"to":
data['recipient_email'],
"subject":
data['subject'],
"text":
data['plaintext'],
"html":
data['html']
}
if data.get('html') is not None:
mailgun_request_data['html'] = data['html']
response = requests.post(
"https://api.mailgun.net/v3/mg.trot.to/messages",
auth=("api", configs.get_config()['mailgun']['general_use_api_key']),
data=mailgun_request_data)
response.raise_for_status()
def main():
"""Main function to run model."""
config = get_config(os.environ)
sys.path.append(os.path.join('tasks', config.task_folder))
# pylint: disable=import-error
from trainer import train
from tester import test
# pylint: enable=import-error
if config.is_distributed:
torch.cuda.set_device(config.local_rank)
torch.distributed.init_process_group(backend='nccl',
init_method='env://')
logger = setup_logger(config.work_dir, distributed_rank=get_rank())
logger.info(f'Using {config.num_gpus} GPUs.')
logger.info(f'Collecting environment info:{get_env_info()}')
logger.info(f'------------------------------')
logger.info(f'Running configurations:')
for key, val in config.__dict__.items():
logger.info(f' {key}: {val}')
logger.info(f'------------------------------')
if config.run_mode == 'train':
train(config, logger)
elif config.run_mode == 'test':
test(config, logger)
def main():
seed_everything()
args = get_config()
current_time = str(datetime.datetime.now()).replace(" ", "_")
exp_name = f"{current_time}_{args.model_name}"
print(args,
file=codecs.open(f"{args.log_dir}/{exp_name}.log", "w", "utf-8"))
logger = create_logger(f"{args.log_dir}", exp_name)
device = torch.device(
f"cuda:{args.device}" if torch.cuda.is_available() else "cpu")
pad_token_id = 0
data_dir = args.data_dir
train_ds = IWSLTDataset(f"{data_dir}/train.de",
f"{data_dir}/train.en",
max_length=128)
src_wd2id, tgt_wd2id = train_ds.src_wd2id, train_ds.tgt_wd2id
tgt_id2wd = {j: i for i, j in tgt_wd2id.items()}
val_ds = IWSLTDataset(f"{data_dir}/valid.de",
f"{data_dir}/valid.en",
src_wd2id=src_wd2id,
tgt_wd2id=tgt_wd2id)
collate_fn = CollateFn(0, True)
if args.debug:
train_ds = train_ds[:400]
train_dl = DataLoader(train_ds,
batch_size=args.batch_size,
num_workers=2,
shuffle=args.shuffle_data,
collate_fn=collate_fn)
val_dl = DataLoader(val_ds,
batch_size=args.batch_size,
num_workers=2,
collate_fn=collate_fn,
shuffle=False)
train_step = math.ceil(len(train_ds) / args.batch_size)
src_vocab_size = len(src_wd2id)
tgt_vocab_size = len(tgt_wd2id)
model = get_model(args, src_vocab_size, tgt_vocab_size)
if args.init_param_width != 0:
init_fn = functools.partial(init_uni_weights,
width=args.init_param_width)
model.apply(init_fn)
count_parameters(model)
model = model.to(device)
optimizer = get_optimizer(
args, model) # optim.Adam(model.parameters(), lr=args.lr)
if args.scheduler == "mutli_step":
milestones = [int(train_step * i) for i in range(5, args.n_epoch, 0.5)]
else:
milestones = None
scheduler = get_scheduler(args, optimizer, milestones)
train_fn(train_dl,
val_dl,
model,
optimizer,
scheduler,
device,
logger,
tgt_id2wd,
args=args)
def runserver(config_path=None):
config = get_config(config_path)
uvicorn.run(
app='scheduleapp:app',
reload=config.FAST_API.RELOAD,
port=config.FAST_API.PORT,
host=str(config.FAST_API.HOST),
)
def main():
"""
Main training function
"""
cfg = get_config(is_train=True)
os.environ['CUDA_VISIBLE_DEVICES'] = str(cfg.gpu_num)
print(cfg)
run(cfg)
def test_nuke_cache(self):
my_config = get_config('test')
my_config._load()
nuke_cache()
for key in CONFIGS.keys():
self.assertFalse(hasattr(CONFIG_CACHE, key))
self.assertFalse(my_config.loaded)
my_config._load()
self.assertTrue(my_config.loaded)
def prepare():
dataset_config: HParams = configs.get_config(FLAGS.dataset_cfgset)().parse(
FLAGS.dataset_cfgs)
log_hparams(dataset_config)
logging.info("Getting and preparing dataset: %s", FLAGS.dataset)
dataset = datasets.get_dataset(FLAGS.dataset)(FLAGS.data_dir,
dataset_config)
dataset.prepare(FLAGS)
def test_register_and_retrieve_config(self):
form_builder = self.instance.get_form_builder()
lazydictionary_post = get_config('test')
form = form_builder({'setting1':'wooot', 'setting2':'2', 'site':'1'}, {})
self.assertTrue(form.is_valid(), form.errors)
form.save()
lazydictionary_post._reset()
self.assertNotEqual(0, len(lazydictionary_post.items()))
self.assertNotEqual(0, len(lazydictionary_post.items()))
def experiment():
model_config: HParams = configs.get_config(FLAGS.model_cfgset)().parse(
FLAGS.model_cfgs)
model = models.get_model(FLAGS.model)(FLAGS.dir, FLAGS.id, model_config)
train_dataset_config: HParams = configs.get_config(
FLAGS.train_dataset_cfgset)().parse(FLAGS.train_dataset_cfgs)
train_dataset = datasets.get_dataset(FLAGS.train_dataset)(
FLAGS.data_dir, train_dataset_config)
train_tf_dataset = train_dataset.load(repeat=True)
if FLAGS.eval_dataset:
eval_dataset_config: HParams = configs.get_config(
FLAGS.eval_dataset_cfgset)().parse(FLAGS.eval_dataset_cfgs)
eval_dataset = datasets.get_dataset(FLAGS.eval_dataset)(
FLAGS.data_dir, eval_dataset_config)
eval_tf_dataset = eval_dataset.load(repeat=False)
else:
eval_dataset_config = None
eval_tf_dataset = None
if (not FLAGS.distributed) or (hvd.rank() == 0):
logging.info(
"Creating Model: %s | Loading Train Dataset: %s | Loading Eval Dataset: %s",
FLAGS.model, FLAGS.train_dataset, FLAGS.eval_dataset)
log_hparams(model_config, train_dataset_config, eval_dataset_config)
logging.info("Beginning training loop")
# Debugging NaN errors.
if FLAGS.check_numerics:
tf.debugging.enable_check_numerics()
while True:
try:
model.train(train_tf_dataset, FLAGS.train_steps, FLAGS.print_freq,
FLAGS.save_freq, eval_tf_dataset, FLAGS.eval_freq)
except tf.errors.AbortedError:
logging.info(
"InvalidArgumentError received from training function. Restarting training."
)
continue
else:
break
def replace_invalid_values(server):
"Replaces values in servers configs that are invalid"
#Replacing main_channel if channel does not exist
main_channel = configs.get_config(configs.server_config, server.id)["MainChannel"]
reset = True
for channel in server.channels:
if channel.id == main_channel:
reset = False
if reset:
configs.set_config(server.id, "Server", "MainChannel / " + get_default_config(server)["MainChannel"])
def main(
dataset='mnist',
seed=0,
num_shards=250,
num_slices=1,
):
rng = random.PRNGKey(seed)
X, y, X_test, y_test = get_dataset(dataset)
config = get_config(dataset)
temp, rng = random.split(rng)
X, y = shuffle(temp, X, y)
print('X: {}, y: {}'.format(X.shape, y.shape))
print('X_test: {}, y_test: {}'.format(X_test.shape, y_test.shape))
# X[0<=i<num_shards][0<=j<num_slices] refers to the j'th slice of the i'th shard
X, y = shard_and_slice(num_shards, num_slices, X, y)
init_params, predict = config['clf']
try:
params = pickle.load(open('private_aggregation.pkl', 'rb'))
except:
print('Training full model (Shards={}, Slices={})...'.format(num_shards, num_slices))
# params[0 <= i < num_shards][0 <= j <= num_slices] refers to the params trained on the first j slices of the i'th shard,
# i.e., j == 0 yields randomly initialized params trained on no data, j == 1 yields params trained on the first slice, etc.
params = get_trained_sharded_and_sliced_params(rng, init_params, predict, X, y, train)
pickle.dump(params, open('private_aggregation.pkl', 'wb'))
targets = np.argmax(y_test, axis=1)
predictions = sharded_and_sliced_predict(params, predict, X_test)
nonprivate_accuracy = np.mean(predictions == targets)
print('Accuracy (nonprivate): {:.4}\n'.format(nonprivate_accuracy))
print('Example votes:')
print(get_votes(params, predict, X_test)[:20])
mechanism_names = ['Exp. Mech.', 'LNMax']
mechanisms = [exponential_mechanism, lnmax]
mechanism_accs = []
per_example_epsilons = [0.001, 0.0025, 0.005, 0.0075, 0.01, 0.025, 0.05, 0.075, 0.1]
for mechanism in mechanisms:
epsilon_accs = []
print('Mechanism: {}'.format(mechanism))
for per_example_epsilon in per_example_epsilons:
temp, rng = random.split(rng)
agg = lambda votes: mechanism(rng, votes, per_example_epsilon)
predictions = sharded_and_sliced_predict(params, predict, X_test, agg)
accuracy = np.mean(predictions == targets)
print('Accuracy (eps={:.4}): {:.4}\n'.format(per_example_epsilon, accuracy))
epsilon_accs.append(accuracy)
mechanism_accs.append(epsilon_accs)
def check_permisson(bot, command_name, member):
"Checks if user has permission for command(commandName)."
has_perm = False
user_config = configs.get_config(configs.user_config, member.server.id)
if member.id in user_config:
if user_config[member.id]["GodMode"]:
has_perm = True
#User specific perms
for role in user_config[member.id]["Permissions"]:
if role == command_name:
has_perm = True
#Role perms
for role in member.roles:
if role.name in configs.get_config(configs.role_config, member.server.id):
if command_name in configs.get_config(configs.role_config, member.server.id)[role.name]["Permissions"]:
has_perm = True
return has_perm
def main():
# Init config.
config = get_config()
logger.info(config)
# Get device.
device = get_device(config.use_cpu, gpu_id=config.gpu_id)
# Set maximum number of threads.
torch.set_num_threads(config.cpu_nums)
# Train or test.
if config.action == 'train':
train(config, device, config.ReinforcementOrSupervised)
elif config.action == 'test':
test(config, device, config.ReinforcementOrSupervised)
def main(argv):
"""Create directories and configure python settings"""
# Setup Directory
experiment_dir = os.path.join(FLAGS.dir, FLAGS.id)
if not os.path.exists(experiment_dir):
os.makedirs(os.path.join(experiment_dir, "logs"), exist_ok=True)
# Setup Logging
FLAGS.alsologtostderr = True
logging.get_absl_handler().use_absl_log_file(FLAGS.logfile, os.path.join(experiment_dir, "logs"))
# Setup seeds
if FLAGS.random_seed:
np.random.seed(FLAGS.random_seed)
tf.random.set_seed(FLAGS.random_seed)
# Log Flags
log_flags(FLAGS)
drawer_id = "05-22_quickdraw_sweep_pixel_weight_28/drawer_enc_tadam_huge_interval0.05_step10000_maxweight0.5-quickdraw_ST1_msl64_28"
drawer_config: HParams = configs.get_config("drawer/huge")().parse("")
drawer: DrawerModel = models.get_model("drawer_enc_tadam")(FLAGS.dir, drawer_id, drawer_config, training=False)
dataset_config: HParams = configs.get_config("quickdraw")().parse("split=T2_msl64_28,shuffle=True,batch_size={}".format(16))
dataset_proto = datasets.get_dataset('quickdraw')(FLAGS.data_dir, dataset_config)
dataset = dataset_proto.load(repeat=False)[0]
# clustering_methods = [sklearn.manifold.TSNE(n_components=2), sklearn.decomposition.PCA(n_components=2), umap.UMAP()]
clustering_method = DBSCAN(eps=4.2)
try:
hyper_embed(drawer, dataset, clustering_method, min_samples=6)
except:
exception = traceback.format_exc()
logging.info(exception)
logging.info("Complete")
def seed_database():
app.config.from_object(get_config())
db.init_app(app)
user_datastore = SQLAlchemyUserDatastore(db, User, Role)
security = Security(app, user_datastore)
user_datastore.create_user(email='*****@*****.**', password='******')
user = db.session.query(User).first()
post = Post()
post.user = user
post.title = "Test Title"
post.content = 'This is the post content.'
post.post_image = 'https://source.unsplash.com/random'
post.post_image_alt = 'Unsplashed Image'
db.session.add(post)
db.session.commit()
def evaluate_df(candidate_df: pd.DataFrame) -> pd.DataFrame:
evaluation_df = copy.copy(candidate_df)
evaluation_df = pd.concat(
[evaluation_df,
pd.DataFrame(columns=["score", "key_frame_labels"])])
test_config = get_config(mode='test')
solver = Solver(test_config)
solver.build()
for indx, row in evaluation_df.iterrows():
video_name = row["unique_clip_name"]
frames, score = solver.test(indx, video_name)
evaluation_df["key_frame_labels"][indx] = frames
evaluation_df["score"][indx] = score
return evaluation_df
def mongo_db():
global current_db
if(current_db==None):
client = MongoClient(
configs.get_config().no_sql.host,
configs.get_config().no_sql.port
)
if (configs.get_config().no_sql.user != ""):
client[configs.get_config().no_sql.name].authenticate(configs.get_config().no_sql.user,
configs.get_config().no_sql.password)
current_db = client[configs.get_config().no_sql.name]
return current_db
def make(self, key):
# ap_meta_path: varchar(512)
# lf_path: varchar(512)
# lf_meta_path: varchar(512)
# TODO: add file length as metadata
config_type = (Experiment() & key).fetch1('config_type')
cfg = configs.get_config(config_type)
base_path = cfg['ephys_path'].format(**key)
path = os.path.join(data_path, base_path)
print(path)
if not os.path.isdir(path):
print('Neural recordings for {session_id} in {experiment_id} are not found'.format(**key))
return
subsession_type, subsession_iter = (Subsession() & key).fetch1('type', 'iteration')
ap_files = [f for f in os.listdir(path) if f.endswith('.ap.bin') and
((f.split('_')[1]==subsession_type and f.split('_')[0] == f"{subsession_iter:04d}") or
(f.split('_')[0]==subsession_type and f.split('_')[1] == f"{subsession_iter:04d}"))] # Flip f.split('_') naming depending on Arnau or Ania's data
ap_files.sort()
print(ap_files)
file = ap_files[0]
# starts, lengths = (EphysRaw() & {'experiment_id': key['experiment_id'], 'mouse_id': key['mouse_id'], 'session_id': key['session_id']}).fetch('start', 'length')
# print("Starts, lengths:", starts, lengths)
# if len(starts) == 0:
# start = 0
# else:
# last_idx = np.argmax(starts)
# start = starts[last_idx] + lengths[last_idx]
rel_file_path = os.path.join(base_path, file)
# id, stimulus_type = file.split('_')[:2]
key['ap_path'] = rel_file_path
key['meta_path'] = rel_file_path[:-3] + 'meta'
key['subsession_type'] = subsession_type
key['subsession_iter'] = subsession_iter
sync_trace = neuropixels_utils.extract_sync(Path(os.path.join(path, file)))
key['sync_trace'] = sync_trace
key['length'] = sync_trace.shape[1]
# key['start'] = start
self.insert1(key)
print('Populated neural recordings for {session_id} in {experiment_id}'.format(**key))
def make(self, key):
stimulus_type, iteration = (Subsession() & key).fetch1('stimulus_type', 'iteration')
print("Importing stimulus", stimulus_type, str(iteration))
config_type = (Experiment() & key).fetch1('config_type')
cfg = configs.get_config(config_type)
recs_path = os.path.join(data_path, cfg['facecam_path'].format(**key), stimulus_type)
if not os.path.exists(recs_path):
print('Face recording for {session_id} {experiment_id}, {stimulus_type} {iteration} NOT found!'.format(stimulus_type=stimulus_type, iteration=iteration, **key))
return
runs = [os.path.splitext(f.split('_')[1])[0] for f in os.listdir(recs_path) if f.endswith('.camlog')]
session_date = (Session() & key).fetch1('session_date')
trial_run = sorted(runs)[0] # Iterations start from index 1
print("{session_date:%Y%m%d}_{trial_run}".format(trial_run=trial_run, session_date=session_date))
tiffs = [f for f in os.listdir(recs_path) if
f.startswith("{session_date:%Y%m%d}_{trial_run}".format(trial_run=trial_run, session_date=session_date)) and f.endswith('.tif')]
tiffs = sorted(tiffs)
print(tiffs)
max_merged_tiffs = 15
i = 0
while i*max_merged_tiffs < len(tiffs):
data_tiff = DataTiff(os.path.join(recs_path, tiffs[i*max_merged_tiffs]))
for tiff in tiffs[i*max_merged_tiffs+1:(i+1)*max_merged_tiffs]:
print(f"Loading {tiff}")
temp = DataTiff(os.path.join(recs_path, tiff))
try:
data_tiff.merge_tiff(temp)
except ValueError as err:
print(f"Was not able to load {tiff} with error: {err}")
print("Merged tiffs")
key['part'] = i
key['recording'] = data_tiff.data
n_frames, height, width = data_tiff.data.shape
key['n_frames'] = n_frames
key['width'] = width
key['height'] = height
print(f"Inserting part {i} for {stimulus_type}, {iteration}")
self.insert1(key)
i += 1
print('Populated a face recording for {session_id} in {experiment_id}'.format(**key))
def make(self, key):
config_type = (Experiment() & key).fetch1('config_type')
cfg = configs.get_config(config_type)
base_path = os.path.join(data_path, cfg['wavesurfer_path'].format(
**key)) # could also be experiment_id/neuropixels/session_id
subsession_files = [
os.path.splitext(f)[0] for f in os.listdir(base_path)
if f.endswith('.h5')
]
for subsession in subsession_files:
key['subsession_id'] = subsession
iteration, type = subsession.split('_')[:2]
try:
key['type'] = type
key['iteration'] = int(iteration)
except:
key['type'] = iteration
key['iteration'] = int(type)
self.insert1(key)
def run_cli(user, gpu, config, checkpoint=None):
if checkpoint:
assert os.path.isfile(checkpoint), \
f"Checkpoint doesn't exist ({checkpoint})"
""" Get config + env setup. """
cfg = configs.get_config(config)
# experiment setup
_set_seed(cfg['experiment']['seed'])
if user == 'yzhang46' and gpu >= 0:
device = f'cuda:{int(gpu)}' if torch.cuda.is_available() else 'cpu'
else:
device = f'cuda:0' if torch.cuda.is_available() else 'cpu'
cfg['experiment']['device'] = device
print(f" > Using device: {device}.")
experiment = experiments.get_module(cfg['experiment']['name'])
experiment.run(cfg, checkpoint=checkpoint)
#
"""
Module for communicating with the servo controller.
"""
# Imports #
# Paths
import sys
sys.path.append("../Libraries")
# Configs
import configs
import configobj
global config, enable
config = configs.get_config("Servo Controller")
enabled = config["enabled"]
mc_name = config["micro_controller"]
# Logging
import logging
log = logging.getLogger(config["logger_name"])
# micro_controller_network
import micro_controller_network
mc = micro_controller_network.get_object(mc_name)
# Static Functions #
def init():
global config, initialized, servos
servos = {}
def test_nuke_cache(self):
get_config('test').items()
nuke_cache()
for key in CONFIGS.keys():
self.assertFalse(hasattr(CONFIG_CACHE, key))
#
# laser.py
#
#
# Created by William Woodall on 2/24/09.
# Copyright (c) 2009 Auburn University. All rights reserved.
#
'''
Module for controlling the Hokuyo URG-04LX Laser Range Finder.
'''
# Imports #
# Configs
import configs
try:
config = configs.get_config('Laser Range Finder')
enabled = config['enabled']
except NameError:
enabled = True
# Logging
import logging
datalog = logging.getLogger("Dashboard")
datalog.propagate = False
try:
log = logging.getLogger(config['logger_name'])
except NameError:
pass
# Networking for dashboard
from Networking import *
# Events
import events
# Threading
# Created by William Woodall on 2/19/09.
# Copyright (c) 2009 Auburn University. All rights reserved.
#
'''
Module for controlling and managing object detection.
'''
# Imports #
# Paths
import sys
sys.path.append("../Libraries")
# Configs
import configs
import configobj
global config, enable
config = configs.get_config('Obj Detection')
enabled = config['enabled']
mc_name = config['micro_controller']
# Logging
import logging
log = logging.getLogger(config['logger_name'])
# Events
import events
# Conditions
from threading import Event
# micro_controller_network
import micro_controller_network
mc = micro_controller_network.get_object(mc_name)
# Static Variables
return_codes = {'\x70':'Micro Switch Triggered',
#
# Created by William Woodall on 12/31/08.
# Copyright (c) 2008 Auburn University. All rights reserved.
#
'''
Module for controlling a sabertooth2x10 motor controller.
TODO:
* Documentation...
'''
# Imports #
# Configs
import configs
global config, enabled
config = configs.get_config('Sabertooth2x10')
enabled = config['enabled']
# Logging
import logging
log = logging.getLogger(config['logger_name'])
# PySerial
if enabled:
try:
from serial import Serial
except Exception as e:
log.error("Serial cannot be imported, you may need to install it: %s" % e)
else:
log.warning("The Motor Controller is disabled! This means the motors will quietly do nothing!")
# Static Functions #
def init():
# antenna_array.py
#
#
# Created by William Woodall on 2/20/09.
# Copyright (c) 2009 Auburn University. All rights reserved.
#
# Imports #
# Paths
import sys
sys.path.append("../Libraries")
# Configs
import configs
import configobj
global config
config = configs.get_config('Antenna Array')
enabled = config['enabled']
mc_name = config['micro_controller']
# Logging
import logging
log = logging.getLogger(config['logger_name'])
# Events
import events
# Event
from threading import Event
# Sleep
from time import sleep
# micro_controller_network
import micro_controller_network
mc = micro_controller_network.get_object(mc_name)
def test_empty_config(self):
lazydictionary_pre = get_config('test')
self.assertEqual(0, len(lazydictionary_pre.items()))
dash_log.propagate = False
serverHandler = LoggingServerHandler('', PORT)
dash_log.addHandler(serverHandler)
dash_log.critical("Loaded Dash Log.")
log.debug("Using logging config file %s" % loggingConfigFile.name)
# Start Configurations
try:
configFile = open('Configurations/robot.cfg')
except IOError:
try:
configFile = open('Configurations/robot.cfg.default')
except:
log.error("No config file could be opened.")
configs.init(configFile)
config = configs.get_config()
log.debug("Using config file %s" % configFile.name)
# Initialize modules #
# Sabertooth2x10
import sabertooth2x10 as saber
saber.init()
mc = saber.get_object()
# Laser Range Finder
import laser
laser.init()
lrf = laser.get_object()
lrf.clear()
# MCN
import micro_controller_network as mcn
#
#
# Created by William Woodall on 1/29/09.
# Copyright (c) 2009 Auburn University. All rights reserved.
#
'''
Module for interfacing with the Micro Controller Network.
TODO:
'''
# Imports #
# Configs
import configs
import configobj
global config, enabled
config = configs.get_config('Micro Controller Network')
enabled = config['enabled']
# Logging
import logging
log = logging.getLogger(config['logger_name'])
# Threading
from threading import Thread, Lock, Timer
# Sleep
from time import sleep
# PySerial
if enabled:
try:
from serial import Serial
except Exception as e:
log.error("Serial cannot be imported, you may need to install it: %s" % e)
else:
# angular_accelerometer.py
#
#
# Created by William Woodall on 2/21/09.
# Copyright (c) 2009 Auburn University. All rights reserved.
#
# Imports #
# Paths
import sys
sys.path.append("../Libraries")
# Configs
import configs
import configobj
global config
config = configs.get_config('Angular Accelerometer')
mc_name = config['micro_controller']
# Logging
import logging
log = logging.getLogger(config['logger_name'])
# Events
import events
# Event
from threading import Event
# Sleep
from time import sleep
# micro_controller_network
import micro_controller_network
mc = micro_controller_network.get_object(mc_name)
# Static Variables
