def _validate_config(self, folder, validate_folder=True):
''' validate config is the primary validation function that checks
for presence and format of required fields.
Parameters
==========
:folder: full path to folder with config.json
:name: if provided, the folder name to check against exp_id
'''
config = "%s/config.json" % folder
name = os.path.basename(folder)
if not os.path.exists(config):
return notvalid("%s: config.json not found." % (folder))
# Load the config
try:
config = read_json(config)
except:
return notvalid("%s: cannot load json, invalid." % (name))
# Config.json should be single dict
if isinstance(config, list):
return notvalid("%s: config.json is a list, not valid." % (name))
# Check over required fields
fields = self.get_validation_fields()
for field, value, ftype in fields:
bot.verbose('field: %s, required: %s' % (field, value))
# Field must be in the keys if required
if field not in config.keys():
if value == 1:
return notvalid(
"%s: config.json is missing required field %s" %
(name, field))
# Field is present, check type
else:
if not isinstance(config[field], ftype):
return notvalid("%s: invalid type, must be %s." %
(name, str(ftype)))
# Expid gets special treatment
if field == "exp_id" and validate_folder is True:
if config[field] != name:
return notvalid(
"%s: exp_id parameter %s does not match folder name." %
(name, config[field]))
# name cannot have special characters, only _ and letters/numbers
if not re.match("^[a-z0-9_-]*$", config[field]):
message = "%s: exp_id parameter %s has invalid characters"
message += "only lowercase [a-z],[0-9], -, and _ allowed."
return notvalid(message % (name, config[field]))
return True
def test_single_experiments(self):
'''test_load_json ensures that all files load
'''
for jsonfile in self.experiments:
print("...%s" % os.path.basename(jsonfile))
config = read_json(jsonfile)
self.assertTrue('github' in config)
self.assertTrue(isinstance(config, dict))
url = config['github']
self.assertTrue(self.ExpValidator.validate(url))
def _validate_config(self, folder, validate_folder=True):
''' validate config is the primary validation function that checks
for presence and format of required fields.
Parameters
==========
:folder: full path to folder with config.json
:name: if provided, the folder name to check against exp_id
'''
config = "%s/config.json" % folder
name = os.path.basename(folder)
if not os.path.exists(config):
return notvalid("%s: config.json not found." %(folder))
# Load the config
try:
config = read_json(config)
except:
return notvalid("%s: cannot load json, invalid." %(name))
# Config.json should be single dict
if isinstance(config, list):
return notvalid("%s: config.json is a list, not valid." %(name))
# Check over required fields
fields = self.get_validation_fields()
for field,value,ftype in fields:
bot.verbose('field: %s, required: %s' %(field,value))
# Field must be in the keys if required
if field not in config.keys():
if value == 1:
return notvalid("%s: config.json is missing required field %s" %(name,field))
# Field is present, check type
else:
if not isinstance(config[field], ftype):
return notvalid("%s: invalid type, must be %s." %(name,str(ftype)))
# Expid gets special treatment
if field == "exp_id" and validate_folder is True:
if config[field] != name:
return notvalid("%s: exp_id parameter %s does not match folder name."
%(name,config[field]))
# name cannot have special characters, only _ and letters/numbers
if not re.match("^[a-z0-9_-]*$", config[field]):
message = "%s: exp_id parameter %s has invalid characters"
message += "only lowercase [a-z],[0-9], -, and _ allowed."
return notvalid(message %(name,config[field]))
return True
def load_experiment(folder, return_path=False):
"""load_experiment:
reads in the config.json for a folder, returns None if not found.
:param folder: full path to experiment folder
:param return_path: if True, don't load the config.json, but return it
"""
fullpath = os.path.abspath(folder)
config = "%s/config.json" % (fullpath)
if not os.path.exists(config):
bot.error("config.json could not be found in %s" % (folder))
config = None
if return_path is False and config is not None:
config = read_json(config)
return config
def load_experiment(folder, return_path=False):
'''load_experiment:
reads in the config.json for a folder, returns None if not found.
:param folder: full path to experiment folder
:param return_path: if True, don't load the config.json, but return it
'''
fullpath = os.path.abspath(folder)
config = "%s/config.json" %(fullpath)
if not os.path.exists(config):
bot.error("config.json could not be found in %s" %(folder))
config = None
if return_path is False and config is not None:
config = read_json(config)
return config
def test_previews(self):
'''assert that each experiment is previewed at the Github page
where served
'''
bot.test('Testing experiment previews...')
for jsonfile in self.experiments:
experiment = os.path.basename(jsonfile)
print("...%s experiment preview?" % experiment)
config = read_json(jsonfile)
self.assertTrue('github' in config)
self.assertTrue(isinstance(config, dict))
url = config['github']
result = self.RuntimeValidator.validate(url)
print(result)
print(url)
self.assertTrue(result)
except AssertionError:
print("[fail] invalid, number of items must be equal.")
raise
# Generate a lookup for each
lookup_x = generate_lookup(x)
lookup_y = generate_lookup(y)
for exp_id, meta in lookup_x.items():
print('\n...%s' % meta['name'])
for key, val in meta.items():
if key == "metadata":
continue
try:
assert (lookup_y[exp_id][key] == val)
print("[pass] %s:%s are matching" % (key, val))
except AssertionError:
print("[fail] mismatch or missing field for %s" % key)
print("%s != %s " % (x[i][key], y[i][key]))
raise
################################################################################
# Tests
x = read_json(index)
y = read_json(comparator)
os.remove(comparator)
compare_dicts(x, y)