def validate_json_model(data):
"""
Checks json model's values are valid
:param data: dictionary where the values are a list of dictionary. Type(dict: [dict])
:return:
"""
try:
# if no sections are found, than the file format must be incorrect
if len(data) == 0:
raise Exception('> [Exception] Invalid model configuration file')
var_cv = os.path.join(io.getAppDataDir(), 'dat/var_cv.dat')
unit_cv = os.path.join(io.getAppDataDir(), 'dat/units_cv.dat')
var = pickle.load(open(var_cv, 'rb'))
unit = pickle.load(open(unit_cv, 'rb'))
ignorecv = int(data["options"][0]["ignorecv"])
for key, value in data.iteritems():
if isinstance(value, list):
for item in value:
for k, v in item.iteritems():
if v == "simulation_start" or v == "simulation_end":
try:
datetime.datetime.strptime(v, getattr(json_types, k))
except ValueError:
raise ValueError("Incorrect data format, should be " + getattr(json_types, k))
else:
if not ignorecv:
if k == "variable_name_cv":
if v not in var:
raise Exception(v + ' is not a valid controlled vocabulary term')
if k == "unit_type_cv":
if v not in unit:
raise Exception(v + ' is not a valid controlled vocabulary term')
software = data["software"][0]
relpath = software["filepath"] # Change name to filepath
basedir = data["basedir"]
abspath = os.path.abspath(os.path.join(basedir, relpath))
sys.path.append(basedir)
if not os.path.isfile(abspath):
raise Exception(abspath + " is not a valid file")
classname = software["classname"]
filename = os.path.basename(abspath)
module = imp.load_source(filename.split(".")[0], abspath)
m = getattr(module, classname)
except Exception, e:
elog.error('Configuration Parsing Error: ' + str(e))
sPrint('Configuration Parsing Error: ' + str(e), MessageType.ERROR)
return 0
def __init__(self):
if not _Log.__monostate:
_Log.__monostate = self.__dict__
app_data = io.getAppDataDir()
LOG_FILENAME = abspath(join(app_data, 'log/engine.log'))
# make sure this path exists
if not exists(dirname(LOG_FILENAME)):
os.mkdir(dirname(LOG_FILENAME))
self.__root = logging.getLogger('EMIT ENGINE')
self.__root.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(funcName)s - %(lineno)s --- %(message)s')
sh_formatter = logging.Formatter('%(asctime)s - [%(levelname)s] --- %(message)s')
sh = logging.StreamHandler(sys.stdout)
sh.setFormatter(sh_formatter)
self.__root.addHandler(sh)
# setup rotating log
rotating_log = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=50000000, backupCount=500)
rotating_log.setFormatter(formatter)
self.__root.addHandler(rotating_log)
else:
self.__dict__ = _Log.__monostate
def __init__(self):
if not _Log.__monostate:
_Log.__monostate = self.__dict__
app_data = io.getAppDataDir()
LOG_FILENAME = abspath(join(app_data, 'log/engine.log'))
# make sure this path exists
if not exists(dirname(LOG_FILENAME)):
os.mkdir(dirname(LOG_FILENAME))
self.__root = logging.getLogger('EMIT ENGINE')
self.__root.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(funcName)s - %(lineno)s --- %(message)s'
)
sh_formatter = logging.Formatter(
'%(asctime)s - [%(levelname)s] --- %(message)s')
sh = logging.StreamHandler(sys.stdout)
sh.setFormatter(sh_formatter)
self.__root.addHandler(sh)
# setup rotating log
rotating_log = logging.handlers.RotatingFileHandler(
LOG_FILENAME, maxBytes=50000000, backupCount=500)
rotating_log.setFormatter(formatter)
self.__root.addHandler(rotating_log)
else:
self.__dict__ = _Log.__monostate
def validate_config_ini(ini_path): # Deprecated. Use utilities.models.validate_json_model
try:
cparser = ConfigParser.ConfigParser(None, multidict)
# parse the ini
cparser.read(ini_path)
# get the ini sections from the parser
parsed_sections = cparser.sections()
# if no sections are found, than the file format must be incorrect
if len(parsed_sections) == 0: raise Exception('> [Exception] Invalid model configuration file')
# load lookup tables
dir = os.path.dirname(__file__)
var_cv = os.path.join(io.getAppDataDir(), 'dat/var_cv.dat')
unit_cv= os.path.join(io.getAppDataDir(), 'dat/units_cv.dat')
var = pickle.load(open(var_cv, 'rb'))
unit= pickle.load(open(unit_cv, 'rb'))
# var = pickle.load(open(os.path.join(dir,'../data/var_cv.dat'),'rb'))
# unit = pickle.load(open(os.path.join(dir,'../data/units_cv.dat'),'rb'))
# check to see if 'ignorecv' option has been provided
ignorecv = False
for p in parsed_sections:
if p.split('^')[0] == 'options':
if cparser.has_option(p,'ignorecv'):
ignorecv = int(cparser.get(p,'ignorecv'))
break
# validate
for section in parsed_sections:
# get ini options
options = cparser.options(section)
if not ignorecv:
# validate units and variables parameters
if section.split('_')[0] == 'output' or section.split('_')[0] == 'input':
# check that variable and unit exist
if 'variable_name_cv' not in options or 'unit_type_cv' not in options:
raise Exception ('Inputs and Outputs must contain "variable_name_cv" and "unit_type_cv" parameters ')
# check each option individually
for option in options:
val = cparser.get(section,option)
# validate date format
if option == 'simulation_start' or option == 'simulation_end':
try:
datetime.datetime.strptime(val, getattr(ini_types, option))
except ValueError:
raise ValueError("Incorrect data format, should be "+getattr(ini_types, option))
else:
# validate data type
#if not isinstance(val,type(getattr(ini_types, option))):
# raise Exception(option+' is not of type '+getattr(ini_types, option))
if not ignorecv:
# check variable cv (i.e. lookup table)
if option == 'variable_name_cv':
if val not in var:
raise Exception (val+' is not a valid controlled vocabulary term')
# check unit type cv (i.e. lookup table)
if option == 'unit_type_cv':
if val not in unit:
raise Exception (val+' is not a valid controlled vocabulary term')
if section.split('^')[0] == 'software':
# check that software filepath is valid
relpath = cparser.get(section,'filepath')
basedir = os.path.realpath(os.path.dirname(ini_path))
abspath = os.path.abspath(os.path.join(basedir,relpath))
# add the base path to the sys.path
sys.path.append(basedir)
if not os.path.isfile(abspath):
raise Exception(abspath+' is not a valid file')
#todo: check that software class name exists
try:
classname = cparser.get(section,'classname')
filename = os.path.basename(abspath)
module = imp.load_source(filename.split('.')[0], abspath)
m = getattr(module, classname)
except Exception, e:
elog.error('Configuration Parsing Error: '+str(e))
sPrint('Configuration Parsing Error: '+str(e), MessageType.ERROR)
except Exception, e:
elog.error('Configuration Parsing Error: '+str(e))
sPrint('Configuration Parsing Error: '+str(e), MessageType.ERROR)
return 0
def validate_config_ini(
ini_path): # Deprecated. Use utilities.models.validate_json_model
try:
cparser = ConfigParser.ConfigParser(None, multidict)
# parse the ini
cparser.read(ini_path)
# get the ini sections from the parser
parsed_sections = cparser.sections()
# if no sections are found, than the file format must be incorrect
if len(parsed_sections) == 0:
raise Exception('> [Exception] Invalid model configuration file')
# load lookup tables
dir = os.path.dirname(__file__)
var_cv = os.path.join(io.getAppDataDir(), 'dat/var_cv.dat')
unit_cv = os.path.join(io.getAppDataDir(), 'dat/units_cv.dat')
var = pickle.load(open(var_cv, 'rb'))
unit = pickle.load(open(unit_cv, 'rb'))
# var = pickle.load(open(os.path.join(dir,'../data/var_cv.dat'),'rb'))
# unit = pickle.load(open(os.path.join(dir,'../data/units_cv.dat'),'rb'))
# check to see if 'ignorecv' option has been provided
ignorecv = False
for p in parsed_sections:
if p.split('^')[0] == 'options':
if cparser.has_option(p, 'ignorecv'):
ignorecv = int(cparser.get(p, 'ignorecv'))
break
# validate
for section in parsed_sections:
# get ini options
options = cparser.options(section)
if not ignorecv:
# validate units and variables parameters
if section.split('_')[0] == 'output' or section.split(
'_')[0] == 'input':
# check that variable and unit exist
if 'variable_name_cv' not in options or 'unit_type_cv' not in options:
raise Exception(
'Inputs and Outputs must contain "variable_name_cv" and "unit_type_cv" parameters '
)
# check each option individually
for option in options:
val = cparser.get(section, option)
# validate date format
if option == 'simulation_start' or option == 'simulation_end':
try:
datetime.datetime.strptime(val,
getattr(ini_types, option))
except ValueError:
raise ValueError("Incorrect data format, should be " +
getattr(ini_types, option))
else:
# validate data type
#if not isinstance(val,type(getattr(ini_types, option))):
# raise Exception(option+' is not of type '+getattr(ini_types, option))
if not ignorecv:
# check variable cv (i.e. lookup table)
if option == 'variable_name_cv':
if val not in var:
raise Exception(
val +
' is not a valid controlled vocabulary term'
)
# check unit type cv (i.e. lookup table)
if option == 'unit_type_cv':
if val not in unit:
raise Exception(
val +
' is not a valid controlled vocabulary term'
)
if section.split('^')[0] == 'software':
# check that software filepath is valid
relpath = cparser.get(section, 'filepath')
basedir = os.path.realpath(os.path.dirname(ini_path))
abspath = os.path.abspath(os.path.join(basedir, relpath))
# add the base path to the sys.path
sys.path.append(basedir)
if not os.path.isfile(abspath):
raise Exception(abspath + ' is not a valid file')
#todo: check that software class name exists
try:
classname = cparser.get(section, 'classname')
filename = os.path.basename(abspath)
module = imp.load_source(filename.split('.')[0], abspath)
m = getattr(module, classname)
except Exception, e:
elog.error('Configuration Parsing Error: ' + str(e))
sPrint('Configuration Parsing Error: ' + str(e),
MessageType.ERROR)
except Exception, e:
elog.error('Configuration Parsing Error: ' + str(e))
sPrint('Configuration Parsing Error: ' + str(e), MessageType.ERROR)
return 0
