def on_start(self):
# print env temporarily to debug cei
import os
log.info('ENV vars: %s' % str(os.environ))
op = self.CFG.get("op", None)
datastore = self.CFG.get("datastore", None)
path = self.CFG.get("path", None)
prefix = self.CFG.get("prefix", get_sys_name()).lower()
log.info("DatastoreLoader: {op=%s, datastore=%s, path=%s, prefix=%s}" %
(op, datastore, path, prefix))
self.da = datastore_admin.DatastoreAdmin()
if op:
if op == "load":
self.da.load_datastore(path, datastore, ignore_errors=False)
elif op == "dump":
self.da.dump_datastore(path, datastore)
elif op == "blame":
# TODO make generic
self.da.get_blame_objects()
elif op == "clear":
self.da.clear_datastore(datastore, prefix)
else:
raise iex.BadRequest("Operation unknown")
else:
raise iex.BadRequest("No operation specified")
def on_start(self):
global DEBUG
if self.CFG.system.force_clean and not self.CFG.system.testing and not DEBUG:
text = "system.force_clean=True. ION Preload does not support this"
log.error(text)
raise iex.BadRequest(text)
op = self.CFG.get("op", None)
path = self.CFG.get("path", None)
scenario = self.CFG.get("scenario", None)
DEBUG = self.CFG.get("debug", False)
self.loadooi = self.CFG.get("loadooi", False)
self.loadui = self.CFG.get("loadui", False)
log.info("IONLoader: {op=%s, path=%s, scenario=%s}" %
(op, path, scenario))
if op:
if op == "load":
self.load_ion(path, scenario)
elif op == "loadooi":
self.extract_ooi_assets(path)
elif op == "loadui":
self.load_ui(path)
else:
raise iex.BadRequest("Operation unknown")
else:
raise iex.BadRequest("No operation specified")
def on_start(self):
op = self.CFG.get("op", None)
log.info("LoadSystemPolicy: {op=%s}" % op)
if op:
if op == "load":
self.op_load_system_policies(self)
else:
raise iex.BadRequest("Operation unknown")
else:
raise iex.BadRequest("No operation specified")
def on_start(self):
op = self.CFG.get("op", None)
log.info("DirectoryAdmin: {op=%s}" % (op))
if op:
if op == "register":
self.op_register_definitions()
elif op == "load":
pass
else:
raise iex.BadRequest("Operation unknown")
else:
raise iex.BadRequest("No operation specified")
def _get_typed_value(self, value, schema_entry=None, targettype=None):
targettype = targettype or schema_entry["type"]
if schema_entry and 'enum_type' in schema_entry:
enum_clzz = getattr(objects, schema_entry['enum_type'])
return enum_clzz._value_map[value]
elif targettype is 'str':
return str(value)
elif targettype is 'bool':
lvalue = value.lower()
if lvalue == 'true':
return True
elif lvalue == 'false' or lvalue == '':
return False
else:
raise iex.BadRequest("Value %s is no bool" % value)
elif targettype is 'int':
try:
return int(value)
except Exception:
log.warn("Value %s is type %s not type %s" %
(value, type(value), targettype))
return ast.literal_eval(value)
elif targettype is 'float':
try:
return float(value)
except Exception:
log.warn("Value %s is type %s not type %s" %
(value, type(value), targettype))
return ast.literal_eval(value)
elif targettype is 'simplelist':
if value.startswith('[') and value.endswith(']'):
value = value[1:len(value) - 1].strip()
return list(value.split(','))
else:
return ast.literal_eval(value)
def _preload_ids(self):
if not DEBUG:
rr_client = self._get_service_client("resource_registry")
org_ids, _ = rr_client.find_resources(name="ION",
restype=RT.Org,
id_only=True)
if not org_ids:
raise iex.BadRequest(
"ION org not found. Was system force_cleaned since bootstrap?"
)
ion_org_id = org_ids[0]
self._register_id(self.ID_ORG_ION, ion_org_id)
def load_ui(self, path):
"""@brief Entry point to the import/generation capabilities from the FileMakerPro database
CVS files to ION resource objects.
"""
if not path:
raise iex.BadRequest("Must provide path")
path = path + "/ui_assets"
log.info("Start parsing UI assets from path=%s" % path)
categories = [
('Internal Resource Type.csv', 'InternalResourceType'),
('Resource.csv', 'ResourceType'),
('Information Level.csv', 'InformationLevel'),
('Attribute.csv', 'Attribute'),
('Block.csv', 'Block'),
('Group.csv', 'Group'),
('Representation.csv', 'Representation'),
('View.csv', 'View'),
('Screen Label.csv', 'ScreenLabel'),
('_jt_Block_Attribute.csv', 'BlockAttribute'),
('_jt_Block_Representation.csv', 'BlockRepresentation'),
('_jt_Group_Block.csv', 'GroupBlock'),
('_jt_View_Group.csv', 'ViewGroup'),
]
self.uiid_prefix = uuid.uuid4().hex[:9] + "_"
self.ui_objs = {}
self.ui_obj_by_id = {}
self.ref_assocs = []
self.ui_assocs = []
for filename, category in categories:
row_do, row_skip = 0, 0
catfunc = getattr(self, "_loadui_%s" % category)
filename = "%s/%s" % (path, filename)
log.info("Loading UI category %s from file %s" %
(category, filename))
try:
with open(filename, "rb") as csvfile:
reader = self._get_csv_reader(csvfile)
for row in reader:
catfunc(row)
row_do += 1
except IOError, ioe:
log.warn("UI category file %s error: %s" %
(filename, str(ioe)))
log.info(
"Loaded UI category %s: %d rows imported, %d rows skipped" %
(category, row_do, row_skip))
def _load_Attachment(self, row):
log.info("Loading Attachment")
res_id = self.resource_ids[row["resource_id"]]
att_obj = self._create_object_from_row("Attachment", row, "att/")
file_path = row["file_path"]
if file_path:
file_path = "%s/attachments/%s" % (self.path, file_path)
try:
with open(file_path, "rb") as attfile:
att_obj.content = attfile.read()
except IOError, ioe:
raise iex.BadRequest("Attachment file_path %s error: %s" %
(file_path, str(ioe)))
def chomp_key_list(out_dict, keys, value):
"""
turn keys like ['a', 'b', 'c', 'd'] and a value into
out_dict['a']['b']['c']['d'] = value
"""
dict_ptr = out_dict
last_ptr = out_dict
for i, key in enumerate(keys):
last_ptr = dict_ptr
if not key in dict_ptr:
dict_ptr[key] = {}
else:
if type(dict_ptr[key]) != type({}):
raise iex.BadRequest(
"Building a dict in %s field, but it exists as %s already"
% (key, type(dict_ptr[key])))
dict_ptr = dict_ptr[key]
last_ptr[keys[-1]] = value
def extract_ooi_assets(self, path):
if not path:
raise iex.BadRequest("Must provide path")
path = path + "/ooi_assets"
log.info("Start parsing OOI assets from path=%s" % path)
categories = [
'Report2_InstrumentTypes',
'Report4_InstrumentsPerSite',
'Report1_InstrumentLocations',
'Report3_InstrumentTypeByLocation',
'Report6_ReferenceDesignatorListWithDepth',
]
self.obs_sites = {}
self.sub_sites = {}
self.platform_models = {}
self.instrument_models = {}
self.logical_platforms = {}
self.logical_instruments = {}
for category in categories:
row_do, row_skip = 0, 0
funcname = "_parse_%s" % category
catfunc = getattr(self, funcname)
filename = "%s/%s.csv" % (path, category)
log.info("Loading category %s from file %s" % (category, filename))
try:
with open(filename, "rb") as csvfile:
for i in xrange(9):
# Skip the first rows, because they are garbage
csvfile.readline()
reader = self._get_csv_reader(csvfile)
for row in reader:
row_do += 1
catfunc(row)
except IOError, ioe:
log.warn("OOI asset file %s error: %s" % (filename, str(ioe)))
log.info("Loaded assets %s: %d rows read" % (category, row_do))
def get_typed_value(value, schema_entry=None, targettype=None):
"""
Performs a value type conversion according to a schema specified target type.
"""
targettype = targettype or schema_entry["type"]
if schema_entry and 'enum_type' in schema_entry:
enum_clzz = getattr(objects, schema_entry['enum_type'])
return enum_clzz._value_map[value]
elif targettype == 'str':
return str(value)
elif targettype == 'bool':
if value in ('TRUE', 'True', 'true', '1', 1, True):
return True
if value in ('FALSE', 'False', 'false', '0', 0, '', None, False):
return False
raise iex.BadRequest("Value %s is no bool" % value)
elif targettype == 'int':
try:
return int(value)
except Exception:
log.warn("Value %s is type %s not type %s" %
(value, type(value), targettype))
return ast.literal_eval(value)
elif targettype == 'float':
try:
return float(value)
except Exception:
log.warn("Value %s is type %s not type %s" %
(value, type(value), targettype))
return ast.literal_eval(value)
elif targettype == 'simplelist':
if value.startswith('[') and value.endswith(']'):
value = value[1:len(value) - 1].strip()
elif not value.strip():
return []
return list(value.split(','))
else:
log.trace('parsing value as %s: %s', targettype, value)
return ast.literal_eval(value)
def get_typed_value(value, schema_entry=None, targettype=None):
"""
Performs a value type conversion according to a schema specified target type.
Supports simplelist and parsedict special type parsing.
"""
targettype = targettype or schema_entry["type"]
if schema_entry and 'enum_type' in schema_entry:
enum_clzz = getattr(objects, schema_entry['enum_type'])
return enum_clzz._value_map[value]
elif targettype == 'str':
return str(value)
elif targettype == 'bool':
if value in ('TRUE', 'True', 'true', '1', 1, True):
return True
if value in ('FALSE', 'False', 'false', '0', 0, '', None, False):
return False
raise iex.BadRequest("Value %s is no bool" % value)
elif targettype == 'int':
try:
return int(value)
except Exception:
log.warn("Value %s is type %s not type %s" %
(value, type(value), targettype))
return ast.literal_eval(value)
elif targettype == 'float':
try:
return float(value)
except Exception:
log.warn("Value %s is type %s not type %s" %
(value, type(value), targettype))
return ast.literal_eval(value)
elif targettype == 'simplelist':
return parse_list(value)
elif targettype == 'parsedict':
return parse_dict(str(value))
else:
log.trace('parsing value as %s: %s', targettype, value)
return ast.literal_eval(value)
def load_ion(self, path, scenario):
if not path:
raise iex.BadRequest("Must provide path")
log.info("Start preloading from path=%s" % path)
categories = [
'User',
'Org',
'UserRole',
'PlatformModel',
'InstrumentModel',
'Observatory',
'Subsite',
'PlatformSite',
'InstrumentSite',
'StreamDefinition',
'PlatformDevice',
'InstrumentDevice',
'InstrumentAgent',
'InstrumentAgentInstance',
'DataProcessDefinition',
#'IngestionConfiguration',
'DataProduct',
'DataProcess',
'DataProductLink',
'Attachment',
]
self.path = path
self.obj_classes = {}
self.resource_ids = {}
self.user_ids = {}
self._preload_ids()
if self.loadooi:
self.extract_ooi_assets(path)
if self.loadui:
self.load_ui(path)
for category in categories:
row_do, row_skip = 0, 0
catfunc_ooi = getattr(self, "_load_%s_OOI" % category, None)
if self.loadooi and catfunc_ooi:
catfunc_ooi()
catfunc = getattr(self, "_load_%s" % category)
filename = "%s/%s.csv" % (path, category)
log.info("Loading category %s from file %s" % (category, filename))
try:
with open(filename, "rb") as csvfile:
reader = self._get_csv_reader(csvfile)
for row in reader:
# Check if scenario applies
rowsc = row[self.COL_SCENARIO]
if not scenario in rowsc:
row_skip += 1
continue
row_do += 1
catfunc(row)
except IOError, ioe:
log.warn("Resource category file %s error: %s" %
(filename, str(ioe)))
log.info("Loaded category %s: %d rows imported, %d rows skipped" %
(category, row_do, row_skip))
def _register_id(self, alias, resid):
if alias in self.resource_ids:
raise iex.BadRequest("ID alias %s used twice" % alias)
self.resource_ids[alias] = resid
log.info("Added resource alias=%s to id=%s" % (alias, resid))
def load_ui(self, path, replace=True, specs_only=True, specs_path=None):
"""
@brief Import UI definitions from the FileMakerPro database exported CVS files
to ION resource objects.
"""
if not path:
raise iex.BadRequest("Must provide path")
if path.startswith("http"):
self._get_ui_files(path)
self.read_from_local = False
log.info("Start parsing UI assets from path=%s" % path)
categories = [
('Graphic Type.csv', 'GraphicType'),
('Graphic.csv', 'Graphic'),
('Information Level.csv', 'InformationLevel'),
('Resource Type.csv', 'ResourceType'),
('Object Type.csv', 'ObjectType'),
('Resource Attribute.csv', 'ResourceAttribute'),
('Object Field.csv', 'ObjectField'),
('Screen Element.csv', 'ScreenElement'),
('Screen Label.csv', 'ScreenLabel'),
('Help Tag.csv', 'HelpTag'),
('Message String.csv', 'MessageString'),
('State.csv', 'State'),
('Widget.csv', 'Widget'),
('_jt_ScreenElement_InformationElement.csv',
'ScreenElementInformationElement'),
('_jt_ScreenElement_ScreenElement.csv', 'EmbeddedScreenElement'),
]
self.uiid_prefix = uuid.uuid4().hex[:9] + "_"
self.ui_objs = {}
self.ui_obj_by_id = {}
self.ref_assocs = []
self.ui_assocs = []
self.warnings = []
self.abort = False
self.counter = 0
for fname, category in categories:
row_do, row_skip = 0, 0
catfunc = getattr(self, "_loadui_%s" % category)
try:
if self.read_from_local:
# Read from file handle
filename = "%s/%s" % (path, fname)
log.info("Loading UI category %s from file %s" %
(category, filename))
try:
with open(filename, "rbU") as csvfile:
reader = csv.DictReader(csvfile, delimiter=',')
for row in reader:
catfunc(row)
self.counter += 1
row_do += 1
except IOError, ioe:
log.warn("UI category file %s error: %s" %
(filename, str(ioe)))
self.abort = True
else:
# Read from string
if fname in self.files:
log.info(
"Loading UI category %s from retrieved file %s" %
(category, fname))
csvfile = self.files[fname]
# This is a hack to be able to read from string
csvfile = csvfile.splitlines()
reader = csv.DictReader(csvfile, delimiter=',')
for row in reader:
catfunc(row)
self.counter += 1
row_do += 1
else:
log.warn("UI category %s has no file %s" %
(category, fname))
self.abort = True
def extract_ooi_assets(self):
"""
Parses SAF Instrument Application export CSV files into intermediate memory structures.
This information can later be loaded in to actual load_ion() function.
"""
if not self.asset_path:
raise iex.BadRequest(
"Must provide path for assets: path=dir or assets=dir")
if self.asset_path.startswith('http'):
raise iex.BadRequest(
'Asset path must be local directory, not URL: ' +
self.asset_path)
log.info("Parsing OOI assets from path=%s", self.asset_path)
categories = [ # Mapping spreadsheet early
'NodeTypes',
# Core concept attributes
'AttributeReportArrays',
'AttributeReportClass',
'AttributeReportDataProducts',
'AttributeReportFamilies',
'AttributeReportMakeModel',
'AttributeReportNodes',
'AttributeReportPorts',
'AttributeReportReferenceDesignator',
'AttributeReportSeries',
'AttributeReportSites',
'AttributeReportSubseries',
'AttributeReportSubsites',
# Additional attributes and links taken from aggregate reports
'InstrumentCatalogFull',
'DataQCLookupTables',
'DataProductSpreadsheet',
'AllSensorTypeCounts',
# Tabs from the mapping spreadsheet
'Arrays',
'Sites',
'Subsites',
'Nodes',
#'Platforms',
'NodeTypes',
'PlatformAgentTypes'
]
# Holds the object representations of parsed OOI assets by type
self.ooi_objects = {}
# Holds a list of attribute names of OOI assets by type
self.ooi_obj_attrs = {}
self.warnings = []
self.csv_files = None
# Load OOIResourceMappings.xlsx
mapping_file = self.asset_path + "/OOIResourceMappings.xlsx"
if os.path.exists(mapping_file):
with open(mapping_file, "rb") as f:
preload_doc_str = f.read()
log.debug("Loaded %s mapping file, size=%s", mapping_file,
len(preload_doc_str))
xls_parser = XLSParser()
self.csv_files = xls_parser.extract_csvs(preload_doc_str)
for category in categories:
row_do, row_skip = 0, 0
catfunc = getattr(self, "_parse_%s" % category)
filename = "%s/%s.csv" % (self.asset_path, category)
log.debug("Loading category %s from file %s", category, filename)
try:
if category in self.csv_files:
csv_doc = self.csv_files[category]
reader = csv.DictReader(csv_doc, delimiter=',')
filename = mapping_file + ":" + category
else:
csvfile = open(filename, "rb")
for i in xrange(9):
# Skip the first rows, because they are garbage
csvfile.readline()
reader = csv.DictReader(csvfile, delimiter=',')
for row in reader:
row_do += 1
catfunc(row)
except IOError as ioe:
log.warn("OOI asset file %s error: %s" % (filename, str(ioe)))
log.debug("Loaded assets %s: %d rows read" % (category, row_do))
# Post processing
self._post_process()
# Do some validation checking
self._perform_ooi_checks()
if self.warnings:
log.warn("WARNINGS:\n%s",
"\n".join(["%s: %s" % (a, b) for a, b in self.warnings]))
for ot, oo in self.ooi_objects.iteritems():
log.info("Type %s has %s entries", ot, len(oo))
