def on_start(self): # print env temporarily to debug cei import os log.info('ENV vars: %s' % str(os.environ)) op = self.CFG.get("op", None) datastore = self.CFG.get("datastore", None) path = self.CFG.get("path", None) prefix = self.CFG.get("prefix", get_sys_name()).lower() log.info("DatastoreLoader: {op=%s, datastore=%s, path=%s, prefix=%s}" % (op, datastore, path, prefix)) self.da = datastore_admin.DatastoreAdmin() if op: if op == "load": self.da.load_datastore(path, datastore, ignore_errors=False) elif op == "dump": self.da.dump_datastore(path, datastore) elif op == "blame": # TODO make generic self.da.get_blame_objects() elif op == "clear": self.da.clear_datastore(datastore, prefix) else: raise iex.BadRequest("Operation unknown") else: raise iex.BadRequest("No operation specified")
def on_start(self): global DEBUG if self.CFG.system.force_clean and not self.CFG.system.testing and not DEBUG: text = "system.force_clean=True. ION Preload does not support this" log.error(text) raise iex.BadRequest(text) op = self.CFG.get("op", None) path = self.CFG.get("path", None) scenario = self.CFG.get("scenario", None) DEBUG = self.CFG.get("debug", False) self.loadooi = self.CFG.get("loadooi", False) self.loadui = self.CFG.get("loadui", False) log.info("IONLoader: {op=%s, path=%s, scenario=%s}" % (op, path, scenario)) if op: if op == "load": self.load_ion(path, scenario) elif op == "loadooi": self.extract_ooi_assets(path) elif op == "loadui": self.load_ui(path) else: raise iex.BadRequest("Operation unknown") else: raise iex.BadRequest("No operation specified")
def on_start(self): op = self.CFG.get("op", None) log.info("LoadSystemPolicy: {op=%s}" % op) if op: if op == "load": self.op_load_system_policies(self) else: raise iex.BadRequest("Operation unknown") else: raise iex.BadRequest("No operation specified")
def on_start(self): op = self.CFG.get("op", None) log.info("DirectoryAdmin: {op=%s}" % (op)) if op: if op == "register": self.op_register_definitions() elif op == "load": pass else: raise iex.BadRequest("Operation unknown") else: raise iex.BadRequest("No operation specified")
def _get_typed_value(self, value, schema_entry=None, targettype=None): targettype = targettype or schema_entry["type"] if schema_entry and 'enum_type' in schema_entry: enum_clzz = getattr(objects, schema_entry['enum_type']) return enum_clzz._value_map[value] elif targettype is 'str': return str(value) elif targettype is 'bool': lvalue = value.lower() if lvalue == 'true': return True elif lvalue == 'false' or lvalue == '': return False else: raise iex.BadRequest("Value %s is no bool" % value) elif targettype is 'int': try: return int(value) except Exception: log.warn("Value %s is type %s not type %s" % (value, type(value), targettype)) return ast.literal_eval(value) elif targettype is 'float': try: return float(value) except Exception: log.warn("Value %s is type %s not type %s" % (value, type(value), targettype)) return ast.literal_eval(value) elif targettype is 'simplelist': if value.startswith('[') and value.endswith(']'): value = value[1:len(value) - 1].strip() return list(value.split(',')) else: return ast.literal_eval(value)
def _preload_ids(self): if not DEBUG: rr_client = self._get_service_client("resource_registry") org_ids, _ = rr_client.find_resources(name="ION", restype=RT.Org, id_only=True) if not org_ids: raise iex.BadRequest( "ION org not found. Was system force_cleaned since bootstrap?" ) ion_org_id = org_ids[0] self._register_id(self.ID_ORG_ION, ion_org_id)
def load_ui(self, path): """@brief Entry point to the import/generation capabilities from the FileMakerPro database CVS files to ION resource objects. """ if not path: raise iex.BadRequest("Must provide path") path = path + "/ui_assets" log.info("Start parsing UI assets from path=%s" % path) categories = [ ('Internal Resource Type.csv', 'InternalResourceType'), ('Resource.csv', 'ResourceType'), ('Information Level.csv', 'InformationLevel'), ('Attribute.csv', 'Attribute'), ('Block.csv', 'Block'), ('Group.csv', 'Group'), ('Representation.csv', 'Representation'), ('View.csv', 'View'), ('Screen Label.csv', 'ScreenLabel'), ('_jt_Block_Attribute.csv', 'BlockAttribute'), ('_jt_Block_Representation.csv', 'BlockRepresentation'), ('_jt_Group_Block.csv', 'GroupBlock'), ('_jt_View_Group.csv', 'ViewGroup'), ] self.uiid_prefix = uuid.uuid4().hex[:9] + "_" self.ui_objs = {} self.ui_obj_by_id = {} self.ref_assocs = [] self.ui_assocs = [] for filename, category in categories: row_do, row_skip = 0, 0 catfunc = getattr(self, "_loadui_%s" % category) filename = "%s/%s" % (path, filename) log.info("Loading UI category %s from file %s" % (category, filename)) try: with open(filename, "rb") as csvfile: reader = self._get_csv_reader(csvfile) for row in reader: catfunc(row) row_do += 1 except IOError, ioe: log.warn("UI category file %s error: %s" % (filename, str(ioe))) log.info( "Loaded UI category %s: %d rows imported, %d rows skipped" % (category, row_do, row_skip))
def _load_Attachment(self, row): log.info("Loading Attachment") res_id = self.resource_ids[row["resource_id"]] att_obj = self._create_object_from_row("Attachment", row, "att/") file_path = row["file_path"] if file_path: file_path = "%s/attachments/%s" % (self.path, file_path) try: with open(file_path, "rb") as attfile: att_obj.content = attfile.read() except IOError, ioe: raise iex.BadRequest("Attachment file_path %s error: %s" % (file_path, str(ioe)))
def chomp_key_list(out_dict, keys, value): """ turn keys like ['a', 'b', 'c', 'd'] and a value into out_dict['a']['b']['c']['d'] = value """ dict_ptr = out_dict last_ptr = out_dict for i, key in enumerate(keys): last_ptr = dict_ptr if not key in dict_ptr: dict_ptr[key] = {} else: if type(dict_ptr[key]) != type({}): raise iex.BadRequest( "Building a dict in %s field, but it exists as %s already" % (key, type(dict_ptr[key]))) dict_ptr = dict_ptr[key] last_ptr[keys[-1]] = value
def extract_ooi_assets(self, path): if not path: raise iex.BadRequest("Must provide path") path = path + "/ooi_assets" log.info("Start parsing OOI assets from path=%s" % path) categories = [ 'Report2_InstrumentTypes', 'Report4_InstrumentsPerSite', 'Report1_InstrumentLocations', 'Report3_InstrumentTypeByLocation', 'Report6_ReferenceDesignatorListWithDepth', ] self.obs_sites = {} self.sub_sites = {} self.platform_models = {} self.instrument_models = {} self.logical_platforms = {} self.logical_instruments = {} for category in categories: row_do, row_skip = 0, 0 funcname = "_parse_%s" % category catfunc = getattr(self, funcname) filename = "%s/%s.csv" % (path, category) log.info("Loading category %s from file %s" % (category, filename)) try: with open(filename, "rb") as csvfile: for i in xrange(9): # Skip the first rows, because they are garbage csvfile.readline() reader = self._get_csv_reader(csvfile) for row in reader: row_do += 1 catfunc(row) except IOError, ioe: log.warn("OOI asset file %s error: %s" % (filename, str(ioe))) log.info("Loaded assets %s: %d rows read" % (category, row_do))
def get_typed_value(value, schema_entry=None, targettype=None): """ Performs a value type conversion according to a schema specified target type. """ targettype = targettype or schema_entry["type"] if schema_entry and 'enum_type' in schema_entry: enum_clzz = getattr(objects, schema_entry['enum_type']) return enum_clzz._value_map[value] elif targettype == 'str': return str(value) elif targettype == 'bool': if value in ('TRUE', 'True', 'true', '1', 1, True): return True if value in ('FALSE', 'False', 'false', '0', 0, '', None, False): return False raise iex.BadRequest("Value %s is no bool" % value) elif targettype == 'int': try: return int(value) except Exception: log.warn("Value %s is type %s not type %s" % (value, type(value), targettype)) return ast.literal_eval(value) elif targettype == 'float': try: return float(value) except Exception: log.warn("Value %s is type %s not type %s" % (value, type(value), targettype)) return ast.literal_eval(value) elif targettype == 'simplelist': if value.startswith('[') and value.endswith(']'): value = value[1:len(value) - 1].strip() elif not value.strip(): return [] return list(value.split(',')) else: log.trace('parsing value as %s: %s', targettype, value) return ast.literal_eval(value)
def get_typed_value(value, schema_entry=None, targettype=None): """ Performs a value type conversion according to a schema specified target type. Supports simplelist and parsedict special type parsing. """ targettype = targettype or schema_entry["type"] if schema_entry and 'enum_type' in schema_entry: enum_clzz = getattr(objects, schema_entry['enum_type']) return enum_clzz._value_map[value] elif targettype == 'str': return str(value) elif targettype == 'bool': if value in ('TRUE', 'True', 'true', '1', 1, True): return True if value in ('FALSE', 'False', 'false', '0', 0, '', None, False): return False raise iex.BadRequest("Value %s is no bool" % value) elif targettype == 'int': try: return int(value) except Exception: log.warn("Value %s is type %s not type %s" % (value, type(value), targettype)) return ast.literal_eval(value) elif targettype == 'float': try: return float(value) except Exception: log.warn("Value %s is type %s not type %s" % (value, type(value), targettype)) return ast.literal_eval(value) elif targettype == 'simplelist': return parse_list(value) elif targettype == 'parsedict': return parse_dict(str(value)) else: log.trace('parsing value as %s: %s', targettype, value) return ast.literal_eval(value)
def load_ion(self, path, scenario): if not path: raise iex.BadRequest("Must provide path") log.info("Start preloading from path=%s" % path) categories = [ 'User', 'Org', 'UserRole', 'PlatformModel', 'InstrumentModel', 'Observatory', 'Subsite', 'PlatformSite', 'InstrumentSite', 'StreamDefinition', 'PlatformDevice', 'InstrumentDevice', 'InstrumentAgent', 'InstrumentAgentInstance', 'DataProcessDefinition', #'IngestionConfiguration', 'DataProduct', 'DataProcess', 'DataProductLink', 'Attachment', ] self.path = path self.obj_classes = {} self.resource_ids = {} self.user_ids = {} self._preload_ids() if self.loadooi: self.extract_ooi_assets(path) if self.loadui: self.load_ui(path) for category in categories: row_do, row_skip = 0, 0 catfunc_ooi = getattr(self, "_load_%s_OOI" % category, None) if self.loadooi and catfunc_ooi: catfunc_ooi() catfunc = getattr(self, "_load_%s" % category) filename = "%s/%s.csv" % (path, category) log.info("Loading category %s from file %s" % (category, filename)) try: with open(filename, "rb") as csvfile: reader = self._get_csv_reader(csvfile) for row in reader: # Check if scenario applies rowsc = row[self.COL_SCENARIO] if not scenario in rowsc: row_skip += 1 continue row_do += 1 catfunc(row) except IOError, ioe: log.warn("Resource category file %s error: %s" % (filename, str(ioe))) log.info("Loaded category %s: %d rows imported, %d rows skipped" % (category, row_do, row_skip))
def _register_id(self, alias, resid): if alias in self.resource_ids: raise iex.BadRequest("ID alias %s used twice" % alias) self.resource_ids[alias] = resid log.info("Added resource alias=%s to id=%s" % (alias, resid))
def load_ui(self, path, replace=True, specs_only=True, specs_path=None): """ @brief Import UI definitions from the FileMakerPro database exported CVS files to ION resource objects. """ if not path: raise iex.BadRequest("Must provide path") if path.startswith("http"): self._get_ui_files(path) self.read_from_local = False log.info("Start parsing UI assets from path=%s" % path) categories = [ ('Graphic Type.csv', 'GraphicType'), ('Graphic.csv', 'Graphic'), ('Information Level.csv', 'InformationLevel'), ('Resource Type.csv', 'ResourceType'), ('Object Type.csv', 'ObjectType'), ('Resource Attribute.csv', 'ResourceAttribute'), ('Object Field.csv', 'ObjectField'), ('Screen Element.csv', 'ScreenElement'), ('Screen Label.csv', 'ScreenLabel'), ('Help Tag.csv', 'HelpTag'), ('Message String.csv', 'MessageString'), ('State.csv', 'State'), ('Widget.csv', 'Widget'), ('_jt_ScreenElement_InformationElement.csv', 'ScreenElementInformationElement'), ('_jt_ScreenElement_ScreenElement.csv', 'EmbeddedScreenElement'), ] self.uiid_prefix = uuid.uuid4().hex[:9] + "_" self.ui_objs = {} self.ui_obj_by_id = {} self.ref_assocs = [] self.ui_assocs = [] self.warnings = [] self.abort = False self.counter = 0 for fname, category in categories: row_do, row_skip = 0, 0 catfunc = getattr(self, "_loadui_%s" % category) try: if self.read_from_local: # Read from file handle filename = "%s/%s" % (path, fname) log.info("Loading UI category %s from file %s" % (category, filename)) try: with open(filename, "rbU") as csvfile: reader = csv.DictReader(csvfile, delimiter=',') for row in reader: catfunc(row) self.counter += 1 row_do += 1 except IOError, ioe: log.warn("UI category file %s error: %s" % (filename, str(ioe))) self.abort = True else: # Read from string if fname in self.files: log.info( "Loading UI category %s from retrieved file %s" % (category, fname)) csvfile = self.files[fname] # This is a hack to be able to read from string csvfile = csvfile.splitlines() reader = csv.DictReader(csvfile, delimiter=',') for row in reader: catfunc(row) self.counter += 1 row_do += 1 else: log.warn("UI category %s has no file %s" % (category, fname)) self.abort = True
def extract_ooi_assets(self): """ Parses SAF Instrument Application export CSV files into intermediate memory structures. This information can later be loaded in to actual load_ion() function. """ if not self.asset_path: raise iex.BadRequest( "Must provide path for assets: path=dir or assets=dir") if self.asset_path.startswith('http'): raise iex.BadRequest( 'Asset path must be local directory, not URL: ' + self.asset_path) log.info("Parsing OOI assets from path=%s", self.asset_path) categories = [ # Mapping spreadsheet early 'NodeTypes', # Core concept attributes 'AttributeReportArrays', 'AttributeReportClass', 'AttributeReportDataProducts', 'AttributeReportFamilies', 'AttributeReportMakeModel', 'AttributeReportNodes', 'AttributeReportPorts', 'AttributeReportReferenceDesignator', 'AttributeReportSeries', 'AttributeReportSites', 'AttributeReportSubseries', 'AttributeReportSubsites', # Additional attributes and links taken from aggregate reports 'InstrumentCatalogFull', 'DataQCLookupTables', 'DataProductSpreadsheet', 'AllSensorTypeCounts', # Tabs from the mapping spreadsheet 'Arrays', 'Sites', 'Subsites', 'Nodes', #'Platforms', 'NodeTypes', 'PlatformAgentTypes' ] # Holds the object representations of parsed OOI assets by type self.ooi_objects = {} # Holds a list of attribute names of OOI assets by type self.ooi_obj_attrs = {} self.warnings = [] self.csv_files = None # Load OOIResourceMappings.xlsx mapping_file = self.asset_path + "/OOIResourceMappings.xlsx" if os.path.exists(mapping_file): with open(mapping_file, "rb") as f: preload_doc_str = f.read() log.debug("Loaded %s mapping file, size=%s", mapping_file, len(preload_doc_str)) xls_parser = XLSParser() self.csv_files = xls_parser.extract_csvs(preload_doc_str) for category in categories: row_do, row_skip = 0, 0 catfunc = getattr(self, "_parse_%s" % category) filename = "%s/%s.csv" % (self.asset_path, category) log.debug("Loading category %s from file %s", category, filename) try: if category in self.csv_files: csv_doc = self.csv_files[category] reader = csv.DictReader(csv_doc, delimiter=',') filename = mapping_file + ":" + category else: csvfile = open(filename, "rb") for i in xrange(9): # Skip the first rows, because they are garbage csvfile.readline() reader = csv.DictReader(csvfile, delimiter=',') for row in reader: row_do += 1 catfunc(row) except IOError as ioe: log.warn("OOI asset file %s error: %s" % (filename, str(ioe))) log.debug("Loaded assets %s: %d rows read" % (category, row_do)) # Post processing self._post_process() # Do some validation checking self._perform_ooi_checks() if self.warnings: log.warn("WARNINGS:\n%s", "\n".join(["%s: %s" % (a, b) for a, b in self.warnings])) for ot, oo in self.ooi_objects.iteritems(): log.info("Type %s has %s entries", ot, len(oo))