def LoadFSDDataInPython(dataResPath, schemaResPath=None, optimize=None, cacheSize=100): import schemaOptimizer if schemaResPath is not None: with open(schemaResPath, 'r') as schemaFile: schema = persistence.LoadSchema(schemaFile.read()) if optimize: schema = persistence.GetUnOptimizedRuntimeSchema(schema) schema = schemaOptimizer.OptimizeSchema(schema, optimize) else: schema = None dataFile = open(dataResPath, 'rb') if schema is None: peekSchema, size = GetEmbeddedSchemaAndSizeFromFile(dataFile) dataFile.seek(0) else: peekSchema = schema if peekSchema['type'] == 'dict' and peekSchema.get('buildIndex', False): return LoadIndexFromFile(dataFile, schema, cacheSize, path=FsdDataPathObject('<file %s>' % dataResPath)) else: s = dataFile.read() log.info('Loading FSD data file %s into memory. %s', dataResPath, sizeof_fmt(len(s))) dataFile.close() return LoadFromString(s, schema, path=FsdDataPathObject('<file %s>' % dataResPath))
def LoadReferencedSchema(reference, pathRoot): schemaFilePath = reference.get('schemaFile') schemaName = reference.get('schema') print 'loading', schemaName, 'from', schemaFilePath if pathRoot is None: raise Exception( 'Could not optimize schema without a path root with which to load referenced schemas' ) with open(os.path.join(pathRoot, schemaFilePath), 'r') as schemaFile: schemas = persistence.LoadSchema(schemaFile)['schemas'] schema = schemas[schemaName] return schema
def LoadFSDDataForCFG(dataResPath, schemaResPath=None, optimize=True, cacheSize=100): import blue import schemaOptimizer res = blue.ResFile() if schemaResPath is not None: if not res.Open(schemaResPath): log.error('Could not load FSD data schema: %s', schemaResPath) else: schema = persistence.LoadSchema(res.Read()) if optimize: schema = persistence.GetUnOptimizedRuntimeSchema(schema) schema = schemaOptimizer.OptimizeSchema( schema, boot.role.capitalize()) else: schema = None if not res.Open(dataResPath): log.error('Could not load FSD data file: %s', dataResPath) else: wrappedRes = BlueResFileIOWrapper(res) if schema is None: peekSchema, size = GetEmbeddedSchemaAndSizeFromFile(wrappedRes) wrappedRes.seek(0) else: peekSchema = schema if peekSchema['type'] == 'dict' and peekSchema.get( 'buildIndex', False): return LoadIndexFromFile(wrappedRes, schema, cacheSize, path=FsdDataPathObject('<file %s>' % dataResPath)) s = res.Read() log.info('Loading FSD data file %s into memory. %s', dataResPath, sizeof_fmt(len(s))) return LoadFromString(s, schema, path=FsdDataPathObject('<file %s>' % dataResPath))
def ValidateReference(schemaNode, o, path, state): referenceInfo = schemaNode.get('reference') if not state.get('validateReferences', False): return [] if 'database' in referenceInfo and dbConnectionForValidationAvailable: serverDB = (referenceInfo['server'], referenceInfo['database']) if serverDB in dbConnections: connection, cursor = dbConnections[serverDB] else: try: connection = pyodbc.connect( 'DRIVER={SQL Server};SERVER=%s;DATABASE=%s;Trusted_Connection=yes' % serverDB) cursor = connection.cursor() dbConnections[serverDB] = (connection, cursor) except pyodbc.Error: print 'ERROR: Could not connect to server %s and database %s to validate external reference' % serverDB connection = None cursor = None dbConnections[serverDB] = (None, None) return [] if connection is None: return [] rows = cursor.execute( 'SELECT * from %(table)s WHERE %(key)s = ?' % referenceInfo, o) if rows.fetchone() is None: errorString = '%(path)s: ' + referenceInfo['errorMessage'] return [ ExternalReferenceError( path, errorString % { 'path': path, 'key': str(o), 'server': referenceInfo['server'], 'database': referenceInfo['database'], 'table': referenceInfo['table'] }) ] elif 'schema' in referenceInfo and 'branchRoot' in state: if 'openSchemas' not in state: state['openSchemas'] = {} state['openData'] = {} otherSchemaPath = os.path.abspath( os.path.join(state['branchRoot'], referenceInfo['schema'])) if 'schemaCache' in state and otherSchemaPath in state['schemaCache']: otherSchema = state['schemaCache'].Get(otherSchemaPath) elif otherSchemaPath not in state['openSchemas']: with open(otherSchemaPath, 'r') as otherSchemaFile: otherSchema = state['openSchemas'][ otherSchemaPath] = persistence.LoadSchema(otherSchemaFile) else: otherSchema = state['openSchemas'][otherSchemaPath] editorSchema = persistence.GetEditorSchema(otherSchema) editorFile = otherSchema['editorFile'] % {'key': o} fullEditorFilePath = os.path.abspath( os.path.join(state['branchRoot'], editorFile)) if 'dataCache' in state and fullEditorFilePath in state['dataCache']: otherData = state['dataCache'].Get(fullEditorFilePath) elif fullEditorFilePath not in state['openData']: with open(fullEditorFilePath, 'r') as otherDataFile: otherData = state['openData'][fullEditorFilePath] = yaml.load( otherDataFile) else: otherData = state['openData'][fullEditorFilePath] if o not in otherData: errorString = '%(path)s: ' + referenceInfo['errorMessage'] return [ ExternalReferenceError( path, errorString % { 'path': path, 'key': str(o), 'dataFile': fullEditorFilePath }) ] return []
def ValidateReference(schemaNode, o, path, state): referenceInfo = schemaNode.get('reference') if not state.get('validateReferences', False): return [] if 'database' in referenceInfo and dbConnectionForValidationAvailable: serverDB = (referenceInfo['server'], referenceInfo['database']) if serverDB in dbConnections: connection, cursor = dbConnections[serverDB] else: try: print 'Connecting to DB' connection = pyodbc.connect( 'DRIVER={SQL Server};SERVER=%s;DATABASE=%s;Trusted_Connection=yes' % serverDB) cursor = connection.cursor() dbConnections[serverDB] = (connection, cursor) except pyodbc.Error: print 'ERROR: Could not connect to server %s and database %s to validate external reference. It may be that you do not have domain account access to read this DB.' % serverDB connection = None cursor = None dbConnections[serverDB] = (None, None) return [] if connection is None: return [] if 'cachedKeySets' not in state: state['cachedKeySets'] = {} cachedKey = (referenceInfo['key'], referenceInfo['table']) if cachedKey not in state['cachedKeySets']: setOfKeys = GetSetOfKeysFromBSDTable(cursor, referenceInfo['key'], referenceInfo['table']) state['cachedKeySets'][cachedKey] = setOfKeys else: setOfKeys = state['cachedKeySets'][cachedKey] if o not in setOfKeys: errorString = '%(path)s: ' + referenceInfo['errorMessage'] return [ ExternalReferenceError( path, errorString % { 'path': path, 'key': str(o), 'server': referenceInfo['server'], 'database': referenceInfo['database'], 'table': referenceInfo['table'] }) ] elif 'schema' in referenceInfo and 'branchRoot' in state: if 'openSchemas' not in state: state['openSchemas'] = {} state['openData'] = {} otherSchemaPath = os.path.abspath( os.path.join(state['branchRoot'], referenceInfo['schema'])) if 'schemaCache' in state and otherSchemaPath in state['schemaCache']: otherSchema = state['schemaCache'].Get(otherSchemaPath) elif otherSchemaPath not in state['openSchemas']: with open(otherSchemaPath, 'r') as otherSchemaFile: otherSchema = state['openSchemas'][ otherSchemaPath] = persistence.LoadSchema(otherSchemaFile) else: otherSchema = state['openSchemas'][otherSchemaPath] editorSchema = persistence.GetEditorSchema(otherSchema) editorFile = otherSchema['editorFile'] % {'key': o} fullEditorFilePath = os.path.abspath( os.path.join(state['branchRoot'], editorFile)) if 'dataCache' in state and fullEditorFilePath in state['dataCache']: otherData = state['dataCache'].Get(fullEditorFilePath) elif fullEditorFilePath not in state['openData']: with open(fullEditorFilePath, 'r') as otherDataFile: otherData = state['openData'][fullEditorFilePath] = yaml.load( otherDataFile) else: otherData = state['openData'][fullEditorFilePath] if o not in otherData: errorString = '%(path)s: ' + referenceInfo['errorMessage'] return [ ExternalReferenceError( path, errorString % { 'path': path, 'key': str(o), 'dataFile': fullEditorFilePath }) ] return []