def getValidConnectionDictionaryFromConnectionString( connectionString ): # first we parse the connection string in order to get the service name and the account name protocolDictionary = service.getProtocolServiceAndAccountFromConnectionString( connectionString ) if not protocolDictionary: raise conditionError.ConditionError( "Invalid connection string: \"%s\"" %( connectionString, ) ) # next, we check whether the connection string is valid connectionDictionary = getValidConnectionDictionary( protocolDictionary ) if not connectionDictionary: #the oracle or frontier service provided was not found raise conditionError.ConditionError( "The service provided in the connection string \"%s\" was not found in the available ones." %( connectionString, ) ) return connectionDictionary
def getValidConnectionDictionaryFromConnectionString(connectionString): # first we parse the connection string in order to get the service name and the account name protocolDictionary = service.getProtocolServiceAndAccountFromConnectionString( connectionString) if not protocolDictionary: raise conditionError.ConditionError( "Invalid connection string: \"%s\"" % (connectionString, )) # next, we check whether the connection string is valid connectionDictionary = getValidConnectionDictionary(protocolDictionary) if not connectionDictionary: #the oracle or frontier service provided was not found raise conditionError.ConditionError( "The service provided in the connection string \"%s\" was not found in the available ones." % (connectionString, )) return connectionDictionary
def checkConnectionString( connectionString, forUpdating = False ): if connectionString.startswith( 'sqlite_file:' ): # todo FIXME: check that the file is existing return True protocolDictionary = service.getProtocolServiceAndAccountFromConnectionString( connectionString ) if not protocolDictionary: raise conditionError.ConditionError( "Invalid connection string: \"%s\"" %( connectionString, ) ) if forUpdating and protocolDictionary[ 'protocol' ] == "frontier": # we are explicitly excluding Frontier services, as they are read-only return False connectionDictionary = getValidConnectionDictionary( protocolDictionary ) if not connectionDictionary: raise conditionError.ConditionError( "The service provided in the connection string \"%s\" was not found in the available ones." %( connectionString, ) ) if forUpdating and ( connectionDictionary[ 'service_type' ] == "OfflineProduction" or connectionDictionary[ 'service_type' ] == "OfflineArchive" ): # we are explicitly excluding read-only services return False return True
def checkConnectionString(connectionString, forUpdating=False): if connectionString.startswith('sqlite_file:'): # todo FIXME: check that the file is existing return True protocolDictionary = service.getProtocolServiceAndAccountFromConnectionString( connectionString) if not protocolDictionary: raise conditionError.ConditionError( "Invalid connection string: \"%s\"" % (connectionString, )) if forUpdating and protocolDictionary['protocol'] == "frontier": # we are explicitly excluding Frontier services, as they are read-only return False connectionDictionary = getValidConnectionDictionary(protocolDictionary) if not connectionDictionary: raise conditionError.ConditionError( "The service provided in the connection string \"%s\" was not found in the available ones." % (connectionString, )) if forUpdating and ( connectionDictionary['service_type'] == "OfflineProduction" or connectionDictionary['service_type'] == "OfflineArchive"): # we are explicitly excluding read-only services return False return True
def checkContents(fileHash, dataPath, metadata, backend): '''Checks whether the data and metadata are correct. data is the filename of the sqlite file. metadata is a string with the metadata file itself. Note: Update the wizard on the upload.py script if the structure changes. ''' logging.debug('check::checkContents(%s, %s, %s)', fileHash, dataPath, repr(metadata)) logging.info('checkContents(): %s: Checking metadata structure...', fileHash) workflows = (u'offline', u'hlt', u'express', u'prompt', u'pcl') structure = { u'destinationDatabase': (True, unicode), u'inputTag': (True, unicode), u'since': (True, (int, type(None))), u'emails': (False, [unicode]), u'userText': (True, unicode), u'destinationTags': (True, { unicode: { u'synchronizeTo': (True, workflows), u'dependencies': (False, { unicode: workflows, }), }, }) } try: typeMatch.match(structure, metadata) except typeMatch.MatchError as e: raise dropBox.DropBoxError('In the metadata, ' + str(e)) logging.info( 'checkContents(): %s: Checking data with respect to metadata...', fileHash) db = conditionDatabase.ConditionDBChecker('sqlite_file:%s' % dataPath, '') try: # Corrupted file try: tags = db.getAllTags() except conditionError.ConditionError as e: raise dropBox.DropBoxError( 'The file is corrupted, as it was not possible to get the list of tags inside it.' ) # Empty file if not tags: raise dropBox.DropBoxError( 'The file does not contain any tags, so it is likely not hosting any Condition payloads.' ) # Wrong input tag if metadata['inputTag'] not in tags: raise dropBox.DropBoxError( 'The input tag "%s" is not in the input SQLite file.' % metadata['inputTag']) # Unsupported service destinationDatabase = metadata['destinationDatabase'] if not destinationDatabase.startswith('oracle:'): raise dropBox.DropBoxError('Oracle is the only supported service.') # Invalid connection string connectionDictionary = service.getProtocolServiceAndAccountFromConnectionString( destinationDatabase) if connectionDictionary is None: raise dropBox.DropBoxError('The connection string is not correct.') # Destination database not supported allowedServices = config.allowedServices[backend] if allowedServices is not None: ok = False for allowedService in allowedServices: if connectionDictionary[ 'service'] in databaseServices.services[ allowedService]['oracle']: ok = True if not ok: raise dropBox.DropBoxError( 'The destination database is not supported.') # Invalid since since = metadata['since'] if since is not None: firstSince = db.iovSequence(metadata['inputTag']).firstSince() if since < firstSince: raise dropBox.DropBoxError( 'The since value "%d" specified in the metadata cannot be smaller than the first IOV since "%d"' % (since, firstSince)) # Invalid synchronizations gtHandle = globalTagHandler.GlobalTagHandler( service.getFrontierConnectionString( service.secrets['connections']['dev']['global_tag']), service.getCxOracleConnectionString( service.secrets['connections']['dev']['run_control']), service.getFrontierConnectionString( service.secrets['connections']['dev']['run_info']), 'runinfo_start_31X_hlt', 'runinfo_31X_hlt', '', 'https://samir-wmcore.cern.ch/t0wmadatasvc/replay', 30, 3, 90, ) productionGTsDict = config.productionGlobalTags for tag, synchronizationDict in metadata['destinationTags'].items(): checkSynchronization(synchronizationDict['synchronizeTo'], destinationDatabase, tag, gtHandle, productionGTsDict) for dependentTag, synchronizeTo in synchronizationDict.get( 'dependencies', {}).items(): checkSynchronization(synchronizeTo, destinationDatabase, dependentTag, gtHandle, productionGTsDict) # firstConditionSafeRun from Tier-0 not available -- only checked # if it is going to be used, i.e. if there is any synchronization # to prompt or pcl. usingFcsr = False for tag, synchronizationDict in metadata['destinationTags'].items(): if synchronizationDict['synchronizeTo'] in ('prompt', 'pcl'): usingFcsr = True break for dependentTag, synchronizeTo in synchronizationDict.get( 'dependencies', {}).items(): if synchronizeTo in ('prompt', 'pcl'): usingFcsr = True break if usingFcsr: try: tier0.Tier0Handler(config.tier0URL, 5, 2, 5, None, False).getFirstSafeRun() except ValueError: # We got an answer but it is invalid. So far this usually means # "None" which is not JSON, when the Tier0 is stopped. raise dropBox.DropBoxError( 'Impossible to upload to synchronize to prompt or pcl while Tier-0 is returning an invalid First Condition Safe Run. %s' % config.fcsrProblemMessage) except tier0.Tier0Error: # Impossible to get anything from the server after retries, # i.e. unreachable, so no data. raise dropBox.DropBoxError( 'Impossible to upload to synchronize to prompt or pcl while Tier-0 is unreachable. Try again after a bit. If this does not help: %s' % config.fcsrProblemMessage) finally: db.close()
def checkContents(fileHash, dataPath, metadata, backend): '''Checks whether the data and metadata are correct. data is the filename of the sqlite file. metadata is a string with the metadata file itself. Note: Update the wizard on the upload.py script if the structure changes. ''' logging.debug('check::checkContents(%s, %s, %s)', fileHash, dataPath, repr(metadata)) logging.info('checkContents(): %s: Checking metadata structure...', fileHash) workflows = (u'offline', u'hlt', u'express', u'prompt', u'pcl') structure = { u'destinationDatabase': (True, unicode), u'inputTag': (True, unicode), u'since': (True, (int, type(None))), u'emails': (False, [unicode]), u'userText': (True, unicode), u'destinationTags': (True, { unicode: { u'synchronizeTo': (True, workflows), u'dependencies': (False, { unicode: workflows, }), }, }) } try: typeMatch.match(structure, metadata) except typeMatch.MatchError as e: raise dropBox.DropBoxError('In the metadata, ' + str(e)) logging.info('checkContents(): %s: Checking data with respect to metadata...', fileHash) db = conditionDatabase.ConditionDBChecker('sqlite_file:%s' % dataPath, '') try: # Corrupted file try: tags = db.getAllTags() except conditionError.ConditionError as e: raise dropBox.DropBoxError('The file is corrupted, as it was not possible to get the list of tags inside it.') # Empty file if not tags: raise dropBox.DropBoxError('The file does not contain any tags, so it is likely not hosting any Condition payloads.') # Wrong input tag if metadata['inputTag'] not in tags: raise dropBox.DropBoxError('The input tag "%s" is not in the input SQLite file.' % metadata['inputTag']) # Unsupported service destinationDatabase = metadata['destinationDatabase'] if not destinationDatabase.startswith('oracle:'): raise dropBox.DropBoxError('Oracle is the only supported service.') # Invalid connection string connectionDictionary = service.getProtocolServiceAndAccountFromConnectionString(destinationDatabase) if connectionDictionary is None: raise dropBox.DropBoxError('The connection string is not correct.') # Destination database not supported allowedServices = config.allowedServices[backend] if allowedServices is not None: ok = False for allowedService in allowedServices: if connectionDictionary['service'] in databaseServices.services[allowedService]['oracle']: ok = True if not ok: raise dropBox.DropBoxError('The destination database is not supported.') # Invalid since since = metadata['since'] if since is not None: firstSince = db.iovSequence(metadata['inputTag']).firstSince() if since < firstSince: raise dropBox.DropBoxError('The since value "%d" specified in the metadata cannot be smaller than the first IOV since "%d"' % (since, firstSince)) # Invalid synchronizations gtHandle = globalTagHandler.GlobalTagHandler( service.getFrontierConnectionString(service.secrets['connections']['dev']['global_tag']), service.getCxOracleConnectionString(service.secrets['connections']['dev']['run_control']), service.getFrontierConnectionString(service.secrets['connections']['dev']['run_info']), 'runinfo_start_31X_hlt', 'runinfo_31X_hlt', '', 'https://samir-wmcore.cern.ch/t0wmadatasvc/replay', 30, 3, 90, ) productionGTsDict = config.productionGlobalTags for tag, synchronizationDict in metadata['destinationTags'].items(): checkSynchronization(synchronizationDict['synchronizeTo'], destinationDatabase, tag, gtHandle, productionGTsDict) for dependentTag, synchronizeTo in synchronizationDict.get('dependencies', {}).items(): checkSynchronization(synchronizeTo, destinationDatabase, dependentTag, gtHandle, productionGTsDict) # firstConditionSafeRun from Tier-0 not available -- only checked # if it is going to be used, i.e. if there is any synchronization # to prompt or pcl. usingFcsr = False for tag, synchronizationDict in metadata['destinationTags'].items(): if synchronizationDict['synchronizeTo'] in ('prompt', 'pcl'): usingFcsr = True break for dependentTag, synchronizeTo in synchronizationDict.get('dependencies', {}).items(): if synchronizeTo in ('prompt', 'pcl'): usingFcsr = True break if usingFcsr: try: tier0.Tier0Handler(config.tier0URL, 5, 2, 5, None, False).getFirstSafeRun() except ValueError: # We got an answer but it is invalid. So far this usually means # "None" which is not JSON, when the Tier0 is stopped. raise dropBox.DropBoxError('Impossible to upload to synchronize to prompt or pcl while Tier-0 is returning an invalid First Condition Safe Run. %s' % config.fcsrProblemMessage) except tier0.Tier0Error: # Impossible to get anything from the server after retries, # i.e. unreachable, so no data. raise dropBox.DropBoxError('Impossible to upload to synchronize to prompt or pcl while Tier-0 is unreachable. Try again after a bit. If this does not help: %s' % config.fcsrProblemMessage) finally: db.close()