def getValidConnectionDictionaryFromConnectionString( connectionString ):
# first we parse the connection string in order to get the service name and the account name
protocolDictionary = service.getProtocolServiceAndAccountFromConnectionString( connectionString )
if not protocolDictionary:
raise conditionError.ConditionError( "Invalid connection string: \"%s\"" %( connectionString, ) )
# next, we check whether the connection string is valid
connectionDictionary = getValidConnectionDictionary( protocolDictionary )
if not connectionDictionary: #the oracle or frontier service provided was not found
raise conditionError.ConditionError( "The service provided in the connection string \"%s\" was not found in the available ones." %( connectionString, ) )
return connectionDictionary
def getValidConnectionDictionaryFromConnectionString(connectionString):
# first we parse the connection string in order to get the service name and the account name
protocolDictionary = service.getProtocolServiceAndAccountFromConnectionString(
connectionString)
if not protocolDictionary:
raise conditionError.ConditionError(
"Invalid connection string: \"%s\"" % (connectionString, ))
# next, we check whether the connection string is valid
connectionDictionary = getValidConnectionDictionary(protocolDictionary)
if not connectionDictionary: #the oracle or frontier service provided was not found
raise conditionError.ConditionError(
"The service provided in the connection string \"%s\" was not found in the available ones."
% (connectionString, ))
return connectionDictionary
def checkConnectionString( connectionString, forUpdating = False ):
if connectionString.startswith( 'sqlite_file:' ):
# todo FIXME: check that the file is existing
return True
protocolDictionary = service.getProtocolServiceAndAccountFromConnectionString( connectionString )
if not protocolDictionary:
raise conditionError.ConditionError( "Invalid connection string: \"%s\"" %( connectionString, ) )
if forUpdating and protocolDictionary[ 'protocol' ] == "frontier":
# we are explicitly excluding Frontier services, as they are read-only
return False
connectionDictionary = getValidConnectionDictionary( protocolDictionary )
if not connectionDictionary:
raise conditionError.ConditionError( "The service provided in the connection string \"%s\" was not found in the available ones." %( connectionString, ) )
if forUpdating and ( connectionDictionary[ 'service_type' ] == "OfflineProduction" or connectionDictionary[ 'service_type' ] == "OfflineArchive" ):
# we are explicitly excluding read-only services
return False
return True
def checkConnectionString(connectionString, forUpdating=False):
if connectionString.startswith('sqlite_file:'):
# todo FIXME: check that the file is existing
return True
protocolDictionary = service.getProtocolServiceAndAccountFromConnectionString(
connectionString)
if not protocolDictionary:
raise conditionError.ConditionError(
"Invalid connection string: \"%s\"" % (connectionString, ))
if forUpdating and protocolDictionary['protocol'] == "frontier":
# we are explicitly excluding Frontier services, as they are read-only
return False
connectionDictionary = getValidConnectionDictionary(protocolDictionary)
if not connectionDictionary:
raise conditionError.ConditionError(
"The service provided in the connection string \"%s\" was not found in the available ones."
% (connectionString, ))
if forUpdating and (
connectionDictionary['service_type'] == "OfflineProduction"
or connectionDictionary['service_type'] == "OfflineArchive"):
# we are explicitly excluding read-only services
return False
return True
def checkContents(fileHash, dataPath, metadata, backend):
'''Checks whether the data and metadata are correct.
data is the filename of the sqlite file.
metadata is a string with the metadata file itself.
Note: Update the wizard on the upload.py script if the structure changes.
'''
logging.debug('check::checkContents(%s, %s, %s)', fileHash, dataPath,
repr(metadata))
logging.info('checkContents(): %s: Checking metadata structure...',
fileHash)
workflows = (u'offline', u'hlt', u'express', u'prompt', u'pcl')
structure = {
u'destinationDatabase': (True, unicode),
u'inputTag': (True, unicode),
u'since': (True, (int, type(None))),
u'emails': (False, [unicode]),
u'userText': (True, unicode),
u'destinationTags': (True, {
unicode: {
u'synchronizeTo': (True, workflows),
u'dependencies': (False, {
unicode: workflows,
}),
},
})
}
try:
typeMatch.match(structure, metadata)
except typeMatch.MatchError as e:
raise dropBox.DropBoxError('In the metadata, ' + str(e))
logging.info(
'checkContents(): %s: Checking data with respect to metadata...',
fileHash)
db = conditionDatabase.ConditionDBChecker('sqlite_file:%s' % dataPath, '')
try:
# Corrupted file
try:
tags = db.getAllTags()
except conditionError.ConditionError as e:
raise dropBox.DropBoxError(
'The file is corrupted, as it was not possible to get the list of tags inside it.'
)
# Empty file
if not tags:
raise dropBox.DropBoxError(
'The file does not contain any tags, so it is likely not hosting any Condition payloads.'
)
# Wrong input tag
if metadata['inputTag'] not in tags:
raise dropBox.DropBoxError(
'The input tag "%s" is not in the input SQLite file.' %
metadata['inputTag'])
# Unsupported service
destinationDatabase = metadata['destinationDatabase']
if not destinationDatabase.startswith('oracle:'):
raise dropBox.DropBoxError('Oracle is the only supported service.')
# Invalid connection string
connectionDictionary = service.getProtocolServiceAndAccountFromConnectionString(
destinationDatabase)
if connectionDictionary is None:
raise dropBox.DropBoxError('The connection string is not correct.')
# Destination database not supported
allowedServices = config.allowedServices[backend]
if allowedServices is not None:
ok = False
for allowedService in allowedServices:
if connectionDictionary[
'service'] in databaseServices.services[
allowedService]['oracle']:
ok = True
if not ok:
raise dropBox.DropBoxError(
'The destination database is not supported.')
# Invalid since
since = metadata['since']
if since is not None:
firstSince = db.iovSequence(metadata['inputTag']).firstSince()
if since < firstSince:
raise dropBox.DropBoxError(
'The since value "%d" specified in the metadata cannot be smaller than the first IOV since "%d"'
% (since, firstSince))
# Invalid synchronizations
gtHandle = globalTagHandler.GlobalTagHandler(
service.getFrontierConnectionString(
service.secrets['connections']['dev']['global_tag']),
service.getCxOracleConnectionString(
service.secrets['connections']['dev']['run_control']),
service.getFrontierConnectionString(
service.secrets['connections']['dev']['run_info']),
'runinfo_start_31X_hlt',
'runinfo_31X_hlt',
'',
'https://samir-wmcore.cern.ch/t0wmadatasvc/replay',
30,
3,
90,
)
productionGTsDict = config.productionGlobalTags
for tag, synchronizationDict in metadata['destinationTags'].items():
checkSynchronization(synchronizationDict['synchronizeTo'],
destinationDatabase, tag, gtHandle,
productionGTsDict)
for dependentTag, synchronizeTo in synchronizationDict.get(
'dependencies', {}).items():
checkSynchronization(synchronizeTo, destinationDatabase,
dependentTag, gtHandle, productionGTsDict)
# firstConditionSafeRun from Tier-0 not available -- only checked
# if it is going to be used, i.e. if there is any synchronization
# to prompt or pcl.
usingFcsr = False
for tag, synchronizationDict in metadata['destinationTags'].items():
if synchronizationDict['synchronizeTo'] in ('prompt', 'pcl'):
usingFcsr = True
break
for dependentTag, synchronizeTo in synchronizationDict.get(
'dependencies', {}).items():
if synchronizeTo in ('prompt', 'pcl'):
usingFcsr = True
break
if usingFcsr:
try:
tier0.Tier0Handler(config.tier0URL, 5, 2, 5, None,
False).getFirstSafeRun()
except ValueError:
# We got an answer but it is invalid. So far this usually means
# "None" which is not JSON, when the Tier0 is stopped.
raise dropBox.DropBoxError(
'Impossible to upload to synchronize to prompt or pcl while Tier-0 is returning an invalid First Condition Safe Run. %s'
% config.fcsrProblemMessage)
except tier0.Tier0Error:
# Impossible to get anything from the server after retries,
# i.e. unreachable, so no data.
raise dropBox.DropBoxError(
'Impossible to upload to synchronize to prompt or pcl while Tier-0 is unreachable. Try again after a bit. If this does not help: %s'
% config.fcsrProblemMessage)
finally:
db.close()
def checkContents(fileHash, dataPath, metadata, backend):
'''Checks whether the data and metadata are correct.
data is the filename of the sqlite file.
metadata is a string with the metadata file itself.
Note: Update the wizard on the upload.py script if the structure changes.
'''
logging.debug('check::checkContents(%s, %s, %s)', fileHash, dataPath, repr(metadata))
logging.info('checkContents(): %s: Checking metadata structure...', fileHash)
workflows = (u'offline', u'hlt', u'express', u'prompt', u'pcl')
structure = {
u'destinationDatabase': (True, unicode),
u'inputTag': (True, unicode),
u'since': (True, (int, type(None))),
u'emails': (False, [unicode]),
u'userText': (True, unicode),
u'destinationTags': (True, {
unicode: {
u'synchronizeTo': (True, workflows),
u'dependencies': (False, {
unicode: workflows,
}),
},
})
}
try:
typeMatch.match(structure, metadata)
except typeMatch.MatchError as e:
raise dropBox.DropBoxError('In the metadata, ' + str(e))
logging.info('checkContents(): %s: Checking data with respect to metadata...', fileHash)
db = conditionDatabase.ConditionDBChecker('sqlite_file:%s' % dataPath, '')
try:
# Corrupted file
try:
tags = db.getAllTags()
except conditionError.ConditionError as e:
raise dropBox.DropBoxError('The file is corrupted, as it was not possible to get the list of tags inside it.')
# Empty file
if not tags:
raise dropBox.DropBoxError('The file does not contain any tags, so it is likely not hosting any Condition payloads.')
# Wrong input tag
if metadata['inputTag'] not in tags:
raise dropBox.DropBoxError('The input tag "%s" is not in the input SQLite file.' % metadata['inputTag'])
# Unsupported service
destinationDatabase = metadata['destinationDatabase']
if not destinationDatabase.startswith('oracle:'):
raise dropBox.DropBoxError('Oracle is the only supported service.')
# Invalid connection string
connectionDictionary = service.getProtocolServiceAndAccountFromConnectionString(destinationDatabase)
if connectionDictionary is None:
raise dropBox.DropBoxError('The connection string is not correct.')
# Destination database not supported
allowedServices = config.allowedServices[backend]
if allowedServices is not None:
ok = False
for allowedService in allowedServices:
if connectionDictionary['service'] in databaseServices.services[allowedService]['oracle']:
ok = True
if not ok:
raise dropBox.DropBoxError('The destination database is not supported.')
# Invalid since
since = metadata['since']
if since is not None:
firstSince = db.iovSequence(metadata['inputTag']).firstSince()
if since < firstSince:
raise dropBox.DropBoxError('The since value "%d" specified in the metadata cannot be smaller than the first IOV since "%d"' % (since, firstSince))
# Invalid synchronizations
gtHandle = globalTagHandler.GlobalTagHandler(
service.getFrontierConnectionString(service.secrets['connections']['dev']['global_tag']),
service.getCxOracleConnectionString(service.secrets['connections']['dev']['run_control']),
service.getFrontierConnectionString(service.secrets['connections']['dev']['run_info']),
'runinfo_start_31X_hlt',
'runinfo_31X_hlt',
'',
'https://samir-wmcore.cern.ch/t0wmadatasvc/replay',
30,
3,
90,
)
productionGTsDict = config.productionGlobalTags
for tag, synchronizationDict in metadata['destinationTags'].items():
checkSynchronization(synchronizationDict['synchronizeTo'], destinationDatabase, tag, gtHandle, productionGTsDict)
for dependentTag, synchronizeTo in synchronizationDict.get('dependencies', {}).items():
checkSynchronization(synchronizeTo, destinationDatabase, dependentTag, gtHandle, productionGTsDict)
# firstConditionSafeRun from Tier-0 not available -- only checked
# if it is going to be used, i.e. if there is any synchronization
# to prompt or pcl.
usingFcsr = False
for tag, synchronizationDict in metadata['destinationTags'].items():
if synchronizationDict['synchronizeTo'] in ('prompt', 'pcl'):
usingFcsr = True
break
for dependentTag, synchronizeTo in synchronizationDict.get('dependencies', {}).items():
if synchronizeTo in ('prompt', 'pcl'):
usingFcsr = True
break
if usingFcsr:
try:
tier0.Tier0Handler(config.tier0URL, 5, 2, 5, None, False).getFirstSafeRun()
except ValueError:
# We got an answer but it is invalid. So far this usually means
# "None" which is not JSON, when the Tier0 is stopped.
raise dropBox.DropBoxError('Impossible to upload to synchronize to prompt or pcl while Tier-0 is returning an invalid First Condition Safe Run. %s' % config.fcsrProblemMessage)
except tier0.Tier0Error:
# Impossible to get anything from the server after retries,
# i.e. unreachable, so no data.
raise dropBox.DropBoxError('Impossible to upload to synchronize to prompt or pcl while Tier-0 is unreachable. Try again after a bit. If this does not help: %s' % config.fcsrProblemMessage)
finally:
db.close()
