def oracleToFrontier(connectionString, offlineServlet=False):
"""
Transforms an Oracle connection string into a frontier one.
Parameters:
connectionString: string for connecting to a condition Database account;
offlineServlet: boolean, default False, if set to True the connection string will contain a frontier servlet in the offline network, even if the connection string point to an online service.
@returns: frontier connection string, raises exception if parsing error or update not allowed.
"""
connectionDictionary = getValidConnectionDictionaryFromConnectionString(
connectionString)
global onlineFrontierConnectionStringTemplate
if connectionDictionary['service_type'] == 'OnlineProduction':
if not offlineServlet:
# we build the online connection string by hand, as there is no site configuration in that cluster
onlineFrontierConnectionStringTemplate = "frontier://(proxyurl=http://localhost:3128)(serverurl=http://localhost:8000/%s)(serverurl=http://localhost:8000/%s)(retrieve-ziplevel=0)(failovertoserver=no)/%s"
return onlineFrontierConnectionStringTemplate % (
(onlineFrontierConnectionStringTemplate.count("%s") - 1) *
(connectionDictionary['frontier_name'], ) +
(connectionDictionary['account'], ))
for serviceDict in serviceList:
if serviceDict['service_type'] == 'OfflineProduction':
connectionDictionary['service_type'] = 'OfflineProduction'
connectionDictionary['db_name'] = serviceDict['oracle'][0]
connectionDictionary['frontier_name'] = serviceDict[
'frontier'][0]
return service.getFrontierConnectionString(connectionDictionary)
def getFrontierConnectionString(account, level, short = False):
'''Returns a connection string from the secrets given its account name
and the production level.
'''
return service.getFrontierConnectionString({
'account': account,
'frontier_name': service.secrets['connections'][level]['frontier_name'],
}, short = short)
def getFrontierConnectionString(account, level, short=False):
'''Returns a connection string from the secrets given its account name
and the production level.
'''
return service.getFrontierConnectionString(
{
'account':
account,
'frontier_name':
service.secrets['connections'][level]['frontier_name'],
},
short=short)
def main():
if service.isAnotherInstanceRunning():
logging.info('Another instance is running, exiting...')
return
connections = service.secrets['connections']
i = 0
for level in connections:
i += 1
connectionStrings = service.getFrontierConnectionStringList(
connections[level])
j = 0
for connectionString in connectionStrings:
j += 1
shortConnectionString = service.getFrontierConnectionString(
{
'account': connections[level]['accounts'][j - 1],
'frontier_name': connections[level]['frontier_name']
},
short=True)
loggingPrefix = '[%s/%s %s %s/%s] %s: ' % (
i, len(connections), level, j, len(connectionStrings),
connectionString.rsplit('/', 1)[1])
try:
logging.info(loggingPrefix + 'Building HTML...')
LastIOVSince(dbName=connectionString).writeTable(
dbName=shortConnectionString)
except Exception as e:
logging.error(loggingPrefix +
'Exception while building HTML: %s.' % str(e))
try:
logging.info(loggingPrefix + 'Building JSON...')
condDB = EcalCondDB(dbName=connectionString)
condDB.listContainers_json_writer(
content=condDB.listContainers(),
dbName=shortConnectionString)
except Exception as e:
logging.error(loggingPrefix +
'Exception while building JSON: %s.' % str(e))
def oracleToFrontier( connectionString, offlineServlet = False ):
"""
Transforms an Oracle connection string into a frontier one.
Parameters:
connectionString: string for connecting to a condition Database account;
offlineServlet: boolean, default False, if set to True the connection string will contain a frontier servlet in the offline network, even if the connection string point to an online service.
@returns: frontier connection string, raises exception if parsing error or update not allowed.
"""
connectionDictionary = getValidConnectionDictionaryFromConnectionString( connectionString )
global onlineFrontierConnectionStringTemplate
if connectionDictionary[ 'service_type' ] == 'OnlineProduction':
if not offlineServlet:
# we build the online connection string by hand, as there is no site configuration in that cluster
onlineFrontierConnectionStringTemplate = "frontier://(proxyurl=http://localhost:3128)(serverurl=http://localhost:8000/%s)(serverurl=http://localhost:8000/%s)(retrieve-ziplevel=0)(failovertoserver=no)/%s"
return onlineFrontierConnectionStringTemplate %( ( onlineFrontierConnectionStringTemplate.count( "%s" ) - 1 ) * ( connectionDictionary[ 'frontier_name' ], ) + ( connectionDictionary[ 'account' ], ) )
for serviceDict in serviceList:
if serviceDict[ 'service_type' ] == 'OfflineProduction':
connectionDictionary[ 'service_type' ] = 'OfflineProduction'
connectionDictionary[ 'db_name' ] = serviceDict[ 'oracle' ][ 0 ]
connectionDictionary[ 'frontier_name' ] = serviceDict[ 'frontier' ][ 0 ]
return service.getFrontierConnectionString( connectionDictionary )
def main():
if service.isAnotherInstanceRunning():
logging.info('Another instance is running, exiting...')
return
connections = service.secrets['connections']
i = 0
for level in connections:
i += 1
connectionStrings = service.getFrontierConnectionStringList(connections[level])
j = 0
for connectionString in connectionStrings:
j += 1
shortConnectionString = service.getFrontierConnectionString({
'account': connections[level]['accounts'][j-1],
'frontier_name': connections[level]['frontier_name']
}, short = True)
loggingPrefix = '[%s/%s %s %s/%s] %s: ' % (i, len(connections), level, j, len(connectionStrings), connectionString.rsplit('/', 1)[1])
try:
logging.info(loggingPrefix + 'Building HTML...')
LastIOVSince(dbName = connectionString).writeTable(dbName = shortConnectionString)
except Exception as e:
logging.error(loggingPrefix + 'Exception while building HTML: %s.' % str(e))
try:
logging.info(loggingPrefix + 'Building JSON...')
condDB = EcalCondDB(dbName = connectionString)
condDB.listContainers_json_writer(content = condDB.listContainers(), dbName = shortConnectionString)
except Exception as e:
logging.error(loggingPrefix + 'Exception while building JSON: %s.' % str(e))
import service
import os
class InfoContainer:
def __init__(self):
pass
baseDBUrl = 'oracle://cms_orcon_adg'
ecalCondDB = service.getFrontierConnectionString({
'account': 'CMS_COND_31X_ECAL',
'frontier_name': 'PromptProd',
})
stripCondDB = baseDBUrl + '/CMS_COND_31X_STRIP'
rpcCondDB = baseDBUrl + '/CMS_COND_31X_RPC'
folders = InfoContainer()
#Directory, where temporary files are stored
folders.tmp_dir = os.environ['PI_file_dir_tmp']
folders.plots_dir = os.environ['PI_file_dir_plot']
folders.histo_dir = '/home/ryzzyj/backend/static_files/histo'
folders.xml_dir = os.environ['PI_file_dir_xml']
folders.table_dir = os.environ['PI_file_dir_html']
folders.iov_table_dir = os.environ['PI_file_dir_html']
folders.iov_table_name = 'iovtable.html'
folders.tag_table_name = 'tagtable.html'
folders.trend_plots_dir = os.environ['PI_file_dir_plot_trend']
folders.json_dir = os.environ['PI_file_dir_json']
db_data = InfoContainer()
def checkContents(fileHash, dataPath, metadata, backend):
'''Checks whether the data and metadata are correct.
data is the filename of the sqlite file.
metadata is a string with the metadata file itself.
Note: Update the wizard on the upload.py script if the structure changes.
'''
logging.debug('check::checkContents(%s, %s, %s)', fileHash, dataPath,
repr(metadata))
logging.info('checkContents(): %s: Checking metadata structure...',
fileHash)
workflows = (u'offline', u'hlt', u'express', u'prompt', u'pcl')
structure = {
u'destinationDatabase': (True, unicode),
u'inputTag': (True, unicode),
u'since': (True, (int, type(None))),
u'emails': (False, [unicode]),
u'userText': (True, unicode),
u'destinationTags': (True, {
unicode: {
u'synchronizeTo': (True, workflows),
u'dependencies': (False, {
unicode: workflows,
}),
},
})
}
try:
typeMatch.match(structure, metadata)
except typeMatch.MatchError as e:
raise dropBox.DropBoxError('In the metadata, ' + str(e))
logging.info(
'checkContents(): %s: Checking data with respect to metadata...',
fileHash)
db = conditionDatabase.ConditionDBChecker('sqlite_file:%s' % dataPath, '')
try:
# Corrupted file
try:
tags = db.getAllTags()
except conditionError.ConditionError as e:
raise dropBox.DropBoxError(
'The file is corrupted, as it was not possible to get the list of tags inside it.'
)
# Empty file
if not tags:
raise dropBox.DropBoxError(
'The file does not contain any tags, so it is likely not hosting any Condition payloads.'
)
# Wrong input tag
if metadata['inputTag'] not in tags:
raise dropBox.DropBoxError(
'The input tag "%s" is not in the input SQLite file.' %
metadata['inputTag'])
# Unsupported service
destinationDatabase = metadata['destinationDatabase']
if not destinationDatabase.startswith('oracle:'):
raise dropBox.DropBoxError('Oracle is the only supported service.')
# Invalid connection string
connectionDictionary = service.getProtocolServiceAndAccountFromConnectionString(
destinationDatabase)
if connectionDictionary is None:
raise dropBox.DropBoxError('The connection string is not correct.')
# Destination database not supported
allowedServices = config.allowedServices[backend]
if allowedServices is not None:
ok = False
for allowedService in allowedServices:
if connectionDictionary[
'service'] in databaseServices.services[
allowedService]['oracle']:
ok = True
if not ok:
raise dropBox.DropBoxError(
'The destination database is not supported.')
# Invalid since
since = metadata['since']
if since is not None:
firstSince = db.iovSequence(metadata['inputTag']).firstSince()
if since < firstSince:
raise dropBox.DropBoxError(
'The since value "%d" specified in the metadata cannot be smaller than the first IOV since "%d"'
% (since, firstSince))
# Invalid synchronizations
gtHandle = globalTagHandler.GlobalTagHandler(
service.getFrontierConnectionString(
service.secrets['connections']['dev']['global_tag']),
service.getCxOracleConnectionString(
service.secrets['connections']['dev']['run_control']),
service.getFrontierConnectionString(
service.secrets['connections']['dev']['run_info']),
'runinfo_start_31X_hlt',
'runinfo_31X_hlt',
'',
'https://samir-wmcore.cern.ch/t0wmadatasvc/replay',
30,
3,
90,
)
productionGTsDict = config.productionGlobalTags
for tag, synchronizationDict in metadata['destinationTags'].items():
checkSynchronization(synchronizationDict['synchronizeTo'],
destinationDatabase, tag, gtHandle,
productionGTsDict)
for dependentTag, synchronizeTo in synchronizationDict.get(
'dependencies', {}).items():
checkSynchronization(synchronizeTo, destinationDatabase,
dependentTag, gtHandle, productionGTsDict)
# firstConditionSafeRun from Tier-0 not available -- only checked
# if it is going to be used, i.e. if there is any synchronization
# to prompt or pcl.
usingFcsr = False
for tag, synchronizationDict in metadata['destinationTags'].items():
if synchronizationDict['synchronizeTo'] in ('prompt', 'pcl'):
usingFcsr = True
break
for dependentTag, synchronizeTo in synchronizationDict.get(
'dependencies', {}).items():
if synchronizeTo in ('prompt', 'pcl'):
usingFcsr = True
break
if usingFcsr:
try:
tier0.Tier0Handler(config.tier0URL, 5, 2, 5, None,
False).getFirstSafeRun()
except ValueError:
# We got an answer but it is invalid. So far this usually means
# "None" which is not JSON, when the Tier0 is stopped.
raise dropBox.DropBoxError(
'Impossible to upload to synchronize to prompt or pcl while Tier-0 is returning an invalid First Condition Safe Run. %s'
% config.fcsrProblemMessage)
except tier0.Tier0Error:
# Impossible to get anything from the server after retries,
# i.e. unreachable, so no data.
raise dropBox.DropBoxError(
'Impossible to upload to synchronize to prompt or pcl while Tier-0 is unreachable. Try again after a bit. If this does not help: %s'
% config.fcsrProblemMessage)
finally:
db.close()
def port(metadata, fileName):
'''Ports metadata into the new format.
'''
# Defaults
outputMetadata = {
'userText': '',
}
allowDependencies = False
for line in metadata.splitlines():
line = line.strip()
# Ignore empty lines
if line == '':
continue
# Ignore comment lines
if line.startswith('#'):
continue
(key, x, value) = line.partition(' ')
key = key.lower()
value = value.strip()
if key == 'destdb':
outputMetadata['destinationDatabase'] = value
elif key == 'tag':
outputMetadata['destinationTags'] = {
value: {}
}
elif key == 'inputtag':
if value == '':
value = outputMetadata['destinationTags'].keys()[0]
outputMetadata['inputTag'] = value
elif key == 'since':
if value != '':
outputMetadata['since'] = int(value)
else:
outputMetadata['since'] = None
elif key == 'iovcheck':
# from dropBox/config.py
productionGTsDict = {
'hlt': 'GR_H_V29',
'express': 'GR_E_V31',
'prompt': 'GR_P_V42'
}
# from dropBox/checkTodo.py
globalTagConnectionString = service.getFrontierConnectionString( service.secrets[ 'connections' ][ 'dev' ][ 'global_tag' ] )
runControlConnectionString = service.getCxOracleConnectionString( service.secrets[ 'connections' ][ 'dev' ][ 'run_control' ] )
runInfoConnectionString = service.getFrontierConnectionString( service.secrets[ 'connections' ][ 'dev' ][ 'run_info' ] )
runInfoStartTag = "runinfo_start_31X_hlt"
runInfoStopTag = "runinfo_31X_hlt"
authPath = ""
tier0DataSvcURI = 'https://samir-wmcore.cern.ch/t0wmadatasvc/replay'
timeOut = 30
retries = 3
retryPeriod = 90
gtHandle = globalTagHandler.GlobalTagHandler( globalTagConnectionString, runControlConnectionString, runInfoConnectionString, runInfoStartTag, runInfoStopTag, authPath, tier0DataSvcURI, timeOut, retries, retryPeriod )
workflow = gtHandle.getWorkflowForTagAndDB( outputMetadata['destinationDatabase'], outputMetadata['destinationTags'].keys()[0], productionGTsDict )
replaceValue = True
if workflow is None:
replaceValue = False #connection string and tag are not in any production Global Tags
elif value == workflow:
replaceValue = False #connection string and tag are in the production Global Tag for the same workflow specified
elif value == 'pcl' and workflow == 'prompt':
replaceValue = False #pcl is a particular case for prompt
# If IOVCheck was 'All', and the tag is in a global tag
# (e.g. workflow is not None, replaceValue == True),
# we replace it with something else (e.g. 'hlt') and therefore
# there will not allow dependencies since this is what the user
# should have used in the first place.
#
# If IOVCheck was 'All', and the tag is not in a global tag
# (e.g. workflow is None, replaceValue == False),
# we do not replace it, so it means it was offline + allow
# dependencies, which we will reach since value still is 'All'.
if replaceValue:
value = workflow
if value == 'All':
allowDependencies = True
value = 'offline'
outputMetadata['destinationTags'][outputMetadata['destinationTags'].keys()[0]] = {
'synchronizeTo': value,
'dependencies': {},
}
elif key in ['duplicatetaghlt', 'duplicatetagexpress', 'duplicatetagprompt', 'duplicatetagpcl']:
if allowDependencies and value != '':
outputMetadata['destinationTags'][outputMetadata['destinationTags'].keys()[0]]['dependencies'][value] = key.partition('duplicatetag')[2]
elif key == 'usertext':
outputMetadata['userText'] = value
# Deprecated stuff
elif key == 'timetype':
continue
# Tier0 stuff that we do not need
elif key == 'source':
continue
elif key == 'fileclass':
continue
# Wrong stuff
elif key == 'es':
# Bad userText in ESRecHitRatioCuts_V03_offline@a57546b2-03d2-11e2-84ae-003048d2bf28.tar.bz2
outputMetadata['userText'] = '%s %s' % (key, value)
elif key == 'gains':
# Bad userText in SiPixelGainCalibrationHLT_2009runs_hlt@429bd0b6-0b6a-11e2-893f-003048f0e2c4.tar.bz2
outputMetadata['userText'] = '%s %s' % (key, value)
elif key == 'pixel':
# Bad userText in SiPixelQuality_v11_bugfix_mc@97ccde9c-1784-11e2-b160-001e4f3da51f.tar.bz2
outputMetadata['userText'] = '%s %s' % (key, value)
elif key == 'es-ee':
# Bad userText in ESEEIntercalibConstants_LG_V03_mc@a7238182-1ad8-11e2-85cc-001e4f3da513.tar.bz2
outputMetadata['userText'] = '%s %s' % (key, value)
elif key == 'lorentz':
# Bad userText in SiPixelLorentzAngle_r194912_v1@4c10ccb4-1ee8-11e2-be80-003048d2bc9a.tar.bz2
outputMetadata['userText'] = '%s %s' % (key, value)
elif key == 'produced':
# Bad userText in SiPixelTemplateDBObject_38T_2012_for_alignment_v1_offline@db41edfc-1fae-11e2-9aab-003048d2bf8c.tar.bz2
# Bad userText in SiPixelTemplateDBObject_38T_IOV1_r194912@efb22762-1fa3-11e2-9fdd-003048d2bf8c.tar.bz2
outputMetadata['userText'] = '%s %s' % (key, value)
elif key == 'fed':
# Bad userText in [email protected]
outputMetadata['userText'] = '%s %s' % (key, value)
elif key == 'bugfix':
# Bad userText in SiPixelQuality_v04_offline@eff63846-247e-11e2-969c-001e4f3e5c33.tar.bz2
outputMetadata['userText'] = '%s %s' % (key, value)
else:
raise Exception('Invalid key: %s', key)
# In the old dropBox we allowed either None or a since which was smaller
# that the one in the data, which made the one in the data being used.
# Since in the new dropBox we only allow None to mean 'take the one from
# the data', we overwrite it here so that we do not discard the file.
if fileName in set([
'[email protected]',
'HcalRespCorrs_v1.02_hlt@d21bd65c-0974-11e2-89b2-003048f0e7a2.tar.bz2',
'HcalRespCorrs_v1.02_express@6c984812-16a6-11e2-bae5-003048d3c892.tar.bz2',
]):
if outputMetadata['since'] != 1:
raise Exception('%s: Expected since == 1 for the manual fix to None.' % fileName)
outputMetadata['since'] = None
return json.dumps(outputMetadata, sort_keys = True, indent = 4)
def port(metadata, fileName):
'''Ports metadata into the new format.
'''
# Defaults
outputMetadata = {
'userText': '',
}
allowDependencies = False
for line in metadata.splitlines():
line = line.strip()
# Ignore empty lines
if line == '':
continue
# Ignore comment lines
if line.startswith('#'):
continue
(key, x, value) = line.partition(' ')
key = key.lower()
value = value.strip()
if key == 'destdb':
outputMetadata['destinationDatabase'] = value
elif key == 'tag':
outputMetadata['destinationTags'] = {value: {}}
elif key == 'inputtag':
if value == '':
value = outputMetadata['destinationTags'].keys()[0]
outputMetadata['inputTag'] = value
elif key == 'since':
if value != '':
outputMetadata['since'] = int(value)
else:
outputMetadata['since'] = None
elif key == 'iovcheck':
# from dropBox/config.py
productionGTsDict = {
'hlt': 'GR_H_V29',
'express': 'GR_E_V31',
'prompt': 'GR_P_V42'
}
# from dropBox/checkTodo.py
globalTagConnectionString = service.getFrontierConnectionString(
service.secrets['connections']['dev']['global_tag'])
runControlConnectionString = service.getCxOracleConnectionString(
service.secrets['connections']['dev']['run_control'])
runInfoConnectionString = service.getFrontierConnectionString(
service.secrets['connections']['dev']['run_info'])
runInfoStartTag = "runinfo_start_31X_hlt"
runInfoStopTag = "runinfo_31X_hlt"
authPath = ""
tier0DataSvcURI = 'https://samir-wmcore.cern.ch/t0wmadatasvc/replay'
timeOut = 30
retries = 3
retryPeriod = 90
gtHandle = globalTagHandler.GlobalTagHandler(
globalTagConnectionString, runControlConnectionString,
runInfoConnectionString, runInfoStartTag, runInfoStopTag,
authPath, tier0DataSvcURI, timeOut, retries, retryPeriod)
workflow = gtHandle.getWorkflowForTagAndDB(
outputMetadata['destinationDatabase'],
outputMetadata['destinationTags'].keys()[0], productionGTsDict)
replaceValue = True
if workflow is None:
replaceValue = False #connection string and tag are not in any production Global Tags
elif value == workflow:
replaceValue = False #connection string and tag are in the production Global Tag for the same workflow specified
elif value == 'pcl' and workflow == 'prompt':
replaceValue = False #pcl is a particular case for prompt
# If IOVCheck was 'All', and the tag is in a global tag
# (e.g. workflow is not None, replaceValue == True),
# we replace it with something else (e.g. 'hlt') and therefore
# there will not allow dependencies since this is what the user
# should have used in the first place.
#
# If IOVCheck was 'All', and the tag is not in a global tag
# (e.g. workflow is None, replaceValue == False),
# we do not replace it, so it means it was offline + allow
# dependencies, which we will reach since value still is 'All'.
if replaceValue:
value = workflow
if value == 'All':
allowDependencies = True
value = 'offline'
outputMetadata['destinationTags'][
outputMetadata['destinationTags'].keys()[0]] = {
'synchronizeTo': value,
'dependencies': {},
}
elif key in [
'duplicatetaghlt', 'duplicatetagexpress', 'duplicatetagprompt',
'duplicatetagpcl'
]:
if allowDependencies and value != '':
outputMetadata['destinationTags'][
outputMetadata['destinationTags'].keys()
[0]]['dependencies'][value] = key.partition(
'duplicatetag')[2]
elif key == 'usertext':
outputMetadata['userText'] = value
# Deprecated stuff
elif key == 'timetype':
continue
# Tier0 stuff that we do not need
elif key == 'source':
continue
elif key == 'fileclass':
continue
# Wrong stuff
elif key == 'es':
# Bad userText in ESRecHitRatioCuts_V03_offline@a57546b2-03d2-11e2-84ae-003048d2bf28.tar.bz2
outputMetadata['userText'] = '%s %s' % (key, value)
elif key == 'gains':
# Bad userText in SiPixelGainCalibrationHLT_2009runs_hlt@429bd0b6-0b6a-11e2-893f-003048f0e2c4.tar.bz2
outputMetadata['userText'] = '%s %s' % (key, value)
elif key == 'pixel':
# Bad userText in SiPixelQuality_v11_bugfix_mc@97ccde9c-1784-11e2-b160-001e4f3da51f.tar.bz2
outputMetadata['userText'] = '%s %s' % (key, value)
elif key == 'es-ee':
# Bad userText in ESEEIntercalibConstants_LG_V03_mc@a7238182-1ad8-11e2-85cc-001e4f3da513.tar.bz2
outputMetadata['userText'] = '%s %s' % (key, value)
elif key == 'lorentz':
# Bad userText in SiPixelLorentzAngle_r194912_v1@4c10ccb4-1ee8-11e2-be80-003048d2bc9a.tar.bz2
outputMetadata['userText'] = '%s %s' % (key, value)
elif key == 'produced':
# Bad userText in SiPixelTemplateDBObject_38T_2012_for_alignment_v1_offline@db41edfc-1fae-11e2-9aab-003048d2bf8c.tar.bz2
# Bad userText in SiPixelTemplateDBObject_38T_IOV1_r194912@efb22762-1fa3-11e2-9fdd-003048d2bf8c.tar.bz2
outputMetadata['userText'] = '%s %s' % (key, value)
elif key == 'fed':
# Bad userText in [email protected]
outputMetadata['userText'] = '%s %s' % (key, value)
elif key == 'bugfix':
# Bad userText in SiPixelQuality_v04_offline@eff63846-247e-11e2-969c-001e4f3e5c33.tar.bz2
outputMetadata['userText'] = '%s %s' % (key, value)
else:
raise Exception('Invalid key: %s', key)
# In the old dropBox we allowed either None or a since which was smaller
# that the one in the data, which made the one in the data being used.
# Since in the new dropBox we only allow None to mean 'take the one from
# the data', we overwrite it here so that we do not discard the file.
if fileName in set([
'[email protected]',
'HcalRespCorrs_v1.02_hlt@d21bd65c-0974-11e2-89b2-003048f0e7a2.tar.bz2',
'HcalRespCorrs_v1.02_express@6c984812-16a6-11e2-bae5-003048d3c892.tar.bz2',
]):
if outputMetadata['since'] != 1:
raise Exception(
'%s: Expected since == 1 for the manual fix to None.' %
fileName)
outputMetadata['since'] = None
return json.dumps(outputMetadata, sort_keys=True, indent=4)
def checkContents(fileHash, dataPath, metadata, backend):
'''Checks whether the data and metadata are correct.
data is the filename of the sqlite file.
metadata is a string with the metadata file itself.
Note: Update the wizard on the upload.py script if the structure changes.
'''
logging.debug('check::checkContents(%s, %s, %s)', fileHash, dataPath, repr(metadata))
logging.info('checkContents(): %s: Checking metadata structure...', fileHash)
workflows = (u'offline', u'hlt', u'express', u'prompt', u'pcl')
structure = {
u'destinationDatabase': (True, unicode),
u'inputTag': (True, unicode),
u'since': (True, (int, type(None))),
u'emails': (False, [unicode]),
u'userText': (True, unicode),
u'destinationTags': (True, {
unicode: {
u'synchronizeTo': (True, workflows),
u'dependencies': (False, {
unicode: workflows,
}),
},
})
}
try:
typeMatch.match(structure, metadata)
except typeMatch.MatchError as e:
raise dropBox.DropBoxError('In the metadata, ' + str(e))
logging.info('checkContents(): %s: Checking data with respect to metadata...', fileHash)
db = conditionDatabase.ConditionDBChecker('sqlite_file:%s' % dataPath, '')
try:
# Corrupted file
try:
tags = db.getAllTags()
except conditionError.ConditionError as e:
raise dropBox.DropBoxError('The file is corrupted, as it was not possible to get the list of tags inside it.')
# Empty file
if not tags:
raise dropBox.DropBoxError('The file does not contain any tags, so it is likely not hosting any Condition payloads.')
# Wrong input tag
if metadata['inputTag'] not in tags:
raise dropBox.DropBoxError('The input tag "%s" is not in the input SQLite file.' % metadata['inputTag'])
# Unsupported service
destinationDatabase = metadata['destinationDatabase']
if not destinationDatabase.startswith('oracle:'):
raise dropBox.DropBoxError('Oracle is the only supported service.')
# Invalid connection string
connectionDictionary = service.getProtocolServiceAndAccountFromConnectionString(destinationDatabase)
if connectionDictionary is None:
raise dropBox.DropBoxError('The connection string is not correct.')
# Destination database not supported
allowedServices = config.allowedServices[backend]
if allowedServices is not None:
ok = False
for allowedService in allowedServices:
if connectionDictionary['service'] in databaseServices.services[allowedService]['oracle']:
ok = True
if not ok:
raise dropBox.DropBoxError('The destination database is not supported.')
# Invalid since
since = metadata['since']
if since is not None:
firstSince = db.iovSequence(metadata['inputTag']).firstSince()
if since < firstSince:
raise dropBox.DropBoxError('The since value "%d" specified in the metadata cannot be smaller than the first IOV since "%d"' % (since, firstSince))
# Invalid synchronizations
gtHandle = globalTagHandler.GlobalTagHandler(
service.getFrontierConnectionString(service.secrets['connections']['dev']['global_tag']),
service.getCxOracleConnectionString(service.secrets['connections']['dev']['run_control']),
service.getFrontierConnectionString(service.secrets['connections']['dev']['run_info']),
'runinfo_start_31X_hlt',
'runinfo_31X_hlt',
'',
'https://samir-wmcore.cern.ch/t0wmadatasvc/replay',
30,
3,
90,
)
productionGTsDict = config.productionGlobalTags
for tag, synchronizationDict in metadata['destinationTags'].items():
checkSynchronization(synchronizationDict['synchronizeTo'], destinationDatabase, tag, gtHandle, productionGTsDict)
for dependentTag, synchronizeTo in synchronizationDict.get('dependencies', {}).items():
checkSynchronization(synchronizeTo, destinationDatabase, dependentTag, gtHandle, productionGTsDict)
# firstConditionSafeRun from Tier-0 not available -- only checked
# if it is going to be used, i.e. if there is any synchronization
# to prompt or pcl.
usingFcsr = False
for tag, synchronizationDict in metadata['destinationTags'].items():
if synchronizationDict['synchronizeTo'] in ('prompt', 'pcl'):
usingFcsr = True
break
for dependentTag, synchronizeTo in synchronizationDict.get('dependencies', {}).items():
if synchronizeTo in ('prompt', 'pcl'):
usingFcsr = True
break
if usingFcsr:
try:
tier0.Tier0Handler(config.tier0URL, 5, 2, 5, None, False).getFirstSafeRun()
except ValueError:
# We got an answer but it is invalid. So far this usually means
# "None" which is not JSON, when the Tier0 is stopped.
raise dropBox.DropBoxError('Impossible to upload to synchronize to prompt or pcl while Tier-0 is returning an invalid First Condition Safe Run. %s' % config.fcsrProblemMessage)
except tier0.Tier0Error:
# Impossible to get anything from the server after retries,
# i.e. unreachable, so no data.
raise dropBox.DropBoxError('Impossible to upload to synchronize to prompt or pcl while Tier-0 is unreachable. Try again after a bit. If this does not help: %s' % config.fcsrProblemMessage)
finally:
db.close()
import service
import os
class InfoContainer:
def __init__(self):
pass
baseDBUrl = 'oracle://cms_orcon_adg'
ecalCondDB = service.getFrontierConnectionString({
'account': 'CMS_COND_31X_ECAL',
'frontier_name': 'PromptProd',
})
stripCondDB = baseDBUrl + '/CMS_COND_31X_STRIP'
rpcCondDB = baseDBUrl + '/CMS_COND_31X_RPC'
folders = InfoContainer()
#Directory, where temporary files are stored
folders.tmp_dir = os.environ['PI_file_dir_tmp']
folders.plots_dir = os.environ['PI_file_dir_plot']
folders.histo_dir = '/home/ryzzyj/backend/static_files/histo'
folders.xml_dir = os.environ['PI_file_dir_xml']
folders.table_dir = os.environ['PI_file_dir_html']
folders.iov_table_dir = os.environ['PI_file_dir_html']
folders.iov_table_name = 'iovtable.html'
folders.tag_table_name = 'tagtable.html'
folders.trend_plots_dir = os.environ['PI_file_dir_plot_trend']
folders.json_dir = os.environ['PI_file_dir_json']
db_data = InfoContainer()
db_data.users_db_conn_str = 'sqlite:///Users.db'
db_data.auth_path = ''
