def reconnect(self): '''Note that this invalidates all cursors as well. ''' logging.info('%s: Connecting to database...', self) self.connection = cx_Oracle.connect( service.getCxOracleConnectionString(self.connectionDictionary), threaded=True)
def checkContents(fileHash, dataPath, metadata, backend): '''Checks whether the data and metadata are correct. data is the filename of the sqlite file. metadata is a string with the metadata file itself. Note: Update the wizard on the upload.py script if the structure changes. ''' logging.debug('check::checkContents(%s, %s, %s)', fileHash, dataPath, repr(metadata)) logging.info('checkContents(): %s: Checking metadata structure...', fileHash) workflows = (u'offline', u'hlt', u'express', u'prompt', u'pcl') structure = { u'destinationDatabase': (True, unicode), u'inputTag': (True, unicode), u'since': (True, (int, type(None))), u'emails': (False, [unicode]), u'userText': (True, unicode), u'destinationTags': (True, { unicode: { u'synchronizeTo': (True, workflows), u'dependencies': (False, { unicode: workflows, }), }, }) } try: typeMatch.match(structure, metadata) except typeMatch.MatchError as e: raise dropBox.DropBoxError('In the metadata, ' + str(e)) logging.info( 'checkContents(): %s: Checking data with respect to metadata...', fileHash) db = conditionDatabase.ConditionDBChecker('sqlite_file:%s' % dataPath, '') try: # Corrupted file try: tags = db.getAllTags() except conditionError.ConditionError as e: raise dropBox.DropBoxError( 'The file is corrupted, as it was not possible to get the list of tags inside it.' ) # Empty file if not tags: raise dropBox.DropBoxError( 'The file does not contain any tags, so it is likely not hosting any Condition payloads.' ) # Wrong input tag if metadata['inputTag'] not in tags: raise dropBox.DropBoxError( 'The input tag "%s" is not in the input SQLite file.' % metadata['inputTag']) # Unsupported service destinationDatabase = metadata['destinationDatabase'] if not destinationDatabase.startswith('oracle:'): raise dropBox.DropBoxError('Oracle is the only supported service.') # Invalid connection string connectionDictionary = service.getProtocolServiceAndAccountFromConnectionString( destinationDatabase) if connectionDictionary is None: raise dropBox.DropBoxError('The connection string is not correct.') # Destination database not supported allowedServices = config.allowedServices[backend] if allowedServices is not None: ok = False for allowedService in allowedServices: if connectionDictionary[ 'service'] in databaseServices.services[ allowedService]['oracle']: ok = True if not ok: raise dropBox.DropBoxError( 'The destination database is not supported.') # Invalid since since = metadata['since'] if since is not None: firstSince = db.iovSequence(metadata['inputTag']).firstSince() if since < firstSince: raise dropBox.DropBoxError( 'The since value "%d" specified in the metadata cannot be smaller than the first IOV since "%d"' % (since, firstSince)) # Invalid synchronizations gtHandle = globalTagHandler.GlobalTagHandler( service.getFrontierConnectionString( service.secrets['connections']['dev']['global_tag']), service.getCxOracleConnectionString( service.secrets['connections']['dev']['run_control']), service.getFrontierConnectionString( service.secrets['connections']['dev']['run_info']), 'runinfo_start_31X_hlt', 'runinfo_31X_hlt', '', 'https://samir-wmcore.cern.ch/t0wmadatasvc/replay', 30, 3, 90, ) productionGTsDict = config.productionGlobalTags for tag, synchronizationDict in metadata['destinationTags'].items(): checkSynchronization(synchronizationDict['synchronizeTo'], destinationDatabase, tag, gtHandle, productionGTsDict) for dependentTag, synchronizeTo in synchronizationDict.get( 'dependencies', {}).items(): checkSynchronization(synchronizeTo, destinationDatabase, dependentTag, gtHandle, productionGTsDict) # firstConditionSafeRun from Tier-0 not available -- only checked # if it is going to be used, i.e. if there is any synchronization # to prompt or pcl. usingFcsr = False for tag, synchronizationDict in metadata['destinationTags'].items(): if synchronizationDict['synchronizeTo'] in ('prompt', 'pcl'): usingFcsr = True break for dependentTag, synchronizeTo in synchronizationDict.get( 'dependencies', {}).items(): if synchronizeTo in ('prompt', 'pcl'): usingFcsr = True break if usingFcsr: try: tier0.Tier0Handler(config.tier0URL, 5, 2, 5, None, False).getFirstSafeRun() except ValueError: # We got an answer but it is invalid. So far this usually means # "None" which is not JSON, when the Tier0 is stopped. raise dropBox.DropBoxError( 'Impossible to upload to synchronize to prompt or pcl while Tier-0 is returning an invalid First Condition Safe Run. %s' % config.fcsrProblemMessage) except tier0.Tier0Error: # Impossible to get anything from the server after retries, # i.e. unreachable, so no data. raise dropBox.DropBoxError( 'Impossible to upload to synchronize to prompt or pcl while Tier-0 is unreachable. Try again after a bit. If this does not help: %s' % config.fcsrProblemMessage) finally: db.close()
def port(metadata, fileName): '''Ports metadata into the new format. ''' # Defaults outputMetadata = { 'userText': '', } allowDependencies = False for line in metadata.splitlines(): line = line.strip() # Ignore empty lines if line == '': continue # Ignore comment lines if line.startswith('#'): continue (key, x, value) = line.partition(' ') key = key.lower() value = value.strip() if key == 'destdb': outputMetadata['destinationDatabase'] = value elif key == 'tag': outputMetadata['destinationTags'] = { value: {} } elif key == 'inputtag': if value == '': value = outputMetadata['destinationTags'].keys()[0] outputMetadata['inputTag'] = value elif key == 'since': if value != '': outputMetadata['since'] = int(value) else: outputMetadata['since'] = None elif key == 'iovcheck': # from dropBox/config.py productionGTsDict = { 'hlt': 'GR_H_V29', 'express': 'GR_E_V31', 'prompt': 'GR_P_V42' } # from dropBox/checkTodo.py globalTagConnectionString = service.getFrontierConnectionString( service.secrets[ 'connections' ][ 'dev' ][ 'global_tag' ] ) runControlConnectionString = service.getCxOracleConnectionString( service.secrets[ 'connections' ][ 'dev' ][ 'run_control' ] ) runInfoConnectionString = service.getFrontierConnectionString( service.secrets[ 'connections' ][ 'dev' ][ 'run_info' ] ) runInfoStartTag = "runinfo_start_31X_hlt" runInfoStopTag = "runinfo_31X_hlt" authPath = "" tier0DataSvcURI = 'https://samir-wmcore.cern.ch/t0wmadatasvc/replay' timeOut = 30 retries = 3 retryPeriod = 90 gtHandle = globalTagHandler.GlobalTagHandler( globalTagConnectionString, runControlConnectionString, runInfoConnectionString, runInfoStartTag, runInfoStopTag, authPath, tier0DataSvcURI, timeOut, retries, retryPeriod ) workflow = gtHandle.getWorkflowForTagAndDB( outputMetadata['destinationDatabase'], outputMetadata['destinationTags'].keys()[0], productionGTsDict ) replaceValue = True if workflow is None: replaceValue = False #connection string and tag are not in any production Global Tags elif value == workflow: replaceValue = False #connection string and tag are in the production Global Tag for the same workflow specified elif value == 'pcl' and workflow == 'prompt': replaceValue = False #pcl is a particular case for prompt # If IOVCheck was 'All', and the tag is in a global tag # (e.g. workflow is not None, replaceValue == True), # we replace it with something else (e.g. 'hlt') and therefore # there will not allow dependencies since this is what the user # should have used in the first place. # # If IOVCheck was 'All', and the tag is not in a global tag # (e.g. workflow is None, replaceValue == False), # we do not replace it, so it means it was offline + allow # dependencies, which we will reach since value still is 'All'. if replaceValue: value = workflow if value == 'All': allowDependencies = True value = 'offline' outputMetadata['destinationTags'][outputMetadata['destinationTags'].keys()[0]] = { 'synchronizeTo': value, 'dependencies': {}, } elif key in ['duplicatetaghlt', 'duplicatetagexpress', 'duplicatetagprompt', 'duplicatetagpcl']: if allowDependencies and value != '': outputMetadata['destinationTags'][outputMetadata['destinationTags'].keys()[0]]['dependencies'][value] = key.partition('duplicatetag')[2] elif key == 'usertext': outputMetadata['userText'] = value # Deprecated stuff elif key == 'timetype': continue # Tier0 stuff that we do not need elif key == 'source': continue elif key == 'fileclass': continue # Wrong stuff elif key == 'es': # Bad userText in ESRecHitRatioCuts_V03_offline@a57546b2-03d2-11e2-84ae-003048d2bf28.tar.bz2 outputMetadata['userText'] = '%s %s' % (key, value) elif key == 'gains': # Bad userText in SiPixelGainCalibrationHLT_2009runs_hlt@429bd0b6-0b6a-11e2-893f-003048f0e2c4.tar.bz2 outputMetadata['userText'] = '%s %s' % (key, value) elif key == 'pixel': # Bad userText in SiPixelQuality_v11_bugfix_mc@97ccde9c-1784-11e2-b160-001e4f3da51f.tar.bz2 outputMetadata['userText'] = '%s %s' % (key, value) elif key == 'es-ee': # Bad userText in ESEEIntercalibConstants_LG_V03_mc@a7238182-1ad8-11e2-85cc-001e4f3da513.tar.bz2 outputMetadata['userText'] = '%s %s' % (key, value) elif key == 'lorentz': # Bad userText in SiPixelLorentzAngle_r194912_v1@4c10ccb4-1ee8-11e2-be80-003048d2bc9a.tar.bz2 outputMetadata['userText'] = '%s %s' % (key, value) elif key == 'produced': # Bad userText in SiPixelTemplateDBObject_38T_2012_for_alignment_v1_offline@db41edfc-1fae-11e2-9aab-003048d2bf8c.tar.bz2 # Bad userText in SiPixelTemplateDBObject_38T_IOV1_r194912@efb22762-1fa3-11e2-9fdd-003048d2bf8c.tar.bz2 outputMetadata['userText'] = '%s %s' % (key, value) elif key == 'fed': # Bad userText in [email protected] outputMetadata['userText'] = '%s %s' % (key, value) elif key == 'bugfix': # Bad userText in SiPixelQuality_v04_offline@eff63846-247e-11e2-969c-001e4f3e5c33.tar.bz2 outputMetadata['userText'] = '%s %s' % (key, value) else: raise Exception('Invalid key: %s', key) # In the old dropBox we allowed either None or a since which was smaller # that the one in the data, which made the one in the data being used. # Since in the new dropBox we only allow None to mean 'take the one from # the data', we overwrite it here so that we do not discard the file. if fileName in set([ '[email protected]', 'HcalRespCorrs_v1.02_hlt@d21bd65c-0974-11e2-89b2-003048f0e7a2.tar.bz2', 'HcalRespCorrs_v1.02_express@6c984812-16a6-11e2-bae5-003048d3c892.tar.bz2', ]): if outputMetadata['since'] != 1: raise Exception('%s: Expected since == 1 for the manual fix to None.' % fileName) outputMetadata['since'] = None return json.dumps(outputMetadata, sort_keys = True, indent = 4)
def port(metadata, fileName): '''Ports metadata into the new format. ''' # Defaults outputMetadata = { 'userText': '', } allowDependencies = False for line in metadata.splitlines(): line = line.strip() # Ignore empty lines if line == '': continue # Ignore comment lines if line.startswith('#'): continue (key, x, value) = line.partition(' ') key = key.lower() value = value.strip() if key == 'destdb': outputMetadata['destinationDatabase'] = value elif key == 'tag': outputMetadata['destinationTags'] = {value: {}} elif key == 'inputtag': if value == '': value = outputMetadata['destinationTags'].keys()[0] outputMetadata['inputTag'] = value elif key == 'since': if value != '': outputMetadata['since'] = int(value) else: outputMetadata['since'] = None elif key == 'iovcheck': # from dropBox/config.py productionGTsDict = { 'hlt': 'GR_H_V29', 'express': 'GR_E_V31', 'prompt': 'GR_P_V42' } # from dropBox/checkTodo.py globalTagConnectionString = service.getFrontierConnectionString( service.secrets['connections']['dev']['global_tag']) runControlConnectionString = service.getCxOracleConnectionString( service.secrets['connections']['dev']['run_control']) runInfoConnectionString = service.getFrontierConnectionString( service.secrets['connections']['dev']['run_info']) runInfoStartTag = "runinfo_start_31X_hlt" runInfoStopTag = "runinfo_31X_hlt" authPath = "" tier0DataSvcURI = 'https://samir-wmcore.cern.ch/t0wmadatasvc/replay' timeOut = 30 retries = 3 retryPeriod = 90 gtHandle = globalTagHandler.GlobalTagHandler( globalTagConnectionString, runControlConnectionString, runInfoConnectionString, runInfoStartTag, runInfoStopTag, authPath, tier0DataSvcURI, timeOut, retries, retryPeriod) workflow = gtHandle.getWorkflowForTagAndDB( outputMetadata['destinationDatabase'], outputMetadata['destinationTags'].keys()[0], productionGTsDict) replaceValue = True if workflow is None: replaceValue = False #connection string and tag are not in any production Global Tags elif value == workflow: replaceValue = False #connection string and tag are in the production Global Tag for the same workflow specified elif value == 'pcl' and workflow == 'prompt': replaceValue = False #pcl is a particular case for prompt # If IOVCheck was 'All', and the tag is in a global tag # (e.g. workflow is not None, replaceValue == True), # we replace it with something else (e.g. 'hlt') and therefore # there will not allow dependencies since this is what the user # should have used in the first place. # # If IOVCheck was 'All', and the tag is not in a global tag # (e.g. workflow is None, replaceValue == False), # we do not replace it, so it means it was offline + allow # dependencies, which we will reach since value still is 'All'. if replaceValue: value = workflow if value == 'All': allowDependencies = True value = 'offline' outputMetadata['destinationTags'][ outputMetadata['destinationTags'].keys()[0]] = { 'synchronizeTo': value, 'dependencies': {}, } elif key in [ 'duplicatetaghlt', 'duplicatetagexpress', 'duplicatetagprompt', 'duplicatetagpcl' ]: if allowDependencies and value != '': outputMetadata['destinationTags'][ outputMetadata['destinationTags'].keys() [0]]['dependencies'][value] = key.partition( 'duplicatetag')[2] elif key == 'usertext': outputMetadata['userText'] = value # Deprecated stuff elif key == 'timetype': continue # Tier0 stuff that we do not need elif key == 'source': continue elif key == 'fileclass': continue # Wrong stuff elif key == 'es': # Bad userText in ESRecHitRatioCuts_V03_offline@a57546b2-03d2-11e2-84ae-003048d2bf28.tar.bz2 outputMetadata['userText'] = '%s %s' % (key, value) elif key == 'gains': # Bad userText in SiPixelGainCalibrationHLT_2009runs_hlt@429bd0b6-0b6a-11e2-893f-003048f0e2c4.tar.bz2 outputMetadata['userText'] = '%s %s' % (key, value) elif key == 'pixel': # Bad userText in SiPixelQuality_v11_bugfix_mc@97ccde9c-1784-11e2-b160-001e4f3da51f.tar.bz2 outputMetadata['userText'] = '%s %s' % (key, value) elif key == 'es-ee': # Bad userText in ESEEIntercalibConstants_LG_V03_mc@a7238182-1ad8-11e2-85cc-001e4f3da513.tar.bz2 outputMetadata['userText'] = '%s %s' % (key, value) elif key == 'lorentz': # Bad userText in SiPixelLorentzAngle_r194912_v1@4c10ccb4-1ee8-11e2-be80-003048d2bc9a.tar.bz2 outputMetadata['userText'] = '%s %s' % (key, value) elif key == 'produced': # Bad userText in SiPixelTemplateDBObject_38T_2012_for_alignment_v1_offline@db41edfc-1fae-11e2-9aab-003048d2bf8c.tar.bz2 # Bad userText in SiPixelTemplateDBObject_38T_IOV1_r194912@efb22762-1fa3-11e2-9fdd-003048d2bf8c.tar.bz2 outputMetadata['userText'] = '%s %s' % (key, value) elif key == 'fed': # Bad userText in [email protected] outputMetadata['userText'] = '%s %s' % (key, value) elif key == 'bugfix': # Bad userText in SiPixelQuality_v04_offline@eff63846-247e-11e2-969c-001e4f3e5c33.tar.bz2 outputMetadata['userText'] = '%s %s' % (key, value) else: raise Exception('Invalid key: %s', key) # In the old dropBox we allowed either None or a since which was smaller # that the one in the data, which made the one in the data being used. # Since in the new dropBox we only allow None to mean 'take the one from # the data', we overwrite it here so that we do not discard the file. if fileName in set([ '[email protected]', 'HcalRespCorrs_v1.02_hlt@d21bd65c-0974-11e2-89b2-003048f0e7a2.tar.bz2', 'HcalRespCorrs_v1.02_express@6c984812-16a6-11e2-bae5-003048d3c892.tar.bz2', ]): if outputMetadata['since'] != 1: raise Exception( '%s: Expected since == 1 for the manual fix to None.' % fileName) outputMetadata['since'] = None return json.dumps(outputMetadata, sort_keys=True, indent=4)
def reconnect(self): '''Note that this invalidates all cursors as well. ''' logging.info('%s: Connecting to database...', self) self.connection = cx_Oracle.connect(service.getCxOracleConnectionString(self.connectionDictionary), threaded = True)
def checkContents(fileHash, dataPath, metadata, backend): '''Checks whether the data and metadata are correct. data is the filename of the sqlite file. metadata is a string with the metadata file itself. Note: Update the wizard on the upload.py script if the structure changes. ''' logging.debug('check::checkContents(%s, %s, %s)', fileHash, dataPath, repr(metadata)) logging.info('checkContents(): %s: Checking metadata structure...', fileHash) workflows = (u'offline', u'hlt', u'express', u'prompt', u'pcl') structure = { u'destinationDatabase': (True, unicode), u'inputTag': (True, unicode), u'since': (True, (int, type(None))), u'emails': (False, [unicode]), u'userText': (True, unicode), u'destinationTags': (True, { unicode: { u'synchronizeTo': (True, workflows), u'dependencies': (False, { unicode: workflows, }), }, }) } try: typeMatch.match(structure, metadata) except typeMatch.MatchError as e: raise dropBox.DropBoxError('In the metadata, ' + str(e)) logging.info('checkContents(): %s: Checking data with respect to metadata...', fileHash) db = conditionDatabase.ConditionDBChecker('sqlite_file:%s' % dataPath, '') try: # Corrupted file try: tags = db.getAllTags() except conditionError.ConditionError as e: raise dropBox.DropBoxError('The file is corrupted, as it was not possible to get the list of tags inside it.') # Empty file if not tags: raise dropBox.DropBoxError('The file does not contain any tags, so it is likely not hosting any Condition payloads.') # Wrong input tag if metadata['inputTag'] not in tags: raise dropBox.DropBoxError('The input tag "%s" is not in the input SQLite file.' % metadata['inputTag']) # Unsupported service destinationDatabase = metadata['destinationDatabase'] if not destinationDatabase.startswith('oracle:'): raise dropBox.DropBoxError('Oracle is the only supported service.') # Invalid connection string connectionDictionary = service.getProtocolServiceAndAccountFromConnectionString(destinationDatabase) if connectionDictionary is None: raise dropBox.DropBoxError('The connection string is not correct.') # Destination database not supported allowedServices = config.allowedServices[backend] if allowedServices is not None: ok = False for allowedService in allowedServices: if connectionDictionary['service'] in databaseServices.services[allowedService]['oracle']: ok = True if not ok: raise dropBox.DropBoxError('The destination database is not supported.') # Invalid since since = metadata['since'] if since is not None: firstSince = db.iovSequence(metadata['inputTag']).firstSince() if since < firstSince: raise dropBox.DropBoxError('The since value "%d" specified in the metadata cannot be smaller than the first IOV since "%d"' % (since, firstSince)) # Invalid synchronizations gtHandle = globalTagHandler.GlobalTagHandler( service.getFrontierConnectionString(service.secrets['connections']['dev']['global_tag']), service.getCxOracleConnectionString(service.secrets['connections']['dev']['run_control']), service.getFrontierConnectionString(service.secrets['connections']['dev']['run_info']), 'runinfo_start_31X_hlt', 'runinfo_31X_hlt', '', 'https://samir-wmcore.cern.ch/t0wmadatasvc/replay', 30, 3, 90, ) productionGTsDict = config.productionGlobalTags for tag, synchronizationDict in metadata['destinationTags'].items(): checkSynchronization(synchronizationDict['synchronizeTo'], destinationDatabase, tag, gtHandle, productionGTsDict) for dependentTag, synchronizeTo in synchronizationDict.get('dependencies', {}).items(): checkSynchronization(synchronizeTo, destinationDatabase, dependentTag, gtHandle, productionGTsDict) # firstConditionSafeRun from Tier-0 not available -- only checked # if it is going to be used, i.e. if there is any synchronization # to prompt or pcl. usingFcsr = False for tag, synchronizationDict in metadata['destinationTags'].items(): if synchronizationDict['synchronizeTo'] in ('prompt', 'pcl'): usingFcsr = True break for dependentTag, synchronizeTo in synchronizationDict.get('dependencies', {}).items(): if synchronizeTo in ('prompt', 'pcl'): usingFcsr = True break if usingFcsr: try: tier0.Tier0Handler(config.tier0URL, 5, 2, 5, None, False).getFirstSafeRun() except ValueError: # We got an answer but it is invalid. So far this usually means # "None" which is not JSON, when the Tier0 is stopped. raise dropBox.DropBoxError('Impossible to upload to synchronize to prompt or pcl while Tier-0 is returning an invalid First Condition Safe Run. %s' % config.fcsrProblemMessage) except tier0.Tier0Error: # Impossible to get anything from the server after retries, # i.e. unreachable, so no data. raise dropBox.DropBoxError('Impossible to upload to synchronize to prompt or pcl while Tier-0 is unreachable. Try again after a bit. If this does not help: %s' % config.fcsrProblemMessage) finally: db.close()