def test_frontier_connect(self): for alias in ["pro", "arc", "int", "dev"]: connection_data = {"db_alias" : alias, "host" : "frontier", "schema" : "cms_conditions"} connection = querying.connect(connection_data) # can cause failure if the database is down close_session_result = connection.close_session() self.assertEqual(close_session_result, True)
def test_orapro_connect(self): connection_data = {"db_alias" : "orapro", "host" : "oracle", "schema" : "cms_conditions", "secrets" : secrets_file} connection = querying.connect(connection_data) self.assertTrue(connection != None) # can cause failure if the database is down close_session_result = connection.close_session() self.assertEqual(close_session_result, True)
def test_frontier_query(self): connection_data = {"db_alias" : "pro", "host" : "frontier", "schema" : "cms_conditions"} connection = querying.connect(connection_data) # query for a tag tag = connection.tag(name="EBAlignment_measured_v06_offline") # can cause failure if the database is down close_session_result = connection.close_session() self.assertEqual(close_session_result, True)
def setUp(self): self.connection_data = { "db_alias": "orapro", "host": "oracle", "schema": "cms_conditions", "secrets": secrets_file } self.connection = querying.connect(self.connection_data)
def test_frontier_connect(self): for alias in ["pro", "arc", "int", "dev"]: connection_data = { "db_alias": alias, "host": "frontier", "schema": "cms_conditions" } connection = querying.connect(connection_data) # can cause failure if the database is down close_session_result = connection.close_session() self.assertEqual(close_session_result, True)
def test_orapro_connect(self): connection_data = { "db_alias": "orapro", "host": "oracle", "schema": "cms_conditions", "secrets": secrets_file } connection = querying.connect(connection_data) self.assertTrue(connection != None) # can cause failure if the database is down close_session_result = connection.close_session() self.assertEqual(close_session_result, True)
def test_frontier_query(self): connection_data = { "db_alias": "pro", "host": "frontier", "schema": "cms_conditions" } connection = querying.connect(connection_data) # query for a tag tag = connection.tag(name="EBAlignment_measured_v06_offline") # can cause failure if the database is down close_session_result = connection.close_session() self.assertEqual(close_session_result, True)
def setUp(self): # set up a connection to oracle self.connection = querying.connect(prod_connection_string, map_blobs=True, secrets=secrets_source) # get a payload self.payload = self.connection.payload(hash="00172cd62d8abae41915978d815ae62cc08ad8b9") if not(os.path.isfile("test_suite.sqlite")): # create file handle = open("test_suite.sqlite", "w") handle.close() # insert schema if os.path.isfile("simple_conditions_schema.sql"): try: process = subprocess.Popen("sqlite3 test_suite.sqlite < simple_conditions_schema.sql") result = process.communicate()[0] except Exception as e: self.test_write_blob_to_sqlite = unittest.skip("Can't setup sqlite database file.")(self.test_write_blob_to_sqlite)
def test_write_blob_to_sqlite(self): import os # open sqlite file in CondDBFW sqlite_con = querying.connect("sqlite://test_suite.sqlite", map_blobs=True) # write to sqlite file sqlite_con.write_and_commit(self.payload) # read payload from sqlite file, check for equality between blobs sqlite_payload = sqlite_con.payload(hash=self.payload.hash) self.assertEqual(sqlite_payload.data, self.payload.data) # delete payload from sqlite file tmp_sqlite_connection = sqlite_con.engine.connect() result = tmp_sqlite_connection.execute("delete from payload where hash=?", self.payload.hash) tmp_sqlite_connection.close() # check that payload isn't in sqlite anymore payload_in_sqlite = sqlite_con.payload(hash=self.payload.hash) self.assertEqual(payload_in_sqlite, None)
def test_write_blob_to_sqlite(self): import os # open sqlite file in CondDBFW sqlite_con = querying.connect("sqlite://test_suite.sqlite", map_blobs=True) # write to sqlite file sqlite_con.write_and_commit(self.payload) # read payload from sqlite file, check for equality between blobs sqlite_payload = sqlite_con.payload(hash=self.payload.hash) self.assertEqual(sqlite_payload.data, self.payload.data) # delete payload from sqlite file tmp_sqlite_connection = sqlite_con.engine.connect() result = tmp_sqlite_connection.execute( "delete from payload where hash=?", self.payload.hash) tmp_sqlite_connection.close() # check that payload isn't in sqlite anymore payload_in_sqlite = sqlite_con.payload(hash=self.payload.hash) self.assertEqual(payload_in_sqlite, None)
def setUp(self): # set up a connection to oracle self.connection = querying.connect(prod_connection_string, map_blobs=True, secrets=secrets_source) # get a payload self.payload = self.connection.payload( hash="00172cd62d8abae41915978d815ae62cc08ad8b9") if not (os.path.isfile("test_suite.sqlite")): # create file handle = open("test_suite.sqlite", "w") handle.close() # insert schema if os.path.isfile("simple_conditions_schema.sql"): try: process = subprocess.Popen( "sqlite3 test_suite.sqlite < simple_conditions_schema.sql") result = process.communicate()[0] except Exception as e: self.test_write_blob_to_sqlite = unittest.skip( "Can't setup sqlite database file.")( self.test_write_blob_to_sqlite)
#!/usr/bin/env python """ Example script to test writing to local sqlite db. """ import sys from CondCore.Utilities.CondDBFW import querying_framework_api from CondCore.Utilities.CondDBFW import querying class query_script(): def script(self, connection): payload = connection.payload(hash=sys.argv[1]) return payload connection_data = {"db_alias" : "orapro", "schema" : "cms_conditions", "host" : "oracle", "secrets" : "/afs/cern.ch/cms/DB/conddb/.cms_cond/netrc"} qf = querying_framework_api(connection_data) data = qf.run_script(query_script()) import pprint pprint.pprint(data.as_dicts()) # test writing to sqlite database sqlite_db_url = "/tmp/jdawes/CMSSW_7_5_2/src/CondCore/Utilities/python/CondDBFW/examples/sqlite_tests.sqlite" sqlite_con = querying.connect({"host" : "sqlite", "db_alias" : sqlite_db_url}) sqlite_con.write(data) sqlite_con.commit()
) (options, arguments) = parser.parse_args() #### Needs cmsrel inside a CMSSW > 80X theRelease = getCMSSWRelease() print "- Getting CondDBFW from release",theRelease connectionString="" if isCMSSWBefore81X( theRelease ): connectionString="frontier://pro/CMS_CONDITIONS" else: connectionString="frontier://FrontierProd/CMS_CONDITIONS" from CondCore.Utilities.CondDBFW import querying connection = querying.connect(connectionString) payload = connection.payload(hash=options.hash) my_dict = payload.parent_tags().as_dicts() fldmap = ( 'name', 's', 'synch', 's', 'insertion', '', ) # Leave these alone for unquoted, tab-delimited record format. head = '\t'.join(fldmap[0:len(fldmap):2]) + '\n' #print head t = PrettyTable(['hash', 'since','tag','synch','insertion time'])
def main(): ############################################## defaultFile='mySqlite.db' defaultTag='myTag' parser = optparse.OptionParser(usage = 'Usage: %prog [options] <file> [<file> ...]\n') parser.add_option('-f', '--file', dest = 'file', default = defaultFile, help = 'file to inspect', ) parser.add_option('-t', '--tag', dest = 'tag', default = defaultTag, help = 'tag to be inspected', ) (options, arguments) = parser.parse_args() sqlite_db_url = options.file sqlite_con = querying.connect('sqlite://'+sqlite_db_url,map_blobs=True) my_dict = sqlite_con.tag(name=options.tag).iovs().as_dicts() sinces = [] for element in my_dict: # print element # print element['hash'] # print element['since'] sinces.append(element['since']) # would be nice to make it work using CondDBFW (sigh!) # payload = sqlite_con.payload(hash=element["payload_hash"]) # sqlite_out = querying.connect('sqlite://testOut.db', map_blobs=True) # sqlite_tag = sqlite_con.tag().all().data()[0] # sqlite_iovs = sqlite_tag.iovs().data() # tag_name = "mytest" # new_tag = sqlite_con.models["tag"](sqlite_tag.as_dicts(convert_timestamps=False), convert_timestamps=False) # new_tag.name = tag_name # sqlite_out.write_and_commit(payload) for i,since in enumerate(sinces): #print i,since print "============================================================" if(i<len(sinces)-1): command = 'conddb_import -c sqlite_file:'+sqlite_db_url.rstrip(".db")+"_IOV_"+str(sinces[i])+'.db -f sqlite_file:'+sqlite_db_url+" -i "+options.tag+" -t "+options.tag+" -b "+str(sinces[i])+" -e "+str(sinces[i+1]-1) print command getCommandOutput(command) else: command = 'conddb_import -c sqlite_file:'+sqlite_db_url.rstrip(".db")+"_IOV_"+str(sinces[i])+'.db -f sqlite_file:'+sqlite_db_url+" -i "+options.tag+" -t "+options.tag+" -b "+str(sinces[i]) print command getCommandOutput(command) # update the trigger bits cmsRunCommand="cmsRun AlCaRecoTriggerBitsRcdUpdate_TEMPL_cfg.py inputDB=sqlite_file:"+sqlite_db_url.rstrip(".db")+"_IOV_"+str(sinces[i])+".db inputTag="+options.tag+" outputDB=sqlite_file:"+sqlite_db_url.rstrip(".db")+"_IOV_"+str(sinces[i])+"_updated.db outputTag="+options.tag+" firstRun="+str(sinces[i]) print cmsRunCommand getCommandOutput(cmsRunCommand) # merge the output mergeCommand = 'conddb_import -f sqlite_file:'+sqlite_db_url.rstrip(".db")+"_IOV_"+str(sinces[i])+'_updated.db -c sqlite_file:'+options.tag+".db -i "+options.tag+" -t "+options.tag+" -b "+str(sinces[i]) print mergeCommand getCommandOutput(mergeCommand) # clean the house cleanCommand = 'rm -fr *updated*.db *IOV_*.db' getCommandOutput(cleanCommand)
) (options, arguments) = parser.parse_args() #### Needs cmsrel inside a CMSSW > 80X theRelease = getCMSSWRelease() print "- Getting CondDBFW from release", theRelease connectionString = "" if isCMSSWBefore81X(theRelease): connectionString = "frontier://pro/CMS_CONDITIONS" else: connectionString = "frontier://FrontierProd/CMS_CONDITIONS" from CondCore.Utilities.CondDBFW import querying connection = querying.connect(connectionString) payload = connection.payload(hash=options.hash) my_dict = payload.parent_tags().as_dicts() fldmap = ( 'name', 's', 'synch', 's', 'insertion', '', ) # Leave these alone for unquoted, tab-delimited record format. head = '\t'.join(fldmap[0:len(fldmap):2]) + '\n'
import getpass import errno import sqlite3 import json import tempfile from prettytable import PrettyTable from datetime import datetime,timedelta def unpack(i): """unpack 64bit unsigned long long into 2 32bit unsigned int, return tuple (high,low) """ high=i>>32 low=i&0xFFFFFFFF return(high,low) #### Needs cmsrel inside a CMSSW > 80X countWrongIOVs=0 from CondCore.Utilities.CondDBFW import querying connection = querying.connect("frontier://pro/CMS_CONDITIONS") TagIOVs = connection.tag(name="EcalLaserAPDPNRatios_prompt_v2").iovs().as_dicts() for iIOV in TagIOVs: since = datetime.fromtimestamp(unpack(int(iIOV["since"]))[0]) #print iIOV["insertion_time"],iIOV["since"],since if( ((iIOV["insertion_time"]-since) > timedelta(days=2) ) and iIOV["insertion_time"] > datetime(2016, 1, 1, 0, 0, 0) ): print "For Since=", since," the delay is > 2 days, i.e.: ====> ",(iIOV["insertion_time"]-since) countWrongIOVs=countWrongIOVs+1 print countWrongIOVs
def setUp(self): self.connection = querying.connect(prod_connection_string, secrets=secrets_source) self.global_tag_name = "74X_dataRun1_HLT_frozen_v2"
def setUp(self): self.connection_data = {"db_alias" : "orapro", "host" : "oracle", "schema" : "cms_conditions", "secrets" : secrets_file} self.connection = querying.connect(self.connection_data)
def main(): ############################################## defaultFile = 'mySqlite.db' defaultTag = 'myTag' parser = optparse.OptionParser( usage='Usage: %prog [options] <file> [<file> ...]\n') parser.add_option( '-f', '--file', dest='file', default=defaultFile, help='file to inspect', ) parser.add_option( '-t', '--tag', dest='tag', default=defaultTag, help='tag to be inspected', ) (options, arguments) = parser.parse_args() sqlite_db_url = options.file sqlite_con = querying.connect('sqlite://' + sqlite_db_url, map_blobs=True) my_dict = sqlite_con.tag(name=options.tag).iovs().as_dicts() sinces = [] for element in my_dict: # print element # print element['hash'] # print element['since'] sinces.append(element['since']) # would be nice to make it work using CondDBFW (sigh!) # payload = sqlite_con.payload(hash=element["payload_hash"]) # sqlite_out = querying.connect('sqlite://testOut.db', map_blobs=True) # sqlite_tag = sqlite_con.tag().all().data()[0] # sqlite_iovs = sqlite_tag.iovs().data() # tag_name = "mytest" # new_tag = sqlite_con.models["tag"](sqlite_tag.as_dicts(convert_timestamps=False), convert_timestamps=False) # new_tag.name = tag_name # sqlite_out.write_and_commit(payload) for i, since in enumerate(sinces): #print i,since print "============================================================" if (i < len(sinces) - 1): command = 'conddb_import -c sqlite_file:' + sqlite_db_url.rstrip( ".db" ) + "_IOV_" + str( sinces[i] ) + '.db -f sqlite_file:' + sqlite_db_url + " -i " + options.tag + " -t " + options.tag + " -b " + str( sinces[i]) + " -e " + str(sinces[i + 1] - 1) print command getCommandOutput(command) else: command = 'conddb_import -c sqlite_file:' + sqlite_db_url.rstrip( ".db" ) + "_IOV_" + str( sinces[i] ) + '.db -f sqlite_file:' + sqlite_db_url + " -i " + options.tag + " -t " + options.tag + " -b " + str( sinces[i]) print command getCommandOutput(command) # update the trigger bits cmsRunCommand = "cmsRun AlCaRecoTriggerBitsRcdUpdate_TEMPL_cfg.py inputDB=sqlite_file:" + sqlite_db_url.rstrip( ".db" ) + "_IOV_" + str( sinces[i] ) + ".db inputTag=" + options.tag + " outputDB=sqlite_file:" + sqlite_db_url.rstrip( ".db") + "_IOV_" + str( sinces[i] ) + "_updated.db outputTag=" + options.tag + " firstRun=" + str( sinces[i]) print cmsRunCommand getCommandOutput(cmsRunCommand) # merge the output mergeCommand = 'conddb_import -f sqlite_file:' + sqlite_db_url.rstrip( ".db" ) + "_IOV_" + str( sinces[i] ) + '_updated.db -c sqlite_file:' + options.tag + ".db -i " + options.tag + " -t " + options.tag + " -b " + str( sinces[i]) print mergeCommand getCommandOutput(mergeCommand) # clean the house cleanCommand = 'rm -fr *updated*.db *IOV_*.db' getCommandOutput(cleanCommand)
def setUp(self): self.connection = querying.connect(prod_connection_string, secrets=secrets_source)
parser = optparse.OptionParser( usage='Usage: %prog [options] <file> [<file> ...]\n') parser.add_option( '-H', '--hash', dest='hash', default="b0e2bc7e4947817d99324ff20f8c3238d06c46fb", help='payload hash to check', ) (options, arguments) = parser.parse_args() #### Needs cmsrel inside a CMSSW > 80X from CondCore.Utilities.CondDBFW import querying connection = querying.connect("frontier://pro/CMS_CONDITIONS") payload = connection.payload(hash=options.hash) my_dict = payload.parent_tags().as_dicts() fldmap = ( 'name', 's', 'synch', 's', 'insertion', '', ) # Leave these alone for unquoted, tab-delimited record format. head = '\t'.join(fldmap[0:len(fldmap):2]) + '\n'