def test_frontier_connect(self):
for alias in ["pro", "arc", "int", "dev"]:
connection_data = {"db_alias" : alias, "host" : "frontier", "schema" : "cms_conditions"}
connection = querying.connect(connection_data)
# can cause failure if the database is down
close_session_result = connection.close_session()
self.assertEqual(close_session_result, True)
def test_orapro_connect(self):
connection_data = {"db_alias" : "orapro", "host" : "oracle", "schema" : "cms_conditions", "secrets" : secrets_file}
connection = querying.connect(connection_data)
self.assertTrue(connection != None)
# can cause failure if the database is down
close_session_result = connection.close_session()
self.assertEqual(close_session_result, True)
def test_frontier_query(self):
connection_data = {"db_alias" : "pro", "host" : "frontier", "schema" : "cms_conditions"}
connection = querying.connect(connection_data)
# query for a tag
tag = connection.tag(name="EBAlignment_measured_v06_offline")
# can cause failure if the database is down
close_session_result = connection.close_session()
self.assertEqual(close_session_result, True)
def setUp(self):
self.connection_data = {
"db_alias": "orapro",
"host": "oracle",
"schema": "cms_conditions",
"secrets": secrets_file
}
self.connection = querying.connect(self.connection_data)
def test_frontier_connect(self):
for alias in ["pro", "arc", "int", "dev"]:
connection_data = {
"db_alias": alias,
"host": "frontier",
"schema": "cms_conditions"
}
connection = querying.connect(connection_data)
# can cause failure if the database is down
close_session_result = connection.close_session()
self.assertEqual(close_session_result, True)
def test_orapro_connect(self):
connection_data = {
"db_alias": "orapro",
"host": "oracle",
"schema": "cms_conditions",
"secrets": secrets_file
}
connection = querying.connect(connection_data)
self.assertTrue(connection != None)
# can cause failure if the database is down
close_session_result = connection.close_session()
self.assertEqual(close_session_result, True)
def test_frontier_query(self):
connection_data = {
"db_alias": "pro",
"host": "frontier",
"schema": "cms_conditions"
}
connection = querying.connect(connection_data)
# query for a tag
tag = connection.tag(name="EBAlignment_measured_v06_offline")
# can cause failure if the database is down
close_session_result = connection.close_session()
self.assertEqual(close_session_result, True)
def setUp(self):
# set up a connection to oracle
self.connection = querying.connect(prod_connection_string, map_blobs=True, secrets=secrets_source)
# get a payload
self.payload = self.connection.payload(hash="00172cd62d8abae41915978d815ae62cc08ad8b9")
if not(os.path.isfile("test_suite.sqlite")):
# create file
handle = open("test_suite.sqlite", "w")
handle.close()
# insert schema
if os.path.isfile("simple_conditions_schema.sql"):
try:
process = subprocess.Popen("sqlite3 test_suite.sqlite < simple_conditions_schema.sql")
result = process.communicate()[0]
except Exception as e:
self.test_write_blob_to_sqlite = unittest.skip("Can't setup sqlite database file.")(self.test_write_blob_to_sqlite)
def test_write_blob_to_sqlite(self):
import os
# open sqlite file in CondDBFW
sqlite_con = querying.connect("sqlite://test_suite.sqlite", map_blobs=True)
# write to sqlite file
sqlite_con.write_and_commit(self.payload)
# read payload from sqlite file, check for equality between blobs
sqlite_payload = sqlite_con.payload(hash=self.payload.hash)
self.assertEqual(sqlite_payload.data, self.payload.data)
# delete payload from sqlite file
tmp_sqlite_connection = sqlite_con.engine.connect()
result = tmp_sqlite_connection.execute("delete from payload where hash=?", self.payload.hash)
tmp_sqlite_connection.close()
# check that payload isn't in sqlite anymore
payload_in_sqlite = sqlite_con.payload(hash=self.payload.hash)
self.assertEqual(payload_in_sqlite, None)
def test_write_blob_to_sqlite(self):
import os
# open sqlite file in CondDBFW
sqlite_con = querying.connect("sqlite://test_suite.sqlite",
map_blobs=True)
# write to sqlite file
sqlite_con.write_and_commit(self.payload)
# read payload from sqlite file, check for equality between blobs
sqlite_payload = sqlite_con.payload(hash=self.payload.hash)
self.assertEqual(sqlite_payload.data, self.payload.data)
# delete payload from sqlite file
tmp_sqlite_connection = sqlite_con.engine.connect()
result = tmp_sqlite_connection.execute(
"delete from payload where hash=?", self.payload.hash)
tmp_sqlite_connection.close()
# check that payload isn't in sqlite anymore
payload_in_sqlite = sqlite_con.payload(hash=self.payload.hash)
self.assertEqual(payload_in_sqlite, None)
def setUp(self):
# set up a connection to oracle
self.connection = querying.connect(prod_connection_string,
map_blobs=True,
secrets=secrets_source)
# get a payload
self.payload = self.connection.payload(
hash="00172cd62d8abae41915978d815ae62cc08ad8b9")
if not (os.path.isfile("test_suite.sqlite")):
# create file
handle = open("test_suite.sqlite", "w")
handle.close()
# insert schema
if os.path.isfile("simple_conditions_schema.sql"):
try:
process = subprocess.Popen(
"sqlite3 test_suite.sqlite < simple_conditions_schema.sql")
result = process.communicate()[0]
except Exception as e:
self.test_write_blob_to_sqlite = unittest.skip(
"Can't setup sqlite database file.")(
self.test_write_blob_to_sqlite)
#!/usr/bin/env python
"""
Example script to test writing to local sqlite db.
"""
import sys
from CondCore.Utilities.CondDBFW import querying_framework_api
from CondCore.Utilities.CondDBFW import querying
class query_script():
def script(self, connection):
payload = connection.payload(hash=sys.argv[1])
return payload
connection_data = {"db_alias" : "orapro", "schema" : "cms_conditions", "host" : "oracle", "secrets" : "/afs/cern.ch/cms/DB/conddb/.cms_cond/netrc"}
qf = querying_framework_api(connection_data)
data = qf.run_script(query_script())
import pprint
pprint.pprint(data.as_dicts())
# test writing to sqlite database
sqlite_db_url = "/tmp/jdawes/CMSSW_7_5_2/src/CondCore/Utilities/python/CondDBFW/examples/sqlite_tests.sqlite"
sqlite_con = querying.connect({"host" : "sqlite", "db_alias" : sqlite_db_url})
sqlite_con.write(data)
sqlite_con.commit()
)
(options, arguments) = parser.parse_args()
#### Needs cmsrel inside a CMSSW > 80X
theRelease = getCMSSWRelease()
print "- Getting CondDBFW from release",theRelease
connectionString=""
if isCMSSWBefore81X( theRelease ):
connectionString="frontier://pro/CMS_CONDITIONS"
else:
connectionString="frontier://FrontierProd/CMS_CONDITIONS"
from CondCore.Utilities.CondDBFW import querying
connection = querying.connect(connectionString)
payload = connection.payload(hash=options.hash)
my_dict = payload.parent_tags().as_dicts()
fldmap = (
'name', 's',
'synch', 's',
'insertion', '',
)
# Leave these alone for unquoted, tab-delimited record format.
head = '\t'.join(fldmap[0:len(fldmap):2]) + '\n'
#print head
t = PrettyTable(['hash', 'since','tag','synch','insertion time'])
def main():
##############################################
defaultFile='mySqlite.db'
defaultTag='myTag'
parser = optparse.OptionParser(usage = 'Usage: %prog [options] <file> [<file> ...]\n')
parser.add_option('-f', '--file',
dest = 'file',
default = defaultFile,
help = 'file to inspect',
)
parser.add_option('-t', '--tag',
dest = 'tag',
default = defaultTag,
help = 'tag to be inspected',
)
(options, arguments) = parser.parse_args()
sqlite_db_url = options.file
sqlite_con = querying.connect('sqlite://'+sqlite_db_url,map_blobs=True)
my_dict = sqlite_con.tag(name=options.tag).iovs().as_dicts()
sinces = []
for element in my_dict:
# print element
# print element['hash']
# print element['since']
sinces.append(element['since'])
# would be nice to make it work using CondDBFW (sigh!)
# payload = sqlite_con.payload(hash=element["payload_hash"])
# sqlite_out = querying.connect('sqlite://testOut.db', map_blobs=True)
# sqlite_tag = sqlite_con.tag().all().data()[0]
# sqlite_iovs = sqlite_tag.iovs().data()
# tag_name = "mytest"
# new_tag = sqlite_con.models["tag"](sqlite_tag.as_dicts(convert_timestamps=False), convert_timestamps=False)
# new_tag.name = tag_name
# sqlite_out.write_and_commit(payload)
for i,since in enumerate(sinces):
#print i,since
print "============================================================"
if(i<len(sinces)-1):
command = 'conddb_import -c sqlite_file:'+sqlite_db_url.rstrip(".db")+"_IOV_"+str(sinces[i])+'.db -f sqlite_file:'+sqlite_db_url+" -i "+options.tag+" -t "+options.tag+" -b "+str(sinces[i])+" -e "+str(sinces[i+1]-1)
print command
getCommandOutput(command)
else:
command = 'conddb_import -c sqlite_file:'+sqlite_db_url.rstrip(".db")+"_IOV_"+str(sinces[i])+'.db -f sqlite_file:'+sqlite_db_url+" -i "+options.tag+" -t "+options.tag+" -b "+str(sinces[i])
print command
getCommandOutput(command)
# update the trigger bits
cmsRunCommand="cmsRun AlCaRecoTriggerBitsRcdUpdate_TEMPL_cfg.py inputDB=sqlite_file:"+sqlite_db_url.rstrip(".db")+"_IOV_"+str(sinces[i])+".db inputTag="+options.tag+" outputDB=sqlite_file:"+sqlite_db_url.rstrip(".db")+"_IOV_"+str(sinces[i])+"_updated.db outputTag="+options.tag+" firstRun="+str(sinces[i])
print cmsRunCommand
getCommandOutput(cmsRunCommand)
# merge the output
mergeCommand = 'conddb_import -f sqlite_file:'+sqlite_db_url.rstrip(".db")+"_IOV_"+str(sinces[i])+'_updated.db -c sqlite_file:'+options.tag+".db -i "+options.tag+" -t "+options.tag+" -b "+str(sinces[i])
print mergeCommand
getCommandOutput(mergeCommand)
# clean the house
cleanCommand = 'rm -fr *updated*.db *IOV_*.db'
getCommandOutput(cleanCommand)
)
(options, arguments) = parser.parse_args()
#### Needs cmsrel inside a CMSSW > 80X
theRelease = getCMSSWRelease()
print "- Getting CondDBFW from release", theRelease
connectionString = ""
if isCMSSWBefore81X(theRelease):
connectionString = "frontier://pro/CMS_CONDITIONS"
else:
connectionString = "frontier://FrontierProd/CMS_CONDITIONS"
from CondCore.Utilities.CondDBFW import querying
connection = querying.connect(connectionString)
payload = connection.payload(hash=options.hash)
my_dict = payload.parent_tags().as_dicts()
fldmap = (
'name',
's',
'synch',
's',
'insertion',
'',
)
# Leave these alone for unquoted, tab-delimited record format.
head = '\t'.join(fldmap[0:len(fldmap):2]) + '\n'
import getpass
import errno
import sqlite3
import json
import tempfile
from prettytable import PrettyTable
from datetime import datetime,timedelta
def unpack(i):
"""unpack 64bit unsigned long long into 2 32bit unsigned int, return tuple (high,low)
"""
high=i>>32
low=i&0xFFFFFFFF
return(high,low)
#### Needs cmsrel inside a CMSSW > 80X
countWrongIOVs=0
from CondCore.Utilities.CondDBFW import querying
connection = querying.connect("frontier://pro/CMS_CONDITIONS")
TagIOVs = connection.tag(name="EcalLaserAPDPNRatios_prompt_v2").iovs().as_dicts()
for iIOV in TagIOVs:
since = datetime.fromtimestamp(unpack(int(iIOV["since"]))[0])
#print iIOV["insertion_time"],iIOV["since"],since
if( ((iIOV["insertion_time"]-since) > timedelta(days=2) ) and iIOV["insertion_time"] > datetime(2016, 1, 1, 0, 0, 0) ):
print "For Since=", since," the delay is > 2 days, i.e.: ====> ",(iIOV["insertion_time"]-since)
countWrongIOVs=countWrongIOVs+1
print countWrongIOVs
def setUp(self):
self.connection = querying.connect(prod_connection_string,
secrets=secrets_source)
self.global_tag_name = "74X_dataRun1_HLT_frozen_v2"
def setUp(self):
self.connection_data = {"db_alias" : "orapro", "host" : "oracle", "schema" : "cms_conditions", "secrets" : secrets_file}
self.connection = querying.connect(self.connection_data)
def main():
##############################################
defaultFile = 'mySqlite.db'
defaultTag = 'myTag'
parser = optparse.OptionParser(
usage='Usage: %prog [options] <file> [<file> ...]\n')
parser.add_option(
'-f',
'--file',
dest='file',
default=defaultFile,
help='file to inspect',
)
parser.add_option(
'-t',
'--tag',
dest='tag',
default=defaultTag,
help='tag to be inspected',
)
(options, arguments) = parser.parse_args()
sqlite_db_url = options.file
sqlite_con = querying.connect('sqlite://' + sqlite_db_url, map_blobs=True)
my_dict = sqlite_con.tag(name=options.tag).iovs().as_dicts()
sinces = []
for element in my_dict:
# print element
# print element['hash']
# print element['since']
sinces.append(element['since'])
# would be nice to make it work using CondDBFW (sigh!)
# payload = sqlite_con.payload(hash=element["payload_hash"])
# sqlite_out = querying.connect('sqlite://testOut.db', map_blobs=True)
# sqlite_tag = sqlite_con.tag().all().data()[0]
# sqlite_iovs = sqlite_tag.iovs().data()
# tag_name = "mytest"
# new_tag = sqlite_con.models["tag"](sqlite_tag.as_dicts(convert_timestamps=False), convert_timestamps=False)
# new_tag.name = tag_name
# sqlite_out.write_and_commit(payload)
for i, since in enumerate(sinces):
#print i,since
print "============================================================"
if (i < len(sinces) - 1):
command = 'conddb_import -c sqlite_file:' + sqlite_db_url.rstrip(
".db"
) + "_IOV_" + str(
sinces[i]
) + '.db -f sqlite_file:' + sqlite_db_url + " -i " + options.tag + " -t " + options.tag + " -b " + str(
sinces[i]) + " -e " + str(sinces[i + 1] - 1)
print command
getCommandOutput(command)
else:
command = 'conddb_import -c sqlite_file:' + sqlite_db_url.rstrip(
".db"
) + "_IOV_" + str(
sinces[i]
) + '.db -f sqlite_file:' + sqlite_db_url + " -i " + options.tag + " -t " + options.tag + " -b " + str(
sinces[i])
print command
getCommandOutput(command)
# update the trigger bits
cmsRunCommand = "cmsRun AlCaRecoTriggerBitsRcdUpdate_TEMPL_cfg.py inputDB=sqlite_file:" + sqlite_db_url.rstrip(
".db"
) + "_IOV_" + str(
sinces[i]
) + ".db inputTag=" + options.tag + " outputDB=sqlite_file:" + sqlite_db_url.rstrip(
".db") + "_IOV_" + str(
sinces[i]
) + "_updated.db outputTag=" + options.tag + " firstRun=" + str(
sinces[i])
print cmsRunCommand
getCommandOutput(cmsRunCommand)
# merge the output
mergeCommand = 'conddb_import -f sqlite_file:' + sqlite_db_url.rstrip(
".db"
) + "_IOV_" + str(
sinces[i]
) + '_updated.db -c sqlite_file:' + options.tag + ".db -i " + options.tag + " -t " + options.tag + " -b " + str(
sinces[i])
print mergeCommand
getCommandOutput(mergeCommand)
# clean the house
cleanCommand = 'rm -fr *updated*.db *IOV_*.db'
getCommandOutput(cleanCommand)
def setUp(self): self.connection = querying.connect(prod_connection_string, secrets=secrets_source) self.global_tag_name = "74X_dataRun1_HLT_frozen_v2"
def setUp(self):
self.connection = querying.connect(prod_connection_string,
secrets=secrets_source)
def setUp(self): self.connection = querying.connect(prod_connection_string, secrets=secrets_source)
parser = optparse.OptionParser(
usage='Usage: %prog [options] <file> [<file> ...]\n')
parser.add_option(
'-H',
'--hash',
dest='hash',
default="b0e2bc7e4947817d99324ff20f8c3238d06c46fb",
help='payload hash to check',
)
(options, arguments) = parser.parse_args()
#### Needs cmsrel inside a CMSSW > 80X
from CondCore.Utilities.CondDBFW import querying
connection = querying.connect("frontier://pro/CMS_CONDITIONS")
payload = connection.payload(hash=options.hash)
my_dict = payload.parent_tags().as_dicts()
fldmap = (
'name',
's',
'synch',
's',
'insertion',
'',
)
# Leave these alone for unquoted, tab-delimited record format.
head = '\t'.join(fldmap[0:len(fldmap):2]) + '\n'
