def setUpClass(cls):
cls.text = """
;
; initial comments in file
;comment line with comment marker not in column 1 not allowed
;
[db-maximal]
USER=maximal_user
PASSWD = maximal_passwd
name = maximal_name
Sid = maximal_sid
type = POSTgres
server = maximal_server
[db-minimal]
USER = Minimal_user
PASSWD = Minimal_passwd
name = Minimal_name
sid = Minimal_sid
server = Minimal_server
[db-extra]
serverr = sevrver
[db-empty]
; empty section
"""
cls.filename = "wellformed.ini"
open(cls.filename, "w").write(cls.text)
protect_file(cls.filename)
cls.maximal = serviceaccess.parse(cls.filename, "db-maximal", "dB")
cls.minimal = serviceaccess.parse(cls.filename, "db-minimal", "db")
cls.empty = serviceaccess.parse(cls.filename, "db-empty", "db")
cls.extra = serviceaccess.parse(cls.filename, "db-extra", "db")
def test_badSection(self):
try:
serviceaccess.parse(self.filename, None, "dB")
except serviceaccess.ServiceaccessException as ex:
self.assertTrue('faulty' in str(ex))
else:
self.assertFalse(True)
def test_via_env_bad(self):
""" test that fault arises when environment names section not in teh file"""
os.environ["DES_DB_SECTION"] = "some-non-existing section"
import ConfigParser
assert_fired = False
try:
serviceaccess.parse(self.filename, None, "db")
except ConfigParser.NoSectionError:
assert_fired = True
self.assertTrue(assert_fired)
def __init__(self,
des_services,
des_http_section,
numtries=5,
secondsBetweenRetries=30):
"""Get password for curl and initialize existing_directories variable.
>>> C = HttpUtils('test_http_utils/.desservices.ini', 'file-http')
>>> len(C.curl_password)
25"""
try:
# Parse the .desservices.ini file:
self.auth_params = serviceaccess.parse(des_services,
des_http_section)
# Create the user/password switch:
self.curl_password = f"{self.auth_params['user']}:{self.auth_params['passwd']}"
except Exception as err:
miscutils.fwdie(f"Unable to get curl password ({err})",
fmdefs.FM_EXIT_FAILURE)
self.curl = pycurl.Curl()
self.curl.setopt(pycurl.USERPWD, self.curl_password)
self.existing_directories = set()
self.numtries = numtries
self.src = None
self.dst = None
self.filesize = None
self.secondsBetweenRetries = secondsBetweenRetries
def test_detect_permission(self):
""" test python API detect mal formed permissions """
d = serviceaccess.parse(self.filename, self.section, "DB")
try:
serviceaccess.check(d, "db")
except serviceaccess.ServiceaccessException:
pass
def __init__(self, section):
jiradict = parse(None, section)
jirauser = jiradict['user']
jirapasswd = jiradict['passwd']
jiraserver = jiradict['server']
jira = JIRA(options={'server': jiraserver},
basic_auth=(jirauser, jirapasswd))
self.jira = jira
self.server = jiraserver
self.user = jirauser
def __init__(self, desfile=None, section=None):
"""
Create an interface object for a DES database.
The DES services file and/or section contained therein may be specified
via the desfile and section arguments. When omitted default values
will be used as defined in DESDM-3. A tag of "db" will be used in all
cases.
"""
self.configdict = serviceaccess.parse(desfile,section,'DB')
self.type = self.configdict['type']
serviceaccess.check (self.configdict, 'DB')
if self.type == 'oracle':
import oracon
conClass = oracon.OracleConnection
elif self.type == 'postgres':
import pgcon
conClass = pgcon.PostgresConnection
else:
raise errors.UnknownDBTypeError (self.type)
MAXTRIES = 5
TRY_DELAY = 10 # seconds
trycnt = 0
done = False
lasterr = ""
while not done and trycnt < MAXTRIES:
trycnt += 1
try:
self.con = conClass (self.configdict)
done = True
except Exception as e:
lasterr = str(e).strip()
timestamp = time.strftime("%x %X", time.localtime())
print "%s: Error when trying to connect to database: %s" % (timestamp,lasterr)
if trycnt < MAXTRIES:
print "\tRetrying...\n"
time.sleep(TRY_DELAY)
if not done:
print "Exechost:", socket.gethostname()
print "Connection information:", str(self)
#for key in ("user", "type", "port", "server"):
# print "\t%s = %s" % (key, self.configdict[key])
print ""
raise Exception("Aborting attempt to connect to database. Last error message: %s" % lasterr)
elif trycnt > 1: # only print success message if we've printed failure message
print "Successfully connected to database after retrying."
def __init__(self, des_services, des_http_section, destination):
try:
# Parse the .desservices.ini file:
self.auth_params = serviceaccess.parse(des_services,
des_http_section)
# set up the connection
self.webdav = webdav.connect(destination,
username=self.auth_params['user'],
password=self.auth_params['passwd'])
except Exception as err:
miscutils.fwdie("Unable to get curl password (%s)" % err,
fmdefs.FM_EXIT_FAILURE)
self.existing_directories = set()
def Wget_data_home(args):
import csv
from despyserviceaccess import serviceaccess
import os
import glob
import sys
catname = """D%08d_%s_%s_r%sp%02d_sextractor_psf.fits""" % (
args.expnum, "%", "%", args.reqnum, args.attnum)
myfile = """D%08d_*_r%sp%02d_sextractor_psf.fits""" % (
args.expnum, args.reqnum, args.attnum)
#Check first if file exists...
if glob.glob(myfile):
#Print '%s does seem to exist... exiting now...' % catname
print "relevant cat files already exist in the current directory... no need to wget..."
#sys.exit(1)
return 1
else:
print "relevant cat files are not in directory... wgetting them from archive..."
queryText = """ select 'https://desar2.cosmology.illinois.edu/DESFiles/desarchive//'||fai.PATH||'/'||fai.FILENAME from prod.file_archive_info fai where filename like '%s' order by fai.filename """ % (
catname)
#new database
if args.verbose > 0: print queryText
creds = serviceaccess.parse(
os.path.join(os.environ['HOME'], '.desservices.ini'), args.section)
USERNAME = creds['user']
PASSWORD = creds['passwd']
dbh = despydb.desdbi.DesDbi(
os.path.join(os.getenv("HOME"), ".desservices.ini"), args.section)
t0 = time.time()
cur = dbh.cursor()
cur.execute(queryText)
if args.verbose > 0: print "query took", time.time() - t0, "seconds"
for line in cur:
#print USERNAME,PASSWORD,line[0]
line2 = '--no-check-certificate --quiet --http-user=%s --http-password=%s %s' % (
USERNAME, PASSWORD, line[0])
#print line2
os.system('wget %s' % line2)
return 0
def setUp(self):
self.text = """
;
; initial comments in file
;comment line with comment marker not in column 1 not allowed
;
[db-maximal]
USER=maximal_user
PASSWD = maximal_passwd
name = maximal_name_1 ; if repeated last name wins
name = maximal_name ; if repeated key, last one wins
Sid = maximal_sid ;comment glued onto value not allowed
type = POSTgres
server = maximal_server
[db-minimal]
USER = Minimal_user
PASSWD = Minimal_passwd
name = Minimal_name
sid = Minimal_sid
server = Minimal_server
[db-extra]
serverr = sevrver ; example of mis-spelled keyword
[db-empty]
; empty section
"""
self.filename = "wellformed.ini"
open(self.filename, "w").write(self.text)
protect_file(self.filename)
self.maximal = serviceaccess.parse(self.filename, "db-maximal", "dB")
self.minimal = serviceaccess.parse(self.filename, "db-minimal", "db")
self.empty = serviceaccess.parse(self.filename, "db-empty", "db")
self.extra = serviceaccess.parse(self.filename, "db-extra", "db")
def get_credentials(
desfile=os.path.join(os.environ['HOME'], '.desservices.ini'),
section='http-desarchive'):
"""
Load the credentials using serviceaccess from a local .desservices file
if possible.
"""
try:
from despyserviceaccess import serviceaccess
creds = serviceaccess.parse(desfile, section)
username = creds['user']
password = creds['passwd']
url = creds.get('url', None)
except Exception:
username = None
password = None
url = None
warning = """WARNING: could not load credentials from .desservices.ini file for section %s
please make sure sections make sense""" % section
print warning
return username, password, url
def get_credentials(
desfile=os.path.join(os.environ['HOME'], '.desservices.ini'),
section='http-desarchive'):
"""
Load the credentials using serviceaccess from a local .desservices file
if possible.
Parameters
----------
desfile : str, optional
The services access file to use, defaults to $HOME/.desservices.ini
section : str, optional
The section to read from `desfile`. Defaults to 'http-desarchive'.
Returns
-------
tuple
The user name, password, and base url from the file.
"""
try:
from despyserviceaccess import serviceaccess
creds = serviceaccess.parse(desfile, section)
username = creds['user']
password = creds['passwd']
url = creds.get('url', None)
except Exception:
username = None
password = None
url = None
warning = """WARNING: could not load credentials from .desservices.ini file for section %s
please make sure sections make sense""" % section
print warning
return username, password, url
def __init__(self,
des_services,
des_http_section,
numtries=5,
secondsBetweenRetries=30):
try:
# Parse the .desservices.ini file:
self.auth_params = serviceaccess.parse(des_services,
des_http_section)
# Create the user/password switch:
self.curl_password = "******" % (self.auth_params['user'],
self.auth_params['passwd'])
except Exception as err:
miscutils.fwdie("Unable to get curl password (%s)" % err,
fmdefs.FM_EXIT_FAILURE)
self.curl = pycurl.Curl()
self.curl.setopt(pycurl.USERPWD, self.curl_password)
self.existing_directories = set()
self.numtries = numtries
self.src = None
self.dst = None
self.filesize = None
self.secondsBetweenRetries = secondsBetweenRetries
def test_via_env_good(self):
"""test python gettion section form the environment"""
os.environ["DES_DB_SECTION"] = self.section
_ = serviceaccess.parse(self.filename, None, "db")
_ = serviceaccess.parse(self.filename, "", "db")
def test_with_env_bad_and_filename_good(self):
"""test that passed in file name trumps environment"""
os.environ["DES_SERVICES"] = "no/file/here"
serviceaccess.parse(self.filename, self.section, "db")
def testbadFileNameWithProcessError(self):
with patch('despyserviceaccess.serviceaccess.subprocess.Popen',
side_effect=Exception()):
with self.assertRaises(IOError):
serviceaccess.parse('blah', 'db-sec', 'db', True)
def test_python_maximal_assert(self):
""" Test that checkign a proper file throws no errors """
with self.assertRaises(serviceaccess.ServiceaccessException):
serviceaccess.check(
serviceaccess.parse(self.filename, 'db-maximal', "db"), 'db')
def test_notDb(self):
self.minimal = serviceaccess.parse(self.filename, "db-minimal")
def test_badFileNameWithRetry(self):
with self.assertRaises(IOError):
serviceaccess.parse('blah', 'db-sec', 'db', True)
def test_with_no_file_HOME(self):
""" test errors is raise if HOME in error"""
os.environ["HOME"] = "no/file/here"
with self.assertRaises(IOError):
serviceaccess.parse(None, self.section, "db")
def test_HOME(self):
""" test file in HOME area is found in python library"""
os.environ["HOME"] = "./"
d = serviceaccess.parse(None, self.section, "DB")
self.assertEqual(d['key'], 'akey')
