def __init__(self, session, stream):
myEnv, status = irods.getRodsEnv()
conn, errMsg = irods.rcConnect(myEnv.getRodsHost(), myEnv.getRodsPort(),
myEnv.getRodsUserName(), myEnv.getRodsZone())
status = irods.clientLogin(conn)
if status:
raise ConfigFileException("Cannot connect to iRODS: (%s) %s" % (status, errMsg))
home = myEnv.getRodsHome()
c = irods.irodsCollection(conn, home)
self.cxn = conn
self.coll = c
instream = stream
# check if abs path to home dir
if stream.startswith(home):
stream = stream[len(home):]
if stream[0] == "/":
stream = stream[1:]
colls = stream.split('/')
for i, cln in enumerate(colls):
exit_status = c.openCollection(cln)
if exit_status < 0:
if (i < len(colls) - 1) or \
(cln not in [obj[0] for obj in c.getObjects()]):
raise IOError("When opening {0}: {1} does not exists in collection {2}".format(instream, cln, c.getCollName()))
def create_irods_connection(username, password, rodsconfig):
from irods import (getRodsEnv, rcConnect, clientLoginWithPassword,
rodsErrorName)
err, rodsEnv = getRodsEnv() # Override all values later
rodsEnv.rodsUserName = username
rodsEnv.rodsHost = rodsconfig[0]
rodsEnv.rodsPort = rodsconfig[1]
rodsEnv.rodsZone = rodsconfig[2]
conn, err = rcConnect(rodsEnv.rodsHost,
rodsEnv.rodsPort,
rodsEnv.rodsUserName,
rodsEnv.rodsZone
)
if err.status != 0:
raise Exception('Connecting to iRODS failed %s'
% rodsErrorName(err.status)[0])
err = clientLoginWithPassword(conn, password)
if err != 0:
raise Exception('Authenticating to iRODS failed %s, user: %, pw: %s'
% rodsErrorName(err.status)[0], username, password)
return conn
def get_default_zone(self):
"""Get the default zone from .irodsEnv"""
status, myEnv = irods.getRodsEnv()
if myEnv and (status == 0):
return myEnv.rodsZone
else:
return ""
def get_default_username(self):
"""Get the default username from .irodsEnv"""
status, myEnv = irods.getRodsEnv()
if myEnv and (status == 0):
return myEnv.rodsUserName
else:
return ""
def get_default_resource(self):
"""Get the default resource from .irodsEnv"""
status, myEnv = irods.getRodsEnv()
if myEnv and (status == 0):
return myEnv.rodsDefResource
else:
return ""
def get_default_port(self):
"""Get the default port from .irodsEnv"""
status, myEnv = irods.getRodsEnv()
if myEnv and (status == 0):
return myEnv.rodsPort
else:
return 0
def get_default_host(self):
"""Get the default host from .irodsEnv"""
status, myEnv = irods.getRodsEnv()
if myEnv and (status == 0):
return myEnv.rodsHost
else:
return ""
def __init__(self, session, stream):
# Check for dependency
if irods is None:
raise MissingDependencyException(
'{0.__module__}.{0.__class__.__name__}'.format(self),
'irods (PyRods)'
)
# Check for URL
if stream.startswith(('irods://', 'rods://')):
myEnv = parse_irodsUrl(stream)
stream = myEnv.relpath
else:
# Get parameters from env
status, myEnv = irods.getRodsEnv()
try:
host = myEnv.getRodsHost()
port = myEnv.getRodsPort()
username = myEnv.getRodsUserName()
zone = myEnv.getRodsZone()
home = myEnv.getRodsHome()
except AttributeError:
host = myEnv.rodsHost
port = myEnv.rodsPort
username = myEnv.rodsUserName
zone = myEnv.rodsZone
home = myEnv.rodsHome
conn, errMsg = irods.rcConnect(host, port, username, zone)
status = irods.clientLogin(conn)
if status:
raise ConfigFileException("Cannot connect to iRODS: ({0}) {1}"
"".format(status, errMsg)
)
c = irods.irodsCollection(conn)
self.cxn = conn
self.coll = c
instream = stream
# Check if abs path to home dir
if stream.startswith(home):
stream = stream[len(home):]
if stream[0] == "/":
stream = stream[1:]
colls = stream.split('/')
for i, cln in enumerate(colls):
exit_status = c.openCollection(cln)
if exit_status < 0:
if (
(i < len(colls) - 1) or
(cln not in [obj[0] for obj in c.getObjects()])
):
raise IOError("When opening {0}: {1} does not exists in "
"collection {2}".format(instream,
cln,
c.getCollName()
)
)
def _open(self, session):
if self.cxn == None:
# connect to iRODS
myEnv, status = irods.getRodsEnv()
host = self.host if self.host else myEnv.getRodsHost()
port = self.port if self.port else myEnv.getRodsPort()
user = self.user if self.user else myEnv.getRodsUserName()
zone = self.zone if self.zone else myEnv.getRodsZone()
conn, errMsg = irods.rcConnect(host, port, user, zone)
if self.passwd:
status = irods.clientLoginWithPassword(conn, self.passwd)
else:
status = irods.clientLogin(conn)
if status:
raise ConfigFileException("Cannot connect to iRODS: (%s) %s" % (status, errMsg.getMsg()))
self.cxn = conn
self.env = myEnv
resources = irods.getResources(self.cxn)
self.resourceHash = {}
for r in resources:
self.resourceHash[r.getName()] = r
if self.coll != None:
# already open, just skip
return None
c = irods.irodsCollection(self.cxn, self.env.getRodsHome())
self.coll = c
# move into cheshire3 section
path = self.get_path(session, 'irodsCollection', 'cheshire3')
dirs = c.getSubCollections()
if not path in dirs:
c.createCollection(path)
c.openCollection(path)
if self.get_setting(session, 'createSubDir', 1):
# now look for object's storage area
# maybe move into database collection
if (isinstance(self.parent, Database)):
sc = self.parent.id
dirs = c.getSubCollections()
if not sc in dirs:
c.createCollection(sc)
c.openCollection(sc)
# move into store collection
dirs = c.getSubCollections()
if not self.id in dirs:
c.createCollection(self.id)
c.openCollection(self.id)
def __init__(self): self.status, self.myEnv = irods.getRodsEnv() self.conn, self.errMsg = irods.rcConnect(self.myEnv.rodsHost, self.myEnv.rodsPort, self.myEnv.rodsUserName, self.myEnv.rodsZone) self.status = irods.clientLogin(self.conn) self.path = makePath(self.myEnv.rodsHome, FOLDER) # addSlash(self.myEnv.rodsHome, reduce(addSlash, FOLDER)) self.coll = irods.irodsCollection(self.conn, self.path) self.popen_opt = dict(shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
def rods_connect():
"""
A basic iRODS connection mechanism that connects using the current iRODS
environment
"""
status, env = irods.getRodsEnv()
assert status == 0, 'connect(): getRodsEnv() failed (%s): %s' % ( status, irods.strerror( status ) )
conn, err = irods.rcConnect( env.rodsHost,
env.rodsPort,
env.rodsUserName,
env.rodsZone )
assert err.status == 0, 'connect(): rcConnect() failed (%s): %s' % ( err.status, err.msg )
status, pw = irods.obfGetPw()
assert status == 0, 'connect(): getting password with obfGetPw() failed (%s): %s' % ( status, irods.strerror( status ) )
status = irods.clientLoginWithObfPassword( conn, pw )
assert status == 0, 'connect(): logging in with clientLoginWithObfPassword() failed (%s): %s' % ( status, irods.strerror( status ) )
return env, conn
def rods_connect():
"""
A basic iRODS connection mechanism that connects using the current iRODS
environment
"""
status, env = irods.getRodsEnv()
assert status == 0, 'connect(): getRodsEnv() failed (%s): %s' % ( status, irods.strerror( status ) )
conn, err = irods.rcConnect( env.rodsHost,
env.rodsPort,
env.rodsUserName,
env.rodsZone )
assert err.status == 0, 'connect(): rcConnect() failed (%s): %s' % ( err.status, err.msg )
status, pw = irods.obfGetPw()
assert status == 0, 'connect(): getting password with obfGetPw() failed (%s): %s' % ( status, irods.strerror( status ) )
status = irods.clientLoginWithObfPassword( conn, pw )
assert status == 0, 'connect(): logging in with clientLoginWithObfPassword() failed (%s): %s' % ( status, irods.strerror( status ) )
return env, conn
def _openIrods(self, session):
if self.cxn is None:
# connect to iRODS
myEnv, status = irods.getRodsEnv()
conn, errMsg = irods.rcConnect(myEnv.getRodsHost(),
myEnv.getRodsPort(),
myEnv.getRodsUserName(),
myEnv.getRodsZone()
)
status = irods.clientLogin(conn)
if status:
raise ConfigFileException("Cannot connect to iRODS: (%s) %s"
"" % (status, errMsg))
self.cxn = conn
self.env = myEnv
if self.coll is not None:
# already open, just skip
return None
c = irods.irodsCollection(self.cxn, self.env.getRodsHome())
self.coll = c
# move into cheshire3 section
path = self.get_path(session, 'irodsCollection', 'cheshire3')
dirs = c.getSubCollections()
if not path in dirs:
c.createCollection(path)
c.openCollection(path)
# now look for object's storage area
# maybe move into database collection
if (isinstance(self.parent, Database)):
sc = self.parent.id
dirs = c.getSubCollections()
if not sc in dirs:
c.createCollection(sc)
c.openCollection(sc)
# move into store collection
dirs = c.getSubCollections()
if not self.id in dirs:
c.createCollection(self.id)
c.openCollection(self.id)
def main(argv=None):
"""Load data into a Cheshire3 database based on parameters in argv."""
global argparser, session, server, db
if argv is None:
args = argparser.parse_args()
else:
args = argparser.parse_args(argv)
if irods is None:
raise MissingDependencyException('icheshire3-load script',
'irods (PyRods)'
)
session = Session()
server = SimpleServer(session, args.serverconfig)
if args.database is None:
try:
dbid = identify_database(session, os.getcwd())
except EnvironmentError as e:
server.log_critical(session, e.message)
return 1
server.log_debug(
session,
"database identifier not specified, discovered: {0}".format(dbid))
else:
dbid = args.database
try:
db = server.get_object(session, dbid)
except ObjectDoesNotExistException:
msg = """Cheshire3 database {0} does not exist.
Please provide a different database identifier using the --database option.
""".format(dbid)
server.log_critical(session, msg)
return 2
else:
# Allow for multiple data arguments
docFac = db.get_object(session, 'defaultDocumentFactory')
for dataArg in args.data:
if dataArg.startswith('irods://'):
parsed = urlsplit(dataArg)
else:
# Examine current environment
status, myEnv = irods.getRodsEnv()
try:
host = myEnv.getRodsHost()
except AttributeError:
host = myEnv.rodsHost
# Port
try:
myEnv.getRodsPort()
except AttributeError:
port = myEnv.rodsPort
# User
try:
username = myEnv.getRodsUserName()
except AttributeError:
username = myEnv.rodsUserName
netloc = '{0}@{1}:{2}'.format(username, host, port)
try:
cqm = myEnv.getRodsCwd()
except AttributeError:
cwd = myEnv.rodsCwd
path = '/'.join([cwd, dataArg])
parsed = SplitResult('irods', netloc, path, None, None)
dataArg = urlunsplit(parsed)
server.log_debug(session, dataArg)
if args.format is None or not args.format.startswith('i'):
fmt = 'irods'
else:
fmt = args.format
server.log_debug(session, fmt)
try:
docFac.load(session, dataArg,
args.cache, fmt, args.tagname, args.codec)
except MissingDependencyException as e:
server.log_critical(session, e.reason)
missingDependencies = e.dependencies
raise MissingDependencyException('cheshire3-load script',
missingDependencies)
wf = db.get_object(session, 'buildIndexWorkflow')
wf.process(session, docFac)
from bq.util.paths import data_path
IRODS_CACHE = data_path('irods_cache')
CONNECTION_POOL = {}
class IrodsError(Exception):
pass
log = logging.getLogger('bq.irods')
PARSE_NET = re.compile(
r'^((?P<user>[^:]+):(?P<password>[\w.#^!;]+)?@)?(?P<host>[^:]+)(?P<port>:\d+)?'
)
irods_env, status = irods.getRodsEnv() # pylint: disable=no-member
def irods_cleanup():
for key, conn in CONNECTION_POOL.items():
print "disconnecting %s" % key
conn.disconnect()
#atexit.register(irods_cleanup)
class IrodsConnection(object):
def __init__(self, url, user=None, host=None, port=None, password=None):
irods_url = urlparse.urlparse(url)
assert irods_url.scheme == 'irods'
def _open(self, session):
if self.cxn == None:
# connect to iRODS
myEnv, status = irods.getRodsEnv()
host = self.host if self.host else myEnv.getRodsHost()
port = self.port if self.port else myEnv.getRodsPort()
user = self.user if self.user else myEnv.getRodsUserName()
zone = self.zone if self.zone else myEnv.getRodsZone()
conn, errMsg = irods.rcConnect(host, port, user, zone)
if self.passwd:
status = irods.clientLoginWithPassword(conn, self.passwd)
else:
status = irods.clientLogin(conn)
if status:
raise ConfigFileException("Cannot connect to iRODS: (%s) %s" % (status, errMsg))
self.cxn = conn
self.env = myEnv
resources = irods.getResources(self.cxn)
self.resourceHash = {}
for r in resources:
self.resourceHash[r.getName()] = r
if self.coll != None:
# already open, just skip
return None
c = irods.irodsCollection(self.cxn, self.env.getRodsHome())
self.coll = c
# move into cheshire3 section
path = self.get_path(session, 'irodsCollection', 'cheshire3')
dirs = c.getSubCollections()
if not path in dirs:
c.createCollection(path)
c.openCollection(path)
if self.get_setting(session, 'createSubDir', 1):
# now look for object's storage area
# maybe move into database collection
if (isinstance(self.parent, Database)):
sc = self.parent.id
dirs = c.getSubCollections()
if not sc in dirs:
c.createCollection(sc)
c.openCollection(sc)
# move into store collection
dirs = c.getSubCollections()
if not self.id in dirs:
c.createCollection(self.id)
c.openCollection(self.id)
# Fetch user metadata
myMetadata = self.get_metadataTypes(session)
umd = c.getUserMetadata()
umdHash = {}
for u in umd:
umdHash[u[0]] = icatValToPy(*u[1:])
for md in myMetadata:
try:
setattr(self, md, umdHash[md])
except KeyError:
# hasn't been set yet
pass
if self.totalItems != 0:
self.meanWordCount = self.totalWordCount / self.totalItems
self.meanByteCount = self.totalByteCount / self.totalItems
else:
self.meanWordCount = 1
self.meanByteCount = 1
def get_current_dir(self):
"""Get the current collection from .irodsEnv"""
status, myEnv = irods.getRodsEnv()
if status < 0:
return None
return myEnv.rodsCwd
