output = out.decode('utf8').strip()
output = output[output.find('\n') + 1:]
raise ConnectionError('Database is locked (%s):\n%s' %
(dburi, output),
given_version=pegasus_version,
db_type=db_type)
else:
raise ConnectionError("%s (%s)" % (e.message, dburi),
given_version=pegasus_version,
db_type=db_type)
if schema_check:
try:
from Pegasus.db.admin.admin_loader import DBAdminError, db_verify
db_verify(db, pegasus_version=pegasus_version, force=force)
except DBAdminError as e:
e.db_type = db_type
raise (e)
return db
def connect_by_submitdir(submit_dir,
db_type,
config_properties=None,
echo=False,
schema_check=True,
create=False,
pegasus_version=None,
force=False,
connect_args = _parse_props(db, props, db_type, connect_args)
if connect_args:
_parse_connect_args(db, connect_args)
# Database creation
if create:
try:
from Pegasus.db.admin.admin_loader import db_create
db_create(dburi, engine, db, pegasus_version=pegasus_version, force=force)
except exc.OperationalError, e:
raise ConnectionError("%s (%s)" % (e.message, dburi))
if schema_check:
from Pegasus.db.admin.admin_loader import db_verify
db_verify(db, pegasus_version=pegasus_version, force=force)
return db
def connect_by_submitdir(submit_dir, db_type, config_properties=None, echo=False, schema_check=True,
create=False, pegasus_version=None, force=False, cl_properties=None):
""" Connect to the database from submit directory and database type """
dburi = url_by_submitdir(submit_dir, db_type, config_properties, cl_properties=cl_properties)
return connect(dburi, echo, schema_check, create=create, pegasus_version=pegasus_version, force=force)
def connect_by_properties(config_properties, db_type, cl_properties=None, echo=False, schema_check=True, create=False,
pegasus_version=None, force=False):
""" Connect to the database from properties file and database type """
dburi = url_by_properties(config_properties, db_type, cl_properties=cl_properties)
def connect(dburi,
echo=False,
schema_check=True,
create=False,
pegasus_version=None,
force=False,
props=None,
db_type=None,
backup=False,
connect_args=None,
verbose=True):
"""
Connect to the provided URL database.
:param dburi: DB URI
:param echo:
:param schema_check: Whether the schema should be checked
:param create: Whether to create/update the database
:param pegasus_version: Pegasus given version to create/check the database
:param force:
:param props:
:param db_type: DB type (JDBCRC, MASTER, or WORKFLOW)
:param backup: Whether to backup the database
:param connect_args:
:param verbose: Whether information log should be printed
:return:
"""
dburi = _parse_jdbc_uri(dburi)
_validate(dburi)
mask = None
if backup:
mask = _backup_db(dburi)
init = _db_is_file(dburi)
try:
log.debug("Connecting to: %s" % dburi)
# parse connection properties
connect_args = _parse_props(dburi, props, db_type, connect_args)
engine = create_engine(dburi,
echo=echo,
pool_recycle=True,
connect_args=connect_args)
engine.connect()
if backup or not init:
_check_db_permissions(dburi, db_type, mask)
except exc.OperationalError as e:
if "mysql" in dburi and "unknown database" in str(e).lower():
raise ConnectionError(
"MySQL database should be previously created: %s (%s)" %
(e.message, dburi),
given_version=pegasus_version,
db_type=db_type)
raise ConnectionError("%s (%s)" % (e.message, dburi),
given_version=pegasus_version,
db_type=db_type)
except Exception as e:
raise ConnectionError("%s (%s)" % (e.message, dburi),
given_version=pegasus_version,
db_type=db_type)
Session = orm.sessionmaker(bind=engine,
autoflush=False,
autocommit=False,
expire_on_commit=False)
db = orm.scoped_session(Session)
# Database creation
if create:
try:
from Pegasus.db.admin.admin_loader import db_create
db_create(dburi,
engine,
db,
pegasus_version=pegasus_version,
force=force,
verbose=verbose)
except exc.OperationalError as e:
if 'database is locked' in str(e).lower():
# database is locked, getting PIDs and commands
p = urlparse(dburi)
out, err = subprocess.Popen('fuser -u %s' % p.path,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
cwd=os.getcwd()).communicate()
pids = out.decode('utf8').strip().replace(' ', ',')
out, err = subprocess.Popen('ps -o "pid user command" -p %s' %
pids,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
cwd=os.getcwd()).communicate()
if err:
raise ConnectionError(
'Database is locked (%s): Unable to run ps command: %s'
% (dburi, err),
given_version=pegasus_version,
db_type=db_type)
output = out.decode('utf8').strip()
output = output[output.find('\n') + 1:]
raise ConnectionError('Database is locked (%s):\n%s' %
(dburi, output),
given_version=pegasus_version,
db_type=db_type)
else:
raise ConnectionError("%s (%s)" % (e.message, dburi),
given_version=pegasus_version,
db_type=db_type)
if schema_check:
try:
from Pegasus.db.admin.admin_loader import DBAdminError, db_verify
db_verify(db, pegasus_version=pegasus_version, force=force)
except DBAdminError as e:
e.db_type = db_type
raise (e)
return db
def connect(
dburi,
echo=False,
schema_check=True,
create=False,
pegasus_version=None,
force=False,
props=None,
db_type=None,
backup=False,
connect_args=None,
verbose=True,
):
"""
Connect to the provided URL database.
:param dburi: DB URI
:param echo:
:param schema_check: Whether the schema should be checked
:param create: Whether to create/update the database
:param pegasus_version: Pegasus given version to create/check the database
:param force:
:param props:
:param db_type: DB type (JDBCRC, MASTER, or WORKFLOW)
:param backup: Whether to backup the database
:param connect_args:
:param verbose: Whether information log should be printed
:return:
"""
dburi = _parse_jdbc_uri(dburi)
_validate(dburi)
mask = None
if backup:
mask = _backup_db(dburi)
init = _db_is_file(dburi)
try:
log.debug(
"Connecting to: %s with connection params as %s" % (dburi, connect_args)
)
# parse connection properties
# PM-898 monitord sends props as None and connect_args has the
# right values for the connection
connect_args = _parse_props(dburi, props, db_type, connect_args)
engine = create_engine(
dburi, echo=echo, pool_recycle=True, connect_args=connect_args
)
engine.connect()
if backup or not init:
_check_db_permissions(dburi, db_type, mask)
except exc.OperationalError as e:
if "mysql" in dburi and "unknown database" in str(e).lower():
raise ConnectionError(
"MySQL database should be previously created: %s (%s)" % (e, dburi),
given_version=pegasus_version,
db_type=db_type,
)
raise ConnectionError(
"%s (%s)" % (e, dburi), given_version=pegasus_version, db_type=db_type
)
except Exception as e:
raise ConnectionError(
"%s (%s)" % (e, dburi), given_version=pegasus_version, db_type=db_type
)
Session = orm.sessionmaker(
bind=engine, autoflush=False, autocommit=False, expire_on_commit=False
)
db = orm.scoped_session(Session)
# Database creation
if create:
try:
from Pegasus.db.admin.admin_loader import db_create
db_create(
dburi,
engine,
db,
pegasus_version=pegasus_version,
force=force,
verbose=verbose,
)
except exc.OperationalError as e:
if "database is locked" in str(e).lower():
# database is locked, getting PIDs and commands
p = urlparse(dburi)
out, err = subprocess.Popen(
"fuser -u %s" % p.path,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
cwd=os.getcwd(),
).communicate()
pids = out.decode("utf8").strip().replace(" ", ",")
out, err = subprocess.Popen(
'ps -o "pid user command" -p %s' % pids,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
cwd=os.getcwd(),
).communicate()
if err:
raise ConnectionError(
"Database is locked (%s): Unable to run ps command: %s"
% (dburi, err),
given_version=pegasus_version,
db_type=db_type,
)
output = out.decode("utf8").strip()
output = output[output.find("\n") + 1 :]
raise ConnectionError(
"Database is locked (%s):\n%s" % (dburi, output),
given_version=pegasus_version,
db_type=db_type,
)
else:
raise ConnectionError(
"%s (%s)" % (e, dburi),
given_version=pegasus_version,
db_type=db_type,
)
if schema_check:
try:
from Pegasus.db.admin.admin_loader import DBAdminError, db_verify
db_verify(db, pegasus_version=pegasus_version, force=force)
except DBAdminError as e:
e.db_type = db_type
raise (e)
return db