def test_create_database(self):
filename = str(uuid.uuid4())
_silentremove(filename)
dburi = "sqlite:///%s" % filename
db = connection.connect(dburi, create=True, verbose=False)
self.assertEqual(db_current_version(db), CURRENT_DB_VERSION)
db.execute("DROP TABLE dbversion")
self.assertRaises(DBAdminError, db_verify, db)
db.close()
db = connection.connect(dburi, create=True, verbose=False)
self.assertEqual(db_current_version(db), CURRENT_DB_VERSION)
db.execute("DELETE FROM dbversion")
db.close()
db = connection.connect(dburi, create=True, verbose=False)
self.assertEqual(db_current_version(db), CURRENT_DB_VERSION)
db.execute("DROP TABLE rc_pfn")
self.assertRaises(DBAdminError, db_verify, db)
db.close()
db = connection.connect(dburi, create=True, verbose=False)
self.assertEqual(db_current_version(db), CURRENT_DB_VERSION)
db.execute("DROP TABLE rc_pfn")
db.execute("DROP TABLE workflow")
db.execute("DROP TABLE master_workflow")
self.assertRaises(DBAdminError, db_verify, db)
self.assertRaises(DBAdminError, db_verify, db, "4.3.0")
db.close()
db = connection.connect(dburi, create=True, verbose=False)
self.assertEqual(db_current_version(db), CURRENT_DB_VERSION)
_remove(filename)
def _get_connection(
dburi=None,
cl_properties=None,
config_properties=None,
submit_dir=None,
db_type=None,
pegasus_version=None,
schema_check=True,
create=False,
force=False,
print_version=True,
):
""" Get connection to the database based on the parameters"""
if dburi:
return connection.connect(
dburi,
pegasus_version=pegasus_version,
schema_check=schema_check,
create=create,
force=force,
db_type=db_type,
print_version=print_version,
)
elif submit_dir:
return connection.connect_by_submitdir(
submit_dir,
db_type,
config_properties,
pegasus_version=pegasus_version,
schema_check=schema_check,
create=create,
force=force,
cl_properties=cl_properties,
print_version=print_version,
)
elif config_properties or _has_connection_properties(cl_properties):
return connection.connect_by_properties(
config_properties,
db_type,
cl_properties=cl_properties,
pegasus_version=pegasus_version,
schema_check=schema_check,
create=create,
force=force,
print_version=print_version,
)
if not db_type:
dburi = connection._get_master_uri()
return connection.connect(
dburi,
pegasus_version=pegasus_version,
schema_check=schema_check,
create=create,
force=force,
db_type=db_type,
print_version=print_version,
)
return None
def test_create_database(self):
filename = str(uuid.uuid4())
_silentremove(filename)
dburi = "sqlite:///%s" % filename
db = connection.connect(dburi, create=True)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
db.execute("DROP TABLE dbversion")
self.assertRaises(DBAdminError, db_verify, db)
db.close()
db = connection.connect(dburi, create=True)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
db.execute("DELETE FROM dbversion")
db.close()
db = connection.connect(dburi, create=True)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
db.execute("DROP TABLE rc_pfn")
self.assertRaises(DBAdminError, db_verify, db)
db.close()
db = connection.connect(dburi, create=True)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
db.execute("DROP TABLE rc_pfn")
db.execute("DROP TABLE workflow")
db.execute("DROP TABLE master_workflow")
self.assertRaises(DBAdminError, db_verify, db)
self.assertRaises(DBAdminError, db_verify, db, "4.3.0")
db.close()
db = connection.connect(dburi, create=True)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
_remove(filename)
def test_malformed_db():
dburi = "sqlite://"
db = connection.connect(dburi, create=True, verbose=False)
assert get_version(db) == CURRENT_DB_VERSION
db.execute("DROP TABLE rc_pfn")
with pytest.raises(DBAdminError):
db_verify(db, check=True)
db.close()
db = connection.connect(dburi, create=True, verbose=False)
assert get_version(db) == CURRENT_DB_VERSION
db.close()
def _get_connection(dburi=None, config_properties=None, submit_dir=None, db_type=None, pegasus_version=None, schema_check=True, create=False, force=False):
""" Get connection to the database based on the parameters"""
if dburi:
return connection.connect(dburi, pegasus_version=pegasus_version, schema_check=schema_check, create=create, force=force)
elif config_properties:
return connection.connect_by_properties(config_properties, db_type, pegasus_version=pegasus_version, schema_check=schema_check, create=create, force=force)
elif submit_dir:
return connection.connect_by_submitdir(submit_dir, db_type, config_properties, pegasus_version=pegasus_version, schema_check=schema_check, create=create, force=force)
if not db_type:
dburi = connection._get_master_uri()
return connection.connect(dburi, pegasus_version=pegasus_version, schema_check=schema_check, create=create, force=force)
return None
def test_malformed_db(self):
filename = str(uuid.uuid4())
_silentremove(filename)
dburi = "sqlite:///%s" % filename
db = connection.connect(dburi, create=True, verbose=False)
self.assertEqual(db_current_version(db), CURRENT_DB_VERSION)
db.execute("DROP TABLE rc_pfn")
self.assertRaises(DBAdminError, db_verify, db)
db.close()
db = connection.connect(dburi, create=True, verbose=False)
self.assertEqual(db_current_version(db), CURRENT_DB_VERSION)
db.close()
_remove(filename)
def test_malformed_db(self):
filename = str(uuid.uuid4())
_silentremove(filename)
dburi = "sqlite:///%s" % filename
db = connection.connect(dburi, create=True)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
db.execute("DROP TABLE rc_lfn")
self.assertRaises(DBAdminError, db_verify, db)
db.close()
db = connection.connect(dburi, create=True)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
db.close()
_remove(filename)
def test_dbs(input, tmp_path):
orig_filename = os.path.dirname(os.path.abspath(__file__)) + "/input/" + input
filename = str(uuid.uuid4())
shutil.copyfile(orig_filename, filename)
dburi = "sqlite:///%s" % filename
db = connection.connect(dburi, create=False, schema_check=False, verbose=False)
with pytest.raises(DBAdminError):
db_verify(db, check=True)
db.close()
db = connection.connect(dburi, create=True, verbose=False)
assert get_version(db) == CURRENT_DB_VERSION
db.close()
def detach(self, wf_uuid=None):
"Remove any master db entries for the given root workflow"
if self.submitdir_exists:
# Verify that we aren't trying to detach a subworkflow
if self.is_subworkflow():
raise SubmitDirException(
"Subworkflows cannot be detached independent of the root workflow"
)
# Connect to master database
mdbsession = connection.connect_by_submitdir(
self.submitdir, connection.DBType.MASTER)
mdb = MasterDatabase(mdbsession)
# Check to see if it even exists
wf = mdb.get_master_workflow(self.wf_uuid)
if wf is None:
print "Workflow is not in master DB"
else:
# Delete the workflow (this will delete the master_workflowstate entries as well)
mdb.delete_master_workflow(self.wf_uuid)
# Update the master db
mdbsession.commit()
mdbsession.close()
else:
# Connect to master database
home = expanduser('~')
mdbsession = connection.connect(
'sqlite:///%s/.pegasus/workflow.db' % home,
db_type=connection.DBType.MASTER)
mdb = MasterDatabase(mdbsession)
try:
if wf_uuid is None:
wfs = mdb.get_master_workflow_for_submitdir(self.submitdir)
if wfs:
msg = "Invalid submit dir: %s, Specify --wf-uuid <WF_UUID> to detach\n" % self.submitdir
msg += "\tWorkflow UUID, DAX Label, Submit Hostname, Submit Dir.\n"
for wf in wfs:
msg += '\t%s, %s, %s, %s\n' % (
wf.wf_uuid, wf.dax_label, wf.submit_hostname,
wf.submit_dir)
raise SubmitDirException(msg)
else:
raise SubmitDirException("Invalid submit dir: %s" %
self.submitdir)
else:
# Delete
mdb.delete_master_workflow(wf_uuid,
submit_dir=self.submitdir)
# Update the master db
mdbsession.commit()
finally:
mdbsession.close()
def delete_workflow(dburi, wf_uuid):
"Expunge workflow from workflow database"
log.info('Expunging %s from workflow database', wf_uuid)
session = connection.connect(dburi, create=True)
try:
query = session.query(Workflow).filter(Workflow.wf_uuid == wf_uuid)
try:
wf = query.one()
except orm.exc.NoResultFound, e:
log.warn('No workflow found with wf_uuid %s - aborting expunge',
wf_uuid)
return
# PM-1218 gather list of descendant workflows with wf_uuid
query = session.query(Workflow).filter(Workflow.root_wf_id == wf.wf_id)
try:
desc_wfs = query.all()
for desc_wf in desc_wfs:
# delete the files from the rc_lfn explicitly as they are
# not associated with workflow table
__delete_workflow_files__(session, desc_wf.wf_uuid,
desc_wf.wf_id)
except orm.exc.NoResultFound, e:
log.warn('No workflow found with root wf_id %s - aborting expunge',
wf.wf_id)
return
def __init__(self, connString=None, expand_workflow=True):
self.log = logging.getLogger("%s.%s" % (self.__module__, self.__class__.__name__))
try:
self.session = connection.connect(connString)
except (connection.ConnectionError, DBAdminError), e:
self.log.exception(e)
raise StampedeDBNotFoundError
def __init__(self, dburi, batch=True, props=None, db_type=None, backup=False, flush_every=1000):
"""Will be overridden by subclasses to take
parameters specific to their function.
"""
self.log = logging.getLogger("%s.%s" % (self.__module__, self.__class__.__name__))
self.dburi = dburi
# PM-898 all props passed should have pegasus prefix stripped off
# so they are more like connect_args to be used for database
connect_args = {}
for key in props.keyset():
# we don't pass url in connect args
if key != "url":
connect_args[key] = props.property(key)
# make sure timeout is an int
if "timeout" in connect_args:
connect_args["timeout"] = int(connect_args["timeout"])
#self.session = connection.connect(dburi, create=True, props=props, db_type=db_type, backup=backup)
self.session = connection.connect(dburi, create=True, connect_args=connect_args, db_type=db_type, backup=backup)
# flags and state for batching
self._batch = batch
self._flush_every = flush_every
self._flush_count = 0
self._last_flush = time.time()
def __init__(self,
conn_string=None,
wf_id=None,
wf_uuid=None,
debug=False):
self._dbg = debug
if conn_string is None:
raise ValueError('Connection string is required')
try:
self.session = connection.connect(conn_string)
except connection.ConnectionError as e:
log.exception(e)
message = e
while isinstance(message, Exception):
message = message.message
if 'attempt to write a readonly database' in message:
raise DBAdminError(message)
raise StampedeDBNotFoundError
self.initialize(wf_id, wf_uuid)
def delete_workflow(dburi, wf_uuid):
"Expunge workflow from workflow database"
log.info('Expunging %s from workflow database', wf_uuid)
session = connection.connect(dburi, create=True)
try:
query = session.query(Workflow).filter(Workflow.wf_uuid == wf_uuid)
try:
wf = query.one()
except orm.exc.NoResultFound as e:
log.warn('No workflow found with wf_uuid %s - aborting expunge', wf_uuid)
return
# PM-1218 gather list of descendant workflows with wf_uuid
query = session.query(Workflow).filter(Workflow.root_wf_id == wf.wf_id )
try:
desc_wfs = query.all()
for desc_wf in desc_wfs:
# delete the files from the rc_lfn explicitly as they are
# not associated with workflow table
__delete_workflow_files__( session, desc_wf.wf_uuid, desc_wf.wf_id )
except orm.exc.NoResultFound as e:
log.warn('No workflow found with root wf_id %s - aborting expunge', wf.wf_id)
return
session.delete(wf)
log.info('Flushing top-level workflow: %s', wf.wf_uuid)
i = time.time()
session.flush()
session.commit()
log.info('Flush took: %f seconds', time.time() - i)
finally:
session.close()
def test_connection_by_uri(self):
filename = str(uuid.uuid4())
_silentremove(filename)
dburi = "sqlite:///%s" % filename
db = connection.connect(dburi, echo=False, schema_check=True, create=True)
db.close()
_remove(filename)
def test_dbs(self):
dbs = ["test-01.db", "test-02.db"]
for db in dbs:
orig_filename = os.path.dirname(os.path.abspath(__file__)) + "/input/" + db
filename = str(uuid.uuid4())
shutil.copyfile(orig_filename, filename)
dburi = "sqlite:///%s" % filename
db = connection.connect(dburi, create=False, schema_check=False, verbose=False)
self.assertRaises(DBAdminError, db_verify, db)
db.close()
db = connection.connect(dburi, create=True, verbose=False)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
db.close()
_remove(filename)
def test_dbs(self):
dbs = ["test-01.db", "test-02.db"]
for db in dbs:
orig_filename = os.path.dirname(os.path.abspath(__file__)) + "/input/" + db
filename = str(uuid.uuid4())
shutil.copyfile(orig_filename, filename)
dburi = "sqlite:///%s" % filename
db = connection.connect(dburi, create=False, schema_check=False)
self.assertRaises(DBAdminError, db_verify, db)
db.close()
db = connection.connect(dburi, create=True)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
db.close()
_remove(filename)
def __init__(self, connString=None, expand_workflow=True):
self.log = logging.getLogger(
"%s.%s" % (self.__module__, self.__class__.__name__))
try:
self.session = connection.connect(connString)
except (connection.ConnectionError, DBAdminError), e:
self.log.exception(e)
raise StampedeDBNotFoundError
def test_version_operations():
dburi = "sqlite://"
db = connection.connect(dburi, create=True, verbose=False)
db_downgrade(db, pegasus_version="4.7.0", verbose=False)
assert get_version(db) == 8
with pytest.raises(DBAdminError):
db_verify(db, check=True)
RCLFN.__table__._set_parent(metadata)
db.close()
db = connection.connect(dburi, create=True, verbose=False)
assert get_version(db) == CURRENT_DB_VERSION
db.close()
dburi2 = "sqlite://"
db2 = connection.connect(dburi2, create=True, verbose=False)
db2.close()
def test_connection_by_uri(self):
filename = str(uuid.uuid4())
_silentremove(filename)
dburi = "sqlite:///%s" % filename
db = connection.connect(dburi,
echo=False,
schema_check=True,
create=True)
db.close()
_remove(filename)
def loop_forever(self):
while True:
u = user.get_user_by_uid(os.getuid())
session = connection.connect(u.get_master_db_url())
try:
dao = Ensembles(session)
self.loop_once(dao)
finally:
session.close()
time.sleep(self.interval)
def loop_forever(self):
while True:
u = user.get_user_by_uid(os.getuid())
session = connection.connect(u.get_master_db_url())
try:
dao = Ensembles(session)
self.loop_once(dao)
finally:
session.close()
time.sleep(self.interval)
def test_version_operations(self):
filename = str(uuid.uuid4())
_silentremove(filename)
dburi = "sqlite:///%s" % filename
db = connection.connect(dburi, create=True)
db_downgrade(db, pegasus_version="4.5.0")
self.assertEquals(db_current_version(db), 4)
self.assertRaises(DBAdminError, db_verify, db)
rc_lfn._set_parent(metadata)
db.close()
db = connection.connect(dburi, create=True)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
db.close()
_remove(filename)
fn = str(uuid.uuid4())
_silentremove(fn)
dburi2 = "sqlite:///%s" % fn
db2 = connection.connect(dburi2, create=True)
def detach(self, wf_uuid=None):
"Remove any master db entries for the given root workflow"
if self.submitdir_exists:
# Verify that we aren't trying to detach a subworkflow
if self.is_subworkflow():
raise SubmitDirException("Subworkflows cannot be detached independent of the root workflow")
# Connect to master database
mdbsession = connection.connect_by_submitdir(self.submitdir, connection.DBType.MASTER)
mdb = MasterDatabase(mdbsession)
# Check to see if it even exists
wf = mdb.get_master_workflow(self.wf_uuid)
if wf is None:
print "Workflow is not in master DB"
else:
# Delete the workflow (this will delete the master_workflowstate entries as well)
mdb.delete_master_workflow(self.wf_uuid)
# Update the master db
mdbsession.commit()
mdbsession.close()
else:
# Connect to master database
home = expanduser('~')
mdbsession = connection.connect('sqlite:///%s/.pegasus/workflow.db' % home,
db_type=connection.DBType.MASTER)
mdb = MasterDatabase(mdbsession)
try:
if wf_uuid is None:
wfs = mdb.get_master_workflow_for_submitdir(self.submitdir)
if wfs:
msg = "Invalid submit dir: %s, Specify --wf-uuid <WF_UUID> to detach\n" % self.submitdir
msg += "\tWorkflow UUID, DAX Label, Submit Hostname, Submit Dir.\n"
for wf in wfs:
msg += '\t%s, %s, %s, %s\n' % (wf.wf_uuid, wf.dax_label, wf.submit_hostname, wf.submit_dir)
raise SubmitDirException(msg)
else:
raise SubmitDirException("Invalid submit dir: %s" % self.submitdir)
else:
# Delete
mdb.delete_master_workflow(wf_uuid, submit_dir=self.submitdir)
# Update the master db
mdbsession.commit()
finally:
mdbsession.close()
def __init__(self, dburi, batch=True, props=None, db_type=None, flush_every=1000):
"""Will be overridden by subclasses to take
parameters specific to their function.
"""
self.log = logging.getLogger("%s.%s" % (self.__module__, self.__class__.__name__))
self.dburi = dburi
self.session = connection.connect(dburi, create=True, props=props, db_type=db_type)
# flags and state for batching
self._batch = batch
self._flush_every = flush_every
self._flush_count = 0
self._last_flush = time.time()
def test_version_operations(self):
filename = str(uuid.uuid4())
_silentremove(filename)
dburi = "sqlite:///%s" % filename
db = connection.connect(dburi, create=True, verbose=False)
db_downgrade(db, pegasus_version="4.5.0", verbose=False)
self.assertEqual(db_current_version(db), 4)
self.assertRaises(DBAdminError, db_verify, db)
rc_lfn._set_parent(metadata)
db.close()
db = connection.connect(dburi, create=True, verbose=False)
self.assertEqual(db_current_version(db), CURRENT_DB_VERSION)
db.close()
_remove(filename)
fn = str(uuid.uuid4())
_silentremove(fn)
dburi2 = "sqlite:///%s" % fn
db2 = connection.connect(dburi2, create=True, verbose=False)
_remove(fn)
def test_jdbc_sqlite(self):
filename = str(uuid.uuid4())
_silentremove(filename)
dburi = "jdbc:sqlite:%s" % filename
db = connection.connect(dburi, create=True, verbose=False)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
db.close()
_remove(filename)
filename = "/tmp/" + str(uuid.uuid4())
_silentremove(filename)
dburi = "jdbc:sqlite:%s" % filename
db = connection.connect(dburi, create=True, verbose=False)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
db.close()
_remove(filename)
dburi = "jdbc:sqlite:/%s" % filename
db = connection.connect(dburi, create=True, verbose=False)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
db.close()
_remove(filename)
def test_create_database():
dburi = "sqlite://"
db = connection.connect(dburi, create=True, verbose=False)
assert get_version(db) == CURRENT_DB_VERSION
db.execute("DROP TABLE dbversion")
with pytest.raises(DBAdminError):
db_verify(db, check=True)
db.close()
db = connection.connect(dburi, create=True, verbose=False)
assert get_version(db) == CURRENT_DB_VERSION
db.execute("DELETE FROM dbversion")
db.close()
db = connection.connect(dburi, create=True, verbose=False)
assert get_version(db) == CURRENT_DB_VERSION
db.execute("DROP TABLE rc_pfn")
with pytest.raises(DBAdminError):
db_verify(db, check=True)
db.close()
db = connection.connect(dburi, create=True, verbose=False)
assert get_version(db) == CURRENT_DB_VERSION
db.execute("DROP TABLE rc_pfn")
db.execute("DROP TABLE master_workflow")
with pytest.raises(DBAdminError):
db_verify(db, check=True)
with pytest.raises(DBAdminError):
db_verify(db, check=True), "4.3.0"
db.close()
db = connection.connect(dburi, create=True, verbose=False)
assert get_version(db) == CURRENT_DB_VERSION
db.close()
def __init__(self, dburi, batch=True, props=None, db_type=None, backup=False, flush_every=1000):
"""Will be overridden by subclasses to take
parameters specific to their function.
"""
self.log = logging.getLogger("%s.%s" % (self.__module__, self.__class__.__name__))
self.dburi = dburi
self.session = connection.connect(dburi, create=True, props=props, db_type=db_type, backup=backup)
# flags and state for batching
self._batch = batch
self._flush_every = flush_every
self._flush_count = 0
self._last_flush = time.time()
def test_jdbc_sqlite(self):
filename = str(uuid.uuid4())
_silentremove(filename)
dburi = "jdbc:sqlite:%s" % filename
db = connection.connect(dburi, create=True)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
db.close()
_remove(filename)
filename = "/tmp/" + str(uuid.uuid4())
_silentremove(filename)
dburi = "jdbc:sqlite:%s" % filename
db = connection.connect(dburi, create=True)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
db.close()
_remove(filename)
dburi = "jdbc:sqlite:/%s" % filename
db = connection.connect(dburi, create=True)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
db.close()
_remove(filename)
def test_version_operations(self):
filename = str(uuid.uuid4())
_silentremove(filename)
dburi = "sqlite:///%s" % filename
db = connection.connect(dburi, create=True)
# db_downgrade(db, "4.4.2")
# self.assertEquals(db_current_version(db), 2)
# self.assertRaises(DBAdminError, db_verify, db)
#
# db_downgrade(db)
# self.assertEquals(db_current_version(db), 1)
# self.assertRaises(DBAdminError, db_verify, db)
# db.close()
#
# db = connection.connect(dburi, create=True, pegasus_version="4.4.0")
# self.assertEquals(db_current_version(db), 2)
# self.assertRaises(DBAdminError, db_verify, db)
# db.close()
db = connection.connect(dburi, create=True)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
self.assertTrue(db_verify(db))
_remove(filename)
def test_version_operations(self):
filename = str(uuid.uuid4())
_silentremove(filename)
dburi = "sqlite:///%s" % filename
db = connection.connect(dburi, create=True)
# db_downgrade(db, "4.4.2")
# self.assertEquals(db_current_version(db), 2)
# self.assertRaises(DBAdminError, db_verify, db)
#
# db_downgrade(db)
# self.assertEquals(db_current_version(db), 1)
# self.assertRaises(DBAdminError, db_verify, db)
# db.close()
#
# db = connection.connect(dburi, create=True, pegasus_version="4.4.0")
# self.assertEquals(db_current_version(db), 2)
# self.assertRaises(DBAdminError, db_verify, db)
# db.close()
db = connection.connect(dburi, create=True)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
self.assertTrue(db_verify(db))
_remove(filename)
def delete(self):
"Delete this submit dir and its entry in the master db"
# Verify that we aren't trying to move a subworkflow
if self.is_subworkflow():
raise SubmitDirException(
"Subworkflows cannot be deleted independent of the root workflow"
)
# Confirm that they want to delete the workflow
while True:
try:
input = raw_input
except NameError:
pass
answer = (input(
"Are you sure you want to delete this workflow? This operation cannot be undone. [y/n]: "
).strip().lower())
if answer == "y":
break
if answer == "n":
return
# Connect to master database
mdbsession = connection.connect_by_submitdir(self.submitdir,
connection.DBType.MASTER)
mdb = MasterDatabase(mdbsession)
# Delete all of the records from the workflow db if they are not using
# an sqlite db that is in the submit dir.
db_url = connection.url_by_submitdir(self.submitdir,
connection.DBType.WORKFLOW)
if self.submitdir not in db_url:
dbsession = connection.connect(db_url)
db = WorkflowDatabase(dbsession)
db.delete_workflow(self.wf_uuid)
dbsession.commit()
dbsession.close()
# Delete the workflow
mdb.delete_master_workflow(self.wf_uuid)
# Remove all the files
shutil.rmtree(self.submitdir)
# Update master db
mdbsession.commit()
mdbsession.close()
def test_partial_database(self):
filename = str(uuid.uuid4())
_silentremove(filename)
dburi = "sqlite:///%s" % filename
db = connection.connect(
dburi, schema_check=False, create=False, verbose=False
)
rc_sequences.create(db.get_bind(), checkfirst=True)
rc_lfn.create(db.get_bind(), checkfirst=True)
rc_pfn.create(db.get_bind(), checkfirst=True)
rc_meta.create(db.get_bind(), checkfirst=True)
self.assertRaises(DBAdminError, db_verify, db)
db.close()
db = connection.connect(dburi, create=True, verbose=False)
self.assertEqual(db_current_version(db), CURRENT_DB_VERSION)
_remove(filename)
db = connection.connect(
dburi, schema_check=False, create=False, verbose=False
)
pg_workflow.create(db.get_bind(), checkfirst=True)
pg_workflowstate.create(db.get_bind(), checkfirst=True)
pg_ensemble.create(db.get_bind(), checkfirst=True)
pg_ensemble_workflow.create(db.get_bind(), checkfirst=True)
self.assertRaises(DBAdminError, db_verify, db)
db.close()
db = connection.connect(dburi, create=True, verbose=False)
self.assertEqual(db_current_version(db), CURRENT_DB_VERSION)
_remove(filename)
db = connection.connect(
dburi, schema_check=False, create=False, verbose=False
)
st_workflow.create(db.get_bind(), checkfirst=True)
st_workflowstate.create(db.get_bind(), checkfirst=True)
st_host.create(db.get_bind(), checkfirst=True)
st_job.create(db.get_bind(), checkfirst=True)
st_job_edge.create(db.get_bind(), checkfirst=True)
st_job_instance.create(db.get_bind(), checkfirst=True)
st_jobstate.create(db.get_bind(), checkfirst=True)
st_task.create(db.get_bind(), checkfirst=True)
st_task_edge.create(db.get_bind(), checkfirst=True)
st_invocation.create(db.get_bind(), checkfirst=True)
self.assertRaises(DBAdminError, db_verify, db)
db.close()
db = connection.connect(dburi, create=True, verbose=False)
self.assertEqual(db_current_version(db), CURRENT_DB_VERSION)
_remove(filename)
def test_upper_version(self):
"""
Test whether DBs created with newer Pegasus version raises an exception.
"""
filename = str(uuid.uuid4())
_silentremove(filename)
dburi = "sqlite:///%s" % filename
db = connection.connect(dburi, create=True)
dbversion = DBVersion()
dbversion.version = CURRENT_DB_VERSION + 1
dbversion.version_number = CURRENT_DB_VERSION + 1
dbversion.version_timestamp = datetime.datetime.now().strftime("%s")
db.add(dbversion)
db.commit()
self.assertRaises(DBAdminError, db_current_version, db)
self.assertRaises(DBAdminError, db_verify, db)
_remove(filename)
def __init__(self, conn_string, debug=False):
self._dbg = debug
if conn_string is None:
raise ValueError("Connection string is required")
try:
self.session = connection.connect(conn_string)
except connection.ConnectionError as e:
log.exception(e)
message = e
while isinstance(message, Exception):
message = message.message
if "attempt to write a readonly database" in message:
raise DBAdminError(message)
raise MasterDBNotFoundError(e)
def test_upper_version(self):
"""
Test whether DBs created with newer Pegasus version raises an exception.
"""
filename = str(uuid.uuid4())
_silentremove(filename)
dburi = "sqlite:///%s" % filename
db = connection.connect(dburi, create=True, verbose=False)
dbversion = DBVersion()
dbversion.version = CURRENT_DB_VERSION + 1
dbversion.version_number = CURRENT_DB_VERSION + 1
dbversion.version_timestamp = datetime.datetime.now().strftime("%s")
db.add(dbversion)
db.commit()
self.assertRaises(DBAdminError, db_current_version, db)
self.assertRaises(DBAdminError, db_verify, db)
_remove(filename)
def __init__(self, connString=None, expand_workflow=True):
self.log = logging.getLogger("%s.%s" % (self.__module__, self.__class__.__name__))
try:
self.session = connection.connect(connString)
except (connection.ConnectionError, DBAdminError) as e:
self.log.exception(e)
raise StampedeDBNotFoundError
self._expand = expand_workflow
self._root_wf_id = []
self._root_wf_uuid = []
self.all_workflows = None
self._job_filter_mode = None
self._time_filter_mode = None
self._host_filter = None
self._xform_filter = {'include':None, 'exclude':None}
self._wfs = []
def __init__(self, conn_string, debug=False):
self._dbg = debug
if conn_string is None:
raise ValueError('Connection string is required')
try:
self.session = connection.connect(conn_string)
except connection.ConnectionError as e:
log.exception(e)
message = e
while isinstance(message, Exception):
message = message.message
if 'attempt to write a readonly database' in message:
raise DBAdminError(message)
raise MasterDBNotFoundError(e)
def __init__(
self,
dburi,
batch=True,
props=None,
db_type=None,
backup=False,
flush_every=1000,
):
"""Will be overridden by subclasses to take
parameters specific to their function.
"""
self.log = logging.getLogger("{}.{}".format(self.__module__,
self.__class__.__name__))
self.dburi = dburi
# PM-898 all props passed should have pegasus prefix stripped off
# so they are more like connect_args to be used for database
connect_args = {}
for key in props.keyset():
# we don't pass url in connect args
if key != "url":
connect_args[key] = props.property(key)
# make sure timeout is an int
if "timeout" in connect_args:
connect_args["timeout"] = int(connect_args["timeout"])
# self.session = connection.connect(dburi, create=True, props=props, db_type=db_type, backup=backup)
self.session = connection.connect(
dburi,
create=True,
connect_args=connect_args,
db_type=db_type,
backup=backup,
)
# flags and state for batching
self._batch = batch
self._flush_every = flush_every
self._flush_count = 0
self._last_flush = time.time()
def delete(self):
"Delete this submit dir and its entry in the master db"
# Verify that we aren't trying to move a subworkflow
if self.is_subworkflow():
raise SubmitDirException("Subworkflows cannot be deleted independent of the root workflow")
# Confirm that they want to delete the workflow
while True:
try:
input = raw_input
except NameError:
pass
answer = input("Are you sure you want to delete this workflow? This operation cannot be undone. [y/n]: ").strip().lower()
if answer == "y":
break
if answer == "n":
return
# Connect to master database
mdbsession = connection.connect_by_submitdir(self.submitdir, connection.DBType.MASTER)
mdb = MasterDatabase(mdbsession)
# Delete all of the records from the workflow db if they are not using
# an sqlite db that is in the submit dir.
db_url = connection.url_by_submitdir(self.submitdir, connection.DBType.WORKFLOW)
if self.submitdir not in db_url:
dbsession = connection.connect(db_url)
db = WorkflowDatabase(dbsession)
db.delete_workflow(self.wf_uuid)
dbsession.commit()
dbsession.close()
# Delete the workflow
mdb.delete_master_workflow(self.wf_uuid)
# Remove all the files
shutil.rmtree(self.submitdir)
# Update master db
mdbsession.commit()
mdbsession.close()
def delete_dashboard_workflow(dburi, wf_uuid):
"Expunge workflow from dashboard database"
log.info('Expunging %s from dashboard database', wf_uuid)
session = connection.connect(dburi, create=True)
try:
query = session.query(DashboardWorkflow).filter(DashboardWorkflow.wf_uuid == wf_uuid)
try:
wf = query.one()
except orm.exc.NoResultFound, e:
log.warn('No workflow found with wf_uuid %s - aborting expunge', wf_uuid)
return
session.delete(wf)
i = time.time()
session.flush()
session.commit()
log.info('Flush took: %f seconds', time.time() - i)
def delete_dashboard_workflow(dburi, wf_uuid):
"Expunge workflow from dashboard database"
log.info("Expunging %s from dashboard database", wf_uuid)
session = connection.connect(dburi, create=True)
try:
query = session.query(DashboardWorkflow).filter(DashboardWorkflow.wf_uuid == wf_uuid)
try:
wf = query.one()
except orm.exc.NoResultFound, e:
log.warn("No workflow found with wf_uuid %s - aborting expunge", wf_uuid)
return
session.delete(wf)
i = time.time()
session.flush()
session.commit()
log.info("Flush took: %f seconds", time.time() - i)
def test_upper_version():
"""
Test whether DBs created with newer Pegasus version raises an exception.
"""
dburi = "sqlite://"
db = connection.connect(dburi, create=True, verbose=False)
dbversion = DBVersion()
dbversion.version = CURRENT_DB_VERSION + 1
dbversion.version_number = CURRENT_DB_VERSION + 1
dbversion.version_timestamp = (
datetime.datetime.now() + datetime.timedelta(seconds=3)
).strftime("%s")
db.add(dbversion)
db.commit()
with pytest.raises(DBAdminError):
get_version(db)
with pytest.raises(DBAdminError):
db_verify(db, check=True)
db.close()
def test_partial_database(self):
filename = str(uuid.uuid4())
_silentremove(filename)
dburi = "sqlite:///%s" % filename
db = connection.connect(dburi, schema_check=False, create=False)
rc_sequences.create(db.get_bind(), checkfirst=True)
rc_lfn.create(db.get_bind(), checkfirst=True)
rc_attr.create(db.get_bind(), checkfirst=True)
self.assertRaises(DBAdminError, db_verify, db)
db.close()
db = connection.connect(dburi, create=True)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
self.assertTrue(db_verify(db))
_remove(filename)
db = connection.connect(dburi, schema_check=False, create=False)
pg_workflow.create(db.get_bind(), checkfirst=True)
pg_workflowstate.create(db.get_bind(), checkfirst=True)
pg_ensemble.create(db.get_bind(), checkfirst=True)
pg_ensemble_workflow.create(db.get_bind(), checkfirst=True)
self.assertRaises(DBAdminError, db_verify, db)
db.close()
db = connection.connect(dburi, create=True)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
self.assertTrue(db_verify(db))
_remove(filename)
db = connection.connect(dburi, schema_check=False, create=False)
st_workflow.create(db.get_bind(), checkfirst=True)
st_workflowstate.create(db.get_bind(), checkfirst=True)
st_host.create(db.get_bind(), checkfirst=True)
st_job.create(db.get_bind(), checkfirst=True)
st_job_edge.create(db.get_bind(), checkfirst=True)
st_job_instance.create(db.get_bind(), checkfirst=True)
st_jobstate.create(db.get_bind(), checkfirst=True)
st_task.create(db.get_bind(), checkfirst=True)
st_task_edge.create(db.get_bind(), checkfirst=True)
st_invocation.create(db.get_bind(), checkfirst=True)
st_file.create(db.get_bind(), checkfirst=True)
self.assertRaises(DBAdminError, db_verify, db)
db.close()
db = connection.connect(dburi, create=True)
self.assertEquals(db_current_version(db), CURRENT_DB_VERSION)
self.assertTrue(db_verify(db))
_remove(filename)
def delete_workflow(dburi, wf_uuid):
"Expunge workflow from workflow database"
log.info('Expunging %s from workflow database', wf_uuid)
session = connection.connect(dburi, create=True)
try:
query = session.query(Workflow).filter(Workflow.wf_uuid == wf_uuid)
try:
wf = query.one()
except orm.exc.NoResultFound as e:
log.warn('No workflow found with wf_uuid %s - aborting expunge',
wf_uuid)
return
# PM-1218 gather list of descendant workflows with wf_uuid
query = session.query(Workflow).filter(Workflow.root_wf_id == wf.wf_id)
try:
desc_wfs = query.all()
for desc_wf in desc_wfs:
# delete the files from the rc_lfn explicitly as they are
# not associated with workflow table
__delete_workflow_files__(session, desc_wf.wf_uuid,
desc_wf.wf_id)
except orm.exc.NoResultFound as e:
log.warn('No workflow found with root wf_id %s - aborting expunge',
wf.wf_id)
return
session.delete(wf)
log.info('Flushing top-level workflow: %s', wf.wf_uuid)
i = time.time()
session.flush()
session.commit()
log.info('Flush took: %f seconds', time.time() - i)
finally:
session.close()
def test_partial_database():
dburi = "sqlite://"
db = connection.connect(dburi, schema_check=False, create=False, verbose=False)
RCLFN.__table__.create(db.get_bind(), checkfirst=True)
RCPFN.__table__.create(db.get_bind(), checkfirst=True)
RCMeta.__table__.create(db.get_bind(), checkfirst=True)
with pytest.raises(DBAdminError):
db_verify(db, check=True)
db.close()
db = connection.connect(dburi, create=True, verbose=False)
assert get_version(db) == CURRENT_DB_VERSION
db.close()
db = connection.connect(dburi, schema_check=False, create=False, verbose=False)
Workflow.__table__.create(db.get_bind(), checkfirst=True)
DashboardWorkflowstate.__table__.create(db.get_bind(), checkfirst=True)
Ensemble.__table__.create(db.get_bind(), checkfirst=True)
EnsembleWorkflow.__table__.create(db.get_bind(), checkfirst=True)
with pytest.raises(DBAdminError):
db_verify(db, check=True)
db.close()
db = connection.connect(dburi, create=True, verbose=False)
assert get_version(db) == CURRENT_DB_VERSION
db.close()
db = connection.connect(dburi, schema_check=False, create=False, verbose=False)
Workflow.__table__.create(db.get_bind(), checkfirst=True)
Workflowstate.__table__.create(db.get_bind(), checkfirst=True)
Host.__table__.create(db.get_bind(), checkfirst=True)
Job.__table__.create(db.get_bind(), checkfirst=True)
JobEdge.__table__.create(db.get_bind(), checkfirst=True)
JobInstance.__table__.create(db.get_bind(), checkfirst=True)
Jobstate.__table__.create(db.get_bind(), checkfirst=True)
Task.__table__.create(db.get_bind(), checkfirst=True)
TaskEdge.__table__.create(db.get_bind(), checkfirst=True)
Invocation.__table__.create(db.get_bind(), checkfirst=True)
with pytest.raises(DBAdminError):
db_verify(db, check=True)
db.close()
db = connection.connect(dburi, create=True, verbose=False)
assert get_version(db) == CURRENT_DB_VERSION
db.close()
def attach(self):
"Add a workflow to the master db"
# Verify that we aren't trying to attach a subworkflow
if self.is_subworkflow():
raise SubmitDirException("Subworkflows cannot be attached independent of the root workflow")
# Connect to master database
mdbsession = connection.connect_by_submitdir(self.submitdir, connection.DBType.MASTER)
mdb = MasterDatabase(mdbsession)
# Check to see if it already exists and just update it
wf = mdb.get_master_workflow(self.wf_uuid)
if wf is not None:
print "Workflow is already in master db"
old_submit_dir = wf.submit_dir
if old_submit_dir != self.submitdir:
print "Updating path..."
wf.submit_dir = self.submitdir
wf.db_url = connection.url_by_submitdir(self.submitdir, connection.DBType.WORKFLOW)
mdbsession.commit()
mdbsession.close()
return
# Connect to workflow db
db_url = connection.url_by_submitdir(self.submitdir, connection.DBType.WORKFLOW)
dbsession = connection.connect(db_url)
db = WorkflowDatabase(dbsession)
# Get workflow record
wf = db.get_workflow(self.wf_uuid)
if wf is None:
print "No database record for that workflow exists"
return
# Update the workflow record
wf.submit_dir = self.submitdir
wf.db_url = db_url
# Insert workflow record into master db
mwf = DashboardWorkflow()
mwf.wf_uuid = wf.wf_uuid
mwf.dax_label = wf.dax_label
mwf.dax_version = wf.dax_version
mwf.dax_file = wf.dax_file
mwf.dag_file_name = wf.dag_file_name
mwf.timestamp = wf.timestamp
mwf.submit_hostname = wf.submit_hostname
mwf.submit_dir = self.submitdir
mwf.planner_arguments = wf.planner_arguments
mwf.user = wf.user
mwf.grid_dn = wf.grid_dn
mwf.planner_version = wf.planner_version
mwf.db_url = wf.db_url
mwf.archived = self.is_archived()
mdbsession.add(mwf)
mdbsession.flush() # We should have the new wf_id after this
# Query states from workflow database
states = db.get_workflow_states(wf.wf_id)
# Insert states into master db
for s in states:
ms = DashboardWorkflowstate()
ms.wf_id = mwf.wf_id
ms.state = s.state
ms.timestamp = s.timestamp
ms.restart_count = s.restart_count
ms.status = s.status
mdbsession.add(ms)
mdbsession.flush()
dbsession.commit()
dbsession.close()
mdbsession.commit()
mdbsession.close()
def move(self, dest):
"Move this submit directory to dest"
dest = os.path.abspath(dest)
if os.path.isfile(dest):
raise SubmitDirException("Destination is a file: %s" % dest)
if os.path.isdir(dest):
if os.path.exists(os.path.join(dest, "braindump.txt")):
raise SubmitDirException("Destination is a submit dir: %s" % dest)
dest = os.path.join(dest, os.path.basename(self.submitdir))
# Verify that we aren't trying to move a subworkflow
if self.is_subworkflow():
raise SubmitDirException("Subworkflows cannot be moved independent of the root workflow")
# Connect to master database
mdbsession = connection.connect_by_submitdir(self.submitdir, connection.DBType.MASTER)
mdb = MasterDatabase(mdbsession)
# Get the workflow record from the master db
db_url = None
wf = mdb.get_master_workflow(self.wf_uuid)
if wf is None:
db_url = connection.url_by_submitdir(self.submitdir, connection.DBType.WORKFLOW)
else:
# We found an mdb record, so we need to update it
# Save the master db's pointer
db_url = wf.db_url
# Update the master db's db_url
# Note that this will only update the URL if it is an sqlite file
# located in the submitdir
log.info("Old master db_url: %s" % wf.db_url)
wf.db_url = db_url.replace(self.submitdir, dest)
log.info("New master db_url: %s" % wf.db_url)
# Change the master db's submit_dir
log.info("Old master submit_dir: %s" % wf.submit_dir)
wf.submit_dir = dest
log.info("New master submit_dir: %s" % wf.submit_dir)
# Update the ensemble record if one exists
ew = mdb.get_ensemble_workflow(self.wf_uuid)
if ew is not None:
log.info("Old ensemble submit dir: %s", ew.submitdir)
ew.submitdir = dest
log.info("New ensemble submit dir: %s", ew.submitdir)
# Update the workflow database if we found one
if db_url is not None:
dbsession = connection.connect(db_url)
db = WorkflowDatabase(dbsession)
root_wf = db.get_workflow(self.wf_uuid)
db.update_submit_dirs(root_wf.wf_id, self.submitdir, dest)
dbsession.commit()
dbsession.close()
# Move all the files
shutil.move(self.submitdir, dest)
# Set new paths in the braindump file
self.braindump["submit_dir"] = dest
self.braindump["basedir"] = os.path.dirname(dest)
utils.write_braindump(os.path.join(dest, "braindump.txt"), self.braindump)
# Note that we do not need to update the properties file even though it
# might contain DB URLs because it cannot contain a DB URL with the submit
# dir in it.
# TODO We might want to update all of the absolute paths in the condor submit files
# if we plan on moving workflows that could be resubmitted in the future
# TODO We might want to update the braindump files for subworkflows
# Update master database
mdbsession.commit()
mdbsession.close()
# Finally, update object
self.submitdir = dest
def all_workflows_db(db, update=True, pegasus_version=None, schema_check=True, force=False):
"""
Update/Downgrade all completed workflow databases listed in master_workflow table.
:param db: DB session object
:param pegasus_version: version of the Pegasus software (e.g., 4.6.0)
:param schema_check: whether a sanity check of the schema should be performed
:param force: whether operations should be performed despite conflicts
"""
# log files
file_prefix = "%s-dbadmin" % time.strftime("%Y%m%dT%H%M%S")
f_out = open("%s.out" % file_prefix, "w")
f_err = open("%s.err" % file_prefix, "w")
data = (
db.query(DashboardWorkflow.db_url, DashboardWorkflowstate.state, func.max(DashboardWorkflowstate.timestamp))
.join(DashboardWorkflowstate)
.group_by(DashboardWorkflow.wf_id)
.all()
)
db_urls = []
for d in data:
if d[1] == "WORKFLOW_TERMINATED":
db_urls.append(d[0])
f_err.write("[ACTIVE] %s\n" % d[0])
counts = {
"total": len(data),
"running": len(data) - len(db_urls),
"success": 0,
"failed": 0,
"unable_to_connect": 0,
}
if update:
msg = ["updating", "Updated"]
else:
msg = ["downgrading", "Downgraded"]
print ""
print "Verifying and %s workflow databases:" % msg[0]
i = counts["running"]
for dburi in db_urls:
log.debug("%s '%s'..." % (msg[0], dburi))
i += 1
sys.stdout.write("\r%d/%d" % (i, counts["total"]))
sys.stdout.flush()
try:
if update:
con = connection.connect(
dburi,
pegasus_version=pegasus_version,
schema_check=schema_check,
create=True,
force=force,
verbose=False,
)
else:
con = connection.connect(dburi, schema_check=schema_check, create=False, verbose=False)
metadata.clear()
warnings.simplefilter("ignore")
metadata.reflect(bind=con.get_bind())
db_downgrade(con, pegasus_version=pegasus_version, force=force, verbose=False)
con.close()
f_out.write("[SUCCESS] %s\n" % dburi)
counts["success"] += 1
except connection.ConnectionError, e:
if "unable to open database file" in str(e):
f_err.write("[UNABLE TO CONNECT] %s\n" % dburi)
counts["unable_to_connect"] += 1
log.debug(e)
else:
f_err.write("[ERROR] %s\n" % dburi)
counts["failed"] += 1
log.debug(e)
except Exception, e:
f_err.write("[ERROR] %s\n" % dburi)
counts["failed"] += 1
log.debug(e)
def all_workflows_db(db,
update=True,
pegasus_version=None,
schema_check=True,
force=False):
"""
Update/Downgrade all completed workflow databases listed in master_workflow table.
:param db: DB session object
:param pegasus_version: version of the Pegasus software (e.g., 4.6.0)
:param schema_check: whether a sanity check of the schema should be performed
:param force: whether operations should be performed despite conflicts
"""
# log files
file_prefix = "%s-dbadmin" % time.strftime("%Y%m%dT%H%M%S")
f_out = open("%s.out" % file_prefix, 'w')
f_err = open("%s.err" % file_prefix, 'w')
data = db.query(DashboardWorkflow.db_url, DashboardWorkflowstate.state,
func.max(DashboardWorkflowstate.timestamp)).join(
DashboardWorkflowstate).group_by(
DashboardWorkflow.wf_id).all()
db_urls = []
for d in data:
if d[1] == "WORKFLOW_TERMINATED":
db_urls.append(d[0])
f_err.write("[ACTIVE] %s\n" % d[0])
counts = {
'total': len(data),
'running': len(data) - len(db_urls),
'success': 0,
'failed': 0,
'unable_to_connect': 0,
}
if update:
msg = ['updating', 'Updated']
else:
msg = ['downgrading', 'Downgraded']
print("")
print("Verifying and %s workflow databases:" % msg[0])
i = counts['running']
for dburi in db_urls:
log.debug("%s '%s'..." % (msg[0], dburi))
i += 1
sys.stdout.write("\r%d/%d" % (i, counts['total']))
sys.stdout.flush()
try:
if update:
con = connection.connect(dburi,
pegasus_version=pegasus_version,
schema_check=schema_check,
create=True,
force=force,
verbose=False)
else:
con = connection.connect(dburi,
schema_check=schema_check,
create=False,
verbose=False)
metadata.clear()
warnings.simplefilter("ignore")
metadata.reflect(bind=con.get_bind())
db_downgrade(con,
pegasus_version=pegasus_version,
force=force,
verbose=False)
con.close()
f_out.write("[SUCCESS] %s\n" % dburi)
counts['success'] += 1
except connection.ConnectionError as e:
if "unable to open database file" in str(e):
f_err.write("[UNABLE TO CONNECT] %s\n" % dburi)
counts['unable_to_connect'] += 1
log.debug(e)
else:
f_err.write("[ERROR] %s\n" % dburi)
counts['failed'] += 1
log.debug(e)
except Exception as e:
f_err.write("[ERROR] %s\n" % dburi)
counts['failed'] += 1
log.debug(e)
f_out.close()
f_err.close()
print("\n\nSummary:")
print(" Verified/%s: %s/%s" %
(msg[1], counts['success'], counts['total']))
print(" Failed: %s/%s" % (counts['failed'], counts['total']))
print(" Unable to connect: %s/%s" %
(counts['unable_to_connect'], counts['total']))
print(" Unable to update (active workflows): %s/%s" %
(counts['running'], counts['total']))
print("\nLog files:")
print(" %s.out (Succeeded operations)" % file_prefix)
print(" %s.err (Failed operations)" % file_prefix)
def __init__(self, dburi, props=None, db_type=None, **kwarg):
self.dburi = dburi
self.session = connection.connect(dburi, create=True, props=props, db_type=db_type)
def __init__(self, dburi, **kwarg):
self.dburi = dburi
self.session = connection.connect(dburi, create=True)
def connect():
log.debug("Connecting to database")
g.session = connection.connect(g.master_db_url)
def attach(self):
"Add a workflow to the master db"
# Verify that we aren't trying to attach a subworkflow
if self.is_subworkflow():
raise SubmitDirException("Subworkflows cannot be attached independent of the root workflow")
# Connect to master database
mdbsession = connection.connect_by_submitdir(self.submitdir, connection.DBType.MASTER)
mdb = MasterDatabase(mdbsession)
# Check to see if it already exists and just update it
wf = mdb.get_master_workflow(self.wf_uuid)
if wf is not None:
print("Workflow is already in master db")
old_submit_dir = wf.submit_dir
if old_submit_dir != self.submitdir:
print("Updating path...")
wf.submit_dir = self.submitdir
wf.db_url = connection.url_by_submitdir(self.submitdir, connection.DBType.WORKFLOW)
mdbsession.commit()
mdbsession.close()
return
# Connect to workflow db
db_url = connection.url_by_submitdir(self.submitdir, connection.DBType.WORKFLOW)
dbsession = connection.connect(db_url)
db = WorkflowDatabase(dbsession)
# Get workflow record
wf = db.get_workflow(self.wf_uuid)
if wf is None:
print("No database record for that workflow exists")
return
# Update the workflow record
wf.submit_dir = self.submitdir
wf.db_url = db_url
# Insert workflow record into master db
mwf = DashboardWorkflow()
mwf.wf_uuid = wf.wf_uuid
mwf.dax_label = wf.dax_label
mwf.dax_version = wf.dax_version
mwf.dax_file = wf.dax_file
mwf.dag_file_name = wf.dag_file_name
mwf.timestamp = wf.timestamp
mwf.submit_hostname = wf.submit_hostname
mwf.submit_dir = self.submitdir
mwf.planner_arguments = wf.planner_arguments
mwf.user = wf.user
mwf.grid_dn = wf.grid_dn
mwf.planner_version = wf.planner_version
mwf.db_url = wf.db_url
mwf.archived = self.is_archived()
mdbsession.add(mwf)
mdbsession.flush() # We should have the new wf_id after this
# Query states from workflow database
states = db.get_workflow_states(wf.wf_id)
# Insert states into master db
for s in states:
ms = DashboardWorkflowstate()
ms.wf_id = mwf.wf_id
ms.state = s.state
ms.timestamp = s.timestamp
ms.restart_count = s.restart_count
ms.status = s.status
mdbsession.add(ms)
mdbsession.flush()
dbsession.commit()
dbsession.close()
mdbsession.commit()
mdbsession.close()