def setup(submit_dir , config_properties):
"""
Setup the populate module
@submit_dir submit directory path of the workflow run
@config_properties path to the propery file
"""
# global reference
global global_base_submit_dir
global global_braindb_submit_dir
global global_db_url
global global_top_wf_uuid
global_base_submit_dir = submit_dir
#Getting values from braindump file
config = utils.slurp_braindb(submit_dir)
if (config.has_key('submit_dir') or config.has_key('run')):
if config.has_key('submit_dir'):
global_braindb_submit_dir = os.path.abspath(config['submit_dir'])
else:
global_braindb_submit_dir = os.path.abspath(config['run'])
else:
logger.error("Submit directory cannot be found in the braindump.txt . ")
sys.exit(1)
# Create the sqllite db url
global_db_url = connection.url_by_submitdir(submit_dir, connection.DBType.WORKFLOW, config_properties)
global_top_wf_uuid = connection.get_wf_uuid(submit_dir)
if global_db_url is None:
sys.exit(1)
def setup(submit_dir, config_properties):
"""
Setup the populate module
@submit_dir submit directory path of the workflow run
@config_properties path to the propery file
"""
# global reference
global global_base_submit_dir
global global_braindb_submit_dir
global global_db_url
global global_top_wf_uuid
global_base_submit_dir = submit_dir
#Getting values from braindump file
config = utils.slurp_braindb(submit_dir)
if ('submit_dir' in config or 'run' in config):
if 'submit_dir' in config:
global_braindb_submit_dir = os.path.abspath(config['submit_dir'])
else:
global_braindb_submit_dir = os.path.abspath(config['run'])
else:
logger.error(
"Submit directory cannot be found in the braindump.txt . ")
sys.exit(1)
# Create the sqllite db url
global_db_url = connection.url_by_submitdir(submit_dir,
connection.DBType.WORKFLOW,
config_properties)
global_top_wf_uuid = connection.get_wf_uuid(submit_dir)
if global_db_url is None:
sys.exit(1)
def delete(self):
"Delete this submit dir and its entry in the master db"
# Verify that we aren't trying to move a subworkflow
if self.is_subworkflow():
raise SubmitDirException(
"Subworkflows cannot be deleted independent of the root workflow"
)
# Confirm that they want to delete the workflow
while True:
try:
input = raw_input
except NameError:
pass
answer = (input(
"Are you sure you want to delete this workflow? This operation cannot be undone. [y/n]: "
).strip().lower())
if answer == "y":
break
if answer == "n":
return
# Connect to master database
mdbsession = connection.connect_by_submitdir(self.submitdir,
connection.DBType.MASTER)
mdb = MasterDatabase(mdbsession)
# Delete all of the records from the workflow db if they are not using
# an sqlite db that is in the submit dir.
db_url = connection.url_by_submitdir(self.submitdir,
connection.DBType.WORKFLOW)
if self.submitdir not in db_url:
dbsession = connection.connect(db_url)
db = WorkflowDatabase(dbsession)
db.delete_workflow(self.wf_uuid)
dbsession.commit()
dbsession.close()
# Delete the workflow
mdb.delete_master_workflow(self.wf_uuid)
# Remove all the files
shutil.rmtree(self.submitdir)
# Update master db
mdbsession.commit()
mdbsession.close()
def delete(self):
"Delete this submit dir and its entry in the master db"
# Verify that we aren't trying to move a subworkflow
if self.is_subworkflow():
raise SubmitDirException("Subworkflows cannot be deleted independent of the root workflow")
# Confirm that they want to delete the workflow
while True:
try:
input = raw_input
except NameError:
pass
answer = input("Are you sure you want to delete this workflow? This operation cannot be undone. [y/n]: ").strip().lower()
if answer == "y":
break
if answer == "n":
return
# Connect to master database
mdbsession = connection.connect_by_submitdir(self.submitdir, connection.DBType.MASTER)
mdb = MasterDatabase(mdbsession)
# Delete all of the records from the workflow db if they are not using
# an sqlite db that is in the submit dir.
db_url = connection.url_by_submitdir(self.submitdir, connection.DBType.WORKFLOW)
if self.submitdir not in db_url:
dbsession = connection.connect(db_url)
db = WorkflowDatabase(dbsession)
db.delete_workflow(self.wf_uuid)
dbsession.commit()
dbsession.close()
# Delete the workflow
mdb.delete_master_workflow(self.wf_uuid)
# Remove all the files
shutil.rmtree(self.submitdir)
# Update master db
mdbsession.commit()
mdbsession.close()
def attach(self):
"Add a workflow to the master db"
# Verify that we aren't trying to attach a subworkflow
if self.is_subworkflow():
raise SubmitDirException("Subworkflows cannot be attached independent of the root workflow")
# Connect to master database
mdbsession = connection.connect_by_submitdir(self.submitdir, connection.DBType.MASTER)
mdb = MasterDatabase(mdbsession)
# Check to see if it already exists and just update it
wf = mdb.get_master_workflow(self.wf_uuid)
if wf is not None:
print "Workflow is already in master db"
old_submit_dir = wf.submit_dir
if old_submit_dir != self.submitdir:
print "Updating path..."
wf.submit_dir = self.submitdir
wf.db_url = connection.url_by_submitdir(self.submitdir, connection.DBType.WORKFLOW)
mdbsession.commit()
mdbsession.close()
return
# Connect to workflow db
db_url = connection.url_by_submitdir(self.submitdir, connection.DBType.WORKFLOW)
dbsession = connection.connect(db_url)
db = WorkflowDatabase(dbsession)
# Get workflow record
wf = db.get_workflow(self.wf_uuid)
if wf is None:
print "No database record for that workflow exists"
return
# Update the workflow record
wf.submit_dir = self.submitdir
wf.db_url = db_url
# Insert workflow record into master db
mwf = DashboardWorkflow()
mwf.wf_uuid = wf.wf_uuid
mwf.dax_label = wf.dax_label
mwf.dax_version = wf.dax_version
mwf.dax_file = wf.dax_file
mwf.dag_file_name = wf.dag_file_name
mwf.timestamp = wf.timestamp
mwf.submit_hostname = wf.submit_hostname
mwf.submit_dir = self.submitdir
mwf.planner_arguments = wf.planner_arguments
mwf.user = wf.user
mwf.grid_dn = wf.grid_dn
mwf.planner_version = wf.planner_version
mwf.db_url = wf.db_url
mwf.archived = self.is_archived()
mdbsession.add(mwf)
mdbsession.flush() # We should have the new wf_id after this
# Query states from workflow database
states = db.get_workflow_states(wf.wf_id)
# Insert states into master db
for s in states:
ms = DashboardWorkflowstate()
ms.wf_id = mwf.wf_id
ms.state = s.state
ms.timestamp = s.timestamp
ms.restart_count = s.restart_count
ms.status = s.status
mdbsession.add(ms)
mdbsession.flush()
dbsession.commit()
dbsession.close()
mdbsession.commit()
mdbsession.close()
def move(self, dest):
"Move this submit directory to dest"
dest = os.path.abspath(dest)
if os.path.isfile(dest):
raise SubmitDirException("Destination is a file: %s" % dest)
if os.path.isdir(dest):
if os.path.exists(os.path.join(dest, "braindump.txt")):
raise SubmitDirException("Destination is a submit dir: %s" % dest)
dest = os.path.join(dest, os.path.basename(self.submitdir))
# Verify that we aren't trying to move a subworkflow
if self.is_subworkflow():
raise SubmitDirException("Subworkflows cannot be moved independent of the root workflow")
# Connect to master database
mdbsession = connection.connect_by_submitdir(self.submitdir, connection.DBType.MASTER)
mdb = MasterDatabase(mdbsession)
# Get the workflow record from the master db
db_url = None
wf = mdb.get_master_workflow(self.wf_uuid)
if wf is None:
db_url = connection.url_by_submitdir(self.submitdir, connection.DBType.WORKFLOW)
else:
# We found an mdb record, so we need to update it
# Save the master db's pointer
db_url = wf.db_url
# Update the master db's db_url
# Note that this will only update the URL if it is an sqlite file
# located in the submitdir
log.info("Old master db_url: %s" % wf.db_url)
wf.db_url = db_url.replace(self.submitdir, dest)
log.info("New master db_url: %s" % wf.db_url)
# Change the master db's submit_dir
log.info("Old master submit_dir: %s" % wf.submit_dir)
wf.submit_dir = dest
log.info("New master submit_dir: %s" % wf.submit_dir)
# Update the ensemble record if one exists
ew = mdb.get_ensemble_workflow(self.wf_uuid)
if ew is not None:
log.info("Old ensemble submit dir: %s", ew.submitdir)
ew.submitdir = dest
log.info("New ensemble submit dir: %s", ew.submitdir)
# Update the workflow database if we found one
if db_url is not None:
dbsession = connection.connect(db_url)
db = WorkflowDatabase(dbsession)
root_wf = db.get_workflow(self.wf_uuid)
db.update_submit_dirs(root_wf.wf_id, self.submitdir, dest)
dbsession.commit()
dbsession.close()
# Move all the files
shutil.move(self.submitdir, dest)
# Set new paths in the braindump file
self.braindump["submit_dir"] = dest
self.braindump["basedir"] = os.path.dirname(dest)
utils.write_braindump(os.path.join(dest, "braindump.txt"), self.braindump)
# Note that we do not need to update the properties file even though it
# might contain DB URLs because it cannot contain a DB URL with the submit
# dir in it.
# TODO We might want to update all of the absolute paths in the condor submit files
# if we plan on moving workflows that could be resubmitted in the future
# TODO We might want to update the braindump files for subworkflows
# Update master database
mdbsession.commit()
mdbsession.close()
# Finally, update object
self.submitdir = dest
def attach(self):
"Add a workflow to the master db"
# Verify that we aren't trying to attach a subworkflow
if self.is_subworkflow():
raise SubmitDirException("Subworkflows cannot be attached independent of the root workflow")
# Connect to master database
mdbsession = connection.connect_by_submitdir(self.submitdir, connection.DBType.MASTER)
mdb = MasterDatabase(mdbsession)
# Check to see if it already exists and just update it
wf = mdb.get_master_workflow(self.wf_uuid)
if wf is not None:
print("Workflow is already in master db")
old_submit_dir = wf.submit_dir
if old_submit_dir != self.submitdir:
print("Updating path...")
wf.submit_dir = self.submitdir
wf.db_url = connection.url_by_submitdir(self.submitdir, connection.DBType.WORKFLOW)
mdbsession.commit()
mdbsession.close()
return
# Connect to workflow db
db_url = connection.url_by_submitdir(self.submitdir, connection.DBType.WORKFLOW)
dbsession = connection.connect(db_url)
db = WorkflowDatabase(dbsession)
# Get workflow record
wf = db.get_workflow(self.wf_uuid)
if wf is None:
print("No database record for that workflow exists")
return
# Update the workflow record
wf.submit_dir = self.submitdir
wf.db_url = db_url
# Insert workflow record into master db
mwf = DashboardWorkflow()
mwf.wf_uuid = wf.wf_uuid
mwf.dax_label = wf.dax_label
mwf.dax_version = wf.dax_version
mwf.dax_file = wf.dax_file
mwf.dag_file_name = wf.dag_file_name
mwf.timestamp = wf.timestamp
mwf.submit_hostname = wf.submit_hostname
mwf.submit_dir = self.submitdir
mwf.planner_arguments = wf.planner_arguments
mwf.user = wf.user
mwf.grid_dn = wf.grid_dn
mwf.planner_version = wf.planner_version
mwf.db_url = wf.db_url
mwf.archived = self.is_archived()
mdbsession.add(mwf)
mdbsession.flush() # We should have the new wf_id after this
# Query states from workflow database
states = db.get_workflow_states(wf.wf_id)
# Insert states into master db
for s in states:
ms = DashboardWorkflowstate()
ms.wf_id = mwf.wf_id
ms.state = s.state
ms.timestamp = s.timestamp
ms.restart_count = s.restart_count
ms.status = s.status
mdbsession.add(ms)
mdbsession.flush()
dbsession.commit()
dbsession.close()
mdbsession.commit()
mdbsession.close()
def move(self, dest):
"Move this submit directory to dest"
dest = os.path.abspath(dest)
if os.path.isfile(dest):
raise SubmitDirException("Destination is a file: %s" % dest)
if os.path.isdir(dest):
if os.path.exists(os.path.join(dest, "braindump.txt")):
raise SubmitDirException("Destination is a submit dir: %s" % dest)
dest = os.path.join(dest, os.path.basename(self.submitdir))
# Verify that we aren't trying to move a subworkflow
if self.is_subworkflow():
raise SubmitDirException("Subworkflows cannot be moved independent of the root workflow")
# Connect to master database
mdbsession = connection.connect_by_submitdir(self.submitdir, connection.DBType.MASTER)
mdb = MasterDatabase(mdbsession)
# Get the workflow record from the master db
db_url = None
wf = mdb.get_master_workflow(self.wf_uuid)
if wf is None:
db_url = connection.url_by_submitdir(self.submitdir, connection.DBType.WORKFLOW)
else:
# We found an mdb record, so we need to update it
# Save the master db's pointer
db_url = wf.db_url
# Update the master db's db_url
# Note that this will only update the URL if it is an sqlite file
# located in the submitdir
log.info("Old master db_url: %s" % wf.db_url)
wf.db_url = db_url.replace(self.submitdir, dest)
log.info("New master db_url: %s" % wf.db_url)
# Change the master db's submit_dir
log.info("Old master submit_dir: %s" % wf.submit_dir)
wf.submit_dir = dest
log.info("New master submit_dir: %s" % wf.submit_dir)
# Update the ensemble record if one exists
ew = mdb.get_ensemble_workflow(self.wf_uuid)
if ew is not None:
log.info("Old ensemble submit dir: %s", ew.submitdir)
ew.submitdir = dest
log.info("New ensemble submit dir: %s", ew.submitdir)
# Update the workflow database if we found one
if db_url is not None:
dbsession = connection.connect(db_url)
db = WorkflowDatabase(dbsession)
root_wf = db.get_workflow(self.wf_uuid)
db.update_submit_dirs(root_wf.wf_id, self.submitdir, dest)
dbsession.commit()
dbsession.close()
# Move all the files
shutil.move(self.submitdir, dest)
# Set new paths in the braindump file
self.braindump["submit_dir"] = dest
self.braindump["basedir"] = os.path.dirname(dest)
utils.write_braindump(os.path.join(dest, "braindump.txt"), self.braindump)
# Note that we do not need to update the properties file even though it
# might contain DB URLs because it cannot contain a DB URL with the submit
# dir in it.
# TODO We might want to update all of the absolute paths in the condor submit files
# if we plan on moving workflows that could be resubmitted in the future
# TODO We might want to update the braindump files for subworkflows
# Update master database
mdbsession.commit()
mdbsession.close()
# Finally, update object
self.submitdir = dest
def get_workflow_uri(submit_dir):
return connection.url_by_submitdir(submit_dir, DBType.WORKFLOW)
