def startCU(self):
print 'Starting Compute Unit submissions'
#p1,p2,p3,p4,p5 - Probes for time calculations
p1 = time.time()
self.compute_data_service = ComputeDataService()
self.compute_data_service.add_pilot_compute_service(self.pilot_compute_service)
for i in range(self.no_jobs):
print 'Submitting job %s on %s'%(i+1,self.pilot["service_url"])
p2 = time.time()
self.compute_unit_description = {
"executable":"/bin/sleep 4",
#"arguments" : ["$MYOUTPUT"],
#"environment" : {'MYOUTPUT':'"Hello from Simple API"'},
"number_of_processes" : 1,
"output" : "stdout.txt",
"error" : "stderr.txt"
}
self.compute_unit = self.compute_data_service.submit_compute_unit(self.compute_unit_description)
p3 = time.time()
self.cu_sub_time = self.cu_sub_time + (p3-p2)
print 'All Compute Units Submitted. Waiting for completion'
p4 = time.time()
self.compute_data_service.wait()
p5 = time.time()
self.total_cu_time = p5 - p1
self.cu_wait_time = p5 - p4
print 'All CU executions completed'
def setupJob(self):
"""
If RE_SETUP='yes' creates and populates subdirectories, one for each
replica called r0, r1, ..., rN in the working directory. Otherwise
reads saved state from the ENGINE_BASENAME.stat file.
To populate each directory calls _buildInpFile(k) to prepare the MD
engine input file for replica k. Also creates soft links to the working
directory for the accessory files specified in ENGINE_INPUT_EXTFILES.
"""
#pilotjob: Initialize PilotJob at given COORDINATION_URL (CU)
self.pj = PilotComputeService(self.keywords.get('COORDINATION_URL'))
#pilotjob: Initialize PilotJob Data service (DU)
self.cds=ComputeDataService()
#pilotjob: Launch the PilotJob at the given COORDINATION_URL
self.launch_pilotjob()
if (self.keywords.get('RE_SETUP') is not None and
self.keywords.get('RE_SETUP').lower() == 'yes'):
# create replicas directories r1, r2, etc.
for k in range(self.nreplicas):
repl_dir = 'r%d'%k
if os.path.exists(repl_dir):
_exit('Replica directories already exist. Either turn off '
'RE_SETUP or remove the directories.')
else:
os.mkdir('r%d'%k)
# create links for external files
if self.extfiles is not None:
for file in self.extfiles:
for k in range(self.nreplicas):
self._linkReplicaFile(file,file,k)
# create status table
self.status = [{'stateid_current': k, 'running_status': 'W',
'cycle_current': 1} for k in range(self.nreplicas)]
# save status tables
self._write_status()
# create input files no. 1
for k in range(self.nreplicas):
self._buildInpFile(k)
self.updateStatus()
else:
self._read_status()
self.updateStatus(restart=True)
# if self.remote:
# self._setup_remote_workdir()
self.print_status()
#at this point all replicas should be in wait state
for k in range(self.nreplicas):
if self.status[k]['running_status'] != 'W':
_exit('Internal error after restart. Not all jobs are in wait '
'state.')
class simple: def __init__(self,no_jobs,pilot,COORD_url=None): self.no_jobs = no_jobs self.pilot = pilot if(COORD_url == None): self.COORD = "redis://[email protected]:6379" else: self.COORD = COORD_url def check(self): print 'Checkup time' print self.COORD def startpilot(self): print 'Start pilot service' self.pilot_compute_service = PilotComputeService(self.COORD) self.pilot_compute_description = { "service_url" : self.pilot["service_url"] } if self.pilot.has_key("number_of_processes"): self.pilot_compute_description["number_of_processes"] = self.pilot["number_of_processes"] if self.pilot.has_key("working_directory"): self.pilot_compute_description["working_directory"] = self.pilot["working_directory"] if self.pilot.has_key("queue"): self.pilot_compute_description["queue"] = self.pilot["queue"] if self.pilot.has_key("walltime"): self.pilot_compute_description["walltime"] = self.pilot["walltime"] self.pilotjob = self.pilot_compute_service.create_pilot(pilot_compute_description=self.pilot_compute_description) print 'Pilot successfully started' def startCU(self): print 'Starting Compute Unit submissions' self.compute_data_service = ComputeDataService() self.compute_data_service.add_pilot_compute_service(self.pilot_compute_service) for i in range(self.no_jobs): print 'Submitting job %s on %s'%(i+1,self.pilot["service_url"]) self.compute_unit_description = { "executable":"/bin/echo", "arguments" : ["$MYOUTPUT"], "environment" : {'MYOUTPUT':'"Hello from Simple API"'}, "number_of_processes" : 1, "output" : "stdout.txt", "error" : "stderr.txt" } self.compute_unit = self.compute_data_service.submit_compute_unit(self.compute_unit_description) print 'All Compute Units Submitted. Waiting for completion' self.compute_data_service.wait() print 'All CU executions completed' def terminate(self): print 'Terminating pilot' self.compute_data_service.cancel() self.pilot_compute_service.cancel()
def startCU(self):
print 'Starting Compute Unit submissions'
self.compute_data_service = ComputeDataService()
self.compute_data_service.add_pilot_compute_service(self.pilot_compute_service)
for i in range(self.no_jobs):
print 'Submitting job %s on %s'%(i+1,self.pilot["service_url"])
self.compute_unit_description = {
"executable":"/bin/echo",
"arguments" : ["$MYOUTPUT"],
"environment" : {'MYOUTPUT':'"Hello from Simple API"'},
"number_of_processes" : 1,
"output" : "stdout.txt",
"error" : "stderr.txt"
}
self.compute_unit = self.compute_data_service.submit_compute_unit(self.compute_unit_description)
print 'All Compute Units Submitted. Waiting for completion'
self.compute_data_service.wait()
print 'All CU executions completed'
def start(self):
# try:
from pilot import PilotComputeService, PilotDataService, ComputeDataService, State
darelogger.info("Create Compute Engine service ")
self.pilot_compute_service = PilotComputeService(coordination_url=COORDINATION_URL)
self.pilot_data_service = PilotDataService()
for compute_pilot, desc in self.workflow.compute_pilot_repo.items():
self.compute_pilot_service_repo.append(self.pilot_compute_service.create_pilot(pilot_compute_description=desc))
#for data_pilot, desc in self.workflow.data_pilot_repo.items():
# self.data_pilot_service_repo.append(self.pilot_data_service.create_pilot(pilot_data_description=desc))
self.compute_data_service = ComputeDataService()
self.compute_data_service.add_pilot_compute_service(self.pilot_compute_service)
# self.compute_data_service.add_pilot_data_service(self.pilot_data_service)
self.step_thread= {}
### run the steps
self.step_start_lock=threading.RLock()
self.step_run_lock=threading.RLock()
for step_id in self.workflow.step_units_repo.keys():
darelogger.info(" Sumitted step %s "%step_id)
self.step_start_lock.acquire()
self.start_thread_step_id =step_id
self.step_start_lock.release()
self.step_thread[step_id] = threading.Thread(target=self.start_step)
self.step_thread[step_id].start()
while(1):
count_step = [v.is_alive() for k,v in self.step_thread.items()]
darelogger.info('count_step %s'%count_step)
if not True in count_step and len(count_step)>0:
break
time.sleep(10)
darelogger.info(" All Steps Done processing")
self.cancel()
"walltime":
10,
"processes_per_node":
1,
"queue":
"small",
"allocation":
"TG-MCB090174",
"working_directory":
"/lustre/scratch/pmantha/agent/",
})
for pcd in pilot_compute_description:
pilotjob = pilot_compute_service.create_pilot(
pilot_compute_description=pcd)
compute_data_service = ComputeDataService()
compute_data_service.add_pilot_compute_service(pilot_compute_service)
# start work unit
compute_unit_description = {
"executable": "/bin/date",
"arguments": [""],
"total_core_count": 1,
"number_of_processes": 1,
"output": "stdout.txt",
"error": "stderr.txt"
}
for i in range(0, 20):
compute_unit = compute_data_service.submit_compute_unit(
compute_unit_description)
class async_re_job(object):
"""
Class to set up and run asynchronous file-based RE calculations
"""
def __init__(self, command_file, options):
self.command_file = command_file
self.cus = {}
self.jobname = os.path.splitext(os.path.basename(command_file))[0]
self.keywords = ConfigObj(self.command_file)
self._checkInput()
self._printStatus()
def _exit(self, message):
_exit(message)
def _openfile(self, name, mode, max_attempts=100):
f = _open(name, mode, max_attempts)
return f
def __getattribute__(self, name):
if name == "replicas_waiting":
# Return a list of replica indices of replicas in a wait state.
# self.updateStatus()
return [k for k in range(self.nreplicas) if self.status[k]["running_status"] == "W"]
elif name == "states_waiting":
# Return a list of state ids of replicas in a wait state.
return [self.status[k]["stateid_current"] for k in self.replicas_waiting]
elif name == "replicas_waiting_to_exchange":
# Return a list of replica indices of replicas in a wait state that
# have ALSO completed at least one cycle.
# self.updateStatus()
return [
k
for k in range(self.nreplicas)
if (self.status[k]["running_status"] == "W" and self.status[k]["cycle_current"] > 1)
]
elif name == "states_waiting_to_exchange":
# Return a list of state ids of replicas in a wait state that have
# ALSO completed at least one cycle.
return [self.status[k]["stateid_current"] for k in self.replicas_waiting_to_exchange]
elif name == "waiting":
return len(self.replicas_waiting)
elif name == "replicas_running":
# Return a list of replica indices of replicas in a running state.
# self.updateStatus()
return [k for k in range(self.nreplicas) if self.status[k]["running_status"] == "R"]
elif name == "running":
return len(self.replicas_running)
else:
return object.__getattribute__(self, name)
def _printStatus(self):
"""Print a report of the input parameters."""
print "command_file =", self.command_file
print "jobname =", self.jobname
for k, v in self.keywords.iteritems():
print k, v
def _checkInput(self):
"""
Check that required parameters are specified. Parse these and other
optional settings.
"""
# Required Options
#
# basename for the job
self.basename = self.keywords.get("ENGINE_INPUT_BASENAME")
if self.basename is None:
self._exit("ENGINE_INPUT_BASENAME needs to be specified")
# execution time in minutes
self.walltime = float(self.keywords.get("WALL_TIME"))
if self.walltime is None:
self._exit("WALL_TIME (in minutes) needs to be specified")
# variables required for PilotJob
if self.keywords.get("COORDINATION_URL") is None:
self._exit("COORDINATION_URL needs to be specified")
if self.keywords.get("RESOURCE_URL") is None:
self._exit("RESOURCE_URL needs to be specified")
if self.keywords.get("QUEUE") is None:
if str(self.keywords.get("RESOURCE_URL")).split(":")[0] != "fork":
self._exit("QUEUE needs to be specified")
if self.keywords.get("BJ_WORKING_DIR") is None:
basedir = os.getcwd()
else:
basedir = self.keywords.get("BJ_WORKING_DIR")
self.bj_working_dir = os.path.join(basedir, "agent")
if not os.path.exists(self.bj_working_dir):
os.mkdir(self.bj_working_dir)
if self.keywords.get("TOTAL_CORES") is None:
self._exit("TOTAL_CORES needs to be specified")
if self.keywords.get("SUBJOB_CORES") is None:
self._exit("SUBJOB_CORES needs to be specified")
# Optional variables
#
env = self.keywords.get("ENGINE_ENVIRONMENT")
if env is not None and env != "":
self.engine_environment = env.split(",")
else:
self.engine_environment = []
# processors per node on this machine (can be auto-detected)
if self.keywords.get("PPN") is not None:
self.ppn = int(self.keywords.get("PPN"))
else:
self.ppn = 1
# spmd_variation for PilotJob (may override this later)
if self.keywords.get("SPMD") is not None:
self.spmd = self.keywords.get("SPMD")
else:
self.spmd = "single"
# number of replicas (may be determined by other means)
self.nreplicas = None
if self.keywords.get("NEXCHG_ROUNDS") is not None:
self.nexchg_rounds = int(self.keywords.get("NEXCHG_ROUNDS"))
else:
self.nexchg_rounds = 1
# examine RESOURCE_URL to see if it's remote (file staging)
# self.remote = self._check_remote_resource(self.keywords.get('RESOURCE_URL'))
# if self.remote:
# print "Use remote execution and file staging"
# if self.keywords.get('REMOTE_WORKING_DIR') is None:
# self._exit("REMOTE_WORKING_DIR needs to be specified")
# if self.keywords.get('REMOTE_DATA_SERVICE') is None: #something like ssh://<user>@<machine>/<datadir>
# self._exit("REMOTE_DATA_SERVICE needs to be specified")
# if self.keywords.get('REMOTE_DATA_SIZE') is None:
# self.remote_data_size = 2048 # 2GB by default
# else:
# self.remote_data_size = self.keywords.get('REMOTE_DATA_SIZE')
if self.keywords.get("NREPLICAS") is not None:
self.nreplicas = int(self.keywords.get("NREPLICAS"))
# extfiles variable for 'setupJob'
self.extfiles = self.keywords.get("ENGINE_INPUT_EXTFILES")
if self.extfiles is not None and self.extfiles != "":
self.extfiles = self.extfiles.split(",")
else:
self.extfiles = None
# verbose printing
if self.keywords.get("VERBOSE").lower() == "yes":
self.verbose = True
else:
self.verbose = False
def _linkReplicaFile(self, link_filename, real_filename, repl):
"""
Link the file at real_filename to the name at link_filename in the
directory belonging to the given replica. If a file is already linked
to this name (e.g. from a previous cycle), remove it first.
"""
os.chdir("r%d" % repl)
# Check that the file to be linked actually exists.
# TODO: This is not robust to absolute path specifications.
real_filename = "../%s" % real_filename
if not os.path.exists(real_filename):
self._exit("No such file: %s" % real_filename)
# Make/re-make the symlink.
if os.path.exists(link_filename):
os.remove(link_filename)
os.symlink(real_filename, link_filename)
os.chdir("..")
def setupJob(self):
"""
If RE_SETUP='yes' creates and populates subdirectories, one for each
replica called r0, r1, ..., rN in the working directory. Otherwise
reads saved state from the ENGINE_BASENAME.stat file.
To populate each directory calls _buildInpFile(k) to prepare the MD
engine input file for replica k. Also creates soft links to the working
directory for the accessory files specified in ENGINE_INPUT_EXTFILES.
"""
# pilotjob: Initialize PilotJob at given COORDINATION_URL (CU)
self.pj = PilotComputeService(self.keywords.get("COORDINATION_URL"))
# pilotjob: Initialize PilotJob Data service (DU)
self.cds = ComputeDataService()
# pilotjob: Launch the PilotJob at the given COORDINATION_URL
self.launch_pilotjob()
if self.keywords.get("RE_SETUP") is not None and self.keywords.get("RE_SETUP").lower() == "yes":
# create replicas directories r1, r2, etc.
for k in range(self.nreplicas):
repl_dir = "r%d" % k
if os.path.exists(repl_dir):
_exit("Replica directories already exist. Either turn off " "RE_SETUP or remove the directories.")
else:
os.mkdir("r%d" % k)
# create links for external files
if self.extfiles is not None:
for file in self.extfiles:
for k in range(self.nreplicas):
self._linkReplicaFile(file, file, k)
# create status table
self.status = [
{"stateid_current": k, "running_status": "W", "cycle_current": 1} for k in range(self.nreplicas)
]
# save status tables
self._write_status()
# create input files no. 1
for k in range(self.nreplicas):
self._buildInpFile(k)
self.updateStatus()
else:
self._read_status()
self.updateStatus(restart=True)
# if self.remote:
# self._setup_remote_workdir()
self.print_status()
# at this point all replicas should be in wait state
for k in range(self.nreplicas):
if self.status[k]["running_status"] != "W":
_exit("Internal error after restart. Not all jobs are in wait " "state.")
def scheduleJobs(self):
# wait until bigjob enters executing
while self.pilotcompute.get_state() != "Running":
time.sleep(10)
# Gets the wall clock time for a replica to complete a cycle
# If unspecified it is estimated as 10% of job wall clock time
# Note
replica_run_time = self.keywords.get("REPLICA_RUN_TIME")
if self.keywords.get("REPLICA_RUN_TIME") is None:
replica_run_time = int(round(self.walltime / 10.0))
else:
replica_run_time = int(self.keywords.get("REPLICA_RUN_TIME"))
# double it to give time for current running processes
# and newly submitted processes to complete
replica_run_time *= 2
# Time in between cycles in seconds
# If unspecified it is set as 30 secs
if self.keywords.get("CYCLE_TIME") is None:
cycle_time = 30.0
else:
cycle_time = float(self.keywords.get("CYCLE_TIME"))
start_time = time.time()
end_time = start_time + 60 * (self.walltime - replica_run_time) - cycle_time - 10
while time.time() < end_time:
time.sleep(1)
self.updateStatus()
self.print_status()
self.launchJobs()
self.updateStatus()
self.print_status()
time.sleep(cycle_time)
self.updateStatus()
self.print_status()
self.doExchanges()
self.updateStatus()
self.print_status()
self.waitJob()
self.cleanJob()
def waitJob(self):
# cancel all not-running submitted subjobs
# for k in range(self.nreplicas):
# if self.status[k]['running_status'] == "R":
# if self.cus[k].get_state() != "Running":
# self.cus[k].cancel()
# self.status[k]['running_status'] = "W"
# update status
# self.updateStatus()
# self.print_status()
# wait until running jobs complete
self.cds.wait()
def cleanJob(self):
self.cds.cancel()
self.pj.cancel()
def launch_pilotjob(self):
# pilotjob: PilotJob description
# pilotjob: Variables defined in command.inp
pcd = {
"service_url": self.keywords.get("RESOURCE_URL"),
"number_of_processes": self.keywords.get("TOTAL_CORES"),
"working_directory": self.bj_working_dir,
"queue": self.keywords.get("QUEUE"),
"processes_per_node": self.ppn,
"project": self.keywords.get("PROJECT"),
"walltime": int(self.keywords.get("WALL_TIME")),
}
if self.keywords.get("SGE_WAYNESS") is not None:
pcd["spmd_variation"] = self.keywords.get("SGE_WAYNESS")
# pilotjob: Create pilot job with above description
self.pj.create_pilot(pilot_compute_description=pcd)
self.cds.add_pilot_compute_service(self.pj)
self.pilotcompute = self.pj.list_pilots()[0]
def _write_status(self):
"""
Pickle the current state of the RE job and write to in BASENAME.stat.
"""
status_file = "%s.stat" % self.basename
f = _open(status_file, "w")
pickle.dump(self.status, f)
f.close()
def _read_status(self):
"""
Unpickle and load the current state of the RE job from BASENAME.stat.
"""
status_file = "%s.stat" % self.basename
f = _open(status_file, "r")
self.status = pickle.load(f)
f.close()
def print_status(self):
"""
Writes to BASENAME_stat.txt a text version of the status of the RE job.
It's fun to follow the progress in real time by doing:
watch cat BASENAME_stat.txt
"""
log = "Replica State Status Cycle \n"
for k in range(self.nreplicas):
log += "%6d %5d %5s %5d \n" % (
k,
self.status[k]["stateid_current"],
self.status[k]["running_status"],
self.status[k]["cycle_current"],
)
log += "Running = %d\n" % self.running
log += "Waiting = %d\n" % self.waiting
logfile = "%s_stat.txt" % self.basename
ofile = _open(logfile, "w")
ofile.write(log)
ofile.close()
def updateStatus(self, restart=False):
"""Scan the replicas and update their states."""
for k in range(self.nreplicas):
self._updateStatus_replica(k, restart)
self._write_status()
def _updateStatus_replica(self, replica, restart):
"""
Update the status of the specified replica. If it has completed a cycle
the input file for the next cycle is prepared and the replica is placed
in the wait state.
"""
this_cycle = self.status[replica]["cycle_current"]
if restart:
if self.status[replica]["running_status"] == "R":
if self._hasCompleted(replica, this_cycle):
self.status[replica]["cycle_current"] += 1
else:
print (
"_updateStatus_replica(): Warning: restarting " "replica %d (cycle %d)" % (replica, this_cycle)
)
self._buildInpFile(replica)
self.status[replica]["running_status"] = "W"
else:
if self.status[replica]["running_status"] == "R":
if self._isDone(replica, this_cycle):
self.status[replica]["running_status"] = "S"
if self._hasCompleted(replica, this_cycle):
self.status[replica]["cycle_current"] += 1
else:
print (
"_updateStatus_replica(): Warning: restarting "
"replica %d (cycle %d)" % (replica, this_cycle)
)
self._buildInpFile(replica)
self.status[replica]["running_status"] = "W"
def _isDone(self, replica, cycle):
"""
Generic function to check if a replica completed a cycle.
Calls in this case pilot-job version.
"""
return self._isDone_PJ(replica, cycle)
def _isDone_PJ(self, replica, cycle):
"""Return true if a replica has exited (done or failed)."""
# pilotjob: Get status of the compute unit
# pilotjob: Query the replica to see if it is in the done state
state = self.cus[replica].get_state()
details = self.cus[replica].get_details()
if state == "Done" or state == "Failed" or state == "Canceled":
if self.verbose:
if details.has_key("start_time"):
if details.has_key("end_time"):
print "*" * 80
print (
"Replica: %d Start Time: %f End Time: %f"
% (replica, float(details["start_time"]), float(details["end_time"]))
)
if details.has_key("end_queue_time"):
print ("End Queue Time: %f\n" % float(details["end_queue_time"]))
return True
else:
return False
def _hasCompleted(self, replica, cycle):
"""
Attempts to check whether a replica has completed successfully from the
bigjob compute unit. This is not expected to work during a restart when
compute units are not available. In the latter case success is assumed.
MD engine modules are recommended to override this default routine with
one that implements a better test of success such as the existence of a
restart file or similar.
"""
try:
state = self.cus[replica].get_state()
except:
print ("_hasCompleted(): Warning: unable to query replica state. " "Assuming success ...")
return True
if state == "Done":
return True
else:
return False
def _njobs_to_run(self):
# size of subjob buffer as a percentage of job slots
# (TOTAL_CORES/SUBJOB_CORES)
subjobs_buffer_size = self.keywords.get("SUBJOBS_BUFFER_SIZE")
if subjobs_buffer_size is None:
subjobs_buffer_size = 0.5
else:
subjobs_buffer_size = float(subjobs_buffer_size)
# launch new replicas if the number of submitted/running subjobs is
# less than the number of available slots
# (total_cores/subjob_cores) + 50%
available_slots = int(self.keywords.get("TOTAL_CORES")) / int(self.keywords.get("SUBJOB_CORES"))
max_njobs_submitted = int((1.0 + subjobs_buffer_size) * available_slots)
nlaunch = self.waiting - max(2, self.nreplicas - max_njobs_submitted)
nlaunch = max(0, nlaunch)
if self.verbose:
print "available_slots: %d" % available_slots
print "max_njobs_submitted: %d" % max_njobs_submitted
print "running/submitted subjobs: %d" % self.running
print "waiting replicas: %d" % self.waiting
print "replicas to launch: %d" % nlaunch
return nlaunch
def launchJobs(self):
"""
Scan the replicas in wait state and randomly launch some of them if
CPU's are available.
"""
jobs_to_launch = self._njobs_to_run()
if jobs_to_launch > 0:
wait = self.replicas_waiting
random.shuffle(wait)
n = min(jobs_to_launch, len(wait))
for k in wait[0:n]:
if self.verbose:
print ("Launching replica %d cycle %d" % (k, self.status[k]["cycle_current"]))
self.cus[k] = self._launchReplica(k, self.status[k]["cycle_current"])
self.status[k]["running_status"] = "R"
def doExchanges(self):
"""Perform exchanges among waiting replicas using Gibbs sampling."""
# NB: asking for self.replicas_waiting_to_exchange UPDATES the list,
# therefore this must be kept static at each repetition.
#
self.updateStatus()
replicas_to_exchange = self.replicas_waiting_to_exchange
states_to_exchange = self.states_waiting_to_exchange
nreplicas_to_exchange = len(replicas_to_exchange)
if nreplicas_to_exchange < 2:
return 0
print "Initiating exchanges amongst %d replicas:" % nreplicas_to_exchange
exchange_start_time = time.time()
# backtrack cycle of waiting replicas
for k in replicas_to_exchange:
self.status[k]["cycle_current"] -= 1
self.status[k]["running_status"] = "E"
# Matrix of replica energies in each state.
# The computeSwapMatrix() function is defined by application
# classes (Amber/US, Impact/BEDAM, etc.)
matrix_start_time = time.time()
swap_matrix = self._computeSwapMatrix(replicas_to_exchange, states_to_exchange)
matrix_time = time.time() - matrix_start_time
sampling_start_time = time.time()
# Perform an exchange for each of the n replicas, m times
if self.nexchg_rounds >= 0:
mreps = self.nexchg_rounds
else:
mreps = nreplicas_to_exchange ** (-self.nexchg_rounds)
accept_count = 0
for reps in range(mreps):
for repl_i in replicas_to_exchange:
sid_i = self.status[repl_i]["stateid_current"]
curr_states = [self.status[repl_j]["stateid_current"] for repl_j in replicas_to_exchange]
repl_j = pairwise_independence_sampling(repl_i, sid_i, replicas_to_exchange, curr_states, swap_matrix)
if repl_j != repl_i:
sid_i = self.status[repl_i]["stateid_current"]
sid_j = self.status[repl_j]["stateid_current"]
self.status[repl_i]["stateid_current"] = sid_j
self.status[repl_j]["stateid_current"] = sid_i
accept_count += 1
# Uncomment to debug Gibbs sampling:
# Actual and observed populations of state permutations should match.
#
# self._debug_collect_state_populations(replicas_to_exchange)
# self._debug_validate_state_populations(replicas_to_exchange,
# states_to_exchange,swap_matrix)
sampling_time = time.time() - sampling_start_time
for k in replicas_to_exchange:
# Place replicas back into "W" (wait) state.
self.status[k]["cycle_current"] += 1
self.status[k]["running_status"] = "W"
total_time = time.time() - exchange_start_time
print "------------------------------------------"
print "Swap matrix computation time: %10.2f s" % matrix_time
print "Gibbs sampling time : %10.2f s" % sampling_time
print "------------------------------------------"
print "Total exchange time : %10.2f s" % total_time
print "%d exchanges accepted" % accept_count
# def _check_remote_resource(self, resource_url):
# """
# check if it's a remote resource. Basically see if 'ssh' is present
# """
# ssh_c = re.compile("(.+)\+ssh://(.*)")
# m = re.match(ssh_c, resource_url)
# if m:
# self.remote_protocol = m.group(1)
# self.remote_server = m.group(2)
# print resource_url + " : yes" + " " + remote_protocol + " " + remote_server
# return 1
# else:
# print resource_url + " : no"
# return 0
# def _setup_remote_workdir(self):
# """
# rsync local working directory with remote working directory
# """
# os.system("ssh %s mkdir -p %s" % (self.remote_server, self.keywords.get('REMOTE_WORKING_DIR')))
# extfiles = " "
# for efile in self.extfiles:
# extfiles = extfiles + " " + efile
# os.system("rsync -av %s %s/%s/" % (extfiles, self.remote_server, self.keywords.get('REMOTE_WORKING_DIR')))
# dirs = ""
# for k in range(self.nreplicas):
# dirs = dirs + " r%d" % k
# setup_script = """
# cd %s ; \
# for i in `seq 0 %d` ; do \
# mkdir -p r$i ; \
# """
def _debug_collect_state_populations(self, replicas):
"""
Calculate the empirically observed distribution of state permutations.
Permutations not observed will NOT be counted and will need to be
added later for proper comparison to the exact distribution.
"""
try:
self.nperm
except (NameError, AttributeError):
self.nperm = {}
curr_states = [self.status[i]["stateid_current"] for i in replicas]
curr_perm = str(zip(replicas, curr_states))
if self.nperm.has_key(curr_perm):
self.nperm[curr_perm] += 1
else:
self.nperm[curr_perm] = 1
def _debug_validate_state_populations(self, replicas, states, U):
"""
Calculate the exact state permutation distribution and compare it to
the observed distribution. The similarity of these distributions is
measured via the Kullback-Liebler divergence.
"""
empirical = sample_to_state_perm_distribution(self.nperm, replicas, states)
exact = state_perm_distribution(replicas, states, U)
print "%8s %-9s %-9s %-s" % ("", "empirical", "exact", "state permutation")
print "-" * 80
if len(empirical.keys()) > len(exact.keys()):
perms = empirical.keys()
else:
perms = exact.keys()
for k, perm in enumerate(perms):
print "%8d %9.4f %9.4f %s" % (k + 1, empirical[perm], exact[perm], perm)
print "-" * 80
dkl = state_perm_divergence(empirical, exact)
print "Kullback-Liebler Divergence = %f" % dkl
print "=" * 80
mbpy.copy(workdir.get_url())
#mbpy = saga.filesystem.File("file://localhost/%s/main_file_futuregrid_newest_localhost.py"%os.getcwd())
#mbpy.copy(workdir.get_url())
pilot_tic = time.time()
pilot_compute_service = PilotComputeService(COORD)
#describe pilot
pilot_compute_description = {
"service_url" :"pbs+ssh://%s" %HOSTNAME,
"number_of_processes" : 8,
"working_directory" : WORKDIR,
"walltime": 10
}
#create pilot
pilotjob = pilot_compute_service.create_pilot(pilot_compute_description=pilot_compute_description)
pilot_toc = time.time()
compute_data_service=ComputeDataService()
compute_data_service.add_pilot_compute_service(pilot_compute_service)
print 'Finished pilot job creation'
print 'Start CU submission'
#submit jobs
# tasks = list()
i=0
job_tic = time.time()
while(i<NUMBER_JOBS):
print 'Submitting job',(i+1)
outputfile = 'file_%s.txt'%(i)
#job description
#task_description = pilot.ComputeUnitDescription()
compute_unit_description= {"executable" : "python","arguments" : [WORKDIR+'/sorter.py',outputfile,str(data[i*(DATA_SIZE/NUMBER_JOBS):(i+1)*(DATA_SIZE/NUMBER_JOBS)])], "number_of_processes" : 1, "output":"stdout.txt","error":"stderror.txt"}
compute_unit =compute_data_service.submit_compute_unit(compute_unit_description)
}
logging.debug("Pilot Data Description 1: \n%s" % str(data_unit_description1))
# What files? Create Pilot Data Description using remote SSH URLs
# make remotete paths
remote_url_list = ["ssh://localhost" + os.path.join(base_dir, i) for i in url_list]
data_unit_description2 = {
"file_urls": remote_url_list,
"affinity_datacenter_label": "eu-de-south",
"affinity_machine_label": "mymachine-2",
}
logging.debug("Pilot Data Description 2: \n%s" % str(data_unit_description2))
# create pilot data service
compute_data_service = ComputeDataService()
# create pilot data service (factory for pilot stores (physical, distributed storage))
pilot_data_service = PilotDataService()
ps1 = pilot_data_service.create_pilot(
{
"service_url": "ssh://localhost/tmp/pilotdata-1/",
"size": 100,
"affinity_datacenter_label": "eu-de-south",
"affinity_machine_label": "mymachine-1",
}
)
ps2 = pilot_data_service.create_pilot(
{
"service_url": "ssh://localhost/tmp/pilotdata-2/",
pilot_compute_service = PilotComputeService(COORDINATION_URL)
pilot_compute_description=[]
pilot_compute_description.append({ "service_url": "ssh://localhost",
"number_of_processes": 12,
"allocation": "TG-MCB090174",
"queue": "normal",
"processes_per_node":12,
"working_directory": os.getcwd()+"/agent",
"walltime":10,
})
for pcd in pilot_compute_description:
pilot_compute_service.create_pilot(pilot_compute_description=pcd)
compute_data_service = ComputeDataService()
compute_data_service.add_pilot_compute_service(pilot_compute_service)
print ("Finished Pilot-Job setup. Submit compute units")
# submit Set A compute units
for i in range(NUMBER_JOBS):
compute_unit_description = {
"executable": "/bin/echo",
"arguments": ["Hello","$ENV1","$ENV2"],
"environment": ['ENV1=env_arg1','ENV2=env_arg2'],
"total_cpu_count": 4,
"spmd_variation":"mpi",
"output": "A_stdout.txt",
"error": "A_stderr.txt",
}
compute_unit = compute_data_service.submit_compute_unit(compute_unit_description)
pilotjob = pilot_compute_service.create_pilot(pilot_compute_description=pilot_compute_description)
while pilotjob.get_state()!="Running":
time.sleep(2)
start = time.time()
# start work unit
compute_unit_description = {
"executable": "/bin/date",
"arguments": [""],
"number_of_processes": 1,
"output": "stdout.txt",
"error": "stderr.txt",
}
cds = ComputeDataService()
cds.add_pilot_compute_service(pilot_compute_service)
unitservice = pilotjob
for i in range(0,NUMBER_CUS):
compute_unit = unitservice.submit_compute_unit(compute_unit_description)
print("Finished setup. Waiting for scheduling of CU")
unitservice.wait()
runtime=time.time()-start
print("Number Slots, Number CUs, Runtime, Throughput")
print("%d,%d,%f,%f"%(NUMBER_SLOTS,NUMBER_CUS,runtime, runtime/NUMBER_CUS))
"mymachine-1"
})
ps2 = pilot_data_service.create_pilot({
'service_url':
"ssh://localhost/tmp/pilotdata-2/",
'size':
100,
'affinity_datacenter_label':
"eu-de-south",
'affinity_machine_label':
"mymachine-2"
})
# create pilot data service
compute_data_service = ComputeDataService()
# add resources to pilot data service
compute_data_service.add_pilot_data_service(pilot_data_service)
###########################################################################
# DU1 should only be scheduled to machine 1
# DU2 should only be scheduled to machine 2
du1 = compute_data_service.submit_data_unit(data_unit_description1)
du2 = compute_data_service.submit_data_unit(data_unit_description2)
logging.debug(
"Finished setup of Pilot Data and Compute Data Service. Waiting for scheduling of Data Units"
)
compute_data_service.wait()
class simple: #Input values from user - Number of jobs, Pilot Description, Redis Server Coordination URL def __init__(self,no_jobs,pilot,COORD_url=None): self.no_jobs = no_jobs self.pilot = pilot if(COORD_url == None): self.COORD = "redis://[email protected]:6379" else: self.COORD = COORD_url #Time variable to analyse timing responses self.pilot_setup_time = 0 self.total_cu_time = 0 self.cu_sub_time = 0 self.cu_wait_time = 0 def startpilot(self): #API currently supports single pilot applications print 'Start pilot service' p1=time.time() self.pilot_compute_service = PilotComputeService(self.COORD) #Mandatory service url,number_of_processes of the Pilot self.pilot_compute_description = { "service_url" : self.pilot["service_url"], "number_of_processes" : self.pilot["number_of_processes"] } #Check for possible keys for Working Directory, #Queue, Walltime. Take default values if not mentioned. if self.pilot.has_key("working_directory"): self.pilot_compute_description["working_directory"] = self.pilot["working_directory"] if self.pilot.has_key("queue"): self.pilot_compute_description["queue"] = self.pilot["queue"] if self.pilot.has_key("walltime"): self.pilot_compute_description["walltime"] = self.pilot["walltime"] self.pilotjob = self.pilot_compute_service.create_pilot(pilot_compute_description=self.pilot_compute_description) p2=time.time() self.pilot_setup_time = p2 - p1 print 'Pilot successfully started' def startCU(self): print 'Starting Compute Unit submissions' #p1,p2,p3,p4,p5 - Probes for time calculations p1 = time.time() self.compute_data_service = ComputeDataService() self.compute_data_service.add_pilot_compute_service(self.pilot_compute_service) for i in range(self.no_jobs): print 'Submitting job %s on %s'%(i+1,self.pilot["service_url"]) p2 = time.time() self.compute_unit_description = { "executable":"/bin/sleep 4", #"arguments" : ["$MYOUTPUT"], #"environment" : {'MYOUTPUT':'"Hello from Simple API"'}, "number_of_processes" : 1, "output" : "stdout.txt", "error" : "stderr.txt" } self.compute_unit = self.compute_data_service.submit_compute_unit(self.compute_unit_description) p3 = time.time() self.cu_sub_time = self.cu_sub_time + (p3-p2) print 'All Compute Units Submitted. Waiting for completion' p4 = time.time() self.compute_data_service.wait() p5 = time.time() self.total_cu_time = p5 - p1 self.cu_wait_time = p5 - p4 print 'All CU executions completed' def terminate(self): #Terminate all CUs and pilot, Display all timing responses print 'Terminating pilot' self.compute_data_service.cancel() self.pilot_compute_service.cancel() print 'No of jobs : ',self.no_jobs print 'Pilot setup time : ',self.pilot_setup_time print 'CU submission time : ',self.cu_sub_time print 'CU wait time : ',self.cu_wait_time print 'Total CU time : ',self.total_cu_time
class DareManager(object):
"""DARE manager:
- reads different configuration files
- submits compute/data units as that in various steps"""
"""Constructor"""
def __init__(self, conffile="/path/to/conf/file"):
"" ""
self.dare_conffile = conffile
self.workflow = PrepareWorkFlow(self.dare_conffile)
self.updater = Updater(self.workflow.update_site_db, self.workflow.dare_web_id)
self.dare_id = "dare-" + str(uuid.uuid1())
self.compute_pilot_service_repo=[]
self.data_pilot_service_repo = []
self.start()
def start(self):
# try:
from pilot import PilotComputeService, PilotDataService, ComputeDataService, State
darelogger.info("Create Compute Engine service ")
self.pilot_compute_service = PilotComputeService(coordination_url=COORDINATION_URL)
self.pilot_data_service = PilotDataService()
for compute_pilot, desc in self.workflow.compute_pilot_repo.items():
self.compute_pilot_service_repo.append(self.pilot_compute_service.create_pilot(pilot_compute_description=desc))
#for data_pilot, desc in self.workflow.data_pilot_repo.items():
# self.data_pilot_service_repo.append(self.pilot_data_service.create_pilot(pilot_data_description=desc))
self.compute_data_service = ComputeDataService()
self.compute_data_service.add_pilot_compute_service(self.pilot_compute_service)
# self.compute_data_service.add_pilot_data_service(self.pilot_data_service)
self.step_thread= {}
### run the steps
self.step_start_lock=threading.RLock()
self.step_run_lock=threading.RLock()
for step_id in self.workflow.step_units_repo.keys():
darelogger.info(" Sumitted step %s "%step_id)
self.step_start_lock.acquire()
self.start_thread_step_id =step_id
self.step_start_lock.release()
self.step_thread[step_id] = threading.Thread(target=self.start_step)
self.step_thread[step_id].start()
while(1):
count_step = [v.is_alive() for k,v in self.step_thread.items()]
darelogger.info('count_step %s'%count_step)
if not True in count_step and len(count_step)>0:
break
time.sleep(10)
darelogger.info(" All Steps Done processing")
self.cancel()
#except:
# self.cancel()
def check_to_start_step(self, step_id):
flags = []
darelogger.info(self.workflow.step_units_repo[step_id].UnitInfo['start_after_steps'])
if self.workflow.step_units_repo[step_id].get_status() == StepUnitStates.New:
for dep_step_id in self.workflow.step_units_repo[step_id].UnitInfo['start_after_steps']:
if self.workflow.step_units_repo[dep_step_id].get_status() != StepUnitStates.Done:
flags.append(False)
darelogger.info(self.workflow.step_units_repo[dep_step_id].get_status())
return False if False in flags else True
def start_step(self):
self.step_start_lock.acquire()
step_id = self.start_thread_step_id
self.step_start_lock.release()
while(1):
darelogger.info(" Checking to start step %s "%step_id)
if self.check_to_start_step(step_id):
self.run_step(step_id)
break
else:
darelogger.info(" Cannot start this step %s sleeping..."%step_id)
time.sleep(10)
def run_step(self, step_id):
#self.step_run_lock.acquire()
starttime = time.time()
#job started update status
this_su = self.workflow.step_units_repo[step_id].UnitInfo
self.updater.update_status( this_su['dare_web_id'], "%s in step %s"%('Running', this_su['name']))
p = []
darelogger.info(" Started running %s "%step_id)
for du_id in self.workflow.step_units_repo[step_id].UnitInfo['transfer_input_data_units']:
#data_unit = self.compute_data_service.submit_data_unit(self.workflow.data_units_repo[du_id])
#darelogger.debug("Pilot Data URL: %s Description: \n%s"%(data_unit, str(self.workflow.data_units_repo[du_id])))
#data_unit.wait()
# self.compute_data_service.wait()
darelogger.debug(" input tranfer for step %s complete"%step_id)
jobs = []
job_start_times = {}
job_states = {}
NUMBER_JOBS = len(self.workflow.step_units_repo[step_id].UnitInfo['compute_units'])
for cu_id in self.workflow.step_units_repo[step_id].UnitInfo['compute_units']:
compute_unit = self.compute_data_service.submit_compute_unit(self.workflow.compute_units_repo[cu_id])
darelogger.info("Compute Unit: Description: \n%s"%(str(self.workflow.compute_units_repo[cu_id])))
jobs.append(compute_unit)
job_start_times[compute_unit]=time.time()
job_states[compute_unit] = compute_unit.get_state()
darelogger.debug("************************ All Jobs submitted ************************")
while 1:
finish_counter=0
result_map = {}
for i in range(0, NUMBER_JOBS):
old_state = job_states[jobs[i]]
state = jobs[i].get_state()
if result_map.has_key(state) == False:
result_map[state]=0
result_map[state] = result_map[state]+1
#print "counter: " + str(i) + " job: " + str(jobs[i]) + " state: " + state
if old_state != state:
darelogger.debug( "Job " + str(jobs[i]) + " changed from: " + old_state + " to " + state)
if old_state != state and self.has_finished(state)==True:
darelogger.info( "%s step Job: "%(self.workflow.step_units_repo[step_id].UnitInfo['name']) + str(jobs[i]) + " Runtime: " + str(time.time()-job_start_times[jobs[i]]) + " s.")
if self.has_finished(state)==True:
finish_counter = finish_counter + 1
job_states[jobs[i]]=state
darelogger.debug( "Current states: " + str(result_map) )
time.sleep(5)
if finish_counter == NUMBER_JOBS:
break
self.workflow.step_units_repo[step_id].set_status(StepUnitStates.Done)
#self.compute_data_service.wait()
darelogger.debug(" Compute jobs for step %s complete"%step_id)
#for du_id in self.workflow.step_units_repo[step_id].UnitInfo['transfer_output_data_units']:
# data_unit = self.compute_data_service.submit_data_unit(self.workflow.data_units_repo[du_id])
# darelogger.debug("Pilot Data URL: %s Description: \n%s"%(data_unit, str(pilot_data_description)))
# data_unit.wait()
#darelogger.debug(" Output tranfer for step %s complete"%step_id)
# self.compute_data_service.wait()
#runtime = time.time()-starttime
#all jobs done update status
self.updater.update_status( this_su['dare_web_id'],"%s is Done" %this_su['name'] )
#self.step_run_lock.release()
def has_finished(self, state):
state = state.lower()
if state=="done" or state=="failed" or state=="canceled":
return True
else:
return False
# def check_to_start_step(self, step_id):
# flags = []
# import pdb; pdb.set_trace()
# print self.workflow.step_units_repo[step_id].UnitInfo['start_after_steps']
# if self.workflow.step_units_repo[step_id].get_status() == StepUnitStates.New:
# for dep_step_id in self.workflow.step_units_repo[step_id].UnitInfo['start_after_steps']:
# if self.workflow.step_units_repo[dep_step_id].get_status() != StepUnitStates.Done:
# flags.append(False)
# print self.workflow.step_units_repo[dep_step_id].get_status()
#return False if False in flags else True
def cancel(self):
darelogger.debug("Terminate Pilot Compute/Data Service")
self.compute_data_service.cancel()
try:
self.pilot_data_service.cancel()
self.pilot_compute_service.cancel()
except:
pass
pilot_compute_description=pilot_compute_description)
while pilotjob.get_state() != "Running":
time.sleep(2)
start = time.time()
# start work unit
compute_unit_description = {
"executable": "/bin/date",
"arguments": [""],
"number_of_processes": 1,
"output": "stdout.txt",
"error": "stderr.txt",
}
cds = ComputeDataService()
cds.add_pilot_compute_service(pilot_compute_service)
unitservice = pilotjob
for i in range(0, NUMBER_CUS):
compute_unit = unitservice.submit_compute_unit(
compute_unit_description)
print("Finished setup. Waiting for scheduling of CU")
unitservice.wait()
runtime = time.time() - start
print("Number Slots, Number CUs, Runtime, Throughput")
print("%d,%d,%f,%f" %
(NUMBER_SLOTS, NUMBER_CUS, runtime, runtime / NUMBER_CUS))
