class dask_controller: #adapted from Charles' code
def __init__(self,n_workers=6,local=True,queue="short",death_timeout=3.,\
walltime='01:30:00',cores=1,processes=1,memory='6GB',\
working_directory="./",job_extra=[]):
self.local = local
self.n_workers = n_workers
self.walltime = walltime
self.queue = queue
self.death_timeout = death_timeout
self.processes = processes
self.memory = memory
self.cores = cores
self.working_directory = working_directory
self.job_extra = job_extra
writedir(working_directory, overwrite=False)
def startdask(self):
if self.local:
self.daskclient = Client()
self.daskclient.cluster.scale(self.n_workers)
else:
self.daskcluster = SLURMCluster(queue=self.queue,death_timeout=self.death_timeout,walltime=self.walltime,\
processes=self.processes,memory=self.memory,\
cores=self.cores,local_directory=self.working_directory,\
log_directory=self.working_directory,job_extra=self.job_extra)
self.workers = self.daskcluster.start_workers(self.n_workers)
self.daskclient = Client(self.daskcluster)
def shutdown(self):
self.daskclient.restart()
if not self.local:
self.daskcluster.stop_all_jobs()
self.daskcluster.close()
for item in os.listdir(self.working_directory):
if "worker-" in item or "slurm-" in item or ".lock" in item:
path = "./" + item
if os.path.isfile(path):
os.remove(path)
elif os.path.isdir(path):
shutil.rmtree(path)
def printprogress(self):
complete = len(
[item for item in self.futures if item.status == "finished"])
print(str(complete) + "/" + str(len(self.futures)))
def displaydashboard(self):
link = self.daskcluster.dashboard_link
display(HTML('<a href="' + link + '">Dashboard</a>'))
def mapfovs(self, function, fov_list, retries=0):
self.function = function
self.retries = retries
def mapallfovs(fov_number, function=function):
function(fov_number)
self.futures = {}
for fov in fov_list:
future = self.daskclient.submit(mapallfovs, fov, retries=retries)
self.futures[fov] = future
def retry_failed(self):
self.failed_fovs = [
fov for fov, future in self.futures.items()
if future.status != 'finished'
]
out = self.daskclient.restart()
self.mapfovs(self.function, self.failed_fovs, retries=self.retries)
def retry_processing(self):
self.proc_fovs = [
fov for fov, future in self.futures.items()
if future.status == 'pending'
]
out = self.daskclient.restart()
self.mapfovs(self.function, self.proc_fovs, retries=self.retries)
class dask_controller: #adapted from Charles' code
def __init__(self,n_workers=6,local=True,queue="short",\
walltime='01:30:00',cores=1,processes=1,memory='6GB',job_extra=[]):
self.local = local
self.n_workers = n_workers
self.walltime = walltime
self.queue = queue
self.processes = processes
self.memory = memory
self.cores = cores
self.job_extra = job_extra
def writedir(self, directory):
if not os.path.exists(directory):
os.makedirs(directory)
def startdask(self):
if self.local:
self.daskclient = Client()
self.daskclient.cluster.scale(self.n_workers)
else:
self.daskcluster = SLURMCluster(queue=self.queue,walltime=self.walltime,\
processes=self.processes,memory=self.memory,
cores=self.cores,job_extra=self.job_extra)
self.workers = self.daskcluster.start_workers(self.n_workers)
self.daskclient = Client(self.daskcluster)
def shutdown(self):
self.daskcluster.stop_all_jobs()
for item in os.listdir("./"):
if "worker-" in item or "slurm-" in item or ".lock" in item:
path = "./" + item
if os.path.isfile(path):
os.remove(path)
elif os.path.isdir(path):
shutil.rmtree(path)
def printprogress(self):
complete = len(
[item for item in self.futures if item.status == "finished"])
print(str(complete) + "/" + str(len(self.futures)))
def mapfovs(self, function, fov_list, retries=0):
self.function = function
self.retries = retries
def mapallfovs(fov_number, function=function):
function(fov_number)
self.futures = {}
for fov in fov_list:
future = self.daskclient.submit(mapallfovs, fov, retries=retries)
self.futures[fov] = future
def retry_failed(self):
self.failed_fovs = [
fov for fov, future in self.futures.items()
if future.status != 'finished'
]
self.daskclient.restart()
time.sleep(5)
self.mapfovs(self.function, self.failed_fovs, retries=self.retries)
def retry_processing(self):
self.proc_fovs = [
fov for fov, future in self.futures.items()
if future.status == 'pending'
]
self.daskclient.restart()
time.sleep(5)
self.mapfovs(self.function, self.proc_fovs, retries=self.retries)
