def run(self, distributable):
_JustCheckExists().input(distributable)
localpath = os.environ["PATH"]
localwd = os.getcwd()
import datetime
now = datetime.datetime.now()
run_dir_rel = os.path.join("runs",_datestamp(appendrandom=True))
run_dir_abs = os.path.join(localwd,run_dir_rel)
create_directory_if_necessary(run_dir_rel, isfile=False)
distributablep_filename = os.path.join(run_dir_rel, "distributable.p")
with open(distributablep_filename, mode='wb') as f:
pickle.dump(distributable, f, pickle.HIGHEST_PROTOCOL)
distributable_py_file = os.path.join(os.path.dirname(__file__),"..","distributable.py")
if not os.path.exists(distributable_py_file): raise Exception("Expect file at " + distributable_py_file + ", but it doesn't exist.")
if not self.just_one_process:
proc_list = []
for taskindex in range(self.taskcount):
environ = self.taskindex_to_environ(taskindex) if self.taskindex_to_environ is not None else None
command_string_list = [sys.executable,
distributable_py_file,
distributablep_filename,
f"LocalInParts({taskindex},{self.taskcount},mkl_num_threads={self.mkl_num_threads},weights={self.weights},environ={environ})".replace(" ","")
]
#logging.info(command_string_list)
proc = subprocess.Popen(command_string_list, cwd=os.getcwd())
proc_list.append(proc)
for taskindex, proc in enumerate(proc_list):
rc = proc.wait()
if not 0 == rc : raise Exception("Running python in python results in non-zero return code in task#{0}".format(taskindex))
else:
from pysnptools.util.mapreduce1.runner import LocalInParts
for taskindex in range(self.taskcount):
environ = self.taskindex_to_environ(taskindex) if self.taskindex_to_environ is not None else None
LocalInParts(taskindex,self.taskcount, mkl_num_threads=self.mkl_num_threads, weights=self.weights, environ=environ).run(distributable)
environ = self.taskindex_to_environ(self.taskcount) if self.taskindex_to_environ is not None else None
result = _run_one_task(distributable, self.taskcount, self.taskcount, distributable.tempdirectory, weights=self.weights, environ=environ)
_JustCheckExists().output(distributable)
return result
def create_run_dir(self):
username = os.environ["USERNAME"]
localwd = os.getcwd()
#!!make an option to specify the full remote WD. Also what is the "\\\\" case for?
if localwd.startswith("\\\\"):
remotewd = self.fileshare + os.path.sep + username +os.path.sep + "\\".join(localwd.split('\\')[4:])
nodelocalwd = "d:\scratch\escience" + os.path.sep + username +os.path.sep + "\\".join(localwd.split('\\')[4:]) #!!!const
else:
remotewd = self.fileshare + os.path.sep + username + os.path.splitdrive(localwd)[1] #using '+' because 'os.path.join' isn't work with shares
nodelocalwd = "d:\scratch\escience" + os.path.sep + username + os.path.splitdrive(localwd)[1] #!!! const
import datetime
now = datetime.datetime.now()
run_dir_rel = os.path.join("runs",pstutil._datestamp(appendrandom=True))
run_dir_abs = os.path.join(remotewd,run_dir_rel)
pstutil.create_directory_if_necessary(run_dir_abs,isfile=False)
return remotewd, run_dir_abs, run_dir_rel, nodelocalwd
def create_run_dir(self):
username = os.environ["USERNAME"]
localwd = os.getcwd()
if localwd.startswith("\\\\"):
remotewd = self.fileshare + os.path.sep + username + os.path.sep + "\\".join(
localwd.split('\\')[4:])
else:
remotewd = self.fileshare + os.path.sep + username + os.path.splitdrive(
localwd
)[1] #using '+' because 'os.path.join' isn't work with shares
remotewd = remotewd.replace("\\", "/")
if remotewd.endswith("/"): # remove trailing /
remotewd = remotewd[:-1]
run_dir_rel = os.path.join("runs",
pstutil._datestamp(appendrandom=True))
pstutil.create_directory_if_necessary("runs", isfile=False)
if not os.path.isfile(".ignoreTgzChange"):
with open("runs" + os.path.sep + ".ignoreTgzChange",
"w") as ignoreFile:
ignoreFile.write("\n")
run_dir_abs = "/user/{0}/{1}".format(username, run_dir_rel)
#!! hadoop_create_directory_if_necessary(run_dir_abs,isfile=False)
return remotewd, run_dir_abs, run_dir_rel
def datestamp(appendrandom=False):
return pstutil._datestamp(appendrandom=appendrandom)