def run(self, distributable):
JustCheckExists().input(distributable)
localpath = os.environ["PATH"]
localwd = os.getcwd()
import datetime
now = datetime.datetime.now()
run_dir_rel = os.path.join("runs", util.datestamp(appendrandom=True))
run_dir_abs = os.path.join(localwd, run_dir_rel)
util.create_directory_if_necessary(run_dir_rel, isfile=False)
distributablep_filename = os.path.join(run_dir_rel, "distributable.p")
with open(distributablep_filename, mode='wb') as f:
pickle.dump(distributable, f, pickle.HIGHEST_PROTOCOL)
distributable_py_file = os.path.join(os.path.dirname(__file__), "..",
"distributable.py")
if not os.path.exists(distributable_py_file):
raise Exception("Expect file at " + distributable_py_file +
", but it doesn't exist.")
command_format_string = sys.executable + " " + distributable_py_file + " " + distributablep_filename + " LocalInParts({0},{1},mkl_num_threads={2})".format(
"{0}", self.taskcount, self.mkl_num_threads)
if not self.just_one_process:
proc_list = []
for taskindex in range(self.taskcount):
command_string = command_format_string.format(taskindex)
proc = subprocess.Popen(
command_string.split(" "), cwd=os.getcwd()
) #!!!bug: If Anaconda is installed in c:\program files\anaconda2 this will fail
proc_list.append(proc)
for taskindex, proc in enumerate(proc_list):
rc = proc.wait()
#for line in proc.stdout.readlines():
# sys.stdout.write(line)
if not 0 == rc:
raise Exception(
"Running python in python results in non-zero return code in task#{0}"
.format(taskindex))
else:
from fastlmm.util.runner import LocalInParts
for taskindex in range(self.taskcount):
LocalInParts(
taskindex,
self.taskcount,
mkl_num_threads=self.mkl_num_threads).run(distributable)
result = run_one_task(distributable, self.taskcount, self.taskcount,
distributable.tempdirectory)
JustCheckExists().output(distributable)
return result
def create_run_dir(self):
username = os.environ["USERNAME"]
localwd = os.getcwd()
#!!make an option to specify the full remote WD. Also what is the "\\\\" case for?
if localwd.startswith("\\\\"):
remotewd = self.fileshare + os.path.sep + username +os.path.sep + "\\".join(localwd.split('\\')[4:])
else:
remotewd = self.fileshare + os.path.sep + username + os.path.splitdrive(localwd)[1] #using '+' because 'os.path.join' isn't work with shares
import datetime
now = datetime.datetime.now()
run_dir_rel = os.path.join("runs",util.datestamp(appendrandom=True))
run_dir_abs = os.path.join(remotewd,run_dir_rel)
util.create_directory_if_necessary(run_dir_abs,isfile=False)
return remotewd, run_dir_abs, run_dir_rel
def run(self, distributable):
JustCheckExists().input(distributable)
localpath = os.environ["PATH"]
localwd = os.getcwd()
import datetime
now = datetime.datetime.now()
run_dir_rel = os.path.join("runs", util.datestamp(appendrandom=True))
run_dir_abs = os.path.join(localwd, run_dir_rel)
util.create_directory_if_necessary(run_dir_rel, isfile=False)
distributablep_filename = os.path.join(run_dir_rel, "distributable.p")
with open(distributablep_filename, mode="wb") as f:
pickle.dump(distributable, f, pickle.HIGHEST_PROTOCOL)
distributable_py_file = os.path.join(os.path.dirname(__file__), "..", "distributable.py")
if not os.path.exists(distributable_py_file):
raise Exception("Expect file at " + distributable_py_file + ", but it doesn't exist.")
command_format_string = (
sys.executable
+ " "
+ distributable_py_file
+ " "
+ distributablep_filename
+ " LocalInParts({0},{1},mkl_num_threads={2})".format("{0}", self.taskcount, self.mkl_num_threads)
)
proc_list = []
for taskindex in xrange(self.taskcount):
command_string = command_format_string.format(taskindex)
proc = subprocess.Popen(command_string.split(" "), cwd=os.getcwd())
proc_list.append(proc)
for taskindex, proc in enumerate(proc_list):
rc = proc.wait()
# for line in proc.stdout.readlines():
# sys.stdout.write(line)
if not 0 == rc:
raise Exception(
"Running python in python results in non-zero return code in task#{0}".format(taskindex)
)
result = run_one_task(distributable, self.taskcount, self.taskcount, distributable.tempdirectory)
JustCheckExists().output(distributable)
return result
def create_run_dir(self):
username = os.environ["USERNAME"]
localwd = os.getcwd()
#!!make an option to specify the full remote WD. Also what is the "\\\\" case for?
if localwd.startswith("\\\\"):
remotewd = self.fileshare + os.path.sep + username + os.path.sep + "\\".join(
localwd.split('\\')[4:])
else:
remotewd = self.fileshare + os.path.sep + username + os.path.splitdrive(
localwd
)[1] #using '+' because 'os.path.join' isn't work with shares
import datetime
now = datetime.datetime.now()
run_dir_rel = os.path.join("runs", util.datestamp(appendrandom=True))
run_dir_abs = os.path.join(remotewd, run_dir_rel)
util.create_directory_if_necessary(run_dir_abs, isfile=False)
return remotewd, run_dir_abs, run_dir_rel
def create_run_dir(self):
username = os.environ["USERNAME"]
localwd = os.getcwd()
if localwd.startswith("\\\\"):
remotewd = self.fileshare + os.path.sep + username + os.path.sep + "\\".join(localwd.split('\\')[4:])
else:
remotewd = self.fileshare + os.path.sep + username + os.path.splitdrive(localwd)[1] #using '+' because 'os.path.join' isn't work with shares
remotewd = remotewd.replace("\\","/")
if remotewd.endswith("/"): # remove trailing /
remotewd = remotewd[:-1]
run_dir_rel = os.path.join("runs",util.datestamp(appendrandom=True))
util.create_directory_if_necessary("runs",isfile=False)
if not os.path.isfile(".ignoreTgzChange"):
with open("runs" + os.path.sep + ".ignoreTgzChange","w") as ignoreFile:
ignoreFile.write("\n")
run_dir_abs = "/user/{0}/{1}".format(username,run_dir_rel)
#!! hadoop_create_directory_if_necessary(run_dir_abs,isfile=False)
return remotewd, run_dir_abs, run_dir_rel
def create_run_dir(self):
username = os.environ["USERNAME"]
localwd = os.getcwd()
if localwd.startswith("\\\\"):
remotewd = self.fileshare + os.path.sep + username + os.path.sep + "\\".join(localwd.split('\\')[4:])
else:
remotewd = self.fileshare + os.path.sep + username + os.path.splitdrive(localwd)[1] #using '+' because 'os.path.join' isn't work with shares
remotewd = remotewd.replace("\\","/")
if remotewd.endswith("/"): # remove trailing /
remotewd = remotewd[:-1]
run_dir_rel = os.path.join("runs",util.datestamp(appendrandom=True))
util.create_directory_if_necessary("runs",isfile=False)
if not os.path.isfile(".ignoreTgzChange"):
with open("runs" + os.path.sep + ".ignoreTgzChange","w") as ignoreFile:
ignoreFile.write("\n")
run_dir_abs = "/user/{0}/{1}".format(username,run_dir_rel)
#!! hadoop_create_directory_if_necessary(run_dir_abs,isfile=False)
return remotewd, run_dir_abs, run_dir_rel
