def run(self, distributable): JustCheckExists().input(distributable) localpath = os.environ["PATH"] localwd = os.getcwd() import datetime now = datetime.datetime.now() run_dir_rel = os.path.join("runs", util.datestamp(appendrandom=True)) run_dir_abs = os.path.join(localwd, run_dir_rel) util.create_directory_if_necessary(run_dir_rel, isfile=False) distributablep_filename = os.path.join(run_dir_rel, "distributable.p") with open(distributablep_filename, mode='wb') as f: pickle.dump(distributable, f, pickle.HIGHEST_PROTOCOL) distributable_py_file = os.path.join(os.path.dirname(__file__), "..", "distributable.py") if not os.path.exists(distributable_py_file): raise Exception("Expect file at " + distributable_py_file + ", but it doesn't exist.") command_format_string = sys.executable + " " + distributable_py_file + " " + distributablep_filename + " LocalInParts({0},{1},mkl_num_threads={2})".format( "{0}", self.taskcount, self.mkl_num_threads) if not self.just_one_process: proc_list = [] for taskindex in range(self.taskcount): command_string = command_format_string.format(taskindex) proc = subprocess.Popen( command_string.split(" "), cwd=os.getcwd() ) #!!!bug: If Anaconda is installed in c:\program files\anaconda2 this will fail proc_list.append(proc) for taskindex, proc in enumerate(proc_list): rc = proc.wait() #for line in proc.stdout.readlines(): # sys.stdout.write(line) if not 0 == rc: raise Exception( "Running python in python results in non-zero return code in task#{0}" .format(taskindex)) else: from fastlmm.util.runner import LocalInParts for taskindex in range(self.taskcount): LocalInParts( taskindex, self.taskcount, mkl_num_threads=self.mkl_num_threads).run(distributable) result = run_one_task(distributable, self.taskcount, self.taskcount, distributable.tempdirectory) JustCheckExists().output(distributable) return result
def create_run_dir(self): username = os.environ["USERNAME"] localwd = os.getcwd() #!!make an option to specify the full remote WD. Also what is the "\\\\" case for? if localwd.startswith("\\\\"): remotewd = self.fileshare + os.path.sep + username +os.path.sep + "\\".join(localwd.split('\\')[4:]) else: remotewd = self.fileshare + os.path.sep + username + os.path.splitdrive(localwd)[1] #using '+' because 'os.path.join' isn't work with shares import datetime now = datetime.datetime.now() run_dir_rel = os.path.join("runs",util.datestamp(appendrandom=True)) run_dir_abs = os.path.join(remotewd,run_dir_rel) util.create_directory_if_necessary(run_dir_abs,isfile=False) return remotewd, run_dir_abs, run_dir_rel
def run(self, distributable): JustCheckExists().input(distributable) localpath = os.environ["PATH"] localwd = os.getcwd() import datetime now = datetime.datetime.now() run_dir_rel = os.path.join("runs", util.datestamp(appendrandom=True)) run_dir_abs = os.path.join(localwd, run_dir_rel) util.create_directory_if_necessary(run_dir_rel, isfile=False) distributablep_filename = os.path.join(run_dir_rel, "distributable.p") with open(distributablep_filename, mode="wb") as f: pickle.dump(distributable, f, pickle.HIGHEST_PROTOCOL) distributable_py_file = os.path.join(os.path.dirname(__file__), "..", "distributable.py") if not os.path.exists(distributable_py_file): raise Exception("Expect file at " + distributable_py_file + ", but it doesn't exist.") command_format_string = ( sys.executable + " " + distributable_py_file + " " + distributablep_filename + " LocalInParts({0},{1},mkl_num_threads={2})".format("{0}", self.taskcount, self.mkl_num_threads) ) proc_list = [] for taskindex in xrange(self.taskcount): command_string = command_format_string.format(taskindex) proc = subprocess.Popen(command_string.split(" "), cwd=os.getcwd()) proc_list.append(proc) for taskindex, proc in enumerate(proc_list): rc = proc.wait() # for line in proc.stdout.readlines(): # sys.stdout.write(line) if not 0 == rc: raise Exception( "Running python in python results in non-zero return code in task#{0}".format(taskindex) ) result = run_one_task(distributable, self.taskcount, self.taskcount, distributable.tempdirectory) JustCheckExists().output(distributable) return result
def create_run_dir(self): username = os.environ["USERNAME"] localwd = os.getcwd() #!!make an option to specify the full remote WD. Also what is the "\\\\" case for? if localwd.startswith("\\\\"): remotewd = self.fileshare + os.path.sep + username + os.path.sep + "\\".join( localwd.split('\\')[4:]) else: remotewd = self.fileshare + os.path.sep + username + os.path.splitdrive( localwd )[1] #using '+' because 'os.path.join' isn't work with shares import datetime now = datetime.datetime.now() run_dir_rel = os.path.join("runs", util.datestamp(appendrandom=True)) run_dir_abs = os.path.join(remotewd, run_dir_rel) util.create_directory_if_necessary(run_dir_abs, isfile=False) return remotewd, run_dir_abs, run_dir_rel
def create_run_dir(self): username = os.environ["USERNAME"] localwd = os.getcwd() if localwd.startswith("\\\\"): remotewd = self.fileshare + os.path.sep + username + os.path.sep + "\\".join(localwd.split('\\')[4:]) else: remotewd = self.fileshare + os.path.sep + username + os.path.splitdrive(localwd)[1] #using '+' because 'os.path.join' isn't work with shares remotewd = remotewd.replace("\\","/") if remotewd.endswith("/"): # remove trailing / remotewd = remotewd[:-1] run_dir_rel = os.path.join("runs",util.datestamp(appendrandom=True)) util.create_directory_if_necessary("runs",isfile=False) if not os.path.isfile(".ignoreTgzChange"): with open("runs" + os.path.sep + ".ignoreTgzChange","w") as ignoreFile: ignoreFile.write("\n") run_dir_abs = "/user/{0}/{1}".format(username,run_dir_rel) #!! hadoop_create_directory_if_necessary(run_dir_abs,isfile=False) return remotewd, run_dir_abs, run_dir_rel