def handleCrossValidations(self): # Start marian adding junk to the code MDSreader = csv.reader(open(self.args_dict['c'], 'r')) header1 = MDSreader.next() header2 = MDSreader.next() header3 = MDSreader.next() if 'Split' in header1: newLine = MDSreader.next() ttList = ["test", "train"] cvList = ["NA"] # find the first line of the mds that isn't na to determine if test/train or cv split while newLine[header1.index("Split")] == "NA": newLine = MDSreader.next() # loop through the mds and fit a maxent model withholding each new cv fold if (not newLine[header1.index("Split")] in ttList): subrun_args = copy.deepcopy(self.args_dict) subrun_args["sub_run"] = 'True' # cvMaxent = copy.deepcopy(self) # cvMaxent.sub_run=True for row in MDSreader: if (not row[header1.index("Split")] in cvList): subrun_args["test_key"] = row[header1.index("Split")] outdir = os.path.join(self.outputdir, "cvSplit" + row[header1.index("Split")]) subrun_args["o"] = outdir stdout_fname = os.path.join(outdir, "ExpandedOutput", "stdOut.txt") stderr_fname = os.path.join(outdir, "ExpandedOutput", "stdErr.txt") if not os.path.exists(outdir): os.makedirs(outdir) if not os.path.exists(os.path.join(outdir, "ExpandedOutput")): os.makedirs(os.path.join(outdir, "ExpandedOutput")) cvList.append(row[header1.index("Split")]) try: cmd = [sys.executable] cmd += sys.argv[:sys.argv.index("--args") + 1] for k, v in subrun_args.iteritems(): if v in [True, False]: cmd.append(k + "=" + str(v).upper()) else: cmd.append(k + "=" + str(v)) cmd.append("maxent_args=" + str(self.maxent_args)) utilities.add_process_to_pool(utilities.launch_cmd, [cmd, stdout_fname, stderr_fname]) except utilities.TrappedError as e: raise RuntimeError(self, e.message) # here we need to run Maxent without the test split csv which breaks it self.test_key = None results = utilities.wait_for_pool_to_finish() for result in results: if "Error" in result[0]: print "!!!!!!!!!!!!!!!!!test!!!!!!!!!!!!!!!!!!!!!!" raise RuntimeError(self, result[0])
def convertEnvironmentalLayers_mc(self, files, outputFolder, type): '''This function has the same functionality as convertEnvironmentalLayers with the addition of utilizing multiple cores to do the processing. ''' results = Queue.Queue() process_count = 0 i = 1 for f in files: f_name = os.path.splitext(os.path.split(f)[1])[0] # args = '-i ' + '"' + f + '"' # args += ' -o ' + '"' + outputFolder + '"' # args += ' -f ' + type args = ['-i', f, '-o', outputFolder, '-f', type] # if self.verbose: # args += " -v" execDir = os.path.split(__file__)[0] executable = os.path.join(execDir, 'singleRasterFormatConverter.py') command_arr = [sys.executable, executable] + args self.logger.writetolog(' '.join(command_arr), False, False) # proc = subprocess.Popen( command_arr ) # thread.start_new_thread(utilities.process_waiter, # (proc, f_name, results)) utilities.add_process_to_pool(utilities.launch_cmd, [command_arr]) utilities.wait_for_pool_to_finish()
def run(self): if self.sub_run: self.maxent_args['plots'] = 'false' self.maxent_args['responsecurves'] = 'false' self.maxent_args['outputgrids'] = 'false' self.validateInputs() self.start_pool() if self.args_dict['c'] != '': self.prepInputs() else: raise Exception, 'No MDS supplied.' if not self.maxent_args.has_key('projectionlayers'): self.maxent_args['projectionlayers'] = '' if self.trainingCSV != '': self.maxent_args['samplesfile'] = self.trainingCSV else: raise Exception, 'No Samples file supplied' if self.testCSV != '': self.maxent_args['testsamplesfile'] = self.testCSV if self.maxent_args['environmentallayers'] != '': self.maxent_args['environmentallayers'] = self.maxent_args['environmentallayers'] if self.maxent_args.has_key('biasfile'): self.maxent_args['biasfile'] = self.maxent_args['biasfile'] elif self.backgroundCSV != '': self.maxent_args['environmentallayers'] = self.backgroundCSV if self.maxent_args.has_key('biasfile'): self.writetolog('The supplied biasfile will be ignored since no environmentallayers were provided', True, True) self.maxent_args.pop('biasfile') self.maxent_args['autorun'] = 'true' if ' ' in self.args_dict['species_name']: self.args_dict['species_name'] = self.args_dict['species_name'].replace(' ', '_') if(not self.test_key): # if there is no testsplit (ie cross validation, remove the empty csv del self.maxent_args['testsamplesfile'] if not self.maxent_path.endswith('.jar'): self.maxent_path = os.path.join(self.maxent_path, 'maxent.jar') cmd = self.gen_maxent_cmd() stdout_fname = os.path.join(self.outputdir, "ExpandedOutput", "stdOut.txt") stderr_fname = os.path.join(self.outputdir, "ExpandedOutput", "stdErr.txt") if not os.path.exists(os.path.join(self.outputdir, "ExpandedOutput")): os.makedirs(os.path.join(self.outputdir, "ExpandedOutput")) self.writetolog(' running command: \n' + utilities.convert_list_to_cmd_str(cmd) + "\n", True, False) if os.path.exists(self.maxent_path): utilities.add_process_to_pool(utilities.launch_cmd, [cmd, stdout_fname, stderr_fname]) utilities.wait_for_pool_to_finish() if not self.sub_run: r_cmd = [sys.executable] r_cmd.extend(sys.argv) r_cmd[1] = r_cmd[1].replace('runMaxent.py', 'runRModel.py') r_cmd.append('lam=' + self.outputdir) self.writetolog(' running command: \n' + utilities.convert_list_to_cmd_str(r_cmd) + "\n", True, False) utilities.add_process_to_pool(utilities.launch_cmd, [r_cmd, stdout_fname, stderr_fname]) utilities.wait_for_pool_to_finish() else: msg = 'The specified Maxent.Jar file ({})\n\t\t'.format(self.maxent_path) msg += "\nNoes not appear to exist on the file system!" self.writetolog(msg, True, True) raise RuntimeError(msg)
def run(self): if self.sub_run: self.maxent_args['plots'] = 'false' self.maxent_args['responsecurves'] = 'false' self.maxent_args['outputgrids'] = 'false' self.validateInputs() self.start_pool() if self.args_dict['c'] != '': self.prepInputs() else: raise Exception, 'No MDS supplied.' if not self.maxent_args.has_key('projectionlayers'): self.maxent_args['projectionlayers'] = '' if self.trainingCSV != '': self.maxent_args['samplesfile'] = self.trainingCSV else: raise Exception, 'No Samples file supplied' if self.testCSV != '': self.maxent_args['testsamplesfile'] = self.testCSV if self.maxent_args['environmentallayers'] != '': self.maxent_args['environmentallayers'] = self.maxent_args['environmentallayers'] if self.maxent_args.has_key('biasfile'): self.maxent_args['biasfile'] = self.maxent_args['biasfile'] elif self.backgroundCSV != '': self.maxent_args['environmentallayers'] = self.backgroundCSV if self.maxent_args.has_key('biasfile'): self.writetolog('The supplied biasfile will be ignored since no environmentallayers were provided', True, True) self.maxent_args.pop('biasfile') self.maxent_args['autorun'] = 'true' if ' ' in self.args_dict['species_name']: self.args_dict['species_name'] = self.args_dict['species_name'].replace(' ', '_') if(not self.test_key): #if there is no testsplit (ie cross validation, remove the empty csv del self.maxent_args['testsamplesfile'] if self.categoricals: catstr = ",".join([cat.replace('_categorical', '') for cat in self.categoricals]) self.maxent_args['togglelayertype'] = catstr if not self.maxent_path.endswith('.jar'): self.maxent_path = os.path.join(self.maxent_path, 'maxent.jar') cmd = self.gen_maxent_cmd() stdout_fname_max = os.path.join(self.outputdir, "stdOut_max.txt") stderr_fname_max = os.path.join(self.outputdir, "stdErr_max.txt") stdout_fname_R = os.path.join(self.outputdir, "stdOut_R.txt") stderr_fname_R = os.path.join(self.outputdir, "stdErr_R.txt") self.writetolog(' running command: \n' + utilities.convert_list_to_cmd_str(cmd) +"\n", True, False) utilities.add_process_to_pool(utilities.launch_cmd, [cmd, stdout_fname_max, stderr_fname_max]) utilities.wait_for_pool_to_finish() r_cmd = [sys.executable] r_cmd.extend(sys.argv) r_cmd[1] = r_cmd[1].replace('runMaxent.py', 'runRModel.py') r_cmd.append('lam='+self.outputdir) self.writetolog(' running command: \n' + utilities.convert_list_to_cmd_str(r_cmd) +"\n", True, False) utilities.add_process_to_pool(utilities.launch_cmd, [r_cmd, stdout_fname_R, stderr_fname_R]) utilities.wait_for_pool_to_finish()