def handleCrossValidations(self):
# Start marian adding junk to the code
MDSreader = csv.reader(open(self.args_dict['c'], 'r'))
header1 = MDSreader.next()
header2 = MDSreader.next()
header3 = MDSreader.next()
if 'Split' in header1:
newLine = MDSreader.next()
ttList = ["test", "train"]
cvList = ["NA"]
# find the first line of the mds that isn't na to determine if test/train or cv split
while newLine[header1.index("Split")] == "NA":
newLine = MDSreader.next()
# loop through the mds and fit a maxent model withholding each new cv fold
if (not newLine[header1.index("Split")] in ttList):
subrun_args = copy.deepcopy(self.args_dict)
subrun_args["sub_run"] = 'True'
# cvMaxent = copy.deepcopy(self)
# cvMaxent.sub_run=True
for row in MDSreader:
if (not row[header1.index("Split")] in cvList):
subrun_args["test_key"] = row[header1.index("Split")]
outdir = os.path.join(self.outputdir,
"cvSplit" + row[header1.index("Split")])
subrun_args["o"] = outdir
stdout_fname = os.path.join(outdir, "ExpandedOutput", "stdOut.txt")
stderr_fname = os.path.join(outdir, "ExpandedOutput", "stdErr.txt")
if not os.path.exists(outdir):
os.makedirs(outdir)
if not os.path.exists(os.path.join(outdir, "ExpandedOutput")):
os.makedirs(os.path.join(outdir, "ExpandedOutput"))
cvList.append(row[header1.index("Split")])
try:
cmd = [sys.executable]
cmd += sys.argv[:sys.argv.index("--args") + 1]
for k, v in subrun_args.iteritems():
if v in [True, False]:
cmd.append(k + "=" + str(v).upper())
else:
cmd.append(k + "=" + str(v))
cmd.append("maxent_args=" + str(self.maxent_args))
utilities.add_process_to_pool(utilities.launch_cmd,
[cmd, stdout_fname, stderr_fname])
except utilities.TrappedError as e:
raise RuntimeError(self, e.message)
# here we need to run Maxent without the test split csv which breaks it
self.test_key = None
results = utilities.wait_for_pool_to_finish()
for result in results:
if "Error" in result[0]:
print "!!!!!!!!!!!!!!!!!test!!!!!!!!!!!!!!!!!!!!!!"
raise RuntimeError(self, result[0])
def convertEnvironmentalLayers_mc(self, files, outputFolder, type):
'''This function has the same functionality as convertEnvironmentalLayers
with the addition of utilizing multiple cores to do the processing.
'''
results = Queue.Queue()
process_count = 0
i = 1
for f in files:
f_name = os.path.splitext(os.path.split(f)[1])[0]
# args = '-i ' + '"' + f + '"'
# args += ' -o ' + '"' + outputFolder + '"'
# args += ' -f ' + type
args = ['-i', f, '-o', outputFolder, '-f', type]
# if self.verbose:
# args += " -v"
execDir = os.path.split(__file__)[0]
executable = os.path.join(execDir,
'singleRasterFormatConverter.py')
command_arr = [sys.executable, executable] + args
self.logger.writetolog(' '.join(command_arr), False, False)
# proc = subprocess.Popen( command_arr )
# thread.start_new_thread(utilities.process_waiter,
# (proc, f_name, results))
utilities.add_process_to_pool(utilities.launch_cmd, [command_arr])
utilities.wait_for_pool_to_finish()
def run(self):
if self.sub_run:
self.maxent_args['plots'] = 'false'
self.maxent_args['responsecurves'] = 'false'
self.maxent_args['outputgrids'] = 'false'
self.validateInputs()
self.start_pool()
if self.args_dict['c'] != '':
self.prepInputs()
else:
raise Exception, 'No MDS supplied.'
if not self.maxent_args.has_key('projectionlayers'):
self.maxent_args['projectionlayers'] = ''
if self.trainingCSV != '':
self.maxent_args['samplesfile'] = self.trainingCSV
else:
raise Exception, 'No Samples file supplied'
if self.testCSV != '':
self.maxent_args['testsamplesfile'] = self.testCSV
if self.maxent_args['environmentallayers'] != '':
self.maxent_args['environmentallayers'] = self.maxent_args['environmentallayers']
if self.maxent_args.has_key('biasfile'):
self.maxent_args['biasfile'] = self.maxent_args['biasfile']
elif self.backgroundCSV != '':
self.maxent_args['environmentallayers'] = self.backgroundCSV
if self.maxent_args.has_key('biasfile'):
self.writetolog('The supplied biasfile will be ignored since no environmentallayers were provided', True, True)
self.maxent_args.pop('biasfile')
self.maxent_args['autorun'] = 'true'
if ' ' in self.args_dict['species_name']:
self.args_dict['species_name'] = self.args_dict['species_name'].replace(' ', '_')
if(not self.test_key): # if there is no testsplit (ie cross validation, remove the empty csv
del self.maxent_args['testsamplesfile']
if not self.maxent_path.endswith('.jar'):
self.maxent_path = os.path.join(self.maxent_path, 'maxent.jar')
cmd = self.gen_maxent_cmd()
stdout_fname = os.path.join(self.outputdir, "ExpandedOutput", "stdOut.txt")
stderr_fname = os.path.join(self.outputdir, "ExpandedOutput", "stdErr.txt")
if not os.path.exists(os.path.join(self.outputdir, "ExpandedOutput")):
os.makedirs(os.path.join(self.outputdir, "ExpandedOutput"))
self.writetolog(' running command: \n' +
utilities.convert_list_to_cmd_str(cmd) + "\n", True, False)
if os.path.exists(self.maxent_path):
utilities.add_process_to_pool(utilities.launch_cmd,
[cmd, stdout_fname, stderr_fname])
utilities.wait_for_pool_to_finish()
if not self.sub_run:
r_cmd = [sys.executable]
r_cmd.extend(sys.argv)
r_cmd[1] = r_cmd[1].replace('runMaxent.py', 'runRModel.py')
r_cmd.append('lam=' + self.outputdir)
self.writetolog(' running command: \n' +
utilities.convert_list_to_cmd_str(r_cmd) + "\n", True, False)
utilities.add_process_to_pool(utilities.launch_cmd,
[r_cmd, stdout_fname, stderr_fname])
utilities.wait_for_pool_to_finish()
else:
msg = 'The specified Maxent.Jar file ({})\n\t\t'.format(self.maxent_path)
msg += "\nNoes not appear to exist on the file system!"
self.writetolog(msg, True, True)
raise RuntimeError(msg)
def run(self):
if self.sub_run:
self.maxent_args['plots'] = 'false'
self.maxent_args['responsecurves'] = 'false'
self.maxent_args['outputgrids'] = 'false'
self.validateInputs()
self.start_pool()
if self.args_dict['c'] != '':
self.prepInputs()
else:
raise Exception, 'No MDS supplied.'
if not self.maxent_args.has_key('projectionlayers'):
self.maxent_args['projectionlayers'] = ''
if self.trainingCSV != '':
self.maxent_args['samplesfile'] = self.trainingCSV
else:
raise Exception, 'No Samples file supplied'
if self.testCSV != '':
self.maxent_args['testsamplesfile'] = self.testCSV
if self.maxent_args['environmentallayers'] != '':
self.maxent_args['environmentallayers'] = self.maxent_args['environmentallayers']
if self.maxent_args.has_key('biasfile'):
self.maxent_args['biasfile'] = self.maxent_args['biasfile']
elif self.backgroundCSV != '':
self.maxent_args['environmentallayers'] = self.backgroundCSV
if self.maxent_args.has_key('biasfile'):
self.writetolog('The supplied biasfile will be ignored since no environmentallayers were provided', True, True)
self.maxent_args.pop('biasfile')
self.maxent_args['autorun'] = 'true'
if ' ' in self.args_dict['species_name']:
self.args_dict['species_name'] = self.args_dict['species_name'].replace(' ', '_')
if(not self.test_key): #if there is no testsplit (ie cross validation, remove the empty csv
del self.maxent_args['testsamplesfile']
if self.categoricals:
catstr = ",".join([cat.replace('_categorical', '')
for cat in self.categoricals])
self.maxent_args['togglelayertype'] = catstr
if not self.maxent_path.endswith('.jar'):
self.maxent_path = os.path.join(self.maxent_path, 'maxent.jar')
cmd = self.gen_maxent_cmd()
stdout_fname_max = os.path.join(self.outputdir, "stdOut_max.txt")
stderr_fname_max = os.path.join(self.outputdir, "stdErr_max.txt")
stdout_fname_R = os.path.join(self.outputdir, "stdOut_R.txt")
stderr_fname_R = os.path.join(self.outputdir, "stdErr_R.txt")
self.writetolog(' running command: \n' +
utilities.convert_list_to_cmd_str(cmd) +"\n", True, False)
utilities.add_process_to_pool(utilities.launch_cmd,
[cmd, stdout_fname_max, stderr_fname_max])
utilities.wait_for_pool_to_finish()
r_cmd = [sys.executable]
r_cmd.extend(sys.argv)
r_cmd[1] = r_cmd[1].replace('runMaxent.py', 'runRModel.py')
r_cmd.append('lam='+self.outputdir)
self.writetolog(' running command: \n' +
utilities.convert_list_to_cmd_str(r_cmd) +"\n", True, False)
utilities.add_process_to_pool(utilities.launch_cmd,
[r_cmd, stdout_fname_R, stderr_fname_R])
utilities.wait_for_pool_to_finish()