def label_images(outputFolder, frameNum, trainingPath, site, yyyymmdd,
numThreads):
'''Apply the labeling algorithm to a single image, then map project the result.'''
print 'Running label for frame: ' + str(frameNum)
# Get required paths
inputFolder = icebridge_common.getJpegFolder(outputFolder)
cameraFolder = icebridge_common.getCameraFolder(outputFolder)
labelFolder = icebridge_common.getLabelFolder(outputFolder)
orthoFolder = icebridge_common.getLabelOrthoFolder(outputFolder)
# Hardcoded paths!!!
toolPath = 'python ~/repo/OSSP/ossp_process.py'
refDemFolder = '/nobackup/smcmich1/icebridge/reference_dems/'
# Run the label tool
NO_SPLITTING = 1 # Plenty of RAM to load these images
ONE_PROCESS = 1
# Figure out the label path
run = run_helper.RunHelper(site, yyyymmdd)
labelName = icebridge_common.makeLabelFileName(run, frameNum)
labelPath = os.path.join(labelFolder, labelName)
#cmd = ('%s %s --output_dir %s --min_frame %d --max_frame %d srgb %s --splits %d --parallel %d' %
# (toolPath, inputFolder, labelFolder, frameNum, frameNum, trainingPath, NO_SPLITTING, ONE_PROCESS))
cmd = (
'time %s %s --output_dir %s --min_frame %d --max_frame %d srgb %s' %
(toolPath, inputFolder, labelFolder, frameNum, frameNum, trainingPath))
print cmd
if icebridge_common.isValidImage(labelPath):
print 'Skipping completed file: ' + labelPath
else:
os.system(cmd)
# Also generate the map projected version of the image
# Figure out the camera and output path
cameraPath = get_camera(cameraFolder, frameNum)
fname = os.path.basename(labelPath).replace('classified',
'classified_ortho')
mapProjPath = os.path.join(orthoFolder, fname)
if not icebridge_common.isValidImage(labelPath):
print 'ERROR: Failed to generate label file: ' + labelPath
return
# Set map projection parameters
toolPath = 'mapproject'
isSouth = (site == 'AN')
srs = projString = icebridge_common.getEpsgCode(isSouth, asString=True)
demPath = 'WGS84' # Map project on to a flat surface, elevation zero.
#demPath = os.path.join(refDemFolder, icebridge_common.getReferenceDemName(site)) # The NSIDC ortho DEM
# Mapproject
cmd = (
'time %s %s %s %s %s -t nadirpinhole --t_srs %s --threads %d --num-processes 1 --ot Byte --nearest-neighbor'
% (toolPath, demPath, labelPath, cameraPath, mapProjPath, srs,
numThreads))
print cmd
if icebridge_common.isValidImage(mapProjPath):
print 'Skipping existing file: ' + mapProjPath
else:
os.system(cmd)
if not os.path.exists(mapProjPath):
print 'ERROR: Failed to generate map projected label file: ' + mapProjPath
return
def main(argsIn):
try:
usage = '''usage: regenerate_summary_images.py <options> '''
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument(
"--data-folder",
dest="dataFolder",
help="Where all the inputs and outputs are stored.")
parser.add_argument("--work-folder",
dest="workFolder",
help="Where working files are stored.")
parser.add_argument("--site", dest="site", help="Site code.")
parser.add_argument("--yyyymmdd", dest="yyyymmdd", help="Date.")
#parser.add_argument("--dem-tarball", dest="demTarball", default=os.getcwd(),
# help="Where all the inputs and outputs are stored.")
#parser.add_argument("--ortho-tarball", dest="orthoTarball", default=None,
# help="Where to unpack the data.")
#parser.add_argument("--summary-tarball", dest="summaryTarball", default=None,
# help="Where to unpack the data.")
#parser.add_argument("--unpack-dir", dest="unpackDir", default=None,
# help="Where to unpack the data.")
parser.add_argument(
"--node-type",
dest="nodeType",
default='san',
help="Node type to use (wes[mfe], san, ivy, has, bro)")
#parser.add_argument("--skip-archive-summary", action="store_true",
# dest="skipArchiveSummary", default=False,
# help="Skip archiving the summary.")
# Debug option
parser.add_argument(
'--minutes-in-devel-queue',
dest='minutesInDevelQueue',
type=int,
default=0,
help="If positive, submit to the devel queue for this many minutes."
)
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
# Check if we are on the right machine
(host, err, status) = asp_system_utils.executeCommand(['uname', '-n'],
suppressOutput=True)
host = host.strip()
if 'pfe' in host and options.nodeType not in PFE_NODES:
raise Exception("From machine " + host + " can only launch on: " +
" ".join(PFE_NODES))
if 'mfe' in host and options.nodeType != 'wes':
raise Exception("From machine " + host + " can only launch on: wes")
# Make sure our paths will work when called from PBS
options.dataFolder = os.path.abspath(options.dataFolder)
#os.system('mkdir -p ' + options.unpackDir)
# TODO: Check folders!
run = run_helper.RunHelper(options.site, options.yyyymmdd,
options.workFolder)
runFolder = os.path.join(options.workFolder, str(run))
os.system('mkdir -p ' + runFolder)
logFolder = os.path.join(runFolder, 'logs')
# Set up logging in the run directory
os.system('mkdir -p ' + logFolder)
logLevel = logging.INFO
logger = icebridge_common.setUpLogger(
logFolder, logLevel, icebridge_common.manager_log_prefix())
logger.info("Logging in: " + logFolder)
checkRequiredTools(
) # Make sure all the needed tools can be found before we start
logger.info("Disabling core dumps.") # these just take a lot of room
os.system("ulimit -c 0")
os.system("umask 022") # enforce files be readable by others
# See how many hours we used so far. I think this counter gets updated once a day.
(out, err, status) = asp_system_utils.executeCommand("acct_ytd",
outputPath=None,
suppressOutput=True,
redo=True,
noThrow=True)
logger.info("Hours used so far:\n" + out + '\n' + err)
try:
# Fetch and extract the tarball files from Lou
localDemFolder = os.path.join(options.dataFolder, run.name() + '_dems')
localOrthoFolder = os.path.join(options.dataFolder,
run.name() + '_orthos')
demSummaryFolder = os.path.join(options.dataFolder,
run.name() + '_dem_summaries')
orthoSummaryFolder = os.path.join(options.dataFolder,
run.name() + '_ortho_summaries')
missingDemFiles = []
missingOrthoFiles = []
for f in os.listdir(localDemFolder):
if 'temp' in f:
raise Exception('Bad file: ' + f)
if ('IODEM3' in f) and (f[-4:] == '.tif'):
inputPath = os.path.join(localDemFolder, f)
outputPath = os.path.join(
demSummaryFolder, f.replace('DEM.tif', 'DEM_browse.tif'))
if not os.path.exists(outputPath):
missingDemFiles.append((inputPath, outputPath))
for f in os.listdir(localOrthoFolder):
if 'temp' in f:
raise Exception('Bad file: ' + f)
if ('IODIM3' in f) and (f[-4:] == '.tif'):
inputPath = os.path.join(localOrthoFolder, f)
outputPath = os.path.join(orthoSummaryFolder,
f.replace('ORTHO.tif', 'ORTHO.jpg'))
if not os.path.exists(outputPath):
missingOrthoFiles.append((inputPath, outputPath))
#print 'Fetching and unpacking tarballs...'
#fetchTarball(options.demTarball, localDemFolder)
#fetchTarball(options.orthoTarball, localOrthoFolder)
#fetchTarball(options.summaryTarball, localSummaryFolder)
# If the summary tarball unpacked to []/summary/summary,
# work with the lower level folder from now on.
#localSummaryFolder = descendIfNeeded(localSummaryFolder)
# Make a list of all input files that are missing their summary file, and
# the desired output path for that file.
#missingDemFiles = getMissingSummaryFiles(localDemFolder, localSummaryFolder, isOrtho=False)
#missingOrthoFiles = getMissingSummaryFiles(localOrthoFolder, localSummaryFolder, isOrtho=True )
# Divide this list into chunks and for each chunk generate a file containing all of
# the gdal_translate commands that need to be executed.
print 'Writing command files...'
commandFileLength = getParallelParams(options.nodeType)[2]
commandFilePrefix = os.path.join(runFolder, 'convert_commands_')
print 'Clearing existing command files.'
os.system('rm ' + commandFilePrefix + '*')
commandFileList = writeCommandFiles(missingDemFiles, missingOrthoFiles,
commandFilePrefix,
commandFileLength)
#raise Exception('DEBUG')
# Get the location to store the logs
pbsLogFolder = run.getPbsLogFolder()
logger.info("Storing logs in: " + pbsLogFolder)
os.system('mkdir -p ' + pbsLogFolder)
# Call multi_process_command_runner.py through PBS for each chunk.
start_time()
(baseName, jobIDs) = submitBatchJobs(commandFileList, options,
pbsLogFolder, run, logger)
# Wait for everything to finish.
pbs_functions.waitForJobCompletion(jobIDs, logger, baseName)
stop_time("pbs_jobs", logger)
# Check that we now have all of the summary files.
# - Both of these should now be empty.
#newMissingDemFiles = getMissingSummaryFiles(localDemFolder, demSummaryFolder, isOrtho=False)
#newMissingOrthoFiles = getMissingSummaryFiles(localOrthoFolder, orthoSummaryFolder, isOrtho=True )
numDemsMissing = 0
numOrthosMissing = 0
for pair in missingDemFiles:
if not os.path.exists(pair[1]):
numDemsMissing += 1
for pair in missingOrthoFiles:
if not os.path.exists(pair[1]):
numOrthosMissing += 1
resultText = (
'After regeneration, missing %d DEM summaries and %d ORTHO summaries'
% (numDemsMissing, numOrthosMissing))
logger.info(resultText)
runWasSuccess = ((numDemsMissing == 0) and (numOrthosMissing == 0))
# If successful, create a new tarball and send it to Lou.
#if runWasSuccess and (not options.skipArchiveSummary):
# start_time()
# archive_functions.packAndSendSummaryFolder(run, localSummaryFolder, logger)
# stop_time("archive summary", logger)
except Exception as e:
resultText = 'Caught exception: ' + str(
e) + '\n' + traceback.format_exc()
runWasSuccess = False
# Send a summary email.
emailAddress = getEmailAddress(icebridge_common.getUser())
logger.info("Sending email to: " + emailAddress)
if runWasSuccess:
sendEmail(emailAddress, 'OIB summary regen passed', resultText)
else:
sendEmail(emailAddress, '"OIB summary regen failed', resultText)
# TODO: Add automated delete command!
#if options.wipeProcessed:
# processedFolder = run.getProcessFolder()
# logger.info("Will delete: " + processedFolder)
# os.system("rm -rf " + processedFolder)
logger.info(
'==== regenerate_summary_images script has finished for run: ' +
str(run) + ' ====')
options.yyyymmdd = m.group(2)
else:
options.site_yyyymmdd = options.site + "_" + options.yyyymmdd
# Read the done file and exit if the current flight is done
done = set()
if options.doneFile != "":
with open(options.doneFile, 'r') as f:
for val in f:
val = val.strip()
done.add(val)
if options.site_yyyymmdd in done:
print("Skipping done flight: " + options.site_yyyymmdd)
return 0
run = run_helper.RunHelper(options.site, options.yyyymmdd, os.getcwd())
# Set up logging in the run directory. Log outside of the run dir,
# as that one we will wipe
logFolder = os.path.abspath(
os.path.join(run.getFolder(), '..', 'push_logs'))
os.system('mkdir -p ' + logFolder)
logLevel = logging.INFO
logger = icebridge_common.setUpLogger(logFolder, logLevel, "push")
logger.info("Logging in: " + logFolder)
# Check the lftp version. On some machines it is too old.
(out, err, status) = asp_system_utils.executeCommand(['lftp', '--version'],
suppressOutput=True)
m = re.match('^.*?LFTP\s+\|\s+Version\s+4.5', out)
if not m:
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument('--stop-frame',
dest='stopFrame',
type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on.')
options = parser.parse_args(argsIn)
except argparse.ArgumentError, msg:
parser.error(msg)
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(
options.site, options.yyyymmdd)
run = run_helper.RunHelper(options.site, options.yyyymmdd,
options.parentFolder)
generateFlightSummary(run, options)
return 0
# Run main function if file used from shell
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
def main(argsIn):
try:
usage = '''usage: push_to_nsidc.py <options> '''
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument(
"--yyyymmdd",
dest="yyyymmdd",
default="",
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument(
"--site",
dest="site",
default="",
help="Name of the location of the images (AN, GR, or AL)")
parser.add_argument(
"--site_yyyymmdd",
dest="site_yyyymmdd",
default="",
help=
"A value like GR_20150330, which will be split into site and yyyymmdd by this script."
)
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument('--stop-frame',
dest='stopFrame',
type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on.')
parser.add_argument("--camera-calibration-folder",
dest="inputCalFolder",
default=None,
help="The folder containing camera calibration.")
parser.add_argument(
"--reference-dem-folder",
dest="refDemFolder",
default=None,
help="The folder containing DEMs that created orthoimages.")
parser.add_argument("--login-info",
dest="loginInfo",
default=None,
help="user,password destination.nsidc.org.")
parser.add_argument("--done-file",
dest="doneFile",
default=None,
help="List of runs that were done by now.")
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
# parse --site_yyyymmdd. Sometimes it is easier to pass this than
# to pass separately --site and --yyyymmdd.
m = re.match('^(\w+)_(\w+)', options.site_yyyymmdd)
if m:
options.site = m.group(1)
options.yyyymmdd = m.group(2)
else:
options.site_yyyymmdd = options.site + "_" + options.yyyymmdd
# Read the done file and exit if the current flight is done
done = set()
if options.doneFile != "":
with open(options.doneFile, 'r') as f:
for val in f:
val = val.strip()
done.add(val)
if options.site_yyyymmdd in done:
print("Skipping done flight: " + options.site_yyyymmdd)
return 0
run = run_helper.RunHelper(options.site, options.yyyymmdd, os.getcwd())
# Set up logging in the run directory. Log outside of the run dir,
# as that one we will wipe
logFolder = os.path.abspath(
os.path.join(run.getFolder(), '..', 'push_logs'))
os.system('mkdir -p ' + logFolder)
logLevel = logging.INFO
logger = icebridge_common.setUpLogger(logFolder, logLevel, "push")
logger.info("Logging in: " + logFolder)
# Check the lftp version. On some machines it is too old.
(out, err, status) = asp_system_utils.executeCommand(['lftp', '--version'],
suppressOutput=True)
m = re.match('^.*?LFTP\s+\|\s+Version\s+4.5', out)
if not m:
raise Exception('Expecting LFTP version 4.5.')
else:
logger.info("Found an acceptable version of LFTP.")
pushByType(run, options, logger, 'DEM')
#pushByType(run, options, logger, 'ORTHO') # need to wait for format decision
# Wipe at the end
cmd = "rm -rf " + run.getFolder()
logger.info(cmd)
os.system(cmd)
def main(argsIn):
'''Parse arguments and call the processing function'''
try:
# Sample usage:
# python generate_flight_summary.py --yyyymmdd 20091016 --site AN
usage = '''generate_flight_summary.py <options>'''
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument(
"--yyyymmdd",
dest="yyyymmdd",
required=True,
help="Specify the year, month, and day in one YYYYMMDD string.")
parser.add_argument(
"--site",
dest="site",
required=True,
help="Name of the location of the images (AN, GR, or AL).")
parser.add_argument("--output-folder", dest="outputFolder", default=None,
help="Name of the output folder. If not specified, " + \
"use something like AN_YYYYMMDD.")
parser.add_argument("--parent-folder",
dest="parentFolder",
default=os.getcwd(),
help="The folder having all the runs.")
parser.add_argument("--skip-kml-gen",
action="store_true",
dest="skipKml",
default=False,
help="Skip combining kml files.")
parser.add_argument(
"--skip-geo-center",
action="store_true",
dest="skipGeo",
default=False,
help="Skip computing the center of the tile, which is slow.")
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument('--stop-frame',
dest='stopFrame',
type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on.')
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
if options.outputFolder is None:
options.outputFolder = icebridge_common.outputFolder(
options.site, options.yyyymmdd)
run = run_helper.RunHelper(options.site, options.yyyymmdd,
options.parentFolder)
generateFlightSummary(run, options)
return 0