def processFile(args):
# File of annotations
annsFilename = args.get('annotations')
# Directory of segmentations
dirname = args.get('input')
rawAnnsByModel = loadAllAnnotations(annsFilename)
annsByModel = convertAnnotations(rawAnnsByModel)
allStats = []
# Compute percentage of annotations
for root, dirs, files in os.walk(dirname):
for name in dirs:
dir = os.path.join(root, name)
segsFile = os.path.join(dir, SEGS_FILE.replace("${name}", name))
if util.is_non_zero_file(segsFile):
try:
anns = loadSegmentsAndCombineAnnotations(
segsFile, annsByModel.get(name))
stats = computeStatistics(anns)
entry = {'id': name}
entry.update(stats)
allStats.append(entry)
# print json.dumps(entry)
except:
log.error('Invalid segmentation file for ' + name)
traceback.print_exc()
else:
log.warn('No segmentation file for ' + name)
if len(allStats) > 0:
if args.get('output'):
with open(args.get('output'), 'wb') as outfile:
saveOutput(args.get('format'), allStats, outfile)
else:
saveOutput(args.get('format'), allStats, sys.stdout)
sys.stdout.flush()
def processDir(args):
dirname = args.get('input')
allLabels = collections.Counter()
allCategories = collections.Counter()
allStats = []
for root, dirs, files in os.walk(dirname):
for name in dirs:
dir = os.path.join(root, name)
segsFile = os.path.join(dir, SEGS_FILE.replace("${name}", name))
annsFile = os.path.join(dir, ANNS_FILE.replace("${name}", name))
# log.info('segsFile=%s,annsFile=%s', segsFile, annsFile)
if util.is_non_zero_file(segsFile):
anns = loadAnnotations(segsFile, annsFile)
stats = computeStatistics(anns, allLabels, allCategories)
entry = {'id': name}
entry.update(stats)
allStats.append(entry)
#print json.dumps(entry)
labelsFile = args.get('labels')
if labelsFile is not None:
with open(labelsFile, 'wb') as labelsOut:
saveCounts(allLabels, labelsOut)
categoriesFile = args.get('categories')
if categoriesFile is not None:
with open(categoriesFile, 'wb') as categoriesOut:
saveCounts(allCategories, categoriesOut)
if len(allStats) > 0:
if args.get('output'):
with open(args.get('output'), 'wb') as outfile:
saveOutput(args.get('format'), allStats, outfile)
else:
saveOutput(args.get('format'), allStats, sys.stdout)
sys.stdout.flush()
def has_scan(dirname):
dirname = strip_dirname(dirname)
id = os.path.basename(dirname)
# If any of these files exists, this directory is likely to be a scan
files = [
id + '.imu', id + '.depth', id + '.h264', id + '.sens', id + '.ply'
]
for file in files:
if util.is_non_zero_file(os.path.join(dirname, file)):
return True
return False
def isMultiBackup(self, listfile, default):
''' Code for checking if you want to take single backup or assigned multiple backups in list file.
'''
multi = False
print "checking for backup names file."
if is_non_zero_file(listfile):
multi = True
print "List file found..."
print "Starting backup listed in " + listfile
else:
multi = False
print "List file not found..."
print "Starting backup of " + default
return multi
def loadAnnotations(segmentsFilename, annotationFilename):
with open(segmentsFilename) as segmentsFile:
segments = json.load(segmentsFile)
segmentGroups = {'segGroups': []}
if util.is_non_zero_file(annotationFilename):
try:
with open(annotationFilename) as annotationFile:
segmentGroups = json.load(annotationFile)
except:
log.error('Error loading annotation file ' + annotationFilename)
traceback.print_exc()
# merge the two
merged = segments.copy()
merged.update(segmentGroups)
return merged
def indexAndSave(args, loadFn, saveFn):
if args.get('output'):
print('Indexing ' + args.get('input') + ' to ' + args.get('output'))
# index into memory
rows = {}
if args.get('single'):
subdir = None
if args.get('root'):
subdir = os.path.relpath(args.get('input'), args.get('root'))
index_single(args.get('input'), subdir,
lambda r: assignItem(rows, r['id'], r), args)
elif args.get('recursive'):
index_all_recursive(args.get('input'),
lambda r: assignItem(rows, r['id'], r), args)
else:
index_all(args.get('input'), lambda r: assignItem(rows, r['id'], r),
args)
# output
allRows = rows
if args.get('output'):
# read existing if there
isNonEmpty = util.is_non_zero_file(args.get('output'))
if args.get('append') and isNonEmpty:
with open(args.get('output')) as data:
existing = loadFn(data)
# combine old with new
allRows = existing.copy()
allRows.update(rows)
output = open(args.get('output'), 'wb')
else:
output = sys.stdout
# dump output
saveFn(allRows, output)
output.flush()
if args.get('output'):
output.close()
return rows
def extract_meta(dirname, subdir, args):
source = args.get('source')
datasets = args.get('datasets')
# Make sure dirname don't end in '/', otherwise our poor basename is not going to work well
dirname = strip_dirname(dirname)
# Extract metadata from csv and create a record
# First check if okay by looking into processed.txt
processedFile = dirname + '/processed.txt'
processed = None
if util.is_non_zero_file(processedFile):
processed = util.read_properties(processedFile, log)
if not processed:
if not args.get('includeAll'):
return False # NOT ACCEPTABLE
else:
if not args.get('includeAll'):
return False # NOT ACCEPTABLE
# Process log
times = None
processLog = dirname + '/process.log'
if os.path.isfile(processLog):
times = timings.computeTimings(processLog)
# Take dirname and extract the final string as the id
id = os.path.basename(dirname)
if subdir is None:
subdir = id
meta1 = {
'fullId': source + '.' + id,
'id': id,
'source': source,
'datasets': datasets,
'path': subdir
}
if times is not None:
total = timings.getTotal(times)
meta1['totalProcessingSecs'] = total.get('secs')
meta1['totalProcessingTime'] = total.get('time')
# Extract create time from id matching: 2016-07-01_04-29-28
date_match = DATE_RE.search(id)
if date_match:
createdAt = date_match.group(0)
# Reformat to be in ISO8601 format
meta1['createdAt'] = createdAt[0:10] + 'T' + createdAt[
11:13] + ':' + createdAt[14:16] + ':' + createdAt[17:19]
# Look for dirname/id.txt and extract fields into our meta
# if there is txt file, read it and append to metadata record
metafile = dirname + '/' + id + '.txt'
if util.is_non_zero_file(metafile):
# Read in txt as dictionary
meta = util.read_properties(metafile, log)
else:
meta = {}
# go through files and find last time as updatedAt time for scan
# Take our basic info and merge it into meta (overwriting what maybe there)
if processed:
meta.update(processed)
meta.update(meta1)
# Check what files we have
meta['files'] = util.list_files(dirname)
lastModified = util.lastModified(meta['files'])
if lastModified:
meta['updatedAt'] = util.millisToIso(
lastModified.get('modifiedAtMillis'))
# Check what stage we are in
# NOTE: This requires meta to be filled in with id and files!!!
if args.get('stages'):
check_stages(args.get('stages'), meta, times)
# Check if we have a ply file and how big it is
filebase = id + '_vh_clean_2' if args.get('checkCleaned') else id
plyfile = dirname + '/' + filebase + '.ply'
plyfileSize = util.filesize(plyfile)
meta['hasCleaned'] = args.get('checkCleaned') and plyfileSize > 0
if plyfileSize > 0:
meta['fileType'] = 'ply'
meta['fileSize'] = plyfileSize
# Check if we have a png file
pngfile = dirname + '/' + filebase + '.png'
pngfileSize = util.filesize(pngfile)
meta['hasScreenshot'] = pngfileSize > 0
# Check if we have a thumbnail file
pngfile = dirname + '/' + filebase + '_thumb.png'
pngfileSize = util.filesize(pngfile)
meta['hasThumbnail'] = pngfileSize > 0
if source == 'nyuv2' and not meta.get('sceneType'):
idParts = meta.get('id').split('_')
meta['sceneType'] = '_'.join(idParts[0:len(idParts - 1)])
if meta.get('sceneLabel'):
# Derive sceneName from sceneLabel
sceneLabel = meta.get('sceneLabel')
match = SCENELABEL_RE.match(sceneLabel)
meta['sceneName'] = match.group(1) if match else sceneLabel
return meta