def extract_features(mode, datasets):
for dataset in datasets:
input = get_sts_input_path(mode, dataset)
print dataset
# print "Generating test files..."
sentence_pairs = load_data(input)
logging.info("Computing features")
doc2features = []
for i, sp in enumerate(sentence_pairs, 1):
if i % 10 == 0:
sys.stdout.write("%d.." % i)
features = compute_features(*sp)
doc2features.append(features)
sys.stdout.write("%d\n" % i)
outdir = os.path.join(FEATURE_PATH, mode, dataset, FEATURE_SET)
make_dirs(outdir)
logging.info("Writing features to: %s" % outdir)
for idx in xrange(len(features)):
outfile = os.path.join(outdir, "%s.txt" % idx)
with open(outfile, 'w') as out:
for doc_features in doc2features:
out.write("%f\n" % doc_features[idx])
def extract_features(mode, datasets):
for dataset in datasets:
input = get_sts_input_path(mode, dataset)
print dataset
# print "Generating test files..."
sentence_pairs = load_data(input)
logging.info("Computing features")
doc2features = []
for i, sp in enumerate(sentence_pairs, 1):
if i % 10 == 0:
sys.stdout.write("%d.." % i)
features = compute_features(*sp)
doc2features.append(features)
sys.stdout.write("%d\n" % i)
outdir = os.path.join(FEATURE_PATH, mode, dataset, FEATURE_SET)
make_dirs(outdir)
logging.info("Writing features to: %s" % outdir)
for idx in xrange(len(features)):
outfile = os.path.join(outdir, "%s.txt" % idx)
with open(outfile, 'w') as out:
for doc_features in doc2features:
out.write("%f\n" % doc_features[idx])
def write_svmfile(mode, datasets):
print "Generating files in %s mode..." % mode
examples = []
outdir = os.path.join(MODEL_PATH, MODEL, mode)
make_dirs(outdir)
for dataset in datasets:
outfile = "%s.svmtk" % dataset
outpath = os.path.join(outdir, outfile)
print outpath
with open(outpath, "w") as out:
ex_generator = ExampleWriter(SVM_FORMAT)
for ex in ex_generator.to_svm(mode, dataset):
out.write(ex)
examples.append(ex)
# if mode == TRAIN:
random.seed(123)
random.shuffle(examples)
outfile = "ALL.svmtk"
outpath = os.path.join(outdir, outfile)
print outpath
with open(outpath, "w") as out:
for ex in examples:
out.write(ex)
def create_cv_folds(self, dir="ova.cv", nfolds=5):
make_dirs(self.ova_path)
# make cv folds for examples from each of the categories
folds = [make_cv_folds(examples, nfolds) for _, examples in self.cat2ex.iteritems()]
# strip (train, test) pairs from each of the folds
for i, fold_data in enumerate(zip(*folds), 1):
print "Fold", i
fold_path = os.path.join(self.ova_path, "fold-{}".format(i))
make_dirs(fold_path)
train_examples = []
test_examples = []
for (train_fold, test_fold) in fold_data:
train_examples.extend(train_fold)
test_examples.extend(test_fold)
labels_fname = os.path.join(fold_path, "labels.test")
with open(labels_fname, "w") as out:
for label, _ in test_examples:
out.write("{}\n".format(label))
for cat in self.categories:
# training
train_fname = os.path.join(fold_path, "{}.train".format(cat))
self.write_ova_examples(train_examples, cat, train_fname)
# testing
test_fname = os.path.join(fold_path, "{}.test".format(cat))
self.write_ova_examples(test_examples, cat, test_fname)
def write_svmfile(mode, datasets):
print "Generating files in %s mode..." % mode
examples = []
outdir = os.path.join(MODEL_PATH, MODEL, mode)
make_dirs(outdir)
for dataset in datasets:
outfile = "%s.svmtk" % dataset
outpath = os.path.join(outdir, outfile)
print outpath
with open(outpath, "w") as out:
ex_generator = ExampleWriter(SVM_FORMAT)
for ex in ex_generator.to_svm(mode, dataset):
out.write(ex)
examples.append(ex)
# if mode == TRAIN:
random.seed(123)
random.shuffle(examples)
outfile = "ALL.svmtk"
outpath = os.path.join(outdir, outfile)
print outpath
with open(outpath, "w") as out:
for ex in examples:
out.write(ex)
def main():
depth_paths, T, pose_paths = getData(args.shapeid)
n = len(depth_paths)
print('found %d clean depth images...' % n)
intrinsic = np.array([[525.0,0,319.5],[0,525.0,239.5],[0,0,1]])
np.random.seed(816)
indices = np.random.permutation(n)
print(indices[:100])
#indices = sorted(indices)
make_dirs(PATH_MAT.format(args.shapeid, 0))
import open3d
pcd_combined = open3d.PointCloud()
for i, idx in enumerate(indices):
import ipdb; ipdb.set_trace()
print('%d / %d' % (i, len(indices)))
mesh = Mesh.read(depth_paths[idx], mode='depth', intrinsic = intrinsic)
pcd = open3d.PointCloud()
pcd.points = open3d.Vector3dVector(mesh.vertex.T)
pcd.transform(inverse(T[idx]))
#pcd = open3d.voxel_down_sample(pcd, voxel_size=0.02)
pcd_combined += pcd
pcd_combined = open3d.voxel_down_sample(pcd_combined, voxel_size=0.02)
sio.savemat(PATH_MAT.format(args.shapeid, i), mdict={
'vertex': mesh.vertex,
'validIdx_rowmajor': mesh.validIdx,
'pose': T[idx],
'depth_path': depth_paths[idx],
'pose_path': pose_paths[idx]})
if i <= 50 and i >= 40:
pcd_combined_down = open3d.voxel_down_sample(pcd_combined, voxel_size=0.02)
open3d.draw_geometries([pcd_combined_down])
pcd_combined_down = open3d.voxel_down_sample(pcd_combined, voxel_size=0.02)
open3d.draw_geometries([pcd_combined_down])
def get_fullname(self):
timestamp = datetime.utcnow()
dirname = path.join(self.directory, self.name,
timestamp.strftime('%Y-%m-%d-%H-00'))
filename = util.set_filename(self.name, timestamp)
util.make_dirs(dirname)
return path.join(dirname, filename)
def run(self):
print("{name} thread running.".format(name = self.name))
util.make_dirs(["./ir_videos"])
while not self._stop:
self.eventWait.wait()
self.event = self.events[0]
del self.events[0]
self.run_event()
if not len(self.events):
self.eventWait.clear()
print("'{name}' stopped.".format(name = self.name))
def process_output(products, producers, outputs, workdir, tile_id,
output_path):
"""Combine the Landsat mosaics into the .tar files."""
status = 0 # Initialize the return status.
util.make_dirs(output_path)
for product_request in sorted(products, reverse=True):
archive = tile_id + '_' + product_request + '.tar'
logging.info('Create product %s', archive)
required = producers['package'][product_request]
included = [
o for o in outputs.values()
if any([r in o for r in required]) and isinstance(o, str)
]
included.append(outputs['XML'][product_request])
local_archive = os.path.join(workdir, archive)
output_archive = os.path.join(output_path, archive)
output_xml = os.path.join(
output_path, os.path.basename(outputs['XML'][product_request]))
util.tar_archive(local_archive, included)
# Copy tarball to output directory.
shutil.copyfile(local_archive, output_archive)
shutil.copyfile(outputs['XML'][product_request], output_xml)
# Create checksums for the output tarballs, and compare with the files
# in the working directory.
if util.process_checksums(tile_id + '*.tar', workdir,
output_path) == 'ERROR':
logger.error('Checksum processing failed.')
# Clean up output files upon failure.
for fullname in glob.glob(os.path.join(output_path, tile_id + '*')):
os.remove(fullname)
status = 1
# Remove local copies of product files.
for fullname in glob.glob(os.path.join(workdir, tile_id + '*.tar')):
os.remove(fullname)
if status == 0:
logger.info(' End product transfer')
else:
logger.info(' Product transfer failed.')
return status
def main():
depth_paths, T, pose_paths = getData(args.shapeid)
n = len(depth_paths)
print('found %d clean depth images...' % n)
intrinsic = parse_info('%s/dataset/scannet/%s/_info.txt' %
(data_path, args.shapeid))
np.random.seed(816)
num_sample = 100
#if n < 5*num_sample:
# stepsize = n // num_sample
#else:
# stepsize = 5
#if stepsize == 0:
# assert False
#indices = [i for i in range(0, n, stepsize)][:num_sample] #np.random.permutation(n)
frame_ids = [
int(depth_paths[i].split('/')[-1].split('.')[0].split('-')[-1])
for i in range(n)
]
indices = pick_frames(frame_ids, max_gap=6, num_sample=num_sample)
if indices is None:
print('not enough valid frames')
exit(0)
print([frame_ids[idx] for idx in indices])
#print(indices[:100])
#indices = sorted(indices)
make_dirs(PATH_MAT.format(args.shapeid, 0))
#import open3d
#pcd_combined = open3d.PointCloud()
for i, idx in enumerate(indices):
#print('%d / %d' % (i, len(indices)))
mesh = Mesh.read(depth_paths[idx], mode='depth', intrinsic=intrinsic)
#pcd = open3d.PointCloud()
#pcd.points = open3d.Vector3dVector(mesh.vertex.T)
#pcd.transform(inverse(T[idx]))
##pcd = open3d.voxel_down_sample(pcd, voxel_size=0.02)
#pcd_combined += pcd
#pcd_combined = open3d.voxel_down_sample(pcd_combined, voxel_size=0.02)
sio.savemat(PATH_MAT.format(args.shapeid, i),
mdict={
'vertex': mesh.vertex,
'validIdx_rowmajor': mesh.validIdx,
'pose': T[idx],
'depth_path': depth_paths[idx],
'pose_path': pose_paths[idx]
})
def train(self, config_dict):
run_id = dt.now().strftime('%Y-%m-%d_%H.%M.%S')
make_dirs(run_id)
create_config(run_id, config_dict)
self.config.read(os.path.join('temp', run_id, 'config', 'config.ini'))
self.df_sized = self.df.iloc[::int(self.config['FORECAST_PARAMS']
['STEP_SIZE']), :]
self.df_sized = proces_data(self.df_sized)
self.features = []
for i in range(self.df_sized.shape[1]):
if self.training:
feature = Feature(self.df_sized.columns[i], run_id,
self.df_sized)
feature.train_models(self.df_sized)
self.features.append(feature)
else:
delete_run('temp', run_id)
return False
move_run(run_id)
return True
def main():
data_path = env()
PATH_MODEL = '%s/processed_dataset/{}/{}/' % data_path
PATH_RELATIVE = '%s/relative_pose/{}' % data_path
models = [
os.path.normpath(p) for p in glob.glob(PATH_MODEL.format(dataset, '*'))
]
make_dirs('%s/tasks' % dataset)
with open('%s/tasks' % dataset, 'w') as fout:
lines = []
for model in models:
objs = glob.glob('%s/*.obj' % model)
modelname = ('/').join(model.split('/')[-2:])
#import ipdb; ipdb.set_trace()
n = len(objs)
basename = [
int(obji.split('/')[-1].split('.')[0]) for obji in objs
]
output_folder = PATH_RELATIVE.format(modelname)
#'%s/relative_pose/%s' % (data_path, modelname)
pathlib.Path(output_folder).mkdir(exist_ok=True, parents=True)
for i in range(n):
for j in range(i + 1, n):
output_file = '{}/{}_{}_super4pcs.txt'.format(
output_folder, basename[i], basename[j])
command = './Super4PCS -i %s %s %s -m %s' % (
objs[i], objs[j], param, output_file)
#print(command)
lines.append(command)
#fout.write('%s\n' % command)
""" ./Super4PCS -i ../datasets/redwood/00021_0.obj ../datasets/redwood/00021_1.obj -o 0.7 -d 0.01 -t 1000 -n 200 -m super4pcs/00021_0_1.txt """
#np.random.shuffle(lines)
for line in lines:
fout.write('%s\n' % line)
def process_tile(current_tile, tile_id, segment, region, tiles_contrib_scenes,
output_path, conf):
"""Process each tile needed for segment.
Args:
current_tile (dict): information about current tile
tile_id (str): tile id string (e.g. LT04_CU_011003_...)
segment (dict): information about a scene
region (str): ARD grid tile area (e.g. CU, AK, HI)
tiles_contrib_scenes (list): neighboring scene details
output_path (str): path to store outputs
conf (dict): runtime configuration options
"""
production_timestamp = landsat.get_production_timestamp()
clip_extents = '{UL_X} {LL_Y} {LR_X} {UR_Y}'.format(**current_tile)
logger.debug("tile_id: %s", tile_id)
logger.debug("clip_extents: %s", clip_extents)
# See if tile_id exists in ARD_COMPLETED_TILES table
tile_rec = db.check_tile_status(db.connect(conf.connstr), tile_id)
logger.debug("Tile status: %s", tile_rec)
if tile_rec:
logger.error('Tile already created! %s', tile_rec)
raise ArdTileException
logger.info("Create Tile %s", tile_id)
# Get file location for scenes that will contribute to the tile
key = 'H{H:03d}V{V:03d}'.format(**current_tile)
contrib_tile_scenes = tiles_contrib_scenes[key]
logger.debug('# Scenes needed for tile %s: %d', tile_id,
len(contrib_tile_scenes))
contributing_scenes = dict()
for contrib_record in contrib_tile_scenes:
wildcard = '{wrspath}{wrsrow}_{acqdate}'.format(**contrib_record)
logger.debug(" Contributing scene: %s", wildcard)
provided_contrib_rec = []
if wildcard in segment['LANDSAT_PRODUCT_ID']:
provided_contrib_rec = {
segment['LANDSAT_PRODUCT_ID']: segment['FILE_LOC']
}
logger.info("Contributing scene from segment: %s",
provided_contrib_rec)
contributing_scenes.update(provided_contrib_rec)
if not provided_contrib_rec:
db_contrib_rec = db.fetch_file_loc(db.connect(conf.connstr),
sat=segment['SATELLITE'],
wildcard=wildcard)
logger.debug('Fetch file locations from DB: %s', db_contrib_rec)
if db_contrib_rec:
db_contrib_rec = {
r['LANDSAT_PRODUCT_ID']: util.ffind(r['FILE_LOC'])
for r in db_contrib_rec
}
# If any of the scenes can't be found, raise an exception.
if None in db_contrib_rec.values():
logger.error("Error finding file for %s",
db_contrib_rec.keys())
raise ArdTileException
logger.info("Contributing scene from db: %s", db_contrib_rec)
contributing_scenes.update(db_contrib_rec)
n_contrib_scenes = len(contributing_scenes)
is_complete_tile = ('Y' if n_contrib_scenes == len(contrib_tile_scenes)
else 'N')
logger.info("Contributing scenes: %s", contributing_scenes)
logger.info('All scenes found to complete tile: %s', is_complete_tile)
logger.info('Tile: %s - Number of contributing scenes: %d', tile_id,
n_contrib_scenes)
invalid_contrib_scenes = ((n_contrib_scenes > conf.maxscenespertile)
or (n_contrib_scenes < conf.minscenespertile))
if invalid_contrib_scenes:
logger.info('Unexpected number of scenes %d', n_contrib_scenes)
raise ArdTileException
if conf.hsmstage:
# Stage files to disk cache for faster access
external.stage_files(contributing_scenes.values(), conf.soap_envelope)
# If each contributing scene is not already unpacked, do it here
for product_id, tar_file_location in contributing_scenes.items():
logger.info('Required Scene: %s', tar_file_location)
try:
directory = os.path.join(conf.workdir, product_id)
util.untar_archive(tar_file_location, directory=directory)
except Exception:
logger.exception('Error staging input data for %s: %s', product_id,
tar_file_location)
raise ArdSceneException
logger.info('Starting to build tile: %s', tile_id)
# Determine which scene(s) will overlay the other scene(s) for this tile.
# North scenes (--row#) will always overlay South scenes (++row#).
# The row values are characters 13 - 15 in the product ID.
stacking = [{
'LANDSAT_PRODUCT_ID': name,
'XML_LOC': util.ffind(conf.workdir, name, '*.xml')
} for name in sorted(contributing_scenes, key=lambda x: x[13:])]
util.make_dirs(os.path.join(conf.workdir, tile_id))
producers = config.read_processing_config(sensor=segment['SATELLITE'])
datatypes = {
"[ TYPE: Int16 ][ RANGE: -100,16000 ][ FILL: -9999 ]":
direct_clip,
"[ TYPE: Int16 ][ RANGE: -2000,16000 ][ FILL: -9999 ]":
direct_clip,
"[ TYPE: Int16 ][ RANGE: -32767,32767 ][ FILL: -32768 ]":
direct_clip,
"[ TYPE: Int16 ][ RANGE: ??? ][ FILL: -9999 ]":
direct_clip,
"[ TYPE: UInt16 ][ RANGE: 0,65535 ][ FILL: 1 ]":
direct_clip,
"[ TYPE: UInt8 ][ RANGE: 0,255 ][ FILL: 1 ]":
direct_clip,
"[ TYPE: UInt8 ][ RANGE: 0,255 ][ FILL: 255 ]":
direct_clip,
"[ TYPE: UInt8 ][ RANGE: 0,255 ][ FILL: NA ][ +LINEAGE ]":
fill_zero_na_lineage,
"[ TYPE: Int16 ][ RANGE: 0,255 ][ FILL: -9999 ][ +LINEAGE ]":
calc_nodata_9999_uint_lineage,
"[ TYPE: Int16 ][ RANGE: ??? ][ FILL: -9999 ][ +LINEAGE ]":
calc_nodata_9999_lineage,
}
outputs = dict()
for product_request in sorted(conf.products, reverse=True):
logging.info('Create product %s', product_request)
required_bands = config.determine_output_products(
producers, product_request)
for band_name, rename in required_bands.items():
dtype = config.datatype_searches(producers, band_name)
logger.info('Requires base band_name %s (Type %s)', band_name,
dtype)
# Process the current dataset type
filename = datatypes[dtype](stacking, band_name, clip_extents,
tile_id, rename, conf.workdir)
outputs[band_name] = filename
# WARNING: Assume LINEAGE will always be present!
if band_name == 'toa_band1':
outputs['LINEAGEQA'] = (process_lineage(
stacking, band_name, clip_extents, tile_id, 'LINEAGEQA',
conf.workdir))
lng_count = process_lineage_contributing(outputs['LINEAGEQA'],
n_contrib_scenes)
outputs['XML'] = (process_metadata(segment, stacking, tile_id,
clip_extents, region, lng_count,
production_timestamp, producers,
conf.workdir))
if process_output(conf.products, producers, outputs, conf.workdir, tile_id,
output_path) != 0:
logger.error('Failed processing products.')
return "ERROR"
if process_browse(producers['browse'], conf.workdir, tile_id,
output_path) != 0:
logger.error('Failed to produce the browse image.')
return "ERROR"
if not conf.debug:
# No errors making this tile, record it in our database
completed_tile_list = [[
tile_id, ",".join(contributing_scenes.keys()), is_complete_tile,
"SUCCESS"
]]
db.insert_tile_record(db.connect(conf.connstr), completed_tile_list)
return "SUCCESS"
args = parser.parse_args()
if args.output_folder.split("/")[0] != args.dir:
args.output_folder = os.path.join(args.dir, args.output_folder)
if not args.run2:
if args.year is None:
configs = json.load(open("config.json"))
args.year = str(configs['year'])
if args.year == "2017":
pbins = [475, 500, 550, 600, 675, 800, 1200]
else:
pbins = [450, 500, 550, 600, 675, 800, 1200]
make_dirs(args.output_folder)
if args.fit is None:
shape_types = ['prefit', 'fit_s']
else:
shape_types = [args.fit]
if args.three_regions:
regions = ['pqq', 'pcc', 'pbb']
else:
regions = ['pass', 'fail']
# f = uproot.open(os.path.join(args.dir, args.input))
f = uproot.open(args.input)
for shape_type in shape_types:
if args.inputonly:
continue
def get_fullname(self):
timestamp = datetime.utcnow()
dirname = path.join(self.directory, self.name, timestamp.strftime('%Y-%m-%d-%H-00'))
filename = util.set_filename(self.name, timestamp)
util.make_dirs(dirname)
return path.join(dirname, filename)
def iterate_assets(
cache,
settings,
asset_folders,
tools,
platform
):
# loop through each asset path and glob
# run the tool associated with each file
logging.info("Running tools on assets...")
total_files = 0
modified_files = 0
for asset in asset_folders:
try:
search_path = os.path.join(
asset.abs_src_folder,
asset.glob
)
#logging.info("Processing: \"%s\"" % search_path)
if tools.has_key(asset.tool):
tool = tools[asset.tool]
else:
raise UnknownToolException(
"Unknown tool \"%s\"" % asset.tool
)
path_created = False
for root, subs, files in os.walk(asset.abs_src_folder, topdown=True):
# filter files based on glob
files[:] = fnmatch.filter(files, asset.glob)
if not path_created:
# make all asset destination folders
make_dirs(asset.abs_dst_folder)
path_created = True
# filter subdirectories based on the glob
matched_subs = fnmatch.filter(subs, asset.glob)
# do not recurse into subdirectories
subs[:] = []
if matched_subs:
# One or more subdirectories matched the glob.
# For now, copy each match over to the destination folder.
# This is to specifically handle the case where directories are entire
# folders (.dSYM, .app). These specific dirs should be
# exceptions where the entire folder is simply copied.
for folder in matched_subs:
copy_tree(source_file_root, dst_file_root)
for file in files:
src_file_path = os.path.join(root, file)
total_files += 1
if not cache.update(src_file_path):
continue
modified_files += 1
execute_commands(
tools,
tool,
settings.paths,
asset,
src_file_path,
platform
)
except UnknownToolException as e:
logging.warn(e.message)
continue
logging.info("Complete.")
logging.info("Modified / Total - %i/%i" % (modified_files, total_files))