コード例 #1
0
ファイル: nlpir_main.py プロジェクト: aseveryn/NLPIR-2013
def extract_features(mode, datasets):
    for dataset in datasets:
        input = get_sts_input_path(mode, dataset)
        print dataset
        # print "Generating test files..."
        sentence_pairs = load_data(input)

        logging.info("Computing features")
        doc2features = []
        for i, sp in enumerate(sentence_pairs, 1):
            if i % 10 == 0:
                sys.stdout.write("%d.." % i)
            features = compute_features(*sp)
            doc2features.append(features)
        sys.stdout.write("%d\n" % i)
        
        outdir = os.path.join(FEATURE_PATH, mode, dataset, FEATURE_SET)
        make_dirs(outdir)
        logging.info("Writing features to: %s" % outdir)

        for idx in xrange(len(features)):
            outfile = os.path.join(outdir, "%s.txt" % idx)
            with open(outfile, 'w') as out:
                for doc_features in doc2features:
                    out.write("%f\n" % doc_features[idx])
コード例 #2
0
ファイル: takelab_main.py プロジェクト: zhuhk/NLPIR-2013
def extract_features(mode, datasets):
    for dataset in datasets:
        input = get_sts_input_path(mode, dataset)
        print dataset
        # print "Generating test files..."
        sentence_pairs = load_data(input)

        logging.info("Computing features")
        doc2features = []
        for i, sp in enumerate(sentence_pairs, 1):
            if i % 10 == 0:
                sys.stdout.write("%d.." % i)
            features = compute_features(*sp)
            doc2features.append(features)
        sys.stdout.write("%d\n" % i)

        outdir = os.path.join(FEATURE_PATH, mode, dataset, FEATURE_SET)
        make_dirs(outdir)
        logging.info("Writing features to: %s" % outdir)

        for idx in xrange(len(features)):
            outfile = os.path.join(outdir, "%s.txt" % idx)
            with open(outfile, 'w') as out:
                for doc_features in doc2features:
                    out.write("%f\n" % doc_features[idx])
コード例 #3
0
def write_svmfile(mode, datasets):
  print "Generating files in %s mode..." % mode
  
  examples = []
  outdir = os.path.join(MODEL_PATH, MODEL, mode)
  make_dirs(outdir)
  for dataset in datasets:
    outfile = "%s.svmtk" % dataset
    outpath = os.path.join(outdir, outfile)
    print outpath
    with open(outpath, "w") as out:
      ex_generator = ExampleWriter(SVM_FORMAT)
      for ex in ex_generator.to_svm(mode, dataset):
        out.write(ex)
        examples.append(ex)

  # if mode == TRAIN:
  random.seed(123)
  random.shuffle(examples)

  outfile = "ALL.svmtk"
  outpath = os.path.join(outdir, outfile)
  print outpath
  with open(outpath, "w") as out:
    for ex in examples:
      out.write(ex)
コード例 #4
0
  def create_cv_folds(self, dir="ova.cv", nfolds=5):
    make_dirs(self.ova_path)
    # make cv folds for examples from each of the categories
    folds = [make_cv_folds(examples, nfolds) for _, examples in self.cat2ex.iteritems()]

    # strip (train, test) pairs from each of the folds
    for i, fold_data in enumerate(zip(*folds), 1):
      print "Fold", i
      fold_path = os.path.join(self.ova_path, "fold-{}".format(i))
      make_dirs(fold_path)

      train_examples = []
      test_examples = []
      for (train_fold, test_fold) in fold_data:
        train_examples.extend(train_fold)
        test_examples.extend(test_fold)

      labels_fname = os.path.join(fold_path, "labels.test")
      with open(labels_fname, "w") as out:
        for label, _ in test_examples:
          out.write("{}\n".format(label))

      for cat in self.categories:
        # training  
        train_fname = os.path.join(fold_path, "{}.train".format(cat))
        self.write_ova_examples(train_examples, cat, train_fname)
        
        # testing
        test_fname = os.path.join(fold_path, "{}.test".format(cat))
        self.write_ova_examples(test_examples, cat, test_fname)
コード例 #5
0
def write_svmfile(mode, datasets):
    print "Generating files in %s mode..." % mode

    examples = []
    outdir = os.path.join(MODEL_PATH, MODEL, mode)
    make_dirs(outdir)
    for dataset in datasets:
        outfile = "%s.svmtk" % dataset
        outpath = os.path.join(outdir, outfile)
        print outpath
        with open(outpath, "w") as out:
            ex_generator = ExampleWriter(SVM_FORMAT)
            for ex in ex_generator.to_svm(mode, dataset):
                out.write(ex)
                examples.append(ex)

    # if mode == TRAIN:
    random.seed(123)
    random.shuffle(examples)

    outfile = "ALL.svmtk"
    outpath = os.path.join(outdir, outfile)
    print outpath
    with open(outpath, "w") as out:
        for ex in examples:
            out.write(ex)
コード例 #6
0
def main():
    depth_paths, T, pose_paths = getData(args.shapeid)
    n = len(depth_paths)
    print('found %d clean depth images...' % n)
    intrinsic = np.array([[525.0,0,319.5],[0,525.0,239.5],[0,0,1]])
    np.random.seed(816)
    indices = np.random.permutation(n)
    print(indices[:100])
    #indices = sorted(indices)
    make_dirs(PATH_MAT.format(args.shapeid, 0))
    import open3d
    pcd_combined = open3d.PointCloud()
    for i, idx in enumerate(indices):
        import ipdb; ipdb.set_trace()
        print('%d / %d' % (i, len(indices)))
        mesh = Mesh.read(depth_paths[idx], mode='depth', intrinsic = intrinsic)
        pcd = open3d.PointCloud()
        pcd.points = open3d.Vector3dVector(mesh.vertex.T)
        pcd.transform(inverse(T[idx]))
        #pcd = open3d.voxel_down_sample(pcd, voxel_size=0.02)
        pcd_combined += pcd
        pcd_combined = open3d.voxel_down_sample(pcd_combined, voxel_size=0.02)
        sio.savemat(PATH_MAT.format(args.shapeid, i), mdict={
            'vertex': mesh.vertex, 
            'validIdx_rowmajor': mesh.validIdx, 
            'pose': T[idx], 
            'depth_path': depth_paths[idx], 
            'pose_path': pose_paths[idx]})
        if i <= 50 and i >= 40:
            pcd_combined_down = open3d.voxel_down_sample(pcd_combined, voxel_size=0.02)
            open3d.draw_geometries([pcd_combined_down])
            

    pcd_combined_down = open3d.voxel_down_sample(pcd_combined, voxel_size=0.02)
    open3d.draw_geometries([pcd_combined_down])
コード例 #7
0
 def get_fullname(self):
     timestamp = datetime.utcnow()
     dirname = path.join(self.directory, self.name,
                         timestamp.strftime('%Y-%m-%d-%H-00'))
     filename = util.set_filename(self.name, timestamp)
     util.make_dirs(dirname)
     return path.join(dirname, filename)
コード例 #8
0
 def run(self):
     print("{name} thread running.".format(name = self.name))
     util.make_dirs(["./ir_videos"])
     while not self._stop:
         self.eventWait.wait()
         self.event = self.events[0]
         del self.events[0]
         self.run_event()
         if not len(self.events):
             self.eventWait.clear()
     print("'{name}' stopped.".format(name = self.name))
コード例 #9
0
ファイル: ARD_Clip.py プロジェクト: ldj01/ard_tile
def process_output(products, producers, outputs, workdir, tile_id,
                   output_path):
    """Combine the Landsat mosaics into the .tar files."""
    status = 0  # Initialize the return status.

    util.make_dirs(output_path)

    for product_request in sorted(products, reverse=True):
        archive = tile_id + '_' + product_request + '.tar'
        logging.info('Create product %s', archive)
        required = producers['package'][product_request]
        included = [
            o for o in outputs.values()
            if any([r in o for r in required]) and isinstance(o, str)
        ]
        included.append(outputs['XML'][product_request])

        local_archive = os.path.join(workdir, archive)
        output_archive = os.path.join(output_path, archive)
        output_xml = os.path.join(
            output_path, os.path.basename(outputs['XML'][product_request]))
        util.tar_archive(local_archive, included)

        # Copy tarball to output directory.
        shutil.copyfile(local_archive, output_archive)
        shutil.copyfile(outputs['XML'][product_request], output_xml)

    # Create checksums for the output tarballs, and compare with the files
    # in the working directory.
    if util.process_checksums(tile_id + '*.tar', workdir,
                              output_path) == 'ERROR':
        logger.error('Checksum processing failed.')

        # Clean up output files upon failure.
        for fullname in glob.glob(os.path.join(output_path, tile_id + '*')):
            os.remove(fullname)

        status = 1

    # Remove local copies of product files.
    for fullname in glob.glob(os.path.join(workdir, tile_id + '*.tar')):
        os.remove(fullname)

    if status == 0:
        logger.info('    End product transfer')
    else:
        logger.info('    Product transfer failed.')

    return status
コード例 #10
0
def main():
    depth_paths, T, pose_paths = getData(args.shapeid)
    n = len(depth_paths)
    print('found %d clean depth images...' % n)
    intrinsic = parse_info('%s/dataset/scannet/%s/_info.txt' %
                           (data_path, args.shapeid))
    np.random.seed(816)
    num_sample = 100
    #if n < 5*num_sample:
    #    stepsize = n // num_sample
    #else:
    #    stepsize = 5
    #if stepsize == 0:
    #    assert False
    #indices = [i for i in range(0, n, stepsize)][:num_sample] #np.random.permutation(n)
    frame_ids = [
        int(depth_paths[i].split('/')[-1].split('.')[0].split('-')[-1])
        for i in range(n)
    ]
    indices = pick_frames(frame_ids, max_gap=6, num_sample=num_sample)
    if indices is None:
        print('not enough valid frames')
        exit(0)
    print([frame_ids[idx] for idx in indices])
    #print(indices[:100])
    #indices = sorted(indices)
    make_dirs(PATH_MAT.format(args.shapeid, 0))
    #import open3d
    #pcd_combined = open3d.PointCloud()
    for i, idx in enumerate(indices):
        #print('%d / %d' % (i, len(indices)))
        mesh = Mesh.read(depth_paths[idx], mode='depth', intrinsic=intrinsic)
        #pcd = open3d.PointCloud()
        #pcd.points = open3d.Vector3dVector(mesh.vertex.T)
        #pcd.transform(inverse(T[idx]))
        ##pcd = open3d.voxel_down_sample(pcd, voxel_size=0.02)
        #pcd_combined += pcd
        #pcd_combined = open3d.voxel_down_sample(pcd_combined, voxel_size=0.02)
        sio.savemat(PATH_MAT.format(args.shapeid, i),
                    mdict={
                        'vertex': mesh.vertex,
                        'validIdx_rowmajor': mesh.validIdx,
                        'pose': T[idx],
                        'depth_path': depth_paths[idx],
                        'pose_path': pose_paths[idx]
                    })
コード例 #11
0
    def train(self, config_dict):
        run_id = dt.now().strftime('%Y-%m-%d_%H.%M.%S')
        make_dirs(run_id)
        create_config(run_id, config_dict)

        self.config.read(os.path.join('temp', run_id, 'config', 'config.ini'))
        self.df_sized = self.df.iloc[::int(self.config['FORECAST_PARAMS']
                                           ['STEP_SIZE']), :]
        self.df_sized = proces_data(self.df_sized)

        self.features = []
        for i in range(self.df_sized.shape[1]):
            if self.training:
                feature = Feature(self.df_sized.columns[i], run_id,
                                  self.df_sized)
                feature.train_models(self.df_sized)
                self.features.append(feature)
            else:
                delete_run('temp', run_id)
                return False
        move_run(run_id)
        return True
コード例 #12
0
def main():
    data_path = env()

    PATH_MODEL = '%s/processed_dataset/{}/{}/' % data_path
    PATH_RELATIVE = '%s/relative_pose/{}' % data_path

    models = [
        os.path.normpath(p) for p in glob.glob(PATH_MODEL.format(dataset, '*'))
    ]
    make_dirs('%s/tasks' % dataset)
    with open('%s/tasks' % dataset, 'w') as fout:
        lines = []
        for model in models:
            objs = glob.glob('%s/*.obj' % model)
            modelname = ('/').join(model.split('/')[-2:])
            #import ipdb; ipdb.set_trace()
            n = len(objs)
            basename = [
                int(obji.split('/')[-1].split('.')[0]) for obji in objs
            ]

            output_folder = PATH_RELATIVE.format(modelname)
            #'%s/relative_pose/%s' % (data_path, modelname)
            pathlib.Path(output_folder).mkdir(exist_ok=True, parents=True)
            for i in range(n):
                for j in range(i + 1, n):
                    output_file = '{}/{}_{}_super4pcs.txt'.format(
                        output_folder, basename[i], basename[j])

                    command = './Super4PCS -i %s %s %s -m %s' % (
                        objs[i], objs[j], param, output_file)
                    #print(command)
                    lines.append(command)
                    #fout.write('%s\n' % command)
                    """ ./Super4PCS -i ../datasets/redwood/00021_0.obj ../datasets/redwood/00021_1.obj -o 0.7 -d 0.01 -t 1000 -n 200 -m super4pcs/00021_0_1.txt """
        #np.random.shuffle(lines)
        for line in lines:
            fout.write('%s\n' % line)
コード例 #13
0
ファイル: ARD_Clip.py プロジェクト: ldj01/ard_tile
def process_tile(current_tile, tile_id, segment, region, tiles_contrib_scenes,
                 output_path, conf):
    """Process each tile needed for segment.

    Args:
        current_tile (dict):  information about current tile
        tile_id (str): tile id string (e.g. LT04_CU_011003_...)
        segment (dict): information about a scene
        region (str): ARD grid tile area (e.g. CU, AK, HI)
        tiles_contrib_scenes (list): neighboring scene details
        output_path (str): path to store outputs
        conf (dict): runtime configuration options

    """
    production_timestamp = landsat.get_production_timestamp()
    clip_extents = '{UL_X} {LL_Y} {LR_X} {UR_Y}'.format(**current_tile)

    logger.debug("tile_id: %s", tile_id)
    logger.debug("clip_extents: %s", clip_extents)

    # See if tile_id exists in ARD_COMPLETED_TILES table
    tile_rec = db.check_tile_status(db.connect(conf.connstr), tile_id)
    logger.debug("Tile status: %s", tile_rec)

    if tile_rec:
        logger.error('Tile already created! %s', tile_rec)
        raise ArdTileException

    logger.info("Create Tile %s", tile_id)

    # Get file location for scenes that will contribute to the tile
    key = 'H{H:03d}V{V:03d}'.format(**current_tile)
    contrib_tile_scenes = tiles_contrib_scenes[key]
    logger.debug('# Scenes needed for tile %s: %d', tile_id,
                 len(contrib_tile_scenes))

    contributing_scenes = dict()
    for contrib_record in contrib_tile_scenes:
        wildcard = '{wrspath}{wrsrow}_{acqdate}'.format(**contrib_record)
        logger.debug("          Contributing scene: %s", wildcard)

        provided_contrib_rec = []
        if wildcard in segment['LANDSAT_PRODUCT_ID']:
            provided_contrib_rec = {
                segment['LANDSAT_PRODUCT_ID']: segment['FILE_LOC']
            }
            logger.info("Contributing scene from segment: %s",
                        provided_contrib_rec)
            contributing_scenes.update(provided_contrib_rec)

        if not provided_contrib_rec:
            db_contrib_rec = db.fetch_file_loc(db.connect(conf.connstr),
                                               sat=segment['SATELLITE'],
                                               wildcard=wildcard)
            logger.debug('Fetch file locations from DB: %s', db_contrib_rec)
            if db_contrib_rec:
                db_contrib_rec = {
                    r['LANDSAT_PRODUCT_ID']: util.ffind(r['FILE_LOC'])
                    for r in db_contrib_rec
                }

                # If any of the scenes can't be found, raise an exception.
                if None in db_contrib_rec.values():
                    logger.error("Error finding file for %s",
                                 db_contrib_rec.keys())
                    raise ArdTileException

                logger.info("Contributing scene from db: %s", db_contrib_rec)
                contributing_scenes.update(db_contrib_rec)

    n_contrib_scenes = len(contributing_scenes)
    is_complete_tile = ('Y' if n_contrib_scenes == len(contrib_tile_scenes)
                        else 'N')
    logger.info("Contributing scenes: %s", contributing_scenes)
    logger.info('All scenes found to complete tile: %s', is_complete_tile)
    logger.info('Tile: %s  - Number of contributing scenes: %d', tile_id,
                n_contrib_scenes)

    invalid_contrib_scenes = ((n_contrib_scenes > conf.maxscenespertile)
                              or (n_contrib_scenes < conf.minscenespertile))
    if invalid_contrib_scenes:
        logger.info('Unexpected number of scenes %d', n_contrib_scenes)
        raise ArdTileException

    if conf.hsmstage:
        # Stage files to disk cache for faster access
        external.stage_files(contributing_scenes.values(), conf.soap_envelope)

    # If each contributing scene is not already unpacked, do it here
    for product_id, tar_file_location in contributing_scenes.items():

        logger.info('Required Scene: %s', tar_file_location)
        try:
            directory = os.path.join(conf.workdir, product_id)
            util.untar_archive(tar_file_location, directory=directory)
        except Exception:
            logger.exception('Error staging input data for %s: %s', product_id,
                             tar_file_location)
            raise ArdSceneException

    logger.info('Starting to build tile: %s', tile_id)

    # Determine which scene(s) will overlay the other scene(s) for this tile.
    # North scenes (--row#) will always overlay South scenes (++row#).
    # The row values are characters 13 - 15 in the product ID.
    stacking = [{
        'LANDSAT_PRODUCT_ID': name,
        'XML_LOC': util.ffind(conf.workdir, name, '*.xml')
    } for name in sorted(contributing_scenes, key=lambda x: x[13:])]

    util.make_dirs(os.path.join(conf.workdir, tile_id))

    producers = config.read_processing_config(sensor=segment['SATELLITE'])

    datatypes = {
        "[ TYPE: Int16 ][ RANGE: -100,16000 ][ FILL: -9999 ]":
        direct_clip,
        "[ TYPE: Int16 ][ RANGE: -2000,16000 ][ FILL: -9999 ]":
        direct_clip,
        "[ TYPE: Int16 ][ RANGE: -32767,32767 ][ FILL: -32768 ]":
        direct_clip,
        "[ TYPE: Int16 ][ RANGE: ??? ][ FILL: -9999 ]":
        direct_clip,
        "[ TYPE: UInt16 ][ RANGE: 0,65535 ][ FILL: 1 ]":
        direct_clip,
        "[ TYPE: UInt8 ][ RANGE: 0,255 ][ FILL: 1 ]":
        direct_clip,
        "[ TYPE: UInt8 ][ RANGE: 0,255 ][ FILL: 255 ]":
        direct_clip,
        "[ TYPE: UInt8 ][ RANGE: 0,255 ][ FILL: NA ][ +LINEAGE ]":
        fill_zero_na_lineage,
        "[ TYPE: Int16 ][ RANGE: 0,255 ][ FILL: -9999 ][ +LINEAGE ]":
        calc_nodata_9999_uint_lineage,
        "[ TYPE: Int16 ][ RANGE: ??? ][ FILL: -9999 ][ +LINEAGE ]":
        calc_nodata_9999_lineage,
    }

    outputs = dict()
    for product_request in sorted(conf.products, reverse=True):
        logging.info('Create product %s', product_request)
        required_bands = config.determine_output_products(
            producers, product_request)
        for band_name, rename in required_bands.items():
            dtype = config.datatype_searches(producers, band_name)
            logger.info('Requires base band_name %s (Type %s)', band_name,
                        dtype)

            # Process the current dataset type
            filename = datatypes[dtype](stacking, band_name, clip_extents,
                                        tile_id, rename, conf.workdir)
            outputs[band_name] = filename

            # WARNING: Assume LINEAGE will always be present!
            if band_name == 'toa_band1':
                outputs['LINEAGEQA'] = (process_lineage(
                    stacking, band_name, clip_extents, tile_id, 'LINEAGEQA',
                    conf.workdir))

    lng_count = process_lineage_contributing(outputs['LINEAGEQA'],
                                             n_contrib_scenes)

    outputs['XML'] = (process_metadata(segment, stacking, tile_id,
                                       clip_extents, region, lng_count,
                                       production_timestamp, producers,
                                       conf.workdir))

    if process_output(conf.products, producers, outputs, conf.workdir, tile_id,
                      output_path) != 0:
        logger.error('Failed processing products.')
        return "ERROR"
    if process_browse(producers['browse'], conf.workdir, tile_id,
                      output_path) != 0:
        logger.error('Failed to produce the browse image.')
        return "ERROR"

    if not conf.debug:
        # No errors making this tile, record it in our database
        completed_tile_list = [[
            tile_id, ",".join(contributing_scenes.keys()), is_complete_tile,
            "SUCCESS"
        ]]
        db.insert_tile_record(db.connect(conf.connstr), completed_tile_list)
    return "SUCCESS"
コード例 #14
0
ファイル: plot.py プロジェクト: andrzejnovak/higgstocharm
    args = parser.parse_args()
    if args.output_folder.split("/")[0] != args.dir:
        args.output_folder = os.path.join(args.dir, args.output_folder)

    if not args.run2:
        if args.year is None:
            configs = json.load(open("config.json"))
            args.year = str(configs['year'])
    
    if args.year == "2017":
        pbins = [475, 500, 550, 600, 675, 800, 1200]
    else:
        pbins = [450, 500, 550, 600, 675, 800, 1200]

    make_dirs(args.output_folder)

    if args.fit is None:
        shape_types = ['prefit', 'fit_s']
    else:
        shape_types = [args.fit]
    if args.three_regions:
        regions = ['pqq', 'pcc', 'pbb']
    else:
        regions = ['pass', 'fail']

    # f = uproot.open(os.path.join(args.dir, args.input))
    f = uproot.open(args.input)
    for shape_type in shape_types:
        if args.inputonly:
            continue
コード例 #15
0
ファイル: pss-capture.py プロジェクト: jmacko/timelapse
 def get_fullname(self):
   timestamp = datetime.utcnow()
   dirname = path.join(self.directory, self.name, timestamp.strftime('%Y-%m-%d-%H-00'))
   filename = util.set_filename(self.name, timestamp)
   util.make_dirs(dirname)
   return path.join(dirname, filename)
コード例 #16
0
ファイル: blacksmith.py プロジェクト: apetrone/blacksmith
def iterate_assets(
		cache, 
		settings, 
		asset_folders,
		tools, 
		platform
	):
	# loop through each asset path and glob
	# run the tool associated with each file
	logging.info("Running tools on assets...")
	total_files = 0
	modified_files = 0
	for asset in asset_folders:
		try:
			search_path = os.path.join(
				asset.abs_src_folder, 
				asset.glob
			)
			#logging.info("Processing: \"%s\"" % search_path)
			
			if tools.has_key(asset.tool):
				tool = tools[asset.tool]
			else:
				raise UnknownToolException(
					"Unknown tool \"%s\"" % asset.tool
				)

			path_created = False

			for root, subs, files in os.walk(asset.abs_src_folder, topdown=True):
				# filter files based on glob
				files[:] = fnmatch.filter(files, asset.glob)

				if not path_created:
					# make all asset destination folders
					make_dirs(asset.abs_dst_folder)
					path_created = True

				# filter subdirectories based on the glob
				matched_subs = fnmatch.filter(subs, asset.glob)

				# do not recurse into subdirectories
				subs[:] = []

				if matched_subs:
					# One or more subdirectories matched the glob.
					# For now, copy each match over to the destination folder.
					# This is to specifically handle the case where directories are entire
					# folders (.dSYM, .app). These specific dirs should be
					# exceptions where the entire folder is simply copied.
					for folder in matched_subs:
						copy_tree(source_file_root, dst_file_root)

				for file in files:
					src_file_path = os.path.join(root, file)

					total_files += 1

					if not cache.update(src_file_path):
						continue

					modified_files += 1

					execute_commands(
						tools, 
						tool, 
						settings.paths,
						asset,
						src_file_path,
						platform
					)

		except UnknownToolException as e:
			logging.warn(e.message)
			continue	


	logging.info("Complete.")
	logging.info("Modified / Total - %i/%i" % (modified_files, total_files))