def friendly_crs_equal(expected, actual, keys=None, use_obj=True, use_wkt=True): """Test if two projection definitions are equal. The main purpose of this function is to help manage differences between pyproj versions. Depending on the version installed and used pyresample may provide a different `proj_dict` or other similar CRS definition. Args: expected (dict, str, pyproj.crs.CRS): Expected CRS definition as a PROJ dictionary or string or CRS object. actual (dict, str, pyproj.crs.CRS): Actual CRS definition keys (list): Specific PROJ parameters to look for. Only takes effect if `use_obj` is `False`. use_obj (bool): Use pyproj's CRS object to test equivalence. Default is True. use_wkt (bool): Increase likely hood of making CRS objects equal by converting WellKnownText before converting to the final CRS object. Requires `use_obj`. Defaults to True. """ if CRS is not None and use_obj: if hasattr(expected, 'crs'): expected = expected.crs if hasattr(actual, 'crs'): actual = actual.crs expected_crs = CRS(expected) actual_crs = CRS(actual) if use_wkt: expected_crs = CRS(expected_crs.to_wkt()) actual_crs = CRS(actual_crs.to_wkt()) return expected_crs == actual_crs raise NotImplementedError("""TODO""")
def configure_database_with_custom_crs(self, index) -> None: """Function to configure database using custom EPSG :return: """ custom_crs = self._load_configfile() for custom_crs_code_and_entity in custom_crs: custom_crs_entity, custom_crs_code = custom_crs_code_and_entity.split(":") resproxy = index._db._engine.execute( f"SELECT * FROM spatial_ref_sys WHERE srid = {custom_crs_code}" ) resset = resproxy.first() if not resset: custom_crs_string = PJCRS(custom_crs[custom_crs_code_and_entity]) index._db._engine.execute( "INSERT INTO spatial_ref_sys VALUES ({}, '{}', '{}', '{}', '{}');".format( custom_crs_code, custom_crs_entity.upper(), custom_crs_code, custom_crs_string.to_wkt(), custom_crs_string.to_proj4(), ) )
def from_pyproj(cls, pyproj_crs: pyproj.CRS): """ Create CRS object from pyproj.CRS object. Parameters ---------- pyproj_crs : pyproj.CRS pyproj.CRS object. Returns ------- CRS CRS instance. """ return cls(projection_string=pyproj_crs.to_wkt(version="WKT1_ESRI"))
def convert(files, outfolder='./3dtiles', overwrite=False, jobs=multiprocessing.cpu_count(), cache_size=int(total_memory_MB / 10), srs_out=None, srs_in=None, fraction=100, benchmark=None, rgb=True, graph=False, color_scale=None, verbose=False, subdivision_limit=2000, length_limit=0.001, merge_limit=200): """convert Convert pointclouds (xyz or las) to 3dtiles tileset containing pnts node :param files: Filenames to process. The file must use the .las or .xyz format. :type files: list of str, or str :param outfolder: The folder where the resulting tileset will be written. :type outfolder: path-like object :param overwrite: Overwrite the ouput folder if it already exists. :type overwrite: bool :param jobs: The number of parallel jobs to start. Default to the number of cpu. :type jobs: int :param cache_size: Cache size in MB. Default to available memory / 10. :type cache_size: int :param srs_out: SRS to convert the output with (numeric part of the EPSG code) :type srs_out: int or str :param srs_in: Override input SRS (numeric part of the EPSG code) :type srs_in: int or str :param fraction: Percentage of the pointcloud to process, between 0 and 100. :type fraction: int :param benchmark: Print summary at the end of the process :type benchmark: str :param rgb: Export rgb attributes. :type rgb: bool :param graph: Produce debug graphes (requires pygal). :type graph: bool :param color_scale: Force color scale :type color_scale: float :raises SrsInMissingException: if py3dtiles couldn't find srs informations in input files and srs_in is not specified """ global globs globs.subdivision_limit = subdivision_limit globs.length_limit = length_limit globs.merge_limit = merge_limit # allow str directly if only one input files = [files] if isinstance(files, str) else files # read all input files headers and determine the aabb/spacing _, ext = os.path.splitext(files[0]) init_reader_fn = las_reader.init if ext == '.las' else xyz_reader.init infos = init_reader_fn(files, color_scale=color_scale, srs_in=srs_in, srs_out=srs_out) avg_min = infos['avg_min'] rotation_matrix = None # srs stuff transformer = None srs_out_wkt = None srs_in_wkt = None if srs_out is not None: crs_out = CRS('epsg:{}'.format(srs_out)) if srs_in is not None: crs_in = CRS('epsg:{}'.format(srs_in)) elif infos['srs_in'] is None: raise SrsInMissingException( 'No SRS informations in the provided files') else: crs_in = CRS(infos['srs_in']) srs_out_wkt = crs_out.to_wkt() srs_in_wkt = crs_in.to_wkt() transformer = Transformer.from_crs(crs_in, crs_out) bl = np.array( list( transformer.transform(infos['aabb'][0][0], infos['aabb'][0][1], infos['aabb'][0][2]))) tr = np.array( list( transformer.transform(infos['aabb'][1][0], infos['aabb'][1][1], infos['aabb'][1][2]))) br = np.array( list( transformer.transform(infos['aabb'][1][0], infos['aabb'][0][1], infos['aabb'][0][2]))) avg_min = np.array( list(transformer.transform(avg_min[0], avg_min[1], avg_min[2]))) x_axis = br - bl bl = bl - avg_min tr = tr - avg_min if srs_out == '4978': # Transform geocentric normal => (0, 0, 1) # and 4978-bbox x axis => (1, 0, 0), # to have a bbox in local coordinates that's nicely aligned with the data rotation_matrix = make_rotation_matrix(avg_min, np.array([0, 0, 1])) rotation_matrix = np.dot( make_rotation_matrix(x_axis, np.array([1, 0, 0])), rotation_matrix) bl = np.dot(bl, rotation_matrix[:3, :3].T) tr = np.dot(tr, rotation_matrix[:3, :3].T) root_aabb = np.array([np.minimum(bl, tr), np.maximum(bl, tr)]) else: # offset root_aabb = infos['aabb'] - avg_min original_aabb = root_aabb if True: base_spacing = compute_spacing(root_aabb) if base_spacing > 10: root_scale = np.array([0.01, 0.01, 0.01]) elif base_spacing > 1: root_scale = np.array([0.1, 0.1, 0.1]) else: root_scale = np.array([1, 1, 1]) root_aabb = root_aabb * root_scale root_spacing = compute_spacing(root_aabb) octree_metadata = OctreeMetadata(aabb=root_aabb, spacing=root_spacing, scale=root_scale[0]) # create folder if os.path.isdir(outfolder): if overwrite: shutil.rmtree(outfolder, ignore_errors=True) else: print('Error, folder \'{}\' already exists'.format(outfolder)) sys.exit(1) os.makedirs(outfolder) working_dir = os.path.join(outfolder, 'tmp') os.makedirs(working_dir) node_store = SharedNodeStore(working_dir) if verbose >= 1: print('Summary:') print(' - points to process: {}'.format(infos['point_count'])) print(' - offset to use: {}'.format(avg_min)) print(' - root spacing: {}'.format(root_spacing / root_scale[0])) print(' - root aabb: {}'.format(root_aabb)) print(' - original aabb: {}'.format(original_aabb)) print(' - scale: {}'.format(root_scale)) startup = time.time() initial_portion_count = len(infos['portions']) if graph: progression_log = open('progression.csv', 'w') def add_tasks_to_process(state, name, task, point_count): assert point_count > 0 tasks_to_process = state.node_process.input if name not in tasks_to_process: tasks_to_process[name] = ([task], point_count) else: tasks, count = tasks_to_process[name] tasks.append(task) tasks_to_process[name] = (tasks, count + point_count) processed_points = 0 points_in_progress = 0 previous_percent = 0 points_in_pnts = 0 max_splitting_jobs_count = max(1, jobs // 2) # zmq setup context = zmq.Context() zmq_skt = context.socket(zmq.ROUTER) zmq_skt.bind('ipc:///tmp/py3dtiles1') zmq_idle_clients = [] state = State(infos['portions']) zmq_processes_killed = -1 zmq_processes = [ multiprocessing.Process(target=zmq_process, args=(graph, srs_out_wkt, srs_in_wkt, node_store, octree_metadata, outfolder, rgb, verbose)) for i in range(jobs) ] for p in zmq_processes: p.start() activities = [p.pid for p in zmq_processes] time_waiting_an_idle_process = 0 while True: # state.print_debug() now = time.time() - startup at_least_one_job_ended = False all_processes_busy = not can_queue_more_jobs(zmq_idle_clients) while all_processes_busy or zmq_skt.poll(timeout=0, flags=zmq.POLLIN): # Blocking read but it's fine because either all our child processes are busy # or we know that there's something to read (zmq.POLLIN) start = time.time() result = zmq_skt.recv_multipart() client_id = result[0] result = result[1:] if len(result) == 1: if len(result[0]) == 0: assert client_id not in zmq_idle_clients zmq_idle_clients += [client_id] if all_processes_busy: time_waiting_an_idle_process += time.time() - start all_processes_busy = False elif result[0] == b'halted': zmq_processes_killed += 1 all_processes_busy = False else: result = pickle.loads(result[0]) processed_points += result['total'] points_in_progress -= result['total'] if 'save' in result and len(result['save']) > 0: node_store.put(result['name'], result['save']) if result['name'][0:4] == b'root': state.reader.active.remove(result['name']) else: del state.node_process.active[result['name']] if len(result['name']) > 0: state.node_process.inactive.append(result['name']) if not state.reader.input and not state.reader.active: if state.node_process.active or state.node_process.input: finished_node = result['name'] if not can_pnts_be_written( finished_node, finished_node, state.node_process.input, state.node_process.active): pass else: state.node_process.inactive.pop(-1) state.to_pnts.input.append( finished_node) for i in range( len(state.node_process.inactive ) - 1, -1, -1): candidate = state.node_process.inactive[ i] if can_pnts_be_written( candidate, finished_node, state.node_process.input, state.node_process.active): state.node_process.inactive.pop( i) state.to_pnts.input.append( candidate) else: for c in state.node_process.inactive: state.to_pnts.input.append(c) state.node_process.inactive.clear() at_least_one_job_ended = True elif result[0] == b'pnts': points_in_pnts += struct.unpack('>I', result[1])[0] state.to_pnts.active.remove(result[2]) else: count = struct.unpack('>I', result[2])[0] add_tasks_to_process(state, result[0], result[1], count) while state.to_pnts.input and can_queue_more_jobs(zmq_idle_clients): node_name = state.to_pnts.input.pop() datas = node_store.get(node_name) assert len(datas) > 0, '{} has no data??'.format(node_name) zmq_send_to_process(zmq_idle_clients, zmq_skt, [b'pnts', node_name, datas]) node_store.remove(node_name) state.to_pnts.active.append(node_name) if can_queue_more_jobs(zmq_idle_clients): potential = sorted([(k, v) for k, v in state.node_process.input.items() if k not in state.node_process.active], key=lambda f: -len(f[0])) while can_queue_more_jobs(zmq_idle_clients) and potential: target_count = 100000 job_list = [] count = 0 idx = len(potential) - 1 while count < target_count and potential and idx >= 0: name, (tasks, point_count) = potential[idx] if name not in state.node_process.active: count += point_count job_list += [name] job_list += [node_store.get(name)] job_list += [struct.pack('>I', len(tasks))] job_list += tasks del potential[idx] del state.node_process.input[name] state.node_process.active[name] = (len(tasks), point_count, now) if name in state.node_process.inactive: state.node_process.inactive.pop( state.node_process.inactive.index(name)) idx -= 1 if job_list: zmq_send_to_process(zmq_idle_clients, zmq_skt, job_list) while (state.reader.input and (points_in_progress < 60000000 or not state.reader.active) and len(state.reader.active) < max_splitting_jobs_count and can_queue_more_jobs(zmq_idle_clients)): if verbose >= 1: print('Submit next portion {}'.format(state.reader.input[-1])) _id = 'root_{}'.format(len(state.reader.input)).encode('ascii') file, portion = state.reader.input.pop() points_in_progress += portion[1] - portion[0] zmq_send_to_process(zmq_idle_clients, zmq_skt, [ pickle.dumps({ 'filename': file, 'offset_scale': (-avg_min, root_scale, rotation_matrix[:3, :3].T if rotation_matrix is not None else None, infos['color_scale']), 'portion': portion, 'id': _id }) ]) state.reader.active.append(_id) # if at this point we have no work in progress => we're done if len(zmq_idle_clients) == jobs or zmq_processes_killed == jobs: if zmq_processes_killed < 0: zmq_send_to_all_process(zmq_idle_clients, zmq_skt, [pickle.dumps(b'shutdown')]) zmq_processes_killed = 0 else: assert points_in_pnts == infos[ 'point_count'], '!!! Invalid point count in the written .pnts (expected: {}, was: {})'.format( infos['point_count'], points_in_pnts) if verbose >= 1: print('Writing 3dtiles {}'.format(infos['avg_min'])) write_tileset(working_dir, outfolder, octree_metadata, avg_min, root_scale, rotation_matrix, rgb) shutil.rmtree(working_dir) if verbose >= 1: print('Done') if benchmark is not None: print('{},{},{},{}'.format( benchmark, ','.join([os.path.basename(f) for f in files]), points_in_pnts, round(time.time() - startup, 1))) for p in zmq_processes: p.terminate() break if at_least_one_job_ended: if verbose >= 3: print('{:^16}|{:^8}|{:^8}'.format('Name', 'Points', 'Seconds')) for name, v in state.node_process.active.items(): print('{:^16}|{:^8}|{:^8}'.format( '{} ({})'.format(name.decode('ascii'), v[0]), v[1], round(now - v[2], 1))) print('') print('Pending:') print(' - root: {} / {}'.format(len(state.reader.input), initial_portion_count)) print(' - other: {} files for {} nodes'.format( sum([len(f[0]) for f in state.node_process.input.values()]), len(state.node_process.input))) print('') elif verbose >= 2: state.print_debug() if verbose >= 1: print('{} % points in {} sec [{} tasks, {} nodes, {} wip]'. format( round(100 * processed_points / infos['point_count'], 2), round(now, 1), jobs - len(zmq_idle_clients), len(state.node_process.active), points_in_progress)) elif verbose >= 0: percent = round(100 * processed_points / infos['point_count'], 2) time_left = (100 - percent) * now / (percent + 0.001) print('\r{:>6} % in {} sec [est. time left: {} sec]'.format( percent, round(now), round(time_left)), end='', flush=True) if False and int(percent) != previous_percent: print('') previous_percent = int(percent) if graph: percent = round(100 * processed_points / infos['point_count'], 3) print('{}, {}'.format(time.time() - startup, percent), file=progression_log) node_store.control_memory_usage(cache_size, verbose) if verbose >= 1: print('destroy', round(time_waiting_an_idle_process, 2)) if graph: progression_log.close() # pygal chart if graph: import pygal dateline = pygal.XY(x_label_rotation=25, secondary_range=(0, 100)) # , show_y_guides=False) for pid in activities: activity = [] filename = 'activity.{}.csv'.format(pid) i = len(activities) - activities.index(pid) - 1 # activities.index(pid) = with open(filename, 'r') as f: content = f.read().split('\n') for line in content[1:]: line = line.split(',') if line[0]: ts = float(line[0]) value = int(line[1]) / 3.0 activity.append((ts, i + value * 0.9)) os.remove(filename) if activity: activity.append((activity[-1][0], activity[0][1])) activity.append(activity[0]) dateline.add(str(pid), activity, show_dots=False, fill=True) with open('progression.csv', 'r') as f: values = [] for line in f.read().split('\n'): if line: line = line.split(',') values += [(float(line[0]), float(line[1]))] os.remove('progression.csv') dateline.add('progression', values, show_dots=False, secondary=True, stroke_style={ 'width': 2, 'color': 'black' }) dateline.render_to_file('activity.svg') context.destroy()
def test_init_from_wkt(): wgs84 = CRS.from_string("+proj=longlat +datum=WGS84 +no_defs") from_wkt = CRS(wgs84.to_wkt()) assert wgs84.to_wkt() == from_wkt.to_wkt()
# %% lat_0 = 50 lon_0 = -120 laea_proj = { 'datum': 'WGS84', 'lat_0': '50', 'lon_0': '-120', 'no_defs': 'None', 'proj': 'laea', 'type': 'crs', 'units': 'm', 'x_0': '0', 'y_0': '0' } p_laea = CRS(laea_proj) p_laea.to_wkt() # %% [markdown] # ### Step 2 is to reproject the extent in the new coordinate system # # Remember that extent order is: [xleft, xright, ybot, ytop]. We need to get these values in the new laea crs. # # Here are the two extents for the image" # %% crs_transform = Transformer.from_crs(p_utm10, p_laea) extent_utm10 = [ul_x_utm10, lr_x_utm10, lr_y_utm10, ul_y_utm10] ul_x_laea, ul_y_laea = crs_transform.transform(ul_x_utm10, ul_y_utm10) lr_x_laea, lr_y_laea = crs_transform.transform(lr_x_utm10, lr_y_utm10) extent_laea = [ul_x_laea, lr_x_laea, lr_y_laea, ul_y_laea] print(f"{extent_utm10=}")
def test_from_wkt(): wgs84 = CRS.from_string("+proj=longlat +datum=WGS84 +no_defs") from_wkt = CRS(wgs84.to_wkt()) assert wgs84.to_wkt() == from_wkt.to_wkt()