def update_projectfile(self): self.loop_projectfile = export_to_projectfile(self.loop_projectfile, self.tmp_path, self.output_path, self.bbox_3d, self.proj_crs) print("PROJECTFILE FOUND AT", self.loop_projectfile)
def load_dtm(self, aus=True): polygon_ll = self.polygon.to_crs(self.dtm_crs) minlong = polygon_ll.total_bounds[0] - self.step_out maxlong = polygon_ll.total_bounds[2] + self.step_out minlat = polygon_ll.total_bounds[1] - self.step_out maxlat = polygon_ll.total_bounds[3] + self.step_out print("Fetching DTM... ", end=" bbox:") print(minlong, maxlong, minlat, maxlat) downloaded = False i = 0 print('Attempt: 0 ', end='') local_dtm = False geotif_file = 'F:/Loop_Data/BGS/terr50_gagg_gb/terr50_gagg_gb_all.tif' while downloaded == False: try: if (aus): m2l_utils.get_dtm(self.dtm_file, minlong, maxlong, minlat, maxlat) elif (local_dtm): bbox = [ self.bbox_3d["minx"], self.bbox_3d["miny"], self.bbox_3d["maxx"], self.bbox_3d["maxy"] ] m2l_utils.get_local_dtm(self.dtm_path, geotif_file, self.dtm_crs, bbox) else: m2l_utils.get_dtm_hawaii(self.dtm_file, minlong, maxlong, minlat, maxlat) downloaded = True except: time.sleep(10) i = i + 1 print(' ', i, end='') if (i == 100): raise NameError( 'map2loop error: Could not access DTM server after 100 attempts' ) print('Done.') if (not local_dtm): geom_rp = m2l_utils.reproject_dtm(self.dtm_file, self.dtm_reproj_file, self.dtm_crs, self.proj_crs) self.dtm = rasterio.open(self.dtm_reproj_file) if self.quiet == 'None': plt.imshow(self.dtm.read(1), cmap='terrain', vmin=0, vmax=1000) plt.title('DTM') plt.show()
def calc_depth_grid(self, dtb): dtm = self.dtm if dtb == "": self.dtb = 0 self.dtb_null = 0 print("dtb and dtb_null set to 0") return # TODO: DTB need to be defined, every function call bellow here that has a False boolean is referencing to the workflow['cover_map'] flag # dtb_grid = os.path.join(data_path,'young_cover_grid.tif') #obviously hard-wired for the moment # dtb_null = '-2147483648' #obviously hard-wired for the moment # cover_map_path = os.path.join(data_path,'Young_Cover_FDS_MGA_clean.shp') #obviously hard-wired for the moment # dtb_clip = os.path.join(output_path,'young_cover_grid_clip.tif') #obviously hard-wired for the moment # cover_dip = 10 # dip of cover away from contact # cover_spacing = 5000 # of contact grid in metres dtb_raw = rasterio.open(dtb_grid) cover = gpd.read_file(cover_map_path) with fiona.open(cover_map_path, "r") as shapefile: shapes = [feature["geometry"] for feature in shapefile] with rasterio.open(dtb_grid) as src: out_image, out_transform = rasterio.mask.mask(src, shapes, crop=True) out_meta = src.meta.copy() out_meta.update({ "driver": "GTiff", "height": out_image.shape[1], "width": out_image.shape[2], "transform": out_transform }) with rasterio.open(dtb_clip, "w", **out_meta) as dest: dest.write(out_image) dtb = rasterio.open(dtb_clip) m2l_geometry.process_cover(output_path, dtm, dtb, dtb_null, cover, workflow['cover_map'], cover_dip, bbox, proj_crs, cover_spacing, contact_decimate=3, use_vector=True, use_grid=True)
def round_vertices(layer_file, precision, output_file): simpledec = re.compile(r"\d*\.\d+") def mround(match): return "{:.{}f}".format(float(match.group()), precision) layer_file.geometry = layer_file.geometry.apply( lambda x: loads(re.sub(simpledec, mround, x.wkt))) layer_file.to_file(output_file) print("rounded file written to ", output_file)
def propagate_contact_dips(self, contact_dip, contact_orientation_decimate): print("Propagating dips along contacts...") orientations = pd.read_csv( os.path.join(self.output_path, 'orientations.csv'), ", ") # This is supposed to be a csv but my csv doesn't have a geometry part contacts = gpd.read_file( os.path.join(self.tmp_path, 'basal_contacts.shp')) # contact_dip = -999 # contact_orientation_decimate = 5 m2l_geometry.save_basal_contacts_orientations_csv( contacts, orientations, self.geol_clip, self.tmp_path, self.output_path, self.dtm, self.dtb, self.dtb_null, False, contact_orientation_decimate, self.c_l, contact_dip, self.dip_grid, self.spacing, self.bbox)
def explode_polylines(indf, c_l, dst_crs): # indf = gpd.GeoDataFrame.from_file(indata) outdf = gpd.GeoDataFrame(columns=indf.columns, crs=dst_crs) for idx, row in indf.iterrows(): if type(row.geometry) == LineString: outdf = outdf.append(row, ignore_index=True) if type(row.geometry) == MultiLineString: multdf = gpd.GeoDataFrame(columns=indf.columns, crs=dst_crs) recs = len(row.geometry) multdf = multdf.append([row] * recs, ignore_index=True) i = 0 for geom in range(recs): multdf.loc[geom, 'geometry'] = row.geometry[geom] multdf.loc[geom, c_l['o']] = str( multdf.loc[geom, c_l['o']]) + '_' + str(i) print('map2loop warning: Fault_' + multdf.loc[geom, c_l['o']], 'is one of a set of duplicates, so renumbering') i = i + 1 outdf = outdf.append(multdf, ignore_index=True) return outdf
def export_png(self): filename = self.loop_projectfile if self.loop_projectfile is None: # TODO: Make sure these user provided paths end with a slash or are joined properly filename = os.path.join(self.project_path, 'GEOLOGY_CLIP') print("Exporting graphical map...") try: self.geology_figure.savefig("{}.png".format(filename)) print("Geology graphic exported to: ", filename) except Exception as e: print(e) print("WARNING: Could not save geology graphic")
def create_cmap(self): # Make colours consistent from map to model formations = sorted([ formation.replace(" ", "_").replace('-', '_') for formation in list(set(self.geol_clip[self.c_l['c']].to_numpy())) ]) temp_colours = [""] * len(formations) self.colour_dict = dict(zip(formations, temp_colours)) try: # Try to retrieve the clut reference colour_ref = pd.read_csv(self.clut_path) for formation in formations: key = formation colour = None try: colour = colour_ref[colour_ref['UNITNAME'] == key]['colour'].to_numpy()[0] except Exception as e: colour = ('#%02X%02X%02X' % (random.randint(0, 255), random.randint( 0, 255), random.randint(0, 255))) self.colour_dict[key] = colour print(key, colour) except Exception as e: # Otherwise, just append a random set self.clut_path = "" random_colours = [ '#%02X%02X%02X' % (random.randint( 0, 255), random.randint(0, 255), random.randint(0, 255)) for i in range(len(formations)) ] i = 0 for key in self.colour_dict.keys(): self.colour_dict[key] = random_colours[i] self.cmap = colors.ListedColormap(self.colour_dict.values(), name='geol_key')
def test_interpolation(self, interpolation_spacing, misorientation, interpolation_scheme): geology_file = self.geol_clip_file structure_file = self.structure_clip_file basal_contacts = os.path.join(self.tmp_path, 'basal_contacts.shp') self.spacing = interpolation_spacing # grid spacing in meters # misorientation = misorientation self.scheme = interpolation_scheme orientations = self.structures quiet_interp = True if self.quiet == "None": quiet_interp = False group_girdle = m2l_utils.plot_bedding_stereonets( orientations, self.geology, self.c_l, quiet_interp) super_groups, self.use_gcode3 = Topology.super_groups_and_groups( group_girdle, self.tmp_path, misorientation) # print(super_groups) # print(self.geology['GROUP_'].unique()) bbox = self.bbox orientation_interp, contact_interp, combo_interp = m2l_interpolation.interpolation_grids( geology_file, structure_file, basal_contacts, bbox, self.spacing, self.proj_crs, self.scheme, super_groups, self.c_l) with open(os.path.join(self.tmp_path, 'interpolated_orientations.csv'), 'w') as f: f.write('X, Y, l, m, n, dip, dip_dir\n') for row in orientation_interp: ostr = '{}, {}, {}, {}, {}, {}, {}\n'.format( row[0], row[1], row[2], row[3], row[4], row[5], row[6]) f.write(ostr) with open(os.path.join(self.tmp_path, 'interpolated_contacts.csv'), 'w') as f: f.write('X, Y, l, m, angle\n') for row in contact_interp: ostr = '{}, {}, {}, {}, {}\n'.format(row[0], row[1], row[2], row[3], row[4]) f.write(ostr) with open(os.path.join(self.tmp_path, 'interpolated_combined.csv'), 'w') as f: f.write('X, Y, l, m, n, dip, dip_dir\n') for row in combo_interp: ostr = '{}, {}, {}, {}, {}, {}, {}\n'.format( row[0], row[1], row[2], row[3], row[4], row[5], row[6]) f.write(ostr) if (self.spacing < 0): self.spacing = -(bbox[2] - bbox[0]) / spacing self.x = int((bbox[2] - bbox[0]) / self.spacing) + 1 self.y = int((bbox[3] - bbox[1]) / self.spacing) + 1 x = self.x y = self.y print(x, y) dip_grid = np.ones((y, x)) dip_grid = dip_grid * -999 dip_dir_grid = np.ones((y, x)) dip_dir_grid = dip_dir_grid * -999 contact_grid = np.ones((y, x)) contact_grid = dip_dir_grid * -999 for row in combo_interp: r = int((row[1] - bbox[1]) / self.spacing) c = int((row[0] - bbox[0]) / self.spacing) dip_grid[r, c] = float(row[5]) dip_dir_grid[r, c] = float(row[6]) for row in contact_interp: r = int((row[1] - bbox[1]) / self.spacing) c = int((row[0] - bbox[0]) / self.spacing) contact_grid[r, c] = float(row[4]) self.dip_grid = dip_grid self.dip_dir_grid = dip_dir_grid if self.quiet == 'None': print('interpolated dips') plt.imshow(self.dip_grid, cmap="hsv", origin='lower', vmin=-90, vmax=90) plt.show() print('interpolated dip directions') plt.imshow(self.dip_dir_grid, cmap="hsv", origin='lower', vmin=0, vmax=360) plt.show() print('interpolated contacts') plt.imshow(contact_grid, cmap="hsv", origin='lower', vmin=-360, vmax=360) plt.show()
def run_map2model(self, deposits, aus): quiet_m2m = False if self.quiet == 'all': quiet_m2m = True if self.mindeps is not None: run_log = map2model.run(self.graph_path, self.geology_file_csv, self.fault_file_csv, self.mindep_file_csv, self.bbox_3d, self.c_l, quiet_m2m, deposits) else: run_log = map2model.run(self.graph_path, self.geology_file_csv, self.fault_file_csv, "", self.bbox_3d, self.c_l, quiet_m2m, deposits) print(run_log) print("Resolving ambiguities using ASUD...", end='\toutput_dir:') if aus: Topology.use_asud(self.strat_graph_file, self.graph_path) self.strat_graph_file = os.path.join(self.graph_path, 'ASUD_strat.gml') print("Done.") print("Generating topology graph display and unit groups...") self.G = nx.read_gml(self.strat_graph_file, label='id') selected_nodes = [n for n, v in self.G.nodes(data=True) if n >= 0] if self.quiet == 'None': nx.draw_networkx(self.G, pos=nx.kamada_kawai_layout(self.G), arrows=True, nodelist=selected_nodes) nlist = list(self.G.nodes.data('LabelGraphics')) nlist.sort() for node in nlist: if node[0] >= 0: elem = str(node[1]).replace("{'text':", "").replace( ", 'fontSize': 14}", "") # second = elem.split(":").replace("'", "") print(node[0], " ", elem) # plt.savefig(os.path.join(self.tmp_path,"topology-fig.png")) print("Topology figure saved to", os.path.join(self.tmp_path, "topology-fig.png")) # Save groups of stratigraphic units groups, self.glabels, G = Topology.get_series(self.strat_graph_file, 'id') quiet_topology = True if self.quiet == 'None': quiet_topology = False Topology.save_units( G, self.tmp_path, self.glabels, Australia=True, asud_strat_file= "https://gist.githubusercontent.com/yohanderose/3b257dc768fafe5aaf70e64ae55e4c42/raw/8598c7563c1eea5c0cd1080f2c418dc975cc5433/ASUD.csv", quiet=quiet_topology) print("Done")
def run(self): if self.quiet == "all": enable_quiet_mode() with tqdm(total=100, position=0) as pbar: pbar.update(0) print("Generating topology analyser input...") self.config.export_csv() self.config.run_map2model(self.run_flags['deposits'], self.run_flags['aus']) pbar.update(10) self.config.load_dtm(self.dtm_file if self.local else self.state) pbar.update(10) self.config.join_features() pbar.update(10) self.config.calc_depth_grid(self.run_flags['dtb']) pbar.update(10) self.config.export_orientations( self.run_flags['orientation_decimate']) pbar.update(10) self.config.export_contacts(self.run_flags['contact_decimate'], self.run_flags['intrusion_mode']) pbar.update(10) self.config.test_interpolation( self.run_flags['interpolation_spacing'], self.run_flags['misorientation'], self.run_flags['interpolation_scheme']) pbar.update(10) # TODO: make all these internal, the config class already has the run_flags dictionary self.config.export_faults(self.run_flags['fault_decimate'], self.run_flags['min_fault_length'], self.run_flags['fault_dip']) self.config.process_plutons(self.run_flags['pluton_dip'], self.run_flags['pluton_form'], self.run_flags['dist_buffer'], self.run_flags['contact_decimate']) pbar.update(20) # Seismic section is in the hamersely model area if (self.workflow['seismic_section']): self.config.extract_section_features(seismic_line_file="", seismic_bbox_file="", seismic_interp_file="") if self.workflow["contact_dips"]: self.config.propagate_contact_dips( self.run_flags['contact_dip'], self.run_flags['contact_orientation_decimate']) if (self.workflow['formation_thickness']): self.config.calc_thickness( self.run_flags['contact_decimate'], self.run_flags['null_scheme'], self.run_flags['thickness_buffer'], self.run_flags['max_thickness_allowed'], self.c_l) if self.workflow["fold_axial_traces"]: self.config.create_fold_axial_trace_points( self.run_flags['fold_decimate'], self.run_flags['fat_step'], self.run_flags['close_dip']) # Prepocess model inputs inputs = ('') if (self.workflow['model_engine'] == 'geomodeller'): inputs = ('invented_orientations', 'intrusive_orientations', 'fat_orientations', 'fault_tip_contacts', 'contact_orientations') elif (self.workflow['model_engine'] == 'loopstructural'): inputs = ('invented_orientations', 'fat_orientations', 'contact_orientations') elif (self.workflow['model_engine'] == 'gempy'): inputs = ('invented_orientations', 'interpolated_orientations', 'fat_orientations', 'contact_orientations') elif (self.workflow['model_engine'] == 'noddy'): inputs = ('') self.config.postprocess(inputs, self.workflow, self.run_flags['use_interpolations'], self.run_flags['use_fat']) pbar.update(10) self.config.save_cmap() self.config.update_projectfile() self.config.export_png() disable_quiet_mode()
def __init__(self, project_path, overwrite, geology_file, fault_file, fold_file, structure_file, mindep_file, bbox_3d, polygon, step_out, dtm_crs, proj_crs, local, quiet, loopFilename, c_l={}, **kwargs): self.project_path = project_path if overwrite is False: print( "WARNING: Overwrite should be a string value {true, in-place} ..." ) self.check_overwrite() if overwrite is True: print( "WARNING: Overwrite should be a string value {true, in-place} ... converting to true." ) overwrite = 'true' if (not os.path.exists(project_path)): # Create proj root dir if doesn't exist os.mkdir(project_path) elif overwrite == "in-place": # Pass if proj root exists and complete overwrite not wanted pass else: # Remove if exists and accept user's direction if overwrite == "true": shutil.rmtree(project_path) while os.path.exists(project_path): pass os.mkdir(project_path) else: self.check_overwrite() self.graph_path = os.path.join(self.project_path, 'graph') self.tmp_path = os.path.join(self.project_path, 'tmp') self.data_path = os.path.join(self.project_path, 'data') self.dtm_path = os.path.join(self.project_path, 'dtm') self.output_path = os.path.join(self.project_path, 'output') self.vtk_path = os.path.join(self.project_path, 'vtk') self.fault_file_csv = os.path.join(self.tmp_path, "faults.csv") self.fault_output_file_csv = os.path.join(self.output_path, "faults.csv") self.structure_file_csv = os.path.join(self.tmp_path, "structure.csv") self.geology_file_csv = os.path.join(self.tmp_path, "geology.csv") self.mindep_file_csv = os.path.join(self.tmp_path, "mindep.csv") self.strat_graph_file = os.path.join(self.graph_path, "graph_strat_NONE.gml") self.dtm_file = os.path.join(self.dtm_path, 'dtm.tif') self.dtm_reproj_file = os.path.join(self.dtm_path, 'dtm_rp.tif') if (not os.path.isdir(self.tmp_path)): os.mkdir(self.tmp_path) if (not os.path.isdir(self.data_path)): os.mkdir(self.data_path) if (not os.path.isdir(self.output_path)): os.mkdir(self.output_path) if (not os.path.isdir(self.dtm_path)): os.mkdir(self.dtm_path) if (not os.path.isdir(self.vtk_path)): os.mkdir(self.vtk_path) if (not os.path.isdir(self.graph_path)): os.mkdir(self.graph_path) self.quiet = quiet if self.quiet == 'all': enable_quiet_mode() self.bbox_3d = bbox_3d self.bbox = tuple([ bbox_3d["minx"], bbox_3d["miny"], bbox_3d["maxx"], bbox_3d["maxy"] ]) self.polygon = polygon self.step_out = step_out self.quiet = quiet self.c_l = c_l self.dtm_crs = dtm_crs self.proj_crs = proj_crs self.loop_projectfile = loopFilename # Check input maps for missing values drift_prefix = kwargs.get('drift_prefix', ['None']) self.local = local # - Check if fold file is always the same as fault or needs to be seperated # TODO: Allow for input as a polygon, not just a bounding box. structure_file, geology_file, fault_file, mindep_file, fold_file, c_l = m2l_map_checker.check_map( structure_file, geology_file, fault_file, mindep_file, fold_file, self.tmp_path, self.bbox, c_l, proj_crs, self.local, drift_prefix) # Process and store workflow params self.geology_file = geology_file self.structure_file = structure_file self.fault_file = fault_file self.fold_file = fold_file self.mindep_file = mindep_file self.clut_path = kwargs['clut_path'] disable_quiet_mode()
def preprocess(self): """[summary] :param command: [description], defaults to "" :type command: str, optional """ if self.quiet == 'all': enable_quiet_mode() geology = gpd.read_file(self.geology_file, bbox=self.bbox) geology[self.c_l['g']].fillna(geology[self.c_l['g2']], inplace=True) geology[self.c_l['g']].fillna(geology[self.c_l['c']], inplace=True) faults = gpd.read_file(self.fault_file, bbox=self.bbox) folds = gpd.read_file(self.fold_file, bbox=self.bbox) structures = gpd.read_file(self.structure_file, bbox=self.bbox) mindeps = None try: mindeps = gpd.read_file(self.mindep_file, bbox=self.bbox) mindeps.crs = self.proj_crs except Exception as e: print("Warning: Valid mineral deposit file missing") # Fix crs to project default and overwrite source geology.crs = self.proj_crs faults.crs = self.proj_crs folds.crs = self.proj_crs structures.crs = self.proj_crs self.mindeps = mindeps self.geology = geology self.faults = faults self.structures = structures # Faults self.faults_clip = faults.copy() self.faults_clip.crs = self.proj_crs self.faults_clip_file = os.path.join(self.tmp_path, "faults_clip.shp") self.faults_clip.to_file(self.faults_clip_file) # Geology self.geol_clip = m2l_utils.explode(self.geology) self.geol_clip.crs = self.proj_crs self.geol_clip_file = os.path.join(self.tmp_path, "geol_clip.shp") self.geol_clip.to_file(self.geol_clip_file) # pd.set_option('display.max_columns', None) # pd.set_option('display.max_rows', None) # Check if bedding data uses the strike convention instead of dip direction if (self.c_l['otype'] == 'strike'): structures['azimuth2'] = structures.apply( lambda row: row[self.c_l['dd']] + 90.0, axis=1) self.c_l['dd'] = 'azimuth2' self.c_l['otype'] = 'dip direction' structures.to_file(self.structure_file) # Structures list1 = [ 'geometry', self.c_l['d'], self.c_l['dd'], self.c_l['sf'], self.c_l['bo'] ] list2 = list(set(list1)) sub_pts = self.structures[list2] structure_code = gpd.sjoin(sub_pts, self.geol_clip, how="left", op="within") minx, miny, maxx, maxy = self.bbox y_point_list = [miny, miny, maxy, maxy, miny] x_point_list = [minx, maxx, maxx, minx, minx] bbox_geom = shapely.geometry.Polygon(zip(x_point_list, y_point_list)) polygo = gpd.GeoDataFrame(index=[0], crs=self.proj_crs, geometry=[bbox_geom]) is_bed = structure_code[self.c_l['sf']].str.contains( self.c_l['bedding'], regex=False) structure_clip = structure_code[is_bed] structure_clip.crs = self.proj_crs if (self.c_l['otype'] == 'strike'): structure_clip['azimuth2'] = structure_clip.apply( lambda row: row[self.c_l['dd']] + 90.0, axis=1) self.c_l['dd'] = 'azimuth2' self.c_l['otype'] = 'dip direction' self.structure_clip = structure_clip[~structure_clip[self.c_l['o']]. isnull()] self.structure_clip_file = os.path.join(self.tmp_path, 'structure_clip.shp') self.structure_clip.to_file(self.structure_clip_file) self.create_cmap() try: fig, ax = plt.subplots() plt.tight_layout() ax.ticklabel_format(axis='both', useOffset=False, style='plain') ax.margins(0.0) fig.set_facecolor("#ffffff00") self.geology_figure = geology.copy().plot( column=self.c_l['c'], ax=ax, figsize=(10, 10), edgecolor='#000000', linewidth=0.2, cmap=self.cmap).get_figure() self.export_png() fig, ax = plt.subplots() base = geology.plot(column=self.c_l['c'], figsize=(10, 10), ax=ax, edgecolor='#000000', linewidth=0.2, legend=True, cmap=self.cmap) leg = base.get_legend() leg.set_bbox_to_anchor((1.04, 1)) structures.plot(ax=base, color='none', edgecolor='black') faults.plot(ax=base, cmap='rainbow', column=self.c_l['f'], figsize=(10, 10), linewidth=0.4) structures[[ 'geometry', self.c_l['gi'], self.c_l['d'], self.c_l['dd'] ]].plot(ax=base) fig = self.polygon.plot(ax=base, color='none', edgecolor='black').get_figure() fig.savefig(os.path.join(self.tmp_path, "input-data.png")) if self.quiet == 'None': plt.show() return except Exception as e: print(e) disable_quiet_mode()
def show_metadata(gdf, name): if (len(gdf) > 0): print("\n", name, " metadata\n--------------------") print(" bbox", gdf.total_bounds) print(" CRS", gdf.crs) print(" # items", len(gdf)) types = [] for i, g in gdf.iterrows(): if (not g.geometry.type in types): types.append(g.geometry.type) print(" Data types", types) else: print("\n", name, " metadata\n--------------------") print(" empty file, check contents")
def check_map(structure_file, geology_file, fault_file, mindep_file, fold_file, tmp_path, bbox, c_l, dst_crs, local_paths, drift_prefix): # print(gpd.read_file(geology_file).columns) # print(gpd.read_file(fault_file).columns) # print(gpd.read_file(fold_file).columns) # print(gpd.read_file(mindep_file).columns) y_point_list = [bbox[1], bbox[1], bbox[3], bbox[3], bbox[1]] x_point_list = [bbox[0], bbox[2], bbox[2], bbox[0], bbox[0]] bbox_geom = Polygon(zip(x_point_list, y_point_list)) polygo = gpd.GeoDataFrame(index=[0], crs=dst_crs, geometry=[bbox_geom]) m2l_errors = [] m2l_warnings = [] for file_name in (structure_file, geology_file, fault_file, fold_file): if not file_name.startswith("http") and not os.path.isfile(file_name): m2l_errors.append('file ' + file_name + ' not found') # Process orientation points if (os.path.isfile(structure_file) or structure_file.startswith("http") or not local_paths): orientations2 = gpd.read_file(structure_file, bbox=bbox) if (c_l['sf'] == c_l['ds']): new_code = 'NEW_' + c_l['sf'] new_code = new_code[:10] orientations = orientations2.rename(columns={c_l['sf']: new_code}, errors="raise") m2l_warnings.append( 'To avoid conflict with geology field of same name, orientation field named "' + str(c_l['sf']) + '" renamed to "' + new_code + '"') c_l['sf'] = new_code else: new_code = '' orientations = orientations2.copy() if (c_l['bo'] == c_l['ds'] and not new_code == ''): c_l['bo'] = new_code if (len(orientations) < 2): m2l_errors.append( 'not enough orientations to complete calculations (need at least 2), projection may be inconsistent' ) orientations = orientations.replace(r'^\s+$', np.nan, regex=True) orientations = orientations[orientations[c_l['d']] != -999] for code in ('sf', 'd', 'dd', 'gi'): if not c_l[code] in orientations.columns: if (code == 'sf'): orientations[c_l[code]] = 'Bed' m2l_warnings.append('field named "' + str(c_l[code]) + '" added with default value "Bed"') elif (not code == 'gi'): m2l_errors.append('"' + c_l[code] + '" field needed') else: m2l_warnings.append('field named "' + str(c_l[code]) + '" added with default value') orientations[c_l[code]] = np.arange(len(orientations)) else: nans = orientations[c_l[code]].isnull().sum() if (nans > 0): m2l_warnings.append( '' + str(nans) + ' NaN/blank found in column "' + str(c_l[code]) + '" of orientations file, replacing with 0') orientations[c_l[code]].fillna("0", inplace=True) unique_o = set(orientations[c_l['gi']]) if (not len(unique_o) == len(orientations)): m2l_warnings.append('duplicate orientation point unique IDs') show_metadata(orientations, "orientations layer") # Process geology polygons if (os.path.isfile(geology_file) or geology_file.startswith("http") or not local_paths): geology = gpd.read_file(geology_file, bbox=bbox) if (not geology.empty): if not c_l['o'] in geology.columns: # print(geology.columns) geology = geology.reset_index() geology[c_l['o']] = geology.index unique_g = set(geology[c_l['o']]) if (not len(unique_g) == len(geology)): m2l_warnings.append('duplicate geology polygon unique IDs') nans = geology[c_l['c']].isnull().sum() if (nans > 0): m2l_errors.append('' + str(nans) + ' NaN/blank found in column "' + str(c_l['c']) + '" of geology file, please fix') if (c_l['g'] == 'No_col' or not c_l['g'] in geology.columns): m2l_warnings.append( 'No secondary strat coding for geology polygons') c_l['g'] = 'group' geology[c_l['g']] = "Top" geology = geology.replace(r'^\s+$', np.nan, regex=True) geology = geology.replace(',', ' ', regex=True) geology[c_l['g']].fillna(geology[c_l['g2']], inplace=True) geology[c_l['g']].fillna(geology[c_l['c']], inplace=True) if (c_l['r1'] == 'No_col' or not c_l['r1'] in geology.columns): m2l_warnings.append('No extra litho for geology polygons') c_l['r1'] = 'r1' geology[c_l['r1']] = 'Nope' if (c_l['r2'] == 'No_col' or not c_l['r2'] in geology.columns): m2l_warnings.append('No more extra litho for geology polygons') c_l['r2'] = 'r2' geology[c_l['r2']] = 'Nope' if (c_l['min'] == 'No_col' or not c_l['min'] in geology.columns): m2l_warnings.append('No min age for geology polygons') c_l['min'] = 'min' geology[c_l['min']] = 0 if (c_l['max'] == 'No_col' or not c_l['max'] in geology.columns): m2l_warnings.append('No max age for geology polygons') c_l['max'] = 'max' geology[c_l['max']] = 100 if (c_l['c'] == 'No_col' or not c_l['c'] in geology.columns): m2l_errors.append( 'Must have primary strat coding field for geology polygons' ) for code in ('c', 'g', 'g2', 'ds', 'u', 'r1'): if (c_l[code] in geology.columns): geology[c_l[code]].str.replace(",", " ") if (code == 'c' or code == 'g' or code == 'g2'): geology[c_l[code]].str.replace(" ", "_") geology[c_l[code]].str.replace("-", "_") nans = geology[c_l[code]].isnull().sum() if (nans > 0): m2l_warnings.append( '' + str(nans) + ' NaN/blank found in column "' + str(c_l[code]) + '" of geology file, replacing with 0') geology[c_l[code]].fillna("0", inplace=True) for drift in drift_prefix: geology = geology[~geology[c_l['u']].str.startswith(drift)] show_metadata(geology, "geology layer") else: print('No geology in area, projection may be inconsistent') # Process fold polylines folds = None if (os.path.isfile(fold_file) or not local_paths): folds = gpd.read_file(fold_file, bbox=bbox) if (len(folds) > 0): if not c_l['o'] in folds.columns: folds = folds.reset_index() folds[c_l['o']] = folds.index unique_g = set(folds[c_l['o']]) if (not len(unique_g) == len(folds)): m2l_warnings.append('duplicate fold polyline unique IDs') folds = folds[folds[c_l['ff']].str.contains(c_l['fold'], case=False)] folds = folds.replace(r'^\s+$', np.nan, regex=True) for code in ('ff', 't'): if (c_l['ff'] == 'No_col' or not c_l['ff'] in folds.columns): m2l_warnings.append('No fold code for fold polylines') c_l['ff'] = 'ff' folds[c_l['ff']] = c_l['fold'] if (c_l['t'] == 'No_col' or not c_l['t'] in folds.columns): m2l_warnings.append('No fold polarity for fold polylines') c_l['t'] = 't' folds[c_l['t']] = 'None' if (c_l[code] in folds.columns): folds[c_l[code]].str.replace(",", " ") nans = folds[c_l[code]].isnull().sum() if (nans > 0): m2l_warnings.append( '' + str(nans) + ' NaN/blank found in column "' + str(c_l[code]) + '" of folds file, replacing with 0') folds[c_l[code]].fillna("0", inplace=True) folds_clip = m2l_utils.clip_shp(folds, polygo) if (len(folds_clip) > 0): folds_explode = explode_polylines(folds_clip, c_l, dst_crs) if (len(folds_explode) > len(folds_clip)): m2l_warnings.append( 'some folds are MultiPolyLines, and have been split') folds_explode.crs = dst_crs show_metadata(folds_clip, "fold layer")
'' + str(nans) + ' NaN/blank found in column "' + str(c_l[code]) + '" of folds file, replacing with 0') folds[c_l[code]].fillna("0", inplace=True) folds_clip = m2l_utils.clip_shp(folds, polygo) if (len(folds_clip) > 0): folds_explode = explode_polylines(folds_clip, c_l, dst_crs) if (len(folds_explode) > len(folds_clip)): m2l_warnings.append( 'some folds are MultiPolyLines, and have been split') folds_explode.crs = dst_crs show_metadata(folds_clip, "fold layer") else: print('No folds in area, projection may be inconsistent') # Process fault polylines if (os.path.isfile(fault_file) or fault_file.startswith("http") or not local_paths): faults_folds = gpd.read_file(fault_file, bbox=bbox) faults = faults_folds[faults_folds[c_l['f']].str.contains(c_l['fault'], case=False)] faults = faults.replace(r'^\s+$', np.nan, regex=True) if not c_l['o'] in faults.columns: m2l_warnings.append('field named "' + str(c_l['o']) + '" added with default value') faults[c_l['o']] = np.arange(len(faults))
def run(self, aus=True, deposits="Fe,Cu,Au,NONE", dtb='', orientation_decimate=0, contact_decimate=5, intrusion_mode=0, interpolation_spacing=500, misorientation=30, interpolation_scheme='scipy_rbf', fault_decimate=5, min_fault_length=5000, fault_dip=90, pluton_dip=45, pluton_form='domes', dist_buffer=10, contact_dip=-999, contact_orientation_decimate=5, null_scheme='null', thickness_buffer=5000, max_thickness_allowed=10000, fold_decimate=5, fat_step=750, close_dip=-999, use_interpolations=True, use_fat=True): """Performs the data processing steps of the map2loop workflow. :param aus: Indicates if area is in Australia for using ASUD. Defaults to True. :type aus: bool :param deposits: Mineral deposit names for focused topology extraction. Defaults to "Fe,Cu,Au,NONE". :type deposits: str :param dtb: Path to depth to basement grid. Defaults to ''. :type dtb: str :param orientation_decimate: Save every nth orientation data point. Defaults to 0. :type orientation_decimate: int :param contact_decimate: Save every nth contact data point. Defaults to 5. :type contact_decimate: int :param intrusion_mode: 1 to exclude all intrusions from basal contacts, 0 to only exclude sills. Defaults to 0. :type intrusion_mode: intrusion_mode: int :param interpolation_spacing: Interpolation grid spacing in meters. Defaults to 500. :type interpolation_spacing: interpolation_spacing: int :param misorientation: Defaults to 30. :type misorientation: int :param interpolation_scheme: What interpolation function to use of scipy_rbf (radial basis) or scipy_idw (inverse distance weighted). Defaults to 'scipy_rbf'. :type interpolation_scheme: str :param fault_decimate: Save every nth fault data point. Defaults to 5. :type fault_decimate: int :param min_fault_length: Min fault length to be considered. Defaults to 5000. :type min_fault_length: int :param fault_dip: Defaults to 90. :type fault_dip: int :param pluton_dip: Defaults to 45. :type pluton_dip: int :param pluton_form: Possible forms from domes, saucers or pendant. Defaults to 'domes'. :type pluton_form: str :param dist_buffer: Buffer for processing plutons. Defaults to 10. :type dist_buffer: int :param contact_dip: Defaults to -999. :type contact_dip: int :param contact_orientation_decimate: Save every nth contact orientation point. Defaults to 5. :type contact_orientation_decimate: int :param null_scheme: How null values present in the dtb. Defaults to 'null'. :type null_scheme: str :param thickness_buffer: Defaults to 5000. :type thickness_buffer: int :param max_thickness_allowed: Defaults to 10000. :type max_thickness_allowed: int :param fold_decimate: Save every nth fold data point. Defaults to 5. :type fold_decimate: int :param fat_step: How much to step out normal to the fold axial trace. Defaults to 750. :type fat_step: int :param close_dip: Dip to assign to all new orientations. Defaults to -999. :type close_dip: int :param use_interpolations: Defaults to True. :type use_interpolations: bool :param use_fat: Defaults to True. :type use_fat: bool """ if self.quiet == 'all': enable_quiet_mode() with tqdm(total=100, position=0) as pbar: pbar.update(0) print("Generating topology analyser input...") self.config.export_csv() self.config.run_map2model(deposits, aus) pbar.update(10) self.config.load_dtm(aus) pbar.update(10) self.config.join_features() pbar.update(10) self.config.calc_depth_grid(dtb) pbar.update(10) self.config.export_orientations(orientation_decimate) pbar.update(10) self.config.export_contacts(contact_decimate, intrusion_mode) pbar.update(10) self.config.test_interpolation(interpolation_spacing, misorientation, interpolation_scheme) pbar.update(10) self.config.export_faults(fault_decimate, min_fault_length, fault_dip) self.config.process_plutons(pluton_dip, pluton_form, dist_buffer, contact_decimate) pbar.update(20) # Seismic section is in the hamersely model area # TODO: Implement this option better and test with Turner if (self.workflow['seismic_section']): self.config.extract_section_features(seismic_line_file="", seismic_bbox_file="", seismic_interp_file="") if (self.workflow['contact_dips']): self.config.propagate_contact_dips( contact_dip, contact_orientation_decimate) if (self.workflow['formation_thickness']): self.config.calc_thickness(contact_decimate, null_scheme, thickness_buffer, max_thickness_allowed, self.c_l) if (self.workflow['fold_axial_traces']): self.config.create_fold_axial_trace_points( fold_decimate, fat_step, close_dip) # Prepocess model inputs inputs = ('') if (self.workflow['model_engine'] == 'geomodeller'): inputs = ('invented_orientations', 'intrusive_orientations', 'fat_orientations', 'fault_tip_contacts', 'contact_orientations') elif (self.workflow['model_engine'] == 'loopstructural'): inputs = ('invented_orientations', 'fat_orientations', 'contact_orientations') elif (self.workflow['model_engine'] == 'gempy'): inputs = ('invented_orientations', 'interpolated_orientations', 'fat_orientations', 'contact_orientations') elif (self.workflow['model_engine'] == 'noddy'): inputs = ('') self.config.postprocess(inputs, self.workflow, use_interpolations, use_fat) pbar.update(10) self.config.save_cmap() if self.loopFilename is not None: self.config.update_projectfile() self.config.export_png() disable_quiet_mode()
def update_config(self, out_dir, overwrite='false', loopFilename=None, bbox_3d={ "minx": 0, "maxx": 0, "maxx": 0, "maxy": 0, "base": -10000, "top": 1200, }, dtm_crs={'init': 'EPSG:4326'}, proj_crs=None, step_out=None, quiet='None', clut_path='', model_engine='loopstructural', run_flags=None, **kwargs): """Creates a 'sub-project' Config object and pre-processes input data for some area. :param out_dir: Path to write output files to. :type out_dir: string :param overwrite: Allow overwriting the given out_dir if it exists, false, true or in-place, defaults to false :type overwrite: string, optional :param bbox_3d: 3D bounding box of coordinates and base/top values defining the area, defaults to { "minx": 0, "maxx": 0, "maxx": 0, "maxy": 0, "base": -10000, "top": 1200, } :type bbox_3d: dict, optional :param dtm_crs: Set the projection of the dtm, defaults to {'init': 'EPSG:4326'} :type dtm_crs: dict, optional :param proj_crs: Set the projection of the input data, defaults to None :type proj_crs: dict, optional :param step_out: How far to consider outside the re-projected dtm, defaults to None :type step_out: int, optional :param quiet: Allow or block print statements and matplotlib figures, 'None' to quiet nothing, 'all' to quiet everything, 'no-figures' to disable plots and allow text output. Defaults to 'None' :type quiet: string, optional :param clut_path: Path to custom map colours file :type clut_path: string, optional :param model_engine: Which modelling engine to use and set associated flags for, defaults to loopstructural :type model_engine: string, optional :param run_flags: Global dictionary that defines custom params such as decimates and fault length see https://github.com/Loop3D/map2loop-2/issues/56 :type run_flags: dict, optional :param **kwargs: """ self.update_workflow(model_engine) self.loopFilename = loopFilename if self.loopFilename is not None: if not os.path.exists(loopFilename): sys.exit("That project file path does not exist.") if bbox_3d["minx"] == 0 and bbox_3d["maxx"] == 0: bbox_3d.update({ "minx": self.proj_bounds[0], "minx": self.proj_bounds[1], "minx": self.proj_bounds[2], "minx": self.proj_bounds[3], }) self.clut_path = "" if proj_crs is None: proj_crs = self.proj_crs if step_out is None: step_out = self.step_out self.quiet = quiet bbox = tuple([ bbox_3d["minx"], bbox_3d["miny"], bbox_3d["maxx"], bbox_3d["maxy"] ]) minx, miny, maxx, maxy = bbox lat_point_list = [miny, miny, maxy, maxy, maxy] lon_point_list = [minx, maxx, maxx, minx, minx] bbox_geom = Polygon(zip(lon_point_list, lat_point_list)) polygon = gpd.GeoDataFrame(index=[0], crs=proj_crs, geometry=[bbox_geom]) # Define the url queries if remote flag is set if self.geology_file is None: self.fetch_sources(bbox) # TODO: Make run flags global vars that can be updated here instead of in run if clut_path != "": self.clut_path = clut_path try: # Check if (perhaps editted) run_flags already exist self.run_flags = self.run_flags except Exception: # Otherwise set them up self.run_flags = { 'aus': True, 'deposits': "Fe,Cu,Au,NONE", 'dtb': '', 'orientation_decimate': 0, 'contact_decimate': 5, 'intrusion_mode': 0, 'interpolation_spacing': 500, 'misorientation': 30, 'interpolation_scheme': 'scipy_rbf', 'fault_decimate': 5, 'min_fault_length': 5000, 'fault_dip': 90, 'pluton_dip': 45, 'pluton_form': 'domes', 'dist_buffer': 10, 'contact_dip': -999, 'contact_orientation_decimate': 5, 'null_scheme': 'null', 'thickness_buffer': 5000, 'max_thickness_allowed': 10000, 'fold_decimate': 5, 'fat_step': 750, 'close_dip': -999, 'use_interpolations': True, 'use_fat': True } # And copy in any new settings from the user if run_flags is not None: try: for key in self.run_flags.keys(): try: self.run_flags[key] = run_flags[key] except Exception: pass except Exception: print('run_flags must be a dictionary, setting defaults.') kwargs = {'clut_path': self.clut_path, 'run_flags': self.run_flags} self.config = Config(out_dir, overwrite, self.geology_file, self.fault_file, self.fold_file, self.structure_file, self.mindep_file, bbox_3d, polygon, step_out, dtm_crs, proj_crs, self.local, self.quiet, self.loopFilename, self.c_l, **kwargs) self.config.preprocess()