def make_layers() -> list[Layer]: return [ Layer( id=key, title=params['title'], description=params['description'], tags=[], input=LayerInput( dataset=dataset, asset=dataset.assets['only'], ), steps=[ *ogr2ogr( input_file='{input_dir}/' + params['input_filename'], output_file='{output_dir}/final.gpkg', boundary_filepath=project.boundaries['background']. filepath, ogr2ogr_args=[ '-where', f'"\"layer\" = \'{params["layer_name"]}\'"', ], ), ], ) for key, params in LAYER_PARAMS.items() ]
def _make_layers() -> Generator[Layer, None, None]: for start_date, end_date in _gravimetric_mass_balance_date_ranges: start_year = start_date.year end_year = end_date.year yield Layer( id=f'esa_cci_gravimetric_mass_balance_dtu_{start_year}_{end_year}', title=f'Mass balance trend {start_year}-{end_year}', description= (f"""Trend derived from the period {start_year}-01-01 to {end_year}-12-31 via gravity measurements. Data is on a ~50 km resolution grid (displayed as points)."""), tags=[], style='gmb_dtu_space', input=LayerInput( dataset=dataset, asset=dataset.assets['only'], ), steps=[ *ogr2ogr( input_file=( '{input_dir}/QGREENLAND_GEOPACKAGES/' f'points_{start_date:%Y-%m-%d}_{end_date:%Y-%m-%d}.gpkg' ), output_file=( '{output_dir}/' f'points_{start_date:%Y-%m-%d}_{end_date:%Y-%m-%d}.gpkg' ), boundary_filepath=project.boundaries['data'].filepath, ), ], )
def _make_layer( *, id: str, title: str, description: str, asset_id: str, partial_filename: str, ) -> Layer: common_description = """The geomagnetic dip poles are positions on the Earth's surface where the geomagnetic field is perpendicular to the ellipsoid, that is, vertical. The north and south dip poles do not have to be (and are not now) antipodal. """ return Layer( id=id, title=title, description=description.format(common_description=common_description), tags=['wmm'], style='geomagnetic_north_pole', input=LayerInput( dataset=wmm.wmm, asset=wmm.wmm.assets[asset_id], ), steps=[ # Add a header to the downloaded txt file so that it can be processed as # 'csv' by `ogr2ogr` CommandStep(args=[ 'sed', '"1i longitude latitude year"', '{input_dir}/' + f'{partial_filename}.xy', '>', '{output_dir}/' + f'{partial_filename}_with_header.xy', ], ), *ogr2ogr( input_file='CSV:{input_dir}/' + f'{partial_filename}_with_header.xy', output_file='{output_dir}/geomagnetic_north_pole.gpkg', ogr2ogr_args=( '-oo', 'X_POSSIBLE_NAMES=longitude', '-oo', 'Y_POSSIBLE_NAMES=latitude', '-s_srs', 'EPSG:4326', ), ), ], )
def _make_layer( *, layer_id: str, title: str, description: str, style: str, where_sql: str, ) -> Layer: return Layer( id=layer_id, title=title, description=description, tags=[], style=style, input=LayerInput( dataset=nunagis_protected_areas, asset=nunagis_protected_areas.assets['only'], ), steps=[ *ogr2ogr( input_file='{input_dir}/fetched.geojson', output_file='{output_dir}/' + f'{layer_id}.gpkg', ogr2ogr_args=[ '-dialect', 'sqlite', '-sql', f"""\"SELECT DATETIME( CAST(created_date AS INTEGER) / 1000, 'unixepoch' ) as created_date, DATETIME( CAST(last_edited_date AS INTEGER) / 1000, 'unixepoch' ) as last_edited_date, * FROM ESRIJSON WHERE {where_sql}\" """, ], ), ], )
def compressed_vector( *, input_file: str, output_file: str, vector_filename: str = '*.shp', decompress_step_kwargs=default_decompress_step_kwargs, ogr2ogr_args: StepArgs = (), boundary_filepath: EvalFilePath = project.boundaries['background']. filepath, ) -> list[CommandStep]: """Unzip a vector data file and reproject.""" return [ decompress_step( input_file=input_file, **decompress_step_kwargs, ), *ogr2ogr( input_file='{input_dir}/' + vector_filename, output_file=output_file, boundary_filepath=boundary_filepath, ogr2ogr_args=ogr2ogr_args, ), ]
from qgreenland.config.datasets.glacier_terminus import glacier_terminus as dataset from qgreenland.config.helpers.layers.glacier_terminus import LAYER_YEARS from qgreenland.config.helpers.layers.glacier_terminus import id_str from qgreenland.config.helpers.steps.ogr2ogr import ogr2ogr from qgreenland.models.config.layer import Layer, LayerInput layers = [ Layer( id=id_str(start=START, end=END), title=f'Glacier termini {START} to {END}', description=f'Glacier terminus during the {START}-{END} winter season.', tags=[], input=LayerInput( dataset=dataset, asset=dataset.assets[f'{START}_{END}'], ), steps=[ *ogr2ogr( input_file='{input_dir}/termini_*.shp', output_file='{output_dir}/boundary.gpkg', ), ], ) for (START, END) in LAYER_YEARS ]
gem_research_stations as dataset, ) from qgreenland.config.helpers.steps.ogr2ogr import ogr2ogr from qgreenland.config.project import project from qgreenland.models.config.layer import Layer, LayerInput gem_research_stations = Layer( id='gem_research_stations', title='GEM research stations', description=( """Location and description of Greenland Ecosystem Monitoring research stations."""), tags=[], style='labeled_point', input=LayerInput( dataset=dataset, asset=dataset.assets['only'], ), steps=[ *ogr2ogr( input_file='{input_dir}/gem_research_stations.geojson', output_file='{output_dir}/final.gpkg', boundary_filepath=project.boundaries['data'].filepath, ogr2ogr_args=( '-sql', ("""'SELECT *, "Station Name" as label FROM "gem_research_stations"'"""), ), ), ], )
*ogr2ogr( input_file='{input_dir}/earthquakes.gpkg', output_file='{output_dir}/earthquakes.gpkg', boundary_filepath=project.boundaries['background'].filepath, ogr2ogr_args=( '-dialect', 'sqlite', '-sql', """\"SELECT geom, id, mag, place, DATETIME(time / 1000, 'unixepoch') as time, DATETIME(updated / 1000, 'unixepoch') as updated, tz, url, detail, felt, cdi, mmi, alert, tsunami, sig, net, code, ids, sources, types, nst, dmin, rms, gap, magType, type, title, title as label FROM merged\"""", ), ),
from qgreenland.models.config.layer import Layer, LayerInput qgr_boundary_data = Layer( id='qgr_boundary_data', title='Greenland-focused boundary', description=("""Polygon representing the tight boundary for QGreenland data/imagery."""), tags=[], style='transparent_shape', input=LayerInput( dataset=dataset, asset=dataset.assets['data'], ), steps=[ *ogr2ogr( input_file='{input_dir}/greenland_rectangle.geojson', output_file='{output_dir}/boundary.gpkg', ), ], ) qgr_boundary_background = Layer( id='qgr_boundary_background', title='Background boundary', description=( """Polygon representing the full boundary of QGreenland background data/imagery."""), tags=[], show=True, style='transparent_shape', input=LayerInput( dataset=dataset,
arctic_sea_routes = Layer( id='arctic_sea_routes', title='Arctic sea routes', description=( """Lines depict the Northern Sea Route, Northwest Passate, and hypothetical Transpolar Route.""" ), tags=[], style='arctic_sea_routes', input=LayerInput( dataset=dataset, asset=dataset.assets['only'], ), steps=[ # TODO: *compressed_vector(...)? CommandStep( args=[ 'unzip', '-d', '{output_dir}', '{input_dir}/Shipping_and_Hydrography-shp.zip', ], ), *ogr2ogr( input_file='{input_dir}/Arctic_Sea_Routes.shp', output_file='{output_dir}/arctic_sea_routes.gpkg', ), ], )
from qgreenland.config.helpers.steps.ogr2ogr import ogr2ogr from qgreenland.models.config.layer import Layer, LayerInput arctic_circle = Layer( id='arctic_circle', title="Arctic Circle (66° 34' North)", description=( """The Arctic Circle is an imaginary line that circles the globe at approximately 66° 34' N and marks the latitude above which the sun does not set on the summer solstice, and does not rise on the winter solstice.""" ), tags=[], style='arctic_circle', input=LayerInput( dataset=dataset, asset=dataset.assets['only'], ), steps=[ *ogr2ogr( input_file='{input_dir}/arctic_circle.geojson', output_file='{output_dir}/arctic_circle.gpkg', ogr2ogr_args=( '-segmentize', '1', '-s_srs', 'EPSG:4326', ), ), ], )
id='populated_places', title='Towns and settlements', description='Points representing towns and settlements in Greenland.', tags=['places'], style='labeled_point', input=LayerInput( dataset=asiaq_private_placenames, asset=asiaq_private_placenames.assets['only'], ), steps=[ *ogr2ogr( input_file='{input_dir}/translations_joined.gpkg', output_file='{output_dir}/final.gpkg', boundary_filepath=project.boundaries['data'].filepath, ogr2ogr_args=( '-sql', ( "'SELECT *, \"New Greenlandic\" as label" ' FROM translations_joined' " WHERE \"Object designation\" IN (\"BY\", \"BYGD\")'" ), ), ), ], ) comprehensive_places = towns_and_settlements.copy(update={ 'id': 'comprehensive_places', 'title': 'Place names database', 'description': 'Points representing named points of interest in Greenland.', 'steps': [ *ogr2ogr( input_file='{input_dir}/translations_joined.gpkg',
title='Ice slab thickness in firn ice 2010-2014', description=( """Ice slab thickness, in meters, in the top 20 meters of firn. Only slabs between 1 and 16 meters are included."""), tags=[], style='firn_ice_points', input=LayerInput( dataset=dataset, asset=dataset.assets['only'], ), steps=[ *ogr2ogr( input_file='{input_dir}/Ice_Layer_Output_Thicknesses.gpkg', output_file='{output_dir}/final.gpkg', ogr2ogr_args=( '-sql', ("""'SELECT geom, fid, Track_name, CAST(Tracenumber AS INTEGER) as Tracenumber, lat, lon, CAST(alongtrack_distance_m AS REAL) as alongtrack_distance_m, CAST("20m_ice_content_m" AS REAL) as "20m_ice_content_m" FROM Ice_Layer_Output_Thicknesses'"""), ), ), ], )
'-d', '{output_dir}', ], ), *ogr2ogr( input_file='{input_dir}/greenland_sgl_s2_20190501_20191001_jakobshavn_merged_v1_1.shp', output_file='{output_dir}/selected.gpkg', boundary_filepath=project.boundaries['data'].filepath, ogr2ogr_args=( '-dialect', 'sqlite', '-sql', """\"SELECT Geometry, id1, DATE( substr(date, 1, 4) || '-' || substr(date, 5, 2) || '-' || substr(date, 7, 2) ) as date, area1, elev, source, tile, row FROM greenland_sgl_s2_20190501_20191001_jakobshavn_merged_v1_1\" """, ), ), ], )
), steps=[ *ogr2ogr( input_file='{input_dir}/locations.gpkg', output_file='{output_dir}/final.gpkg', boundary_filepath=project.boundaries['background'].filepath, ogr2ogr_args=( '-s_srs', 'EPSG:4326', '-sql', ("""'SELECT _ogr_geometry_, fid, glacier_id, glacier_name, glacier_type, glacier_lat, glacier_lon, time_period, sources_data, sources_background, CAST("#_points" AS INTEGER) as "#_points", CAST("#_readings" AS INTEGER) as "#_readings", CAST("#_readings_final" AS INTEGER) as "#_readings_final", finished, comments, label FROM foo'"""), ), ), ], )
from qgreenland.config.datasets.ice_cores import ice_cores as dataset from qgreenland.config.helpers.steps.ogr2ogr import ogr2ogr from qgreenland.config.project import project from qgreenland.models.config.layer import Layer, LayerInput ice_cores = Layer( id='ice_cores', title='Ice cores', description=("""Point locations of ice cores sampled in Greenland."""), tags=[], style='labeled_point', input=LayerInput( dataset=dataset, asset=dataset.assets['only'], ), steps=[ *ogr2ogr( input_file='{input_dir}/paleo_icecore.kmz', output_file='{output_dir}/final.gpkg', boundary_filepath=project.boundaries['data'].filepath, ogr2ogr_args=( '-sql', ("""'SELECT *, Name as label FROM "Ice Core"'"""), ), ), ], )
'-e', r'"s/^\s\+//g"', # Replace all other whitespace with ',' '-e', r'"s/\s\+/,/g"', # Add a header '-e', '1i"longitude,latitude,magnetic_anomaly,index,long_wavelength"', '{input_dir}/full_wdmam.xyz', '>', '{output_dir}/full_wdmam.csv', ], ), *ogr2ogr( input_file='{input_dir}/full_wdmam.csv', output_file='{output_dir}/wdmam_greenland.gpkg', boundary_filepath=project.boundaries['data'].filepath, ogr2ogr_args=[ '-oo', 'X_POSSIBLE_NAMES=longitude', '-oo', 'Y_POSSIBLE_NAMES=latitude', '-oo', 'AUTODETECT_TYPE=True', '-s_srs', 'EPSG:4326', ], ), ], )
from qgreenland.config.datasets.seismograph_stations import ( seismograph_stations as dataset, ) from qgreenland.config.helpers.steps.ogr2ogr import ogr2ogr from qgreenland.config.project import project from qgreenland.models.config.layer import Layer, LayerInput seismograph_stations = Layer( id='seismograph_stations', title='Seismograph stations', description=( """Location and details of Greenland seismograph stations."""), tags=[], style='seismograph_stations', input=LayerInput( dataset=dataset, asset=dataset.assets['only'], ), steps=[ *ogr2ogr( input_file='{input_dir}/stations.kmz', output_file='{output_dir}/ogr2ogr.gpkg', boundary_filepath=project.boundaries['data'].filepath, ), ], )
from qgreenland.config.datasets.glacier_terminus import glacier_terminus as dataset from qgreenland.config.helpers.steps.ogr2ogr import ogr2ogr from qgreenland.models.config.layer import Layer, LayerInput glacier_terminus_glacier_ids = Layer( id='glacier_terminus_glacier_ids', title='Glacier IDs', description='Glacier location for termini with matching ID.', tags=[], style='glacier_ids', input=LayerInput( dataset=dataset, asset=dataset.assets['glacier_ids'], ), steps=[ *ogr2ogr( input_file='{input_dir}/GlacierIDs_v02.0.shp', output_file='{output_dir}/boundary.gpkg', ), ], )
layers = [ Layer( id=id, title=params['title'], description=params['description'], tags=[], style='labeled_point', input=LayerInput( dataset=dataset, asset=dataset.assets[params['asset_id']], ), steps=[ *ogr2ogr( # This CSV data is tab-delimeted, but ogr2ogr can # auto-detect that. input_file='{input_dir}/*.csv', output_file='{output_dir}/final.gpkg', ogr2ogr_args=( '-s_srs', 'EPSG:4326', '-oo', 'X_POSSIBLE_NAMES=lon', '-oo', 'Y_POSSIBLE_NAMES=lat', '-sql', fr'"SELECT *, name as label from \"{params["table_name"]}\""', ), ), ], ) for id, params in promice_layer_params.items() ]
soil_types = Layer( id='soil_types', title='Soil characteristics', description=( """Polygons representing dominant soil characteristics with percentage polygon area for each soil type. Data coverage limited to Greenland.""" ), tags=[], style='soil_types', input=LayerInput( dataset=dataset, asset=dataset.assets['only'], ), steps=[ decompress_step( decompress_type='gzip', input_file='{input_dir}/*.gz', ), *ogr2ogr( input_file='{input_dir}/ggd602_soils_greenland.shp', output_file='{output_dir}/soil_types.gpkg', boundary_filepath=project.boundaries['data'].filepath, ogr2ogr_args=( '-s_srs', '"+proj=laea +a=6370997.00 +b=6370997.00 +lat_0=90 +lon_0=180 +x_0=0 +y_0=0"', ), ), ], )
' as ice/ocean by regional climate models (filled).'), 'input_filename': 'basins_filled.gpkg', }, } layers = [ Layer( id=layer_id, title=layer_id.replace('_', ' ').capitalize(), description=("""Calculated locations for subglacial hydrologic basin ice-margin-terminating outlets."""), tags=[], style=layer_id.replace('_filled', ''), input=LayerInput( dataset=dataset, asset=dataset.assets[layer_id], ), steps=[ *ogr2ogr( input_file='{input_dir}/' + params['input_filename'], output_file='{output_dir}/' + f'{layer_id}.gpkg', ogr2ogr_args=(_stream_selection_ogr2ogr_args if 'streams' in layer_id else []), ), ], ) for layer_id, params in _layer_params.items() ] ORDERED_LAYER_IDS = list(_layer_params.keys())
nunagis_municipalities_population = Layer( id='nunagis_municipalities_population', title='Greenland municipalities and population 2019', description=( """Polygons representing municipalities of Greenland and associated population numbers for 2019."""), tags=[], style='nunagis_municipalities_population', input=LayerInput( dataset=dataset.nunagis_pop2019_municipalities, asset=dataset.nunagis_pop2019_municipalities.assets['only'], ), steps=[ *ogr2ogr( input_file='{input_dir}/fetched.geojson', output_file='{output_dir}/nunagis_municipalities_population.gpkg', ), ], ) ne_states_provinces = Layer( id='ne_states_provinces', title='Global administrative divisions', description=("""Polygons representing countries' internal administrative boundaries."""), tags=[], style='administrative_divisions', input=LayerInput( dataset=dataset.ne_states_provinces, asset=dataset.ne_states_provinces.assets['only'], ),