def make_layers() -> list[Layer]:
return [
Layer(
id=key,
title=params['title'],
description=params['description'],
tags=[],
input=LayerInput(
dataset=dataset,
asset=dataset.assets['only'],
),
steps=[
*ogr2ogr(
input_file='{input_dir}/' + params['input_filename'],
output_file='{output_dir}/final.gpkg',
boundary_filepath=project.boundaries['background'].
filepath,
ogr2ogr_args=[
'-where',
f'"\"layer\" = \'{params["layer_name"]}\'"',
],
),
],
) for key, params in LAYER_PARAMS.items()
]
def _make_layers() -> Generator[Layer, None, None]:
for start_date, end_date in _gravimetric_mass_balance_date_ranges:
start_year = start_date.year
end_year = end_date.year
yield Layer(
id=f'esa_cci_gravimetric_mass_balance_dtu_{start_year}_{end_year}',
title=f'Mass balance trend {start_year}-{end_year}',
description=
(f"""Trend derived from the period {start_year}-01-01 to {end_year}-12-31 via gravity
measurements. Data is on a ~50 km resolution grid (displayed as
points)."""),
tags=[],
style='gmb_dtu_space',
input=LayerInput(
dataset=dataset,
asset=dataset.assets['only'],
),
steps=[
*ogr2ogr(
input_file=(
'{input_dir}/QGREENLAND_GEOPACKAGES/'
f'points_{start_date:%Y-%m-%d}_{end_date:%Y-%m-%d}.gpkg'
),
output_file=(
'{output_dir}/'
f'points_{start_date:%Y-%m-%d}_{end_date:%Y-%m-%d}.gpkg'
),
boundary_filepath=project.boundaries['data'].filepath,
),
],
)
def _make_layer(
*,
id: str,
title: str,
description: str,
asset_id: str,
partial_filename: str,
) -> Layer:
common_description = """The geomagnetic dip poles are positions on the
Earth's surface where the geomagnetic field is perpendicular to the ellipsoid,
that is, vertical. The north and south dip poles do not have to be (and are not
now) antipodal.
"""
return Layer(
id=id,
title=title,
description=description.format(common_description=common_description),
tags=['wmm'],
style='geomagnetic_north_pole',
input=LayerInput(
dataset=wmm.wmm,
asset=wmm.wmm.assets[asset_id],
),
steps=[
# Add a header to the downloaded txt file so that it can be processed as
# 'csv' by `ogr2ogr`
CommandStep(args=[
'sed',
'"1i longitude latitude year"',
'{input_dir}/' + f'{partial_filename}.xy',
'>',
'{output_dir}/' + f'{partial_filename}_with_header.xy',
], ),
*ogr2ogr(
input_file='CSV:{input_dir}/' +
f'{partial_filename}_with_header.xy',
output_file='{output_dir}/geomagnetic_north_pole.gpkg',
ogr2ogr_args=(
'-oo',
'X_POSSIBLE_NAMES=longitude',
'-oo',
'Y_POSSIBLE_NAMES=latitude',
'-s_srs',
'EPSG:4326',
),
),
],
)
def _make_layer(
*,
layer_id: str,
title: str,
description: str,
style: str,
where_sql: str,
) -> Layer:
return Layer(
id=layer_id,
title=title,
description=description,
tags=[],
style=style,
input=LayerInput(
dataset=nunagis_protected_areas,
asset=nunagis_protected_areas.assets['only'],
),
steps=[
*ogr2ogr(
input_file='{input_dir}/fetched.geojson',
output_file='{output_dir}/' + f'{layer_id}.gpkg',
ogr2ogr_args=[
'-dialect', 'sqlite',
'-sql',
f"""\"SELECT
DATETIME(
CAST(created_date AS INTEGER) / 1000, 'unixepoch'
) as created_date,
DATETIME(
CAST(last_edited_date AS INTEGER) / 1000, 'unixepoch'
) as last_edited_date,
*
FROM ESRIJSON
WHERE {where_sql}\" """,
],
),
],
)
def compressed_vector(
*,
input_file: str,
output_file: str,
vector_filename: str = '*.shp',
decompress_step_kwargs=default_decompress_step_kwargs,
ogr2ogr_args: StepArgs = (),
boundary_filepath: EvalFilePath = project.boundaries['background'].
filepath,
) -> list[CommandStep]:
"""Unzip a vector data file and reproject."""
return [
decompress_step(
input_file=input_file,
**decompress_step_kwargs,
),
*ogr2ogr(
input_file='{input_dir}/' + vector_filename,
output_file=output_file,
boundary_filepath=boundary_filepath,
ogr2ogr_args=ogr2ogr_args,
),
]
from qgreenland.config.datasets.glacier_terminus import glacier_terminus as dataset
from qgreenland.config.helpers.layers.glacier_terminus import LAYER_YEARS
from qgreenland.config.helpers.layers.glacier_terminus import id_str
from qgreenland.config.helpers.steps.ogr2ogr import ogr2ogr
from qgreenland.models.config.layer import Layer, LayerInput
layers = [
Layer(
id=id_str(start=START, end=END),
title=f'Glacier termini {START} to {END}',
description=f'Glacier terminus during the {START}-{END} winter season.',
tags=[],
input=LayerInput(
dataset=dataset,
asset=dataset.assets[f'{START}_{END}'],
),
steps=[
*ogr2ogr(
input_file='{input_dir}/termini_*.shp',
output_file='{output_dir}/boundary.gpkg',
),
],
) for (START, END) in LAYER_YEARS
]
gem_research_stations as dataset, )
from qgreenland.config.helpers.steps.ogr2ogr import ogr2ogr
from qgreenland.config.project import project
from qgreenland.models.config.layer import Layer, LayerInput
gem_research_stations = Layer(
id='gem_research_stations',
title='GEM research stations',
description=(
"""Location and description of Greenland Ecosystem Monitoring research
stations."""),
tags=[],
style='labeled_point',
input=LayerInput(
dataset=dataset,
asset=dataset.assets['only'],
),
steps=[
*ogr2ogr(
input_file='{input_dir}/gem_research_stations.geojson',
output_file='{output_dir}/final.gpkg',
boundary_filepath=project.boundaries['data'].filepath,
ogr2ogr_args=(
'-sql',
("""'SELECT *, "Station Name" as label
FROM "gem_research_stations"'"""),
),
),
],
)
*ogr2ogr(
input_file='{input_dir}/earthquakes.gpkg',
output_file='{output_dir}/earthquakes.gpkg',
boundary_filepath=project.boundaries['background'].filepath,
ogr2ogr_args=(
'-dialect',
'sqlite',
'-sql',
"""\"SELECT
geom,
id,
mag,
place,
DATETIME(time / 1000, 'unixepoch') as time,
DATETIME(updated / 1000, 'unixepoch') as updated,
tz,
url,
detail,
felt,
cdi,
mmi,
alert,
tsunami,
sig,
net,
code,
ids,
sources,
types,
nst,
dmin,
rms,
gap,
magType,
type,
title,
title as label
FROM merged\"""",
),
),
from qgreenland.models.config.layer import Layer, LayerInput
qgr_boundary_data = Layer(
id='qgr_boundary_data',
title='Greenland-focused boundary',
description=("""Polygon representing the tight boundary for QGreenland
data/imagery."""),
tags=[],
style='transparent_shape',
input=LayerInput(
dataset=dataset,
asset=dataset.assets['data'],
),
steps=[
*ogr2ogr(
input_file='{input_dir}/greenland_rectangle.geojson',
output_file='{output_dir}/boundary.gpkg',
),
],
)
qgr_boundary_background = Layer(
id='qgr_boundary_background',
title='Background boundary',
description=(
"""Polygon representing the full boundary of QGreenland background
data/imagery."""),
tags=[],
show=True,
style='transparent_shape',
input=LayerInput(
dataset=dataset,
arctic_sea_routes = Layer(
id='arctic_sea_routes',
title='Arctic sea routes',
description=(
"""Lines depict the Northern Sea Route, Northwest Passate, and
hypothetical Transpolar Route."""
),
tags=[],
style='arctic_sea_routes',
input=LayerInput(
dataset=dataset,
asset=dataset.assets['only'],
),
steps=[
# TODO: *compressed_vector(...)?
CommandStep(
args=[
'unzip',
'-d', '{output_dir}',
'{input_dir}/Shipping_and_Hydrography-shp.zip',
],
),
*ogr2ogr(
input_file='{input_dir}/Arctic_Sea_Routes.shp',
output_file='{output_dir}/arctic_sea_routes.gpkg',
),
],
)
from qgreenland.config.helpers.steps.ogr2ogr import ogr2ogr
from qgreenland.models.config.layer import Layer, LayerInput
arctic_circle = Layer(
id='arctic_circle',
title="Arctic Circle (66° 34' North)",
description=(
"""The Arctic Circle is an imaginary line that circles the globe at
approximately 66° 34' N and marks the latitude above which the sun does
not set on the summer solstice, and does not rise on the winter
solstice."""
),
tags=[],
style='arctic_circle',
input=LayerInput(
dataset=dataset,
asset=dataset.assets['only'],
),
steps=[
*ogr2ogr(
input_file='{input_dir}/arctic_circle.geojson',
output_file='{output_dir}/arctic_circle.gpkg',
ogr2ogr_args=(
'-segmentize', '1',
'-s_srs', 'EPSG:4326',
),
),
],
)
id='populated_places',
title='Towns and settlements',
description='Points representing towns and settlements in Greenland.',
tags=['places'],
style='labeled_point',
input=LayerInput(
dataset=asiaq_private_placenames,
asset=asiaq_private_placenames.assets['only'],
),
steps=[
*ogr2ogr(
input_file='{input_dir}/translations_joined.gpkg',
output_file='{output_dir}/final.gpkg',
boundary_filepath=project.boundaries['data'].filepath,
ogr2ogr_args=(
'-sql', (
"'SELECT *, \"New Greenlandic\" as label"
' FROM translations_joined'
" WHERE \"Object designation\" IN (\"BY\", \"BYGD\")'"
),
),
),
],
)
comprehensive_places = towns_and_settlements.copy(update={
'id': 'comprehensive_places',
'title': 'Place names database',
'description': 'Points representing named points of interest in Greenland.',
'steps': [
*ogr2ogr(
input_file='{input_dir}/translations_joined.gpkg',
title='Ice slab thickness in firn ice 2010-2014',
description=(
"""Ice slab thickness, in meters, in the top 20 meters of firn. Only
slabs between 1 and 16 meters are included."""),
tags=[],
style='firn_ice_points',
input=LayerInput(
dataset=dataset,
asset=dataset.assets['only'],
),
steps=[
*ogr2ogr(
input_file='{input_dir}/Ice_Layer_Output_Thicknesses.gpkg',
output_file='{output_dir}/final.gpkg',
ogr2ogr_args=(
'-sql',
("""'SELECT
geom,
fid,
Track_name,
CAST(Tracenumber AS INTEGER) as Tracenumber,
lat,
lon,
CAST(alongtrack_distance_m AS REAL) as alongtrack_distance_m,
CAST("20m_ice_content_m" AS REAL) as "20m_ice_content_m"
FROM Ice_Layer_Output_Thicknesses'"""),
),
),
],
)
'-d', '{output_dir}',
],
),
*ogr2ogr(
input_file='{input_dir}/greenland_sgl_s2_20190501_20191001_jakobshavn_merged_v1_1.shp',
output_file='{output_dir}/selected.gpkg',
boundary_filepath=project.boundaries['data'].filepath,
ogr2ogr_args=(
'-dialect', 'sqlite',
'-sql',
"""\"SELECT
Geometry,
id1,
DATE(
substr(date, 1, 4)
|| '-'
|| substr(date, 5, 2)
|| '-'
|| substr(date, 7, 2)
) as date,
area1,
elev,
source,
tile,
row
FROM greenland_sgl_s2_20190501_20191001_jakobshavn_merged_v1_1\" """,
),
),
],
)
),
steps=[
*ogr2ogr(
input_file='{input_dir}/locations.gpkg',
output_file='{output_dir}/final.gpkg',
boundary_filepath=project.boundaries['background'].filepath,
ogr2ogr_args=(
'-s_srs',
'EPSG:4326',
'-sql',
("""'SELECT
_ogr_geometry_,
fid,
glacier_id,
glacier_name,
glacier_type,
glacier_lat,
glacier_lon,
time_period,
sources_data,
sources_background,
CAST("#_points" AS INTEGER) as "#_points",
CAST("#_readings" AS INTEGER) as "#_readings",
CAST("#_readings_final" AS INTEGER) as "#_readings_final",
finished,
comments,
label
FROM foo'"""),
),
),
],
)
from qgreenland.config.datasets.ice_cores import ice_cores as dataset
from qgreenland.config.helpers.steps.ogr2ogr import ogr2ogr
from qgreenland.config.project import project
from qgreenland.models.config.layer import Layer, LayerInput
ice_cores = Layer(
id='ice_cores',
title='Ice cores',
description=("""Point locations of ice cores sampled in Greenland."""),
tags=[],
style='labeled_point',
input=LayerInput(
dataset=dataset,
asset=dataset.assets['only'],
),
steps=[
*ogr2ogr(
input_file='{input_dir}/paleo_icecore.kmz',
output_file='{output_dir}/final.gpkg',
boundary_filepath=project.boundaries['data'].filepath,
ogr2ogr_args=(
'-sql',
("""'SELECT *, Name as label
FROM "Ice Core"'"""),
),
),
],
)
'-e',
r'"s/^\s\+//g"',
# Replace all other whitespace with ','
'-e',
r'"s/\s\+/,/g"',
# Add a header
'-e',
'1i"longitude,latitude,magnetic_anomaly,index,long_wavelength"',
'{input_dir}/full_wdmam.xyz',
'>',
'{output_dir}/full_wdmam.csv',
],
),
*ogr2ogr(
input_file='{input_dir}/full_wdmam.csv',
output_file='{output_dir}/wdmam_greenland.gpkg',
boundary_filepath=project.boundaries['data'].filepath,
ogr2ogr_args=[
'-oo',
'X_POSSIBLE_NAMES=longitude',
'-oo',
'Y_POSSIBLE_NAMES=latitude',
'-oo',
'AUTODETECT_TYPE=True',
'-s_srs',
'EPSG:4326',
],
),
],
)
from qgreenland.config.datasets.seismograph_stations import (
seismograph_stations as dataset, )
from qgreenland.config.helpers.steps.ogr2ogr import ogr2ogr
from qgreenland.config.project import project
from qgreenland.models.config.layer import Layer, LayerInput
seismograph_stations = Layer(
id='seismograph_stations',
title='Seismograph stations',
description=(
"""Location and details of Greenland seismograph stations."""),
tags=[],
style='seismograph_stations',
input=LayerInput(
dataset=dataset,
asset=dataset.assets['only'],
),
steps=[
*ogr2ogr(
input_file='{input_dir}/stations.kmz',
output_file='{output_dir}/ogr2ogr.gpkg',
boundary_filepath=project.boundaries['data'].filepath,
),
],
)
from qgreenland.config.datasets.glacier_terminus import glacier_terminus as dataset
from qgreenland.config.helpers.steps.ogr2ogr import ogr2ogr
from qgreenland.models.config.layer import Layer, LayerInput
glacier_terminus_glacier_ids = Layer(
id='glacier_terminus_glacier_ids',
title='Glacier IDs',
description='Glacier location for termini with matching ID.',
tags=[],
style='glacier_ids',
input=LayerInput(
dataset=dataset,
asset=dataset.assets['glacier_ids'],
),
steps=[
*ogr2ogr(
input_file='{input_dir}/GlacierIDs_v02.0.shp',
output_file='{output_dir}/boundary.gpkg',
),
],
)
layers = [
Layer(
id=id,
title=params['title'],
description=params['description'],
tags=[],
style='labeled_point',
input=LayerInput(
dataset=dataset,
asset=dataset.assets[params['asset_id']],
),
steps=[
*ogr2ogr(
# This CSV data is tab-delimeted, but ogr2ogr can
# auto-detect that.
input_file='{input_dir}/*.csv',
output_file='{output_dir}/final.gpkg',
ogr2ogr_args=(
'-s_srs', 'EPSG:4326',
'-oo', 'X_POSSIBLE_NAMES=lon',
'-oo', 'Y_POSSIBLE_NAMES=lat',
'-sql', fr'"SELECT *, name as label from \"{params["table_name"]}\""',
),
),
],
)
for id, params in promice_layer_params.items()
]
soil_types = Layer(
id='soil_types',
title='Soil characteristics',
description=(
"""Polygons representing dominant soil characteristics with percentage
polygon area for each soil type. Data coverage limited to Greenland."""
),
tags=[],
style='soil_types',
input=LayerInput(
dataset=dataset,
asset=dataset.assets['only'],
),
steps=[
decompress_step(
decompress_type='gzip',
input_file='{input_dir}/*.gz',
),
*ogr2ogr(
input_file='{input_dir}/ggd602_soils_greenland.shp',
output_file='{output_dir}/soil_types.gpkg',
boundary_filepath=project.boundaries['data'].filepath,
ogr2ogr_args=(
'-s_srs',
'"+proj=laea +a=6370997.00 +b=6370997.00 +lat_0=90 +lon_0=180 +x_0=0 +y_0=0"',
),
),
],
)
' as ice/ocean by regional climate models (filled).'),
'input_filename':
'basins_filled.gpkg',
},
}
layers = [
Layer(
id=layer_id,
title=layer_id.replace('_', ' ').capitalize(),
description=("""Calculated locations for subglacial hydrologic basin
ice-margin-terminating outlets."""),
tags=[],
style=layer_id.replace('_filled', ''),
input=LayerInput(
dataset=dataset,
asset=dataset.assets[layer_id],
),
steps=[
*ogr2ogr(
input_file='{input_dir}/' + params['input_filename'],
output_file='{output_dir}/' + f'{layer_id}.gpkg',
ogr2ogr_args=(_stream_selection_ogr2ogr_args
if 'streams' in layer_id else []),
),
],
) for layer_id, params in _layer_params.items()
]
ORDERED_LAYER_IDS = list(_layer_params.keys())
nunagis_municipalities_population = Layer(
id='nunagis_municipalities_population',
title='Greenland municipalities and population 2019',
description=(
"""Polygons representing municipalities of Greenland and associated
population numbers for 2019."""),
tags=[],
style='nunagis_municipalities_population',
input=LayerInput(
dataset=dataset.nunagis_pop2019_municipalities,
asset=dataset.nunagis_pop2019_municipalities.assets['only'],
),
steps=[
*ogr2ogr(
input_file='{input_dir}/fetched.geojson',
output_file='{output_dir}/nunagis_municipalities_population.gpkg',
),
],
)
ne_states_provinces = Layer(
id='ne_states_provinces',
title='Global administrative divisions',
description=("""Polygons representing countries' internal administrative
boundaries."""),
tags=[],
style='administrative_divisions',
input=LayerInput(
dataset=dataset.ne_states_provinces,
asset=dataset.ne_states_provinces.assets['only'],
),