def output(self): return PostgresTarget( 'tiger{year}'.format(year=self.year), '{name}_state{state}{suffix}'.format( name=SUMLEVELS[self.geography]['table'], state=self.state, suffix=SIMPLIFIED_SUFFIX))
def output(self): return PostgresTarget( 'us.census.tiger', 'UnionTigerWaterGeoms_{name}_state{state}{suffix}'.format( name=SUMLEVELS[self.geography]['table'], state=self.state, suffix=SIMPLIFIED_SUFFIX))
def output(self): tables = [ 'obs_meta', 'obs_meta_numer', 'obs_meta_denom', 'obs_meta_geom', 'obs_meta_timespan', 'obs_meta_geom_numer_timespan' ] return [ PostgresTarget('observatory', t, non_empty=False) for t in tables ] + [UpdatedMetaTarget()]
def output(self): targets = [] for config in self.config_data: tables_data = self._get_table_names(config) for table_data in tables_data: targets.append( PostgresTarget(table_data['schema'], table_data['table'], where='z = {}'.format(self.zoom_level))) return targets
def output(self): tables = ['obs_meta', 'obs_meta_geom'] if not all([ PostgresTarget('observatory', t, non_empty=False).exists() for t in tables ]): return [] targets = {} session = current_session() resp = session.execute(''' WITH subquery AS (SELECT foo.geom_id, CASE WHEN foo.key LIKE 'section%' THEN foo.key ELSE NULL END section, CASE WHEN foo.key LIKE 'subsection%' THEN foo.key ELSE NULL END subsection FROM observatory.obs_meta_geom, LATERAL (SELECT geom_id, * FROM jsonb_each(geom_tags)) foo), subquery2 as (SELECT geom_id, REPLACE(MAX(section), 'section/', '') section, REPLACE(MAX(subsection), 'subsection/', '') subsection FROM subquery GROUP BY geom_id) SELECT DISTINCT UNNEST(section_tags), unnested.subsection_tags FROM observatory.obs_meta, LATERAL (SELECT UNNEST(subsection_tags) AS subsection_tags) unnested UNION ALL SELECT DISTINCT section, subsection FROM subquery2 WHERE section IS NOT NULL AND subsection IS NOT NULL ''') for section_id, subsection_id in resp: if self.section: if not section_id.startswith(self.section): continue targets[(section_id, subsection_id)] = LocalTarget( 'catalog/source/{section}/{subsection}.rst'.format( section=strip_tag_id(section_id), subsection=strip_tag_id(subsection_id))) targets[('licenses', None)] = LocalTarget('catalog/source/licenses.rst') targets[('sources', None)] = LocalTarget('catalog/source/sources.rst') return targets
def output(self): session = current_session() for table in session.query(OBSTable): split = table.id.split('.') schema, task_id = split[0:-1], split[-1] modname = 'tasks.' + '.'.join(schema) module = __import__(modname, fromlist=['*']) exists = False for name in dir(module): kls = getattr(module, name) if not isinstance(kls, Register): continue if task_id.startswith(underscore_slugify(name)): exists = True if exists is True: LOGGER.info('{table} exists'.format(table=table)) else: # TODO drop table LOGGER.info(table) yield PostgresTarget(schema='observatory', tablename=table.tablename)
def output(self): return PostgresTarget('us.census.tiger', 'UnionTigerWaterGeoms_{name}_state{state}'.format(name=SUMLEVELS[self.geography]['table'], state=self.state))
def output(self): return PostgresTarget('tiger{year}'.format(year=self.year), SUMLEVELS[self.geography]['table'] + SIMPLIFIED_SUFFIX)
def output(self): return PostgresTarget('tiger{year}'.format(year=self.year), '{name}_state{state}'.format(name=SUMLEVELS[self.geography]['table'], state=self.state))
def output(self): schema = 'tiger{year}'.format(year=self.year) return PostgresTarget(schema, 'blocks_interpolation')
def output(self): return PostgresTarget(self.mc_schema, 'mc_' + self.geography)
def output(self): return PostgresTarget('tiler', '{country}_dz_hierarchy_geonames_{year}'.format(country=self._country, year=self.year))
def output(self): targets = [] for config in self.config_data: targets.append( PostgresTarget(config['schema'], self._get_table_name(config))) return targets
def output(self): return PostgresTarget(self.schema, self.tablename)
def output(self): return PostgresTarget('us.census.tiger', 'UnionTigerWaterGeom_' + SUMLEVELS[self.geography]['table'] + SIMPLIFIED_SUFFIX)
def output(self): return PostgresTarget('tiler', 'xyz_us_do_geoms')
def test_postgres_target_existencess(): ''' PostgresTarget existenceness should be 0 if table DNE, 1 if it exists sans \ rows, and 2 if it has rows in it. ''' session = current_session() target = PostgresTarget('public', 'to_be') assert_equals(target._existenceness(), 0) assert_equals(target.empty(), False) assert_equals(target.exists(), False) session.execute('CREATE TABLE to_be (id INT)') session.commit() assert_equals(target._existenceness(), 1) assert_equals(target.empty(), True) assert_equals(target.exists(), False) session.execute('INSERT INTO to_be VALUES (1)') session.commit() assert_equals(target._existenceness(), 2) assert_equals(target.empty(), False) assert_equals(target.exists(), True)
def output(self): config_data = self._get_config_data(self.config_file) return PostgresTarget( 'tiler', 'xyz_{country}_do_geoms'.format(country=config_data["country"]))