def update_meta(metatable, table):
"""
After ingest/update, update the metatable registry to reflect table information.
:param metatable: MetaTable instance to update.
:param table: Table instance to update from.
:returns: None
"""
metatable.update_date_added()
metatable.obs_from, metatable.obs_to = session.query(
func.min(table.c.point_date),
func.max(table.c.point_date)
).first()
metatable.bbox = session.query(
func.ST_SetSRID(
func.ST_Envelope(func.ST_Union(table.c.geom)),
4326
)
).first()[0]
session.add(metatable)
try:
session.commit()
except:
session.rollback()
raise
def approve_dataset(source_url_hash):
# get the MetaTable row and change the approved_status and bounce back to view-datasets.
meta = session.query(MetaTable).get(source_url_hash)
json_data_types = None
if ((not meta.is_socrata_source) and meta.contributed_data_types):
json_data_types = json.loads(meta.contributed_data_types)
add_dataset_task.delay(source_url_hash, data_types=json_data_types)
upd = {'approved_status': 'true'}
meta.approved_status = 'true'
session.commit()
# Email the user who submitted that their dataset has been approved.
# email the response to somebody
msg_body = """Hello %s,\r\n
\r\n
Your dataset has been approved and added to Plenar.io:\r\n
\r\n
%s\r\n
\r\n
It should appear on http://plenar.io within 24 hours.\r\n
\r\n
Thank you!\r\n
The Plenario Team\r\n
http://plenar.io""" % (meta.contributor_name, meta.human_name)
send_mail(subject="Your dataset has been added to Plenar.io",
recipient=meta.contributor_email,
body=msg_body)
def point_meta_from_submit_form(form, is_approved):
column_names, labels = form_columns(form)
name = slugify(form['dataset_name'], delim=u'_')[:50]
md = MetaTable(
url=form['file_url'],
view_url=form.get('view_url'),
dataset_name=name,
human_name=form['dataset_name'],
attribution=form.get('dataset_attribution'),
description=form.get('dataset_description'),
update_freq=form['update_frequency'],
contributor_name=form['contributor_name'],
contributor_organization=form.get('contributor_organization'),
contributor_email=form['contributor_email'],
approved_status=is_approved,
observed_date=labels['observed_date'],
latitude=labels.get('latitude', None),
longitude=labels.get('longitude', None),
location=labels.get('location', None),
column_names=column_names
)
session.add(md)
session.commit()
return md
def approve_dataset(source_url_hash):
# get the MetaTable row and change the approved_status and bounce back to view-datasets.
meta = session.query(MetaTable).get(source_url_hash)
json_data_types = None
if ((not meta.is_socrata_source) and meta.contributed_data_types):
json_data_types = json.loads(meta.contributed_data_types)
add_dataset_task.delay(source_url_hash, data_types=json_data_types)
upd = { 'approved_status': 'true' }
meta.approved_status = 'true'
session.commit()
# Email the user who submitted that their dataset has been approved.
# email the response to somebody
msg_body = """Hello %s,\r\n
\r\n
Your dataset has been approved and added to Plenar.io:\r\n
\r\n
%s\r\n
\r\n
It should appear on http://plenar.io within 24 hours.\r\n
\r\n
Thank you!\r\n
The Plenario Team\r\n
http://plenar.io""" % (meta.contributor_name, meta.human_name)
send_mail(subject="Your dataset has been added to Plenar.io",
recipient=meta.contributor_email, body=msg_body)
def edit_shape(dataset_name):
form = EditShapeForm()
meta = session.query(ShapeMetadata).get(dataset_name)
if form.validate_on_submit():
upd = {
'human_name': form.human_name.data,
'description': form.description.data,
'attribution': form.attribution.data,
'update_freq': form.update_freq.data,
}
session.query(ShapeMetadata)\
.filter(ShapeMetadata.dataset_name == meta.dataset_name)\
.update(upd)
session.commit()
if not meta.approved_status:
approve_shape(dataset_name)
flash('%s updated successfully!' % meta.human_name, 'success')
return redirect(url_for('views.view_datasets'))
else:
pass
context = {
'form': form,
'meta': meta,
}
return render_template('admin/edit-shape.html', **context)
def ingest_from_fixture(fixture_meta, fname):
md = MetaTable(**fixture_meta)
session.add(md)
session.commit()
path = os.path.join(fixtures_path, fname)
point_etl = PlenarioETL(md, source_path=path)
point_etl.add()
def edit_dataset(source_url_hash):
form = EditDatasetForm()
meta = session.query(MetaTable).get(source_url_hash)
table = Table('dat_%s' % meta.dataset_name,
Base.metadata,
autoload=True,
autoload_with=engine)
fieldnames = table.columns.keys()
if form.validate_on_submit():
upd = {
'human_name': form.human_name.data,
'description': form.description.data,
'attribution': form.attribution.data,
'obs_from': form.obs_from.data,
'obs_to': form.obs_to.data,
'update_freq': form.update_freq.data,
'business_key': form.business_key.data,
'latitude': form.latitude.data,
'longitude': form.longitude.data,
'location': form.location.data,
'observed_date': form.observed_date.data,
}
session.query(MetaTable)\
.filter(MetaTable.source_url_hash == meta.source_url_hash)\
.update(upd)
session.commit()
flash('%s updated successfully!' % meta.human_name, 'success')
return redirect(url_for('views.view_datasets'))
context = {
'form': form,
'meta': meta,
'fieldnames': fieldnames,
}
return render_template('edit-dataset.html', **context)
def edit_shape(dataset_name):
form = EditShapeForm()
meta = session.query(ShapeMetadata).get(dataset_name)
if form.validate_on_submit():
upd = {
'human_name': form.human_name.data,
'description': form.description.data,
'attribution': form.attribution.data,
'update_freq': form.update_freq.data,
}
session.query(ShapeMetadata)\
.filter(ShapeMetadata.dataset_name == meta.dataset_name)\
.update(upd)
session.commit()
if not meta.approved_status:
approve_shape(dataset_name)
flash('%s updated successfully!' % meta.human_name, 'success')
return redirect(url_for('views.view_datasets'))
else:
pass
context = {
'form': form,
'meta': meta,
}
return render_template('admin/edit-shape.html', **context)
def ingest_from_fixture(fixture_meta, fname):
md = MetaTable(**fixture_meta)
session.add(md)
session.commit()
path = os.path.join(fixtures_path, fname)
point_etl = PlenarioETL(md, source_path=path)
point_etl.add()
def ingest_fixture(fixture):
# Add the fixture to the metadata first
shape_meta = ShapeMetadata.add(caller_session=session, human_name=fixture.human_name, source_url=None)
session.commit()
# Bypass the celery task and call on a ShapeETL directly
ShapeETL(meta=shape_meta, source_path=fixture.path).import_shapefile()
return shape_meta
def setUp(self):
session.rollback()
# Ensure we have metadata loaded into the database
# to mimic the behavior of metadata ingestion preceding file ingestion.
drop_meta('dog_park_permits')
drop_meta('community_radio_events')
drop_meta('public_opera_performances')
# Make new MetaTable objects
self.unloaded_meta = MetaTable(url='nightvale.gov/events.csv',
human_name='Community Radio Events',
business_key='Event Name',
observed_date='Date',
latitude='lat',
longitude='lon',
approved_status=True)
self.existing_meta = MetaTable(url='nightvale.gov/dogpark.csv',
human_name='Dog Park Permits',
business_key='Hooded Figure ID',
observed_date='Date',
latitude='lat',
longitude='lon',
approved_status=False)
self.opera_meta = MetaTable(url='nightvale.gov/opera.csv',
human_name='Public Opera Performances',
business_key='Event Name',
observed_date='Date',
location='Location',
approved_status=False)
session.add_all(
[self.existing_meta, self.opera_meta, self.unloaded_meta])
session.commit()
# Also, let's have one table pre-loaded...
self.existing_table = sa.Table(
'dog_park_permits', MetaData(), Column('hooded_figure_id',
Integer),
Column('point_date', TIMESTAMP, nullable=False),
Column('date', Date, nullable=True),
Column('lat', Float, nullable=False),
Column('lon', Float, nullable=False),
Column('hash', String(32), primary_key=True),
Column('geom', Geometry('POINT', srid=4326), nullable=True))
drop_if_exists(self.existing_table.name)
self.existing_table.create(bind=app_engine)
# ... with some pre-existing data
ins = self.existing_table.insert().values(
hooded_figure_id=1,
point_date=date(2015, 1, 2),
lon=-87.6495076896,
lat=41.7915865543,
geom=None,
hash='addde9be7f59e95fc08e54e29b2a947f')
app_engine.execute(ins)
def approve_dataset(source_url_hash):
# Approve it
meta = session.query(MetaTable).get(source_url_hash)
meta.approved_status = True
session.commit()
# Ingest it
add_dataset_task.delay(source_url_hash)
send_approval_email(meta.human_name, meta.contributor_name,
meta.contributor_email)
def approve_dataset(source_url_hash):
# Approve it
meta = session.query(MetaTable).get(source_url_hash)
meta.approved_status = True
session.commit()
# Ingest it
add_dataset_task.delay(source_url_hash)
send_approval_email(meta.human_name, meta.contributor_name,
meta.contributor_email)
def approve_shape(dataset_name):
# Approve it
meta = session.query(ShapeMetadata).get(dataset_name)
meta.approved_status = True
session.commit()
# Ingest it
add_shape_task.delay(dataset_name)
send_approval_email(meta.human_name, meta.contributor_name,
meta.contributor_email)
def approve_shape(dataset_name):
# Approve it
meta = session.query(ShapeMetadata).get(dataset_name)
meta.approved_status = True
session.commit()
# Ingest it
add_shape_task.delay(dataset_name)
send_approval_email(meta.human_name, meta.contributor_name,
meta.contributor_email)
def edit_dataset(source_url_hash):
form = EditDatasetForm()
meta = session.query(MetaTable).get(source_url_hash)
fieldnames = meta.column_names
num_rows = 0
if meta.approved_status:
try:
table_name = meta.dataset_name
table = Table(table_name,
Base.metadata,
autoload=True,
autoload_with=engine)
# Would prefer to just get the names from the metadata
# without needing to reflect.
fieldnames = table.columns.keys()
pk_name = [p.name for p in table.primary_key][0]
pk = table.c[pk_name]
num_rows = session.query(pk).count()
except sqlalchemy.exc.NoSuchTableError:
# dataset has been approved, but perhaps still processing.
pass
if form.validate_on_submit():
upd = {
'human_name': form.human_name.data,
'description': form.description.data,
'attribution': form.attribution.data,
'update_freq': form.update_freq.data,
'latitude': form.latitude.data,
'longitude': form.longitude.data,
'location': form.location.data,
'observed_date': form.observed_date.data,
}
session.query(MetaTable)\
.filter(MetaTable.source_url_hash == meta.source_url_hash)\
.update(upd)
session.commit()
if not meta.approved_status:
approve_dataset(source_url_hash)
flash('%s updated successfully!' % meta.human_name, 'success')
return redirect(url_for('views.view_datasets'))
else:
pass
context = {
'form': form,
'meta': meta,
'fieldnames': fieldnames,
'num_rows': num_rows,
}
return render_template('admin/edit-dataset.html', **context)
def ingest_fixture(fixture):
# Add the fixture to the metadata first
shape_meta = ShapeMetadata.add(human_name=fixture.human_name,
source_url=None,
update_freq=fixture.update_freq,
approved_status=False)
session.commit()
# Bypass the celery task and call on a ShapeETL directly
ShapeETL(meta=shape_meta, source_path=fixture.path).add()
return shape_meta
def init_user():
if plenario.settings.DEFAULT_USER:
print 'creating default user %s' % plenario.settings.DEFAULT_USER['name']
user = plenario.models.User(**plenario.settings.DEFAULT_USER)
session.add(user)
try:
session.commit()
except Exception as e:
session.rollback()
raise e
def ingest_fixture(fixture):
# Add the fixture to the metadata first
shape_meta = ShapeMetadata.add(human_name=fixture.human_name,
source_url=None,
update_freq=fixture.update_freq,
approved_status=False)
session.commit()
# Bypass the celery task and call on a ShapeETL directly
ShapeETL(meta=shape_meta, source_path=fixture.path).add()
return shape_meta
def init_user():
if plenario.settings.DEFAULT_USER:
print 'creating default user %s' % plenario.settings.DEFAULT_USER['name']
user = plenario.models.User(**plenario.settings.DEFAULT_USER)
session.add(user)
try:
session.commit()
except Exception as e:
session.rollback()
raise e
def init_master_meta_user():
print 'creating master, meta and user tables'
Base.metadata.create_all(bind=app_engine)
if plenario.settings.DEFAULT_USER:
print 'creating default user %s' % plenario.settings.DEFAULT_USER['name']
user = plenario.models.User(**plenario.settings.DEFAULT_USER)
session.add(user)
try:
session.commit()
except IntegrityError:
pass
def edit_dataset(source_url_hash):
form = EditDatasetForm()
meta = session.query(MetaTable).get(source_url_hash)
fieldnames = meta.column_names
num_rows = 0
if meta.approved_status:
try:
table_name = meta.dataset_name
table = Table(table_name, Base.metadata,
autoload=True, autoload_with=engine)
# Would prefer to just get the names from the metadata
# without needing to reflect.
fieldnames = table.columns.keys()
pk_name = [p.name for p in table.primary_key][0]
pk = table.c[pk_name]
num_rows = session.query(pk).count()
except sqlalchemy.exc.NoSuchTableError:
# dataset has been approved, but perhaps still processing.
pass
if form.validate_on_submit():
upd = {
'human_name': form.human_name.data,
'description': form.description.data,
'attribution': form.attribution.data,
'update_freq': form.update_freq.data,
'latitude': form.latitude.data,
'longitude': form.longitude.data,
'location': form.location.data,
'observed_date': form.observed_date.data,
}
session.query(MetaTable)\
.filter(MetaTable.source_url_hash == meta.source_url_hash)\
.update(upd)
session.commit()
if not meta.approved_status:
approve_dataset(source_url_hash)
flash('%s updated successfully!' % meta.human_name, 'success')
return redirect(url_for('views.view_datasets'))
else:
pass
context = {
'form': form,
'meta': meta,
'fieldnames': fieldnames,
'num_rows': num_rows,
}
return render_template('admin/edit-dataset.html', **context)
def add_dataset_to_metatable(request, url, dataset_id, dataset_info,
socrata_source, approved_status):
data_types = []
business_key = None
observed_date = None
latitude = None
longitude = None
location = None
for k, v in request.form.iteritems():
if k.startswith('data_type_'):
key = k.replace("data_type_", "")
data_types.append({"field_name": key, "data_type": v})
if k.startswith('key_type_'):
key = k.replace("key_type_", "")
if (v == "business_key"): business_key = key
if (v == "observed_date"): observed_date = key
if (v == "latitude"): latitude = key
if (v == "longitude"): longitude = key
if (v == "location"): location = key
if socrata_source:
data_types = dataset_info['columns']
url = dataset_info['source_url']
d = {
'dataset_name': slugify(request.form.get('dataset_name'),
delim=u'_')[:50],
'human_name': request.form.get('dataset_name'),
'attribution': request.form.get('dataset_attribution'),
'description': request.form.get('dataset_description'),
'source_url': url,
'source_url_hash': dataset_id,
'update_freq': request.form.get('update_frequency'),
'business_key': business_key,
'observed_date': observed_date,
'latitude': latitude,
'longitude': longitude,
'location': location,
'contributor_name': request.form.get('contributor_name'),
'contributor_organization':
request.form.get('contributor_organization'),
'contributor_email': request.form.get('contributor_email'),
'contributed_data_types': json.dumps(data_types),
'approved_status': approved_status,
'is_socrata_source': socrata_source
}
# add this to meta_master
md = MetaTable(**d)
session.add(md)
session.commit()
return md
def update_shape(self, table_name):
# Associate the dataset with this celery task
# so we can check on the task's status
meta = session.query(ShapeMetadata).get(table_name)
meta.celery_task_id = self.request.id
session.commit()
# Update the shapefile
ShapeETL(meta=meta).update()
return 'Finished updating shape dataset {} from {}.'.\
format(meta.dataset_name, meta.source_url)
def update_shape(self, table_name):
# Associate the dataset with this celery task
# so we can check on the task's status
meta = session.query(ShapeMetadata).get(table_name)
meta.celery_task_id = self.request.id
session.commit()
# Update the shapefile
ShapeETL(meta=meta).update()
return 'Finished updating shape dataset {} from {}.'.\
format(meta.dataset_name, meta.source_url)
def setUp(self):
session.rollback()
# Ensure we have metadata loaded into the database
# to mimic the behavior of metadata ingestion preceding file ingestion.
drop_meta('dog_park_permits')
drop_meta('community_radio_events')
drop_meta('public_opera_performances')
# Make new MetaTable objects
self.unloaded_meta = MetaTable(url='nightvale.gov/events.csv',
human_name='Community Radio Events',
business_key='Event Name',
observed_date='Date',
latitude='lat', longitude='lon',
approved_status=True)
self.existing_meta = MetaTable(url='nightvale.gov/dogpark.csv',
human_name='Dog Park Permits',
business_key='Hooded Figure ID',
observed_date='Date',
latitude='lat', longitude='lon',
approved_status=False)
self.opera_meta = MetaTable(url='nightvale.gov/opera.csv',
human_name='Public Opera Performances',
business_key='Event Name',
observed_date='Date',
location='Location',
approved_status=False)
session.add_all([self.existing_meta, self.opera_meta, self.unloaded_meta])
session.commit()
# Also, let's have one table pre-loaded...
self.existing_table = sa.Table('dog_park_permits', MetaData(),
Column('hooded_figure_id', Integer),
Column('point_date', TIMESTAMP, nullable=False),
Column('date', Date, nullable=True),
Column('lat', Float, nullable=False),
Column('lon', Float, nullable=False),
Column('hash', String(32), primary_key=True),
Column('geom', Geometry('POINT', srid=4326), nullable=True))
drop_if_exists(self.existing_table.name)
self.existing_table.create(bind=app_engine)
# ... with some pre-existing data
ins = self.existing_table.insert().values(hooded_figure_id=1,
point_date=date(2015, 1, 2),
lon=-87.6495076896,
lat=41.7915865543,
geom=None,
hash='addde9be7f59e95fc08e54e29b2a947f')
app_engine.execute(ins)
def init_census():
print 'initializing and populating US Census blocks'
print 'this will *also* take a few minutes ...'
census_settings = plenario.settings.CENSUS_BLOCKS
# Only try to cache to AWS if we've specified a key
save_to_s3 = (plenario.settings.AWS_ACCESS_KEY != '')
census_meta = plenario.models.ShapeMetadata.add(source_url=census_settings['source_url'],
human_name=census_settings['human_name'],
caller_session=session)
session.commit()
ShapeETL(meta=census_meta, save_to_s3=save_to_s3).import_shapefile()
def delete_dataset(self, source_url_hash):
md = session.query(MetaTable).get(source_url_hash)
try:
dat_table = md.point_table
dat_table.drop(engine, checkfirst=True)
except NoSuchTableError:
# Move on so we can get rid of the metadata
pass
session.delete(md)
try:
session.commit()
except InternalError, e:
raise delete_dataset.retry(exc=e)
def delete_dataset(self, source_url_hash):
md = session.query(MetaTable).get(source_url_hash)
try:
dat_table = md.point_table
dat_table.drop(engine, checkfirst=True)
except NoSuchTableError:
# Move on so we can get rid of the metadata
pass
session.delete(md)
try:
session.commit()
except InternalError, e:
raise delete_dataset.retry(exc=e)
def add_dataset_to_metatable(request, url, dataset_id, dataset_info, socrata_source, approved_status):
data_types = []
business_key = None
observed_date = None
latitude = None
longitude = None
location = None
for k, v in request.form.iteritems():
if k.startswith('data_type_'):
key = k.replace("data_type_", "")
data_types.append({"field_name": key, "data_type": v})
if k.startswith('key_type_'):
key = k.replace("key_type_", "")
if (v == "business_key"): business_key = key
if (v == "observed_date"): observed_date = key
if (v == "latitude"): latitude = key
if (v == "longitude"): longitude = key
if (v == "location"): location = key
if socrata_source:
data_types = dataset_info['columns']
url = dataset_info['source_url']
d = {
'dataset_name': slugify(request.form.get('dataset_name'), delim=u'_')[:50],
'human_name': request.form.get('dataset_name'),
'attribution': request.form.get('dataset_attribution'),
'description': request.form.get('dataset_description'),
'source_url': url,
'source_url_hash': dataset_id,
'update_freq': request.form.get('update_frequency'),
'business_key': business_key,
'observed_date': observed_date,
'latitude': latitude,
'longitude': longitude,
'location': location,
'contributor_name': request.form.get('contributor_name'),
'contributor_organization': request.form.get('contributor_organization'),
'contributor_email': request.form.get('contributor_email'),
'contributed_data_types': json.dumps(data_types),
'approved_status': approved_status,
'is_socrata_source': socrata_source
}
# add this to meta_master
md = MetaTable(**d)
session.add(md)
session.commit()
return md
def reset_password():
form = ResetPasswordForm()
errors = []
if form.validate_on_submit():
user = db_session.query(User).get(flask_session['user_id'])
check = user.check_password(user.name, form.old_password.data)
if check:
user.password = form.new_password.data
db_session.add(user)
db_session.commit()
flash('Password reset successful!', 'success')
else:
errors.append('Password is not correct')
return render_template('reset-password.html', form=form, errors=errors)
def import_shapefile(self):
if self.meta.is_ingested:
raise PlenarioETLError("Table {} has already been ingested.".format(self.table_name))
# NB: this function is not atomic.
# update_after_ingest could fail after _ingest_shapefile succeeds, leaving us with inaccurate metadata.
# If this becomes a problem, we can tweak the ogr2ogr import to return a big SQL string
# rather than just going ahead and importing the shapefile.
# Then we could put both operations in the same transaction.
self._ingest_shapefile()
self.meta.update_after_ingest(session)
session.commit()
def reset_password():
form = ResetPasswordForm()
errors = []
if form.validate_on_submit():
user = db_session.query(User).get(flask_session['user_id'])
check = user.check_password(user.name, form.old_password.data)
if check:
user.password = form.new_password.data
db_session.add(user)
db_session.commit()
flash('Password reset successful!', 'success')
else:
errors.append('Password is not correct')
return render_template('reset-password.html', form=form, errors=errors)
def add(self):
if self.meta.is_ingested:
raise PlenarioETLError("Table {} has already been ingested.".
format(self.table_name))
new = HashedShape(self.table_name, self.source_url, self.source_path)
try:
new.ingest()
self.meta.update_after_ingest()
session.commit()
except:
# In case ingestion failed partway through,
# be sure to leave no trace.
new.drop()
raise
def init_user():
if DEFAULT_USER['name']:
print 'Creating default user %s' % DEFAULT_USER['name']
if session.query(plenario.models.User).count() > 0:
print 'Users already exist. Skipping this step.'
return
user = plenario.models.User(**DEFAULT_USER)
session.add(user)
try:
session.commit()
except Exception as e:
session.rollback()
print "Problem while creating default user: ", e
else:
print 'No default user specified. Skipping this step.'
def shape_meta_from_submit_form(form, is_approved):
md = ShapeMetadata.add(
human_name=form['dataset_name'],
source_url=form['file_url'],
view_url=form.get('view_url'),
attribution=form.get('dataset_attribution'),
description=form.get('dataset_description'),
update_freq=form['update_frequency'],
contributor_name=form['contributor_name'],
contributor_organization=form.get('contributor_organization'),
contributor_email=form['contributor_email'],
approved_status=is_approved)
session.commit()
return md
def shape_meta_from_submit_form(form, is_approved):
md = ShapeMetadata.add(
human_name=form['dataset_name'],
source_url=form['file_url'],
view_url=form.get('view_url'),
attribution=form.get('dataset_attribution'),
description=form.get('dataset_description'),
update_freq=form['update_frequency'],
contributor_name=form['contributor_name'],
contributor_organization=form.get('contributor_organization'),
contributor_email=form['contributor_email'],
approved_status=is_approved)
session.commit()
return md
def add(self):
if self.meta.is_ingested:
raise PlenarioETLError(
"Table {} has already been ingested.".format(self.table_name))
new = HashedShape(self.table_name, self.source_url, self.source_path)
try:
new.ingest()
self.meta.update_after_ingest()
session.commit()
except:
# In case ingestion failed partway through,
# be sure to leave no trace.
new.drop()
raise
def add_user():
form = AddUserForm()
if form.validate_on_submit():
user_info = {
'name': form.name.data,
'email': form.email.data,
'password': form.password.data
}
user = User(**user_info)
db_session.add(user)
db_session.commit()
context = {
'form': form,
'name': form.name.data,
'email': form.email.data,
'users': db_session.query(User).all()
}
return render_template('admin/add-user.html', **context)
def add_user():
form = AddUserForm()
if form.validate_on_submit():
user_info = {
'name': form.name.data,
'email': form.email.data,
'password': form.password.data
}
user = User(**user_info)
db_session.add(user)
db_session.commit()
context = {
'form': form,
'name': form.name.data,
'email': form.email.data,
'users': db_session.query(User).all()
}
return render_template('add-user.html', **context)
def setUpClass(cls, shutdown=False):
# Remove tables that we're about to recreate.
# This doesn't happen in teardown because I find it helpful
# to inspect them in the DB after running the tests.
meta_table_names = ['meta_shape']
fixture_table_names = [fixture.table_name for key, fixture in fixtures.iteritems()]
drop_tables(meta_table_names + fixture_table_names)
# Re-add meta tables
init_meta()
# Fully ingest the fixtures
BasePlenarioTest.ingest_fixture(fixtures['city'])
BasePlenarioTest.ingest_fixture(fixtures['streets'])
BasePlenarioTest.ingest_fixture(fixtures['zips'])
BasePlenarioTest.ingest_fixture(fixtures['neighborhoods'])
# Add a dummy dataset to the metadata without ingesting a shapefile for it
cls.dummy_name = ShapeMetadata.add(human_name=u'Dummy Name',
source_url=None,
update_freq='yearly',
approved_status=False).dataset_name
session.commit()
tables_to_drop = [
'flu_shot_clinics',
'landmarks',
'crimes',
'meta_master'
]
drop_tables(tables_to_drop)
init_meta()
ingest_from_fixture(flu_shot_meta, flu_path)
ingest_from_fixture(landmarks_meta, landmarks_path)
ingest_from_fixture(crime_meta, crime_path)
cls.app = create_app().test_client()
'''/detail'''
def setUpClass(cls, shutdown=False):
# Remove tables that we're about to recreate.
# This doesn't happen in teardown because I find it helpful
# to inspect them in the DB after running the tests.
meta_table_names = ['meta_shape']
fixture_table_names = [
fixture.table_name for key, fixture in fixtures.iteritems()
]
drop_tables(meta_table_names + fixture_table_names)
# Re-add meta tables
init_meta()
# Fully ingest the fixtures
BasePlenarioTest.ingest_fixture(fixtures['city'])
BasePlenarioTest.ingest_fixture(fixtures['streets'])
BasePlenarioTest.ingest_fixture(fixtures['zips'])
BasePlenarioTest.ingest_fixture(fixtures['neighborhoods'])
# Add a dummy dataset to the metadata without ingesting a shapefile for it
cls.dummy_name = ShapeMetadata.add(human_name=u'Dummy Name',
source_url=None,
update_freq='yearly',
approved_status=False).dataset_name
session.commit()
tables_to_drop = [
'flu_shot_clinics', 'landmarks', 'crimes', 'meta_master'
]
drop_tables(tables_to_drop)
init_meta()
ingest_from_fixture(flu_shot_meta, flu_path)
ingest_from_fixture(landmarks_meta, landmarks_path)
ingest_from_fixture(crime_meta, crime_path)
cls.app = create_app().test_client()
'''/detail'''
def update_meta(metadata, table):
"""
After ingest/update, update the metadata registry to reflect
:param metadata:
:param table:
"""
metadata.update_date_added()
metadata.obs_from, metadata.obs_to =\
session.query(func.min(table.c.point_date),
func.max(table.c.point_date)).first()
bbox = session.query(
func.ST_SetSRID(func.ST_Envelope(func.ST_Union(table.c.geom)),
4326)).first()[0]
metadata.bbox = bbox
session.add(metadata)
try:
session.commit()
except:
session.rollback()
raise
def test_delete_shape(self):
# Can we remove a shape that's fully ingested?
city_meta = session.query(ShapeMetadata).get(
fixtures['city'].table_name)
self.assertIsNotNone(city_meta)
city_meta.remove_table()
session.commit()
city_meta = session.query(ShapeMetadata).get(
fixtures['city'].table_name)
self.assertIsNone(city_meta)
# Can we remove a shape that's only in the metadata?
dummy_meta = session.query(ShapeMetadata).get(self.dummy_name)
self.assertIsNotNone(dummy_meta)
dummy_meta.remove_table()
session.commit()
dummy_meta = session.query(ShapeMetadata).get(self.dummy_name)
self.assertIsNone(dummy_meta)
# Add them back to return to original test state
ShapeTests.ingest_fixture(fixtures['city'])
ShapeMetadata.add(human_name=u'Dummy Name',
source_url=None,
update_freq='yearly',
approved_status=False)
session.commit()
def test_delete_shape(self):
# Can we remove a shape that's fully ingested?
city_meta = session.query(ShapeMetadata).get(fixtures['city'].table_name)
self.assertIsNotNone(city_meta)
city_meta.remove_table()
session.commit()
city_meta = session.query(ShapeMetadata).get(fixtures['city'].table_name)
self.assertIsNone(city_meta)
# Can we remove a shape that's only in the metadata?
dummy_meta = session.query(ShapeMetadata).get(self.dummy_name)
self.assertIsNotNone(dummy_meta)
dummy_meta.remove_table()
session.commit()
dummy_meta = session.query(ShapeMetadata).get(self.dummy_name)
self.assertIsNone(dummy_meta)
# Add them back to return to original test state
ShapeTests.ingest_fixture(fixtures['city'])
ShapeMetadata.add(human_name=u'Dummy Name',
source_url=None,
update_freq='yearly',
approved_status=False)
session.commit()
def update_meta(metadata, table):
"""
After ingest/update, update the metadata registry to reflect
:param metadata:
:param table:
"""
metadata.update_date_added()
metadata.obs_from, metadata.obs_to =\
session.query(func.min(table.c.point_date),
func.max(table.c.point_date)).first()
bbox = session.query(func.ST_SetSRID(
func.ST_Envelope(func.ST_Union(table.c.geom)),
4326
)).first()[0]
metadata.bbox = bbox
session.add(metadata)
try:
session.commit()
except:
session.rollback()
raise
def setUpClass(cls):
# Remove tables that we're about to recreate.
# This doesn't happen in teardown because I find it helpful to inspect them in the DB after running the tests.
meta_table_names = ["dat_master", "meta_shape", "meta_master", "plenario_user"]
fixture_table_names = [fixture.table_name for key, fixture in fixtures.iteritems()]
drop_tables(meta_table_names + fixture_table_names)
# Re-add meta tables
init_master_meta_user()
# Fully ingest the fixtures
ShapeTests.ingest_fixture(fixtures["city"])
ShapeTests.ingest_fixture(fixtures["streets"])
ShapeTests.ingest_fixture(fixtures["zips"])
# Add a dummy dataset to the metadata without ingesting a shapefile for it
cls.dummy_name = ShapeMetadata.add(
caller_session=session, human_name=u"Dummy Name", source_url=None
).dataset_name
session.commit()
cls.app = create_app().test_client()
def update_meta(metatable, table):
"""
After ingest/update, update the metatable registry to reflect table information.
:param metatable: MetaTable instance to update.
:param table: Table instance to update from.
:returns: None
"""
try:
metatable.update_date_added()
metatable.obs_from, metatable.obs_to = session.query(
func.min(table.c.point_date),
func.max(table.c.point_date)
).first()
metatable.bbox = session.query(
func.ST_SetSRID(
func.ST_Envelope(func.ST_Union(table.c.geom)),
4326
)
).first()[0]
metatable.column_names = {
c.name: str(c.type) for c in metatable.column_info()
if c.name not in {u'geom', u'point_date', u'hash'}
}
session.add(metatable)
session.commit()
except:
session.rollback()
raise
def point_meta_from_submit_form(form, is_approved):
column_names, labels = form_columns(form)
name = slugify(form['dataset_name'], delim=u'_')[:50]
md = MetaTable(
url=form['file_url'],
view_url=form.get('view_url'),
dataset_name=name,
human_name=form['dataset_name'],
attribution=form.get('dataset_attribution'),
description=form.get('dataset_description'),
update_freq=form['update_frequency'],
contributor_name=form['contributor_name'],
contributor_organization=form.get('contributor_organization'),
contributor_email=form['contributor_email'],
approved_status=is_approved,
observed_date=labels['observed_date'],
latitude=labels.get('latitude', None),
longitude=labels.get('longitude', None),
location=labels.get('location', None),
column_names=column_names)
session.add(md)
session.commit()
return md
def test_delete_shape(self):
# Can we remove a shape that's fully ingested?
city_meta = session.query(ShapeMetadata).get(fixtures["city"].table_name)
self.assertIsNotNone(city_meta)
city_meta.remove_table(caller_session=session)
session.commit()
city_meta = session.query(ShapeMetadata).get(fixtures["city"].table_name)
self.assertIsNone(city_meta)
# Can we remove a shape that's only in the metadata?
dummy_meta = session.query(ShapeMetadata).get(self.dummy_name)
self.assertIsNotNone(dummy_meta)
dummy_meta.remove_table(caller_session=session)
session.commit()
dummy_meta = session.query(ShapeMetadata).get(self.dummy_name)
self.assertIsNone(dummy_meta)
# Add them back to return to original test state
ShapeTests.ingest_fixture(fixtures["city"])
ShapeMetadata.add(caller_session=session, human_name=u"Dummy Name", source_url=None)
session.commit()
if form.validate_on_submit():
upd = {
'human_name': form.human_name.data,
'description': form.description.data,
'attribution': form.attribution.data,
'update_freq': form.update_freq.data,
'business_key': form.business_key.data,
'latitude': form.latitude.data,
'longitude': form.longitude.data,
'location': form.location.data,
'observed_date': form.observed_date.data,
}
session.query(MetaTable)\
.filter(MetaTable.source_url_hash == meta.source_url_hash)\
.update(upd)
session.commit()
if (meta.approved_status != 'true'):
approve_dataset(source_url_hash)
flash('%s updated successfully!' % meta.human_name, 'success')
return redirect(url_for('views.view_datasets'))
else:
pass
context = {
'form': form,
'meta': meta,
'fieldnames': fieldnames,
'num_rows': num_rows,
'num_weather_observations': num_weather_observations,
def drop_tables(table_names):
drop_template = 'DROP TABLE IF EXISTS {};'
command = ''.join([drop_template.format(table_name) for table_name in table_names])
session.execute(command)
session.commit()
def submit_dataset():
# Slightly dumb way to make sure that POSTs are only coming from
# originating domain for the time being
referer = request.headers.get('Referer')
if referer:
referer = urlparse(referer).netloc
req_url = urlparse(request.url).netloc
if referer != req_url:
abort(401)
else:
abort(401)
resp = {'status': 'ok', 'message': ''}
status_code = 200
errors = []
post = request.form.get('data')
if not post:
try:
post = request.form.keys()[0]
except IndexError:
resp['status'] = 'error'
resp['message'] = 'Unable to decode POST data'
status_code = 400
if status_code == 200:
post = json.loads(post)
if post.get('view_url'):
if post.get('socrata'):
source_domain = urlparse(post['view_url']).netloc
four_by_four = re.findall(r'/([a-z0-9]{4}-[a-z0-9]{4})',
post['view_url'])[-1]
view_url = 'http://%s/api/views/%s' % (source_domain,
four_by_four)
dataset_info, errors, status_code = get_socrata_data_info(
view_url)
source_url = '%s/rows.csv?accessType=DOWNLOAD' % view_url
else:
dataset_info = {
'attribution': '',
'description': '',
}
source_url = post['view_url']
dataset_info['name'] = urlparse(source_url).path.split('/')[-1]
if errors:
resp['message'] = ', '.join([e for e in errors])
resp['status'] = 'error'
status_code = 400
else:
dataset_id = md5(source_url).hexdigest()
md = session.query(MetaTable).get(dataset_id)
if not md:
d = {
'dataset_name': slugify(dataset_info['name'],
delim=u'_'),
'human_name': dataset_info['name'],
'attribution': dataset_info['attribution'],
'description': dataset_info['description'],
'source_url': source_url,
'source_url_hash': dataset_id,
'update_freq': post['update_frequency'],
'business_key': post['field_definitions']['id_field'],
'observed_date':
post['field_definitions']['date_field'],
'latitude': post['field_definitions'].get('latitude'),
'longitude':
post['field_definitions'].get('longitude'),
'location': post['field_definitions'].get('location')
}
if len(d['dataset_name']) > 49:
d['dataset_name'] = d['dataset_name'][:50]
md = MetaTable(**d)
session.add(md)
session.commit()
add_dataset.delay(md.source_url_hash,
data_types=post.get('data_types'))
resp[
'message'] = 'Dataset %s submitted successfully' % dataset_info[
'name']
else:
resp['status'] = 'error'
resp['message'] = 'Must provide a url where data can be downloaded'
status_code = 400
resp = make_response(json.dumps(resp, default=dthandler), status_code)
resp.headers['Content-Type'] = 'application/json'
return resp
def drop_tables(table_names):
drop_template = 'DROP TABLE IF EXISTS {};'
command = ''.join(
[drop_template.format(table_name) for table_name in table_names])
session.execute(command)
session.commit()
def delete_shape(self, table_name):
shape_meta = session.query(ShapeMetadata).get(table_name)
shape_meta.remove_table()
session.commit()
return 'Removed {}'.format(table_name)
