def frequency_update(frequency):
# hourly, daily, weekly, monthly, yearly
md = session.query(MetaTable)\
.filter(MetaTable.update_freq == frequency).all()
for m in md:
update_dataset.delay(m.source_url_hash)
return '%s update complete' % frequency
def hourly_update():
md = session.query(MetaTable)\
.filter(MetaTable.update_freq == 'hourly').all()
for m in md:
update_dataset.delay(m.source_url_hash)
print 'Updating %s' % m.human_name
return 'yay'
def frequency_update(frequency):
# hourly, daily, weekly, monthly, yearly
md = session.query(MetaTable)\
.filter(MetaTable.update_freq == frequency).all()
for m in md:
update_dataset.delay(m.source_url_hash)
return '%s update complete' % frequency
def hourly_update():
md = session.query(MetaTable)\
.filter(MetaTable.update_freq == 'hourly').all()
for m in md:
update_dataset.delay(m.source_url_hash)
print 'Updating %s' % m.human_name
return 'yay'
def _update_stations(self):
reader = UnicodeCSVDictReader(self.clean_station_info)
conn = engine.connect()
for row in reader:
station = session.query(self.station_table).filter(self.station_table.c.wban_code == row['wban_code']).all()
if not station:
ins = self.station_table.insert().values(**row)
conn.execute(ins)
def _update_stations(self):
reader = UnicodeCSVDictReader(self.clean_station_info)
conn = engine.connect()
for row in reader:
station = session.query(self.station_table).filter(
self.station_table.c.wban_code == row['wban_code']).all()
if not station:
ins = self.station_table.insert().values(**row)
conn.execute(ins)
def add_shape(self, table_name):
# Associate the dataset with this celery task so we can check on the task's status
meta = session.query(ShapeMetadata).get(table_name)
meta.celery_task_id = self.request.id
session.commit()
# Ingest the shapefile
ShapeETL(meta=meta).import_shapefile()
return "Finished adding shape dataset {} from {}.".format(meta.dataset_name, meta.source_url)
def update_dataset(self, source_url_hash, s3_path=None):
md = session.query(MetaTable).get(source_url_hash)
if md.result_ids:
ids = md.result_ids
ids.append(self.request.id)
else:
ids = [self.request.id]
with engine.begin() as c:
c.execute(
MetaTable.__table__.update().where(MetaTable.source_url_hash == source_url_hash).values(result_ids=ids)
)
etl = PlenarioETL(md.as_dict())
etl.update(s3_path=s3_path)
return "Finished updating {0} ({1})".format(md.human_name, md.source_url_hash)
def update_dataset(self, source_url_hash, s3_path=None):
md = session.query(MetaTable).get(source_url_hash)
if md.result_ids:
ids = md.result_ids
ids.append(self.request.id)
else:
ids = [self.request.id]
with engine.begin() as c:
c.execute(MetaTable.__table__.update()\
.where(MetaTable.source_url_hash == source_url_hash)\
.values(result_ids=ids))
etl = PlenarioETL(md.as_dict())
etl.update(s3_path=s3_path)
return 'Finished updating {0} ({1})'.format(md.human_name, md.source_url_hash)
def delete_dataset(self, source_url_hash):
md = session.query(MetaTable).get(source_url_hash)
try:
dat_table = Table(
"dat_%s" % md.dataset_name, Base.metadata, autoload=True, autoload_with=engine, keep_existing=True
)
dat_table.drop(engine, checkfirst=True)
except NoSuchTableError:
pass
master_table = MasterTable.__table__
delete = master_table.delete().where(master_table.c.dataset_name == md.dataset_name)
conn = engine.contextual_connect()
try:
conn.execute(delete)
session.delete(md)
session.commit()
except InternalError, e:
raise delete_dataset.retry(exc=e)
def delete_dataset(self, source_url_hash):
md = session.query(MetaTable).get(source_url_hash)
try:
dat_table = Table('dat_%s' % md.dataset_name, Base.metadata,
autoload=True, autoload_with=engine, keep_existing=True)
dat_table.drop(engine, checkfirst=True)
except NoSuchTableError:
pass
master_table = MasterTable.__table__
delete = master_table.delete()\
.where(master_table.c.dataset_name == md.dataset_name)
conn = engine.contextual_connect()
try:
conn.execute(delete)
session.delete(md)
session.commit()
except InternalError, e:
raise delete_dataset.retry(exc=e)
def update_dataset(source_url_hash, s3_path=None):
md = session.query(MetaTable).get(source_url_hash)
etl = PlenarioETL(md.as_dict())
etl.update(s3_path=s3_path)
return 'Finished updating %s' % md.human_name
def add_dataset(source_url_hash, s3_path=None, data_types=None):
md = session.query(MetaTable).get(source_url_hash)
etl = PlenarioETL(md.as_dict(), data_types=data_types)
etl.add(s3_path=s3_path)
return 'Finished adding %s' % md.human_name
def delete_shape(self, table_name):
shape_meta = session.query(ShapeMetadata).get(table_name)
shape_meta.remove_table(caller_session=session)
session.commit()
return "Removed {}".format(table_name)
def update_dataset(source_url_hash, s3_path=None):
md = session.query(MetaTable).get(source_url_hash)
etl = PlenarioETL(md.as_dict())
etl.update(s3_path=s3_path)
return 'Finished updating %s' % md.human_name
def add_dataset(source_url_hash, s3_path=None, data_types=None):
md = session.query(MetaTable).get(source_url_hash)
etl = PlenarioETL(md.as_dict(), data_types=data_types)
etl.add(s3_path=s3_path)
return 'Finished adding %s' % md.human_name