def import_data(self, from_url):
"""
Imports catchment and annual maximum flow data into the database
:param from_url: URL to pull data from (only zip files supported)
:type from_url: str
"""
self.update_state(state='PROGRESS', meta={'message': ''})
assert from_url.lower().endswith('.zip')
logger.info("Starting to import from url {}".format(from_url))
try:
db_session = db.Session()
data_fn = 'data.zip'
r = requests.get(from_url, stream=True)
with tempfile.TemporaryDirectory() as work_folder:
with open(os.path.join(work_folder, data_fn), 'wb') as data_f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
data_f.write(chunk)
logger.info("Data downloaded to {}".format(os.path.join(work_folder, data_fn)))
with zipfile.ZipFile(os.path.join(work_folder, data_fn), 'r') as data_f:
cd3_and_am = [m for m in data_f.infolist()
if os.path.splitext(m.filename)[1].lower() in ['.cd3', '.am']]
for member in cd3_and_am:
member.filename = os.path.basename(member.filename) # strip folder info, extract all files to root
data_f.extract(member, path=work_folder)
logger.info("{} Files extracted to {}".format(len(cd3_and_am), work_folder))
loaders.folder_to_db(work_folder, db_session, method='update', incl_pot=False)
logger.info("Catchment files parsed")
db_session.commit()
logger.info("Catchments committed to database")
except Exception as e:
db_session.rollback()
raise # Celery handles errors
finally:
if db_session:
db_session.close()
def build_all_stations():
session = db.Session()
# Load catchments
db.empty_db_tables()
loaders.folder_to_db('./data', session, autocommit=True, incl_pot=False)
catchments = session.query(Catchment).order_by(Catchment.id).all()
# Add WGS84 transformed points
for catchment in catchments:
lon, lat = pyproj.transform(COORDS[catchment.country], COORDS['wgs84'],
catchment.descriptors.ihdtm_ngr.x, catchment.descriptors.ihdtm_ngr.y)
catchment.point_wgs84 = Point(lon, lat)
# Put through jinja template
env = Environment(loader=FileSystemLoader('./json'))
template = env.get_template('stations.txt')
content = template.render(catchments=catchments)
with open('./json/updated_stations.geojson', 'w', encoding='utf-8') as f:
f.write(content)
session.close()
