def import_data(self, from_url): """ Imports catchment and annual maximum flow data into the database :param from_url: URL to pull data from (only zip files supported) :type from_url: str """ self.update_state(state='PROGRESS', meta={'message': ''}) assert from_url.lower().endswith('.zip') logger.info("Starting to import from url {}".format(from_url)) try: db_session = db.Session() data_fn = 'data.zip' r = requests.get(from_url, stream=True) with tempfile.TemporaryDirectory() as work_folder: with open(os.path.join(work_folder, data_fn), 'wb') as data_f: for chunk in r.iter_content(chunk_size=1024): if chunk: data_f.write(chunk) logger.info("Data downloaded to {}".format(os.path.join(work_folder, data_fn))) with zipfile.ZipFile(os.path.join(work_folder, data_fn), 'r') as data_f: cd3_and_am = [m for m in data_f.infolist() if os.path.splitext(m.filename)[1].lower() in ['.cd3', '.am']] for member in cd3_and_am: member.filename = os.path.basename(member.filename) # strip folder info, extract all files to root data_f.extract(member, path=work_folder) logger.info("{} Files extracted to {}".format(len(cd3_and_am), work_folder)) loaders.folder_to_db(work_folder, db_session, method='update', incl_pot=False) logger.info("Catchment files parsed") db_session.commit() logger.info("Catchments committed to database") except Exception as e: db_session.rollback() raise # Celery handles errors finally: if db_session: db_session.close()
def build_all_stations(): session = db.Session() # Load catchments db.empty_db_tables() loaders.folder_to_db('./data', session, autocommit=True, incl_pot=False) catchments = session.query(Catchment).order_by(Catchment.id).all() # Add WGS84 transformed points for catchment in catchments: lon, lat = pyproj.transform(COORDS[catchment.country], COORDS['wgs84'], catchment.descriptors.ihdtm_ngr.x, catchment.descriptors.ihdtm_ngr.y) catchment.point_wgs84 = Point(lon, lat) # Put through jinja template env = Environment(loader=FileSystemLoader('./json')) template = env.get_template('stations.txt') content = template.render(catchments=catchments) with open('./json/updated_stations.geojson', 'w', encoding='utf-8') as f: f.write(content) session.close()