def get_bedcounts(self, max_ts=None, latest=False, preprocess=True):
if isinstance(max_ts, str) and max_ts.isnumeric():
max_ts = datetime.datetime.fromtimestamp(int(max_ts))
result = None
if latest:
result = self.db.get_visible_bed_counts_for_user(
user_id=None, force=True, max_date=max_ts
)
else:
result = self.db.get_bed_counts(max_date=max_ts)
result = store.to_pandas(result, max_depth=2)
if result.shape[0] == 0:
return result
to_drop = set(result.columns).intersection([
'icu_bed_counts', 'icu_users', 'icu_managers'
])
result = result.drop(columns=to_drop)
if preprocess:
result = preprocessing.preprocess_bedcounts(result)
result = result.sort_values(by=["create_date", "icu_name"])
return result
def export_icus(self) -> TextIO:
db_cols = copy.copy(ICU_COLUMNS)
db_cols.remove('region')
icus_pd = store.to_pandas(self._store.get_icus(), max_depth=1)
out_pd = icus_pd[db_cols].copy()
out_pd['region'] = icus_pd['region_name']
out_pd = out_pd[ICU_COLUMNS]
return out_pd.to_csv(index=False)
async def generate_plots(self):
db = self.db_factory.create()
df_bedcounts = to_pandas(db.get_bed_counts())
df_bedcounts = normalize_colum_names(df_bedcounts)
logging.info(
'[periodic callback] Starting plots generation with predicu')
cached_data = {'bedcounts': df_bedcounts}
generate_plots(cached_data=cached_data,
output_dir=self.config.backoffice.extra_plots_dir)
def get(self):
"""Serves a page with a table gathering current bedcount data with some extra information."""
arg_region = self.get_query_argument('region', default=None)
locale = self.get_user_locale()
current_region = None
current_region_name = locale.translate("All regions")
if arg_region is not None and arg_region.isdigit():
try:
current_region = self.db.get_region(int(arg_region))
current_region_name = current_region.name
except Exception:
logging.warning(f"Invalid query argument: region={arg_region} "
f"Falling back to all regions.")
figures = []
bed_counts = self.db.get_visible_bed_counts_for_user(
self.current_user.user_id)
if bed_counts:
bed_counts = to_pandas(bed_counts)
else:
# when no data, make sure the resulting dataframe has
# correct column names.
columns = [key for key in dir(BedCount) if not key.startswith('_')]
columns += ['icu_dept']
bed_counts = pd.DataFrame([], columns=columns)
if current_region is not None:
mask = bed_counts['icu_region_id'] == current_region.region_id
bed_counts = bed_counts[mask]
df, metrics_layout = _prepare_data(bed_counts)
p = _make_bar_plot(df, locale)
script, div = components(p)
figures.append(dict(script=script, div=div))
plots_extra = _list_extra_plots(
Path(self.config.backoffice.extra_plots_dir))
# stack two columns per row
plots_extra = _grouper(plots_extra, 2)
regions = [{
'name': el.name,
'id': el.region_id
} for el in self.db.get_regions()]
regions = list(sorted(regions, key=lambda x: x['name']))
return self.render("operational-dashboard.html",
figures=figures,
regions=regions,
current_region_name=current_region_name,
metrics_layout=metrics_layout,
plots_extra=plots_extra)
def get(self, collection='bedcounts', max_ts=None, preprocess=None):
"""Returns the proper pandas dataframe."""
if collection in ['bedcounts', 'all_bedcounts']:
latest = collection == 'bedcounts'
preprocess = not latest if preprocess is None else preprocess
return self.get_bedcounts(max_ts, latest=latest, preprocess=preprocess)
if collection == 'icus':
result = self.db.get_icus()
else:
result = self.db.get_regions()
return store.to_pandas(result, max_depth=0)
def test_integration_generate_plots(name, integration_config, tmpdir):
store = db_store.create_store_factory_for_sqlite_db(
integration_config).create()
cached_data = load_test_data()
cached_data['bedcounts'] = to_pandas(store.get_bed_counts())
output_dir = str(tmpdir.mkdir("sub"))
generate_plots(plots=[name],
output_dir=output_dir,
cached_data=cached_data)
assert (Path(output_dir) / (name + ".png")).exists()
def get(self, collection):
file_format = self.get_query_argument('format', default=None)
max_ts = self.get_query_argument('max_ts', default=None)
data = None
get_fn = self.get_fns.get(collection, None)
if get_fn is None:
logging.debug("API called with incorrect endpoint: {collection}.")
self.redirect(home.HomeHandler.ROUTE)
return
if collection in ['bedcounts', 'all_bedcounts']:
if isinstance(max_ts, str) and max_ts.isnumeric():
max_ts = datetime.datetime.fromtimestamp(int(max_ts))
get_fn = functools.partial(get_fn, max_date=max_ts)
data = store.to_pandas(get_fn(), max_depth=1)
else:
data = store.to_pandas(get_fn(), max_depth=0)
for k, v in _get_headers(collection, file_format).items():
self.set_header(k, v)
if file_format == 'csv':
stream = io.StringIO()
data.to_csv(stream, index=False)
self.write(stream.getvalue())
elif file_format == 'hdf':
with tempfile.NamedTemporaryFile() as f:
tmp_path = f.name
data.to_hdf(
tmp_path,
key='data',
complib='blosc:lz4',
complevel=9,
)
with open(tmp_path, 'rb') as f:
self.write(f.read())
os.remove(tmp_path)
else:
self.write(data.to_html())
def get(self):
"""Serves a page with a table gathering current bedcount data with some extra information."""
arg_region = self.get_query_argument('region', default=None)
current_region = None
current_region_name = "Toutes les régions"
if arg_region is not None and arg_region.isdigit():
try:
current_region = self.db.get_region(int(arg_region))
current_region_name = current_region.name
except Exception:
logging.warning(f"Invalid query argument: region={arg_region} "
f"Falling back to all regions.")
figures = []
bed_counts = self.db.get_visible_bed_counts_for_user(self.user.user_id)
bed_counts = to_pandas(bed_counts)
if current_region is not None:
mask = bed_counts['icu_region_id'] == current_region.region_id
bed_counts = bed_counts[mask]
df, metrics_layout = _prepare_data(bed_counts)
p = _make_bar_plot(df)
script, div = components(p)
figures.append(dict(script=script, div=div))
plots_extra = _list_extra_plots(
Path(self.config.backoffice.extra_plots_dir))
# stack two columns per row
plots_extra = _grouper(plots_extra, 2)
regions = [{
'name': el.name,
'id': el.region_id
} for el in self.db.get_regions()]
regions = list(sorted(regions, key=lambda x: x['name']))
return self.render("operational-dashboard.html",
figures=figures,
regions=regions,
current_region_name=current_region_name,
metrics_layout=metrics_layout,
plots_extra=plots_extra)
def test_load_test_data():
"""Ensure consitency between test data used in plots and BD schema
e.g. between load_test_data and db.get_bed_counts
"""
# TODO: cache generation of in memory DB with fake data into a pytest fixture
df = load_test_data()['bedcounts']
# duplicate columns with "date"
del df['datetime']
store = StoreFactory(create_engine("sqlite:///:memory:")).create()
populate_store_fake(store)
df_db = to_pandas(store.get_bed_counts())
df_db = normalize_colum_names(df_db, 'bedcounts')
# All bedcount columns in tests data used for plots must belong to the DB schema
missing_columns = set(df.columns).difference(df_db.columns)
assert not missing_columns
def get(self, collection):
if self.current_user.access_type not in self.GET_ACCESS:
logging.error(
f"API called with incorrect access_type: {self.current_user.access_type}."
)
self.set_status(403)
return
file_format = self.get_query_argument('format', default=None)
max_ts = self.get_query_argument('max_ts', default=None)
# should_preprocess: whether preprocessing should be applied to the data
# the raw data of ICUBAM contains inputs errors and this preprocessing will
# attempt to fix them
# it should be used cautiously because it alters the data in ways that are
# useful for analysis purposes but not necessarily reflect the exact/real
# bed count values
# whenever a query argument named 'preprocess' is present, we enable this
# preprocessing (it can be 'preprocess=<anything>' or simply 'preprocess')
should_preprocess = (
self.get_query_argument('preprocess', default=None) is not None
)
data = None
get_fn = self.get_fns.get(collection, None)
if get_fn is None:
logging.debug("API called with incorrect endpoint: {collection}.")
self.redirect(home.HomeHandler.ROUTE)
return
if collection in ['bedcounts', 'all_bedcounts']:
if isinstance(max_ts, str) and max_ts.isnumeric():
max_ts = datetime.fromtimestamp(int(max_ts))
get_fn = functools.partial(get_fn, max_date=max_ts)
data = store.to_pandas(get_fn(), max_depth=1)
if collection == 'all_bedcounts' and should_preprocess:
# this cached_data dict is just a way to tell predicu not to load the
# data from ICUBAM and use this already loaded data instead
cached_data = {'raw_icubam': data}
data = predicu.data.load_bedcounts(
cached_data=cached_data,
clean=True,
)
else:
data = store.to_pandas(get_fn(), max_depth=0)
for k, v in _get_headers(collection, file_format).items():
self.set_header(k, v)
if file_format == 'csv':
stream = io.StringIO()
data.to_csv(stream, index=False)
self.write(stream.getvalue())
elif file_format == 'hdf':
with tempfile.NamedTemporaryFile() as f:
tmp_path = f.name
data.to_hdf(
tmp_path,
key='data',
complib='blosc:lz4',
complevel=9,
)
with open(tmp_path, 'rb') as f:
self.write(f.read())
os.remove(tmp_path)
else:
self.write(data.to_html())
def make(user_id,
db,
region=None,
locale=None,
extra_plots_dir="",
external=False):
current_region = None
if locale is not None:
current_region_name = locale.translate("All regions")
else:
current_region_name = "All regions"
current_region_id = None
if region is not None and region.isdigit():
try:
current_region = db.get_region(int(region))
current_region_name = current_region.name
current_region_id = int(region)
except Exception:
logging.warning(f"Invalid query argument: region={region} "
f"Falling back to all regions.")
if not external:
get_counts_fn = db.get_visible_bed_counts_for_user
else:
get_counts_fn = functools.partial(
db.get_bed_counts_for_external_client, latest=True)
figures = []
bed_counts = get_counts_fn(user_id)
if bed_counts:
bed_counts = to_pandas(bed_counts)
region_id_seen = bed_counts.icu_region_id.unique().tolist()
if current_region is not None:
mask = bed_counts['icu_region_id'] == current_region.region_id
bed_counts = bed_counts[mask]
else:
# when no data, make sure the resulting dataframe has
# correct column names.
columns = [key for key in dir(BedCount) if not key.startswith('_')]
columns += ['icu_dept']
bed_counts = pd.DataFrame([], columns=columns)
region_id_seen = []
df, metrics_layout = _prepare_data(bed_counts)
p = _make_bar_plot(df, locale)
script, div = components(p)
figures.append(dict(script=script, div=div))
plots_extra = _list_extra_plots(Path(extra_plots_dir))
img_map = ImageURLMapper(plots_extra)
regions = [{
'name': el.name,
'id': el.region_id
} for el in db.get_regions() if el.region_id in region_id_seen]
regions = list(sorted(regions, key=lambda x: x['name']))
return {
'figures': figures,
'regions': regions,
'region_name': current_region_name,
'region_id': current_region_id,
'metrics_layout': metrics_layout,
'img_map': img_map,
'_grouper': _grouper,
}