def write_org_data_to_minio(org_sector_name, org_df, org_gdf, minio_access, minio_secrety):
sector_name_slug = org_sector_name.lower().strip().replace(" ", "-")
dataset_name = "-".join([COMMUNITY_ORGANISATION_DATASET_PREFIX, sector_name_slug])
logging.debug(f"Writing to '{org_df.shape[0]}' values to {dataset_name}")
# First, the CSV
result = minio_utils.dataframe_to_minio(
org_df,
minio_bucket=COMMUNITY_ORGANISATION_BUCKET_NAME,
filename_prefix_override=dataset_name,
minio_key=minio_access,
minio_secret=minio_secrety,
data_classification=minio_utils.DataClassification.EDGE,
file_format="csv",
data_versioning=False
)
assert result, "Writing CSV to minio failed"
# Then, the GeoJSON
with tempfile.TemporaryDirectory() as temp_dir:
geojson_filename = dataset_name + ".geojson"
local_path = os.path.join(temp_dir, geojson_filename)
org_gdf.to_file(local_path, driver="GeoJSON")
result = minio_utils.file_to_minio(
local_path,
minio_bucket=COMMUNITY_ORGANISATION_BUCKET_NAME,
minio_key=minio_access,
minio_secret=minio_secrety,
data_classification=minio_utils.DataClassification.EDGE,
)
assert result, "Writing GeoJSON to minio failed"
def write_to_minio(data, minio_filename, filename_prefix_override,
minio_bucket, minio_key, minio_secret):
"""
write data to minio
:param data: (obj) the data
:param minio_filename: (str) the filename
:param filename_prefix_override: (str) minio override string (prefix and file name)
:param minio_bucket: (str) minio bucket name
:param minio_key: (str) the minio access key
:param minio_secret: (str) the minio key secret
:return:
"""
with tempfile.TemporaryDirectory() as tempdir:
local_path = os.path.join(tempdir, minio_filename)
logging.debug(f"Writing out data to '{local_path}'")
with open(local_path, "w") as plot_file:
plot_file.write(data)
logging.debug(f"Uploading '{local_path}' to Minio")
result = minio_utils.file_to_minio(
filename=local_path,
filename_prefix_override=filename_prefix_override,
minio_bucket=minio_bucket,
minio_key=minio_key,
minio_secret=minio_secret,
data_classification=EDGE_CLASSIFICATION,
)
assert result
def send_email(account, email_message_dict, attachments, message_uuid,
dry_run):
logging.debug(f"Saving {message_uuid}.json to Minio")
with tempfile.TemporaryDirectory() as tempdir:
local_path = os.path.join(tempdir, f"{message_uuid}.json")
with open(local_path, "w") as message_file:
json.dump(email_message_dict, message_file)
minio_utils.file_to_minio(
filename=local_path,
minio_bucket=EMAILS_BUCKET,
minio_key=secrets["minio"]["edge"]["access"],
minio_secret=secrets["minio"]["edge"]["secret"],
data_classification=EDGE_CLASSIFICATION,
)
message = Message(account=account, **email_message_dict)
logging.debug("Attaching logo")
logo_path = os.path.join(RESOURCES_PATH, CITY_LOGO_FILENAME)
with open(logo_path, "rb") as logo_file:
message.attach(
FileAttachment(name=CITY_LOGO_FILENAME, content=logo_file.read()))
logging.debug(f"Attaching data files")
for attachment_name, attachment_file in attachments:
logging.debug(f"attachment_name='{attachment_name}'")
with open(attachment_file, "rb") as plot_file:
message.attach(
FileAttachment(name=attachment_name, content=plot_file.read()))
logging.debug(f"Sending {message_uuid} email")
# Turning down various exchange loggers for the send - they're a bit noisy
exchangelib_loggers = [
logging.getLogger(name) for name in logging.root.manager.loggerDict
if name.startswith("exchangelib")
]
for logger in exchangelib_loggers:
logger.setLevel(logging.INFO)
if not dry_run:
message.send(save_copy=True)
else:
logging.warning("**--dry_run flag set, hence not sending emails...**")
return True
def write_map_to_minio(
city_map,
map_file_prefix,
tempdir,
minio_access,
minio_secret,
js_libs,
css_libs,
map_suffix=MAP_FILENAME,
map_minio_prefix=epi_map_case_layers_to_minio.CASE_MAP_PREFIX):
# Uploading JS and CSS dependencies
for key, new_path in js_libs + css_libs:
*_, local_filename = os.path.split(new_path)
local_path = os.path.join(tempdir, local_filename)
logging.debug(f"Uploading '{key}' from '{local_path}'")
result = minio_utils.file_to_minio(
filename=local_path,
filename_prefix_override=f"{map_minio_prefix}{DEP_DIR}/",
minio_bucket=MINIO_BUCKET,
minio_key=minio_access,
minio_secret=minio_secret,
data_classification=MINIO_CLASSIFICATION,
)
assert result, f"Failed to upload {local_filename}"
map_filename = f"{map_file_prefix}_{map_suffix}"
local_path = os.path.join(tempdir, map_filename)
folium.folium._default_js = js_libs
folium.folium._default_css = css_libs
city_map.save(local_path)
result = minio_utils.file_to_minio(
filename=local_path,
filename_prefix_override=map_minio_prefix,
minio_bucket=MINIO_BUCKET,
minio_key=minio_access,
minio_secret=minio_secret,
data_classification=MINIO_CLASSIFICATION,
)
assert result
def write_table_widget_file_to_minio(localpath, minio_access, minio_secret):
result = minio_utils.file_to_minio(
filename=localpath,
filename_prefix_override=STATS_WIDGET_PREFIX,
minio_bucket=epi_map_case_layers_to_minio.MINIO_COVID_BUCKET,
minio_key=minio_access,
minio_secret=minio_secret,
data_classification=epi_map_case_layers_to_minio.EDGE_MINIO_CLASSIFICATION,
)
assert result
def write_layers_to_minio(layers_dict,
minio_access,
minio_secret,
prefix=CASE_MAP_PREFIX):
for layer_name, (layer_local_path, _) in layers_dict.items():
result = minio_utils.file_to_minio(
filename=layer_local_path,
filename_prefix_override=prefix,
minio_bucket=MINIO_COVID_BUCKET,
minio_key=minio_access,
minio_secret=minio_secret,
data_classification=MINIO_EDGE_CLASSIFICATION,
)
assert result
def write_metadata_to_minio(metadata_dict, tempdir, metadata_filename, minio_access, minio_secret, minio_prefix=CASE_MAP_PREFIX):
local_path = os.path.join(tempdir, metadata_filename)
with open(local_path, "w") as not_spatial_case_count_file:
json.dump(metadata_dict, not_spatial_case_count_file)
result = minio_utils.file_to_minio(
filename=local_path,
filename_prefix_override=minio_prefix,
minio_bucket=MINIO_COVID_BUCKET,
minio_key=minio_access,
minio_secret=minio_secret,
data_classification=EDGE_MINIO_CLASSIFICATION,
)
assert result
def get_leaflet_dep_file(url, tempdir, http_session, minio_access,
minio_secret):
filename = os.path.basename(urlparse(url).path)
local_path = os.path.join(tempdir, filename)
resp = http_session.get(url)
with open(local_path, "wb") as dep_file:
dep_file.write(resp.content)
minio_utils.file_to_minio(
filename=local_path,
filename_prefix_override=(
f"{service_request_map_layers_to_minio.SERVICE_REQUEST_MAP_PREFIX}"
f"{DEP_DIR}/"),
minio_bucket=MINIO_BUCKET,
minio_key=minio_access,
minio_secret=minio_secret,
data_classification=MINIO_CLASSIFICATION,
)
new_path = (f"{DEP_DIR}/{filename}")
return new_path
def write_to_minio(html, minio_filename, minio_access, minio_secret):
with tempfile.TemporaryDirectory() as tempdir:
local_path = os.path.join(tempdir, minio_filename)
logging.debug(f"Writing out HTML to '{local_path}'")
with open(local_path, "w") as line_plot_file:
line_plot_file.write(html)
logging.debug(f"Uploading '{local_path}' to Minio")
result = minio_utils.file_to_minio(
filename=local_path,
filename_prefix_override=WIDGETS_RESTRICTED_PREFIX,
minio_bucket=MINIO_BUCKET,
minio_key=minio_access,
minio_secret=minio_secret,
data_classification=MINIO_CLASSIFICATION,
)
assert result
def write_map_to_minio(map, directorate_file_prefix, time_period_prefix,
map_suffix, tempdir, minio_access, minio_secret,
js_libs, css_libs):
map_filename = f"{time_period_prefix}_{directorate_file_prefix}_{map_suffix}"
local_path = os.path.join(tempdir, map_filename)
folium.folium._default_js = js_libs
folium.folium._default_css = css_libs
map.save(local_path)
result = minio_utils.file_to_minio(
filename=local_path,
filename_prefix_override=service_request_map_layers_to_minio.
SERVICE_REQUEST_MAP_PREFIX,
minio_bucket=MINIO_BUCKET,
minio_key=minio_access,
minio_secret=minio_secret,
data_classification=MINIO_CLASSIFICATION,
)
assert result
def write_html_to_minio(plotly_fig, outfile, prefix, bucket, secret_access, secret_secret):
with tempfile.TemporaryDirectory() as tempdir:
local_path = os.path.join(tempdir, outfile)
logging.debug(f"Push[ing] '{local_path}' to Minio")
with open(local_path, "w") as plot_file:
plotly_fig.write_html(
plot_file,
include_plotlyjs='directory',
default_height='100%',
default_width='100%'
)
result = minio_utils.file_to_minio(
filename=local_path,
filename_prefix_override=prefix,
minio_bucket=bucket,
minio_key=secret_access,
minio_secret=secret_secret,
data_classification=EDGE_CLASSIFICATION,
)
assert result
logging.debug(f"Push[ed] '{local_path}' to Minio")
return result
def plot_risk(plot_df_willing, label_name, out_name):
label_name_cln = label_name.replace("_", " ").capitalize()
formatter = mdates.DateFormatter('%y/%m/%d')
locator = mdates.WeekdayLocator(interval=1)
fig, (ax1, ax2) = plt.subplots(ncols=2,
nrows=1,
sharex=True,
figsize=(20, 8))
sns.lineplot(data=plot_df_willing,
x=VAX_DATE,
y=VACCINATED_CUMSUM,
hue=AGG_LEVEL_NAME,
ax=ax1)
sns.lineplot(data=plot_df_willing,
x=VAX_DATE,
y=VACCINATED_PERCENT,
hue=AGG_LEVEL_NAME,
ax=ax2)
legend_list = []
for ax in [ax1, ax2]:
ax.set_xlabel("")
ax.xaxis.set_major_formatter(formatter)
ax.xaxis.set_major_locator(locator)
ax.tick_params(axis='x', rotation=45)
if label_name != TOP_LEVEL:
for leg_labels in ax.get_legend_handles_labels():
legend_list.append(leg_labels)
new_leg_list = []
for label in legend_list[1]:
if "," in label:
new_lab = (label.strip("\(\)").split(",")[1]).strip()
else:
new_lab = label
new_leg_list.append(new_lab)
ax.legend(handles=legend_list[0],
labels=new_leg_list,
loc=2,
title=label_name_cln)
else:
ax.legend(loc=2)
ax1.set_ylabel(WILLNIG_COUNT_LAB)
ax2.set_ylabel(WILLNIG_PERC_LAB)
ax2.set_ylim(0, 100)
plt.suptitle(f"{TITLE_PREFIX} {label_name_cln}")
with tempfile.TemporaryDirectory() as tempdir:
outfile_name = pathlib.Path(tempdir, f"{out_name}.png")
plt.savefig(str(outfile_name),
dpi=96,
format='png',
facecolor='white',
bbox_inches='tight')
logging.debug(f"writing {outfile_name} to minio")
result = minio_utils.file_to_minio(
filename=outfile_name,
minio_bucket=COVID_BUCKET,
filename_prefix_override=VACC_PLOT_PREFIX,
data_classification=EDGE_CLASSIFICATION,
)
return result
for logger in exchangelib_loggers:
logger.setLevel(logging.INFO)
SUBJECT_FILTER = 'MCSH-Billing_Invoicing'
filtered_items = exchange_utils.filter_account(account, SUBJECT_FILTER)
CURRENT_FILENAME = "MCSH-Billing_Invoicing.xlsx"
BUCKET = 'covid'
BUCKET_PREFIX = "data/staging/"
for attachment_path in exchange_utils.get_latest_attachment_file(
filtered_items):
logging.debug("Uploading '{}' to minio://covid/{}".format(
attachment_path, BUCKET_PREFIX))
print(attachment_path)
new_path = os.path.join(os.path.dirname(attachment_path),
CURRENT_FILENAME)
print(new_path)
if attachment_path.endswith(".xlsx"):
os.rename(attachment_path, new_path)
minio_utils.file_to_minio(
filename=new_path,
filename_prefix_override=BUCKET_PREFIX,
minio_bucket=BUCKET,
minio_key=secrets["minio"]["edge"]["access"],
minio_secret=secrets["minio"]["edge"]["secret"],
data_classification=minio_utils.DataClassification.EDGE,
)
# Delete filtered items because we feel sorry for our poor inbox
filtered_items.delete()
temp_path = pathlib.Path(tempfile.TemporaryDirectory().name)
temp_path.mkdir()
directorate_file_prefix = directorate_file_prefix.replace("_", "-")
out_points_geojson = pathlib.Path(
temp_path, f"{directorate_file_prefix}-{POINT_OUTPUT_FILENAME_SUFFIX}")
out_hex_geojson = pathlib.Path(
temp_path, f"{directorate_file_prefix}-{HEX_OUTPUT_FILENAME_SUFFIX}")
if out_points_geojson.exists():
os.unlink(out_points_geojson)
if out_hex_geojson.exists():
os.unlink(out_hex_geojson)
latest_work_status_counts_point.to_file(out_points_geojson,
driver="GeoJSON")
latest_work_status_counts_hex.to_file(out_hex_geojson, driver="GeoJSON")
file_tuples = [(out_hex_geojson, latest_work_status_counts_hex),
(out_points_geojson, latest_work_status_counts_point)]
for (file_name, df) in file_tuples:
minio_utils.file_to_minio(
filename=file_name,
minio_bucket=MINIO_BUCKET,
filename_prefix_override=DATA_RESTRICTED_PREFIX,
minio_key=secrets["minio"]["edge"]["access"],
minio_secret=secrets["minio"]["edge"]["secret"],
data_classification=MINIO_CLASSIFICATION,
)
logging.debug(f"Done")
password=receiver_dict["password"],
dashboard_link="ds3.capetown.gov.za/covid-dash/ct-covid-dash-city.html",
request_id=message_uuid,
iso8601_timestamp=iso8601_timestamp)
message_dict = dict(
subject=subject,
body=HTMLBody(body),
to_recipients=[receiver_dict["email_address"]],
)
message = Message(account=account, **message_dict)
message.attach(city_logo_attachment)
logging.info(f"Saving {message_uuid}.json to Minio")
with tempfile.TemporaryDirectory() as tempdir:
local_path = os.path.join(tempdir, f"{message_uuid}.json")
with open(local_path, "w") as message_file:
json.dump(message_dict, message_file)
minio_utils.file_to_minio(
filename=local_path,
minio_bucket="covid-19-dash-user-cred-emails",
minio_key=secrets["minio"]["edge"]["access"],
minio_secret=secrets["minio"]["edge"]["secret"],
data_classification=minio_utils.DataClassification.EDGE,
)
if not dry_run:
logging.info(f"Sending email to {receiver_dict['email_address']}")
message.send(save_copy=False)
def send_email(account, email_message_dict, attachments, message_uuid,
dry_run):
logging.debug(f"Saving {message_uuid}.json to Minio")
with tempfile.TemporaryDirectory() as tempdir:
local_path = os.path.join(tempdir, f"{message_uuid}.json")
with open(local_path, "w") as message_file:
json.dump(email_message_dict, message_file)
minio_utils.file_to_minio(
filename=local_path,
minio_bucket="covid-19-dash-hr-bp-emails",
minio_key=secrets["minio"]["edge"]["access"],
minio_secret=secrets["minio"]["edge"]["secret"],
data_classification=minio_utils.DataClassification.EDGE,
)
message = Message(account=account, **email_message_dict)
logging.debug("Attaching logo")
logo_path = os.path.join(RESOURCES_PATH, CITY_LOGO_FILENAME)
with open(logo_path, "rb") as logo_file:
message.attach(
FileAttachment(name=CITY_LOGO_FILENAME, content=logo_file.read()))
logging.debug(f"Attaching data files")
for attachment_name, attachment_file in attachments:
logging.debug(f"attachment_name='{attachment_name}'")
attachment_content = attachment_file.read()
logging.debug("Backing up attachment file to Minio...")
with tempfile.TemporaryDirectory() as tempdir:
local_path = os.path.join(tempdir, attachment_name)
with open(local_path, "wb") as attachment_temp_file:
attachment_temp_file.write(attachment_content)
minio_utils.file_to_minio(
filename=local_path,
filename_prefix_override=message_uuid + "/",
minio_bucket="covid-19-dash-hr-bp-emails",
minio_key=secrets["minio"]["edge"]["access"],
minio_secret=secrets["minio"]["edge"]["secret"],
data_classification=minio_utils.DataClassification.EDGE,
)
logging.debug("Attaching the content")
message.attach(
FileAttachment(name=attachment_name, content=attachment_content))
logging.debug(f"Sending {message_uuid} email")
# Turning down various exchange loggers for the send - they're a bit noisy
exchangelib_loggers = [
logging.getLogger(name) for name in logging.root.manager.loggerDict
if name.startswith("exchangelib")
]
for logger in exchangelib_loggers:
logger.setLevel(logging.INFO)
if not dry_run:
message.send(save_copy=True)
else:
logging.warning(
"**--not-a-drill flag not set, hence not sending emails...**")
return True
