def write_org_data_to_minio(org_sector_name, org_df, org_gdf, minio_access, minio_secrety): sector_name_slug = org_sector_name.lower().strip().replace(" ", "-") dataset_name = "-".join([COMMUNITY_ORGANISATION_DATASET_PREFIX, sector_name_slug]) logging.debug(f"Writing to '{org_df.shape[0]}' values to {dataset_name}") # First, the CSV result = minio_utils.dataframe_to_minio( org_df, minio_bucket=COMMUNITY_ORGANISATION_BUCKET_NAME, filename_prefix_override=dataset_name, minio_key=minio_access, minio_secret=minio_secrety, data_classification=minio_utils.DataClassification.EDGE, file_format="csv", data_versioning=False ) assert result, "Writing CSV to minio failed" # Then, the GeoJSON with tempfile.TemporaryDirectory() as temp_dir: geojson_filename = dataset_name + ".geojson" local_path = os.path.join(temp_dir, geojson_filename) org_gdf.to_file(local_path, driver="GeoJSON") result = minio_utils.file_to_minio( local_path, minio_bucket=COMMUNITY_ORGANISATION_BUCKET_NAME, minio_key=minio_access, minio_secret=minio_secrety, data_classification=minio_utils.DataClassification.EDGE, ) assert result, "Writing GeoJSON to minio failed"
def write_to_minio(data, minio_filename, filename_prefix_override, minio_bucket, minio_key, minio_secret): """ write data to minio :param data: (obj) the data :param minio_filename: (str) the filename :param filename_prefix_override: (str) minio override string (prefix and file name) :param minio_bucket: (str) minio bucket name :param minio_key: (str) the minio access key :param minio_secret: (str) the minio key secret :return: """ with tempfile.TemporaryDirectory() as tempdir: local_path = os.path.join(tempdir, minio_filename) logging.debug(f"Writing out data to '{local_path}'") with open(local_path, "w") as plot_file: plot_file.write(data) logging.debug(f"Uploading '{local_path}' to Minio") result = minio_utils.file_to_minio( filename=local_path, filename_prefix_override=filename_prefix_override, minio_bucket=minio_bucket, minio_key=minio_key, minio_secret=minio_secret, data_classification=EDGE_CLASSIFICATION, ) assert result
def send_email(account, email_message_dict, attachments, message_uuid, dry_run): logging.debug(f"Saving {message_uuid}.json to Minio") with tempfile.TemporaryDirectory() as tempdir: local_path = os.path.join(tempdir, f"{message_uuid}.json") with open(local_path, "w") as message_file: json.dump(email_message_dict, message_file) minio_utils.file_to_minio( filename=local_path, minio_bucket=EMAILS_BUCKET, minio_key=secrets["minio"]["edge"]["access"], minio_secret=secrets["minio"]["edge"]["secret"], data_classification=EDGE_CLASSIFICATION, ) message = Message(account=account, **email_message_dict) logging.debug("Attaching logo") logo_path = os.path.join(RESOURCES_PATH, CITY_LOGO_FILENAME) with open(logo_path, "rb") as logo_file: message.attach( FileAttachment(name=CITY_LOGO_FILENAME, content=logo_file.read())) logging.debug(f"Attaching data files") for attachment_name, attachment_file in attachments: logging.debug(f"attachment_name='{attachment_name}'") with open(attachment_file, "rb") as plot_file: message.attach( FileAttachment(name=attachment_name, content=plot_file.read())) logging.debug(f"Sending {message_uuid} email") # Turning down various exchange loggers for the send - they're a bit noisy exchangelib_loggers = [ logging.getLogger(name) for name in logging.root.manager.loggerDict if name.startswith("exchangelib") ] for logger in exchangelib_loggers: logger.setLevel(logging.INFO) if not dry_run: message.send(save_copy=True) else: logging.warning("**--dry_run flag set, hence not sending emails...**") return True
def write_map_to_minio( city_map, map_file_prefix, tempdir, minio_access, minio_secret, js_libs, css_libs, map_suffix=MAP_FILENAME, map_minio_prefix=epi_map_case_layers_to_minio.CASE_MAP_PREFIX): # Uploading JS and CSS dependencies for key, new_path in js_libs + css_libs: *_, local_filename = os.path.split(new_path) local_path = os.path.join(tempdir, local_filename) logging.debug(f"Uploading '{key}' from '{local_path}'") result = minio_utils.file_to_minio( filename=local_path, filename_prefix_override=f"{map_minio_prefix}{DEP_DIR}/", minio_bucket=MINIO_BUCKET, minio_key=minio_access, minio_secret=minio_secret, data_classification=MINIO_CLASSIFICATION, ) assert result, f"Failed to upload {local_filename}" map_filename = f"{map_file_prefix}_{map_suffix}" local_path = os.path.join(tempdir, map_filename) folium.folium._default_js = js_libs folium.folium._default_css = css_libs city_map.save(local_path) result = minio_utils.file_to_minio( filename=local_path, filename_prefix_override=map_minio_prefix, minio_bucket=MINIO_BUCKET, minio_key=minio_access, minio_secret=minio_secret, data_classification=MINIO_CLASSIFICATION, ) assert result
def write_table_widget_file_to_minio(localpath, minio_access, minio_secret): result = minio_utils.file_to_minio( filename=localpath, filename_prefix_override=STATS_WIDGET_PREFIX, minio_bucket=epi_map_case_layers_to_minio.MINIO_COVID_BUCKET, minio_key=minio_access, minio_secret=minio_secret, data_classification=epi_map_case_layers_to_minio.EDGE_MINIO_CLASSIFICATION, ) assert result
def write_layers_to_minio(layers_dict, minio_access, minio_secret, prefix=CASE_MAP_PREFIX): for layer_name, (layer_local_path, _) in layers_dict.items(): result = minio_utils.file_to_minio( filename=layer_local_path, filename_prefix_override=prefix, minio_bucket=MINIO_COVID_BUCKET, minio_key=minio_access, minio_secret=minio_secret, data_classification=MINIO_EDGE_CLASSIFICATION, ) assert result
def write_metadata_to_minio(metadata_dict, tempdir, metadata_filename, minio_access, minio_secret, minio_prefix=CASE_MAP_PREFIX): local_path = os.path.join(tempdir, metadata_filename) with open(local_path, "w") as not_spatial_case_count_file: json.dump(metadata_dict, not_spatial_case_count_file) result = minio_utils.file_to_minio( filename=local_path, filename_prefix_override=minio_prefix, minio_bucket=MINIO_COVID_BUCKET, minio_key=minio_access, minio_secret=minio_secret, data_classification=EDGE_MINIO_CLASSIFICATION, ) assert result
def get_leaflet_dep_file(url, tempdir, http_session, minio_access, minio_secret): filename = os.path.basename(urlparse(url).path) local_path = os.path.join(tempdir, filename) resp = http_session.get(url) with open(local_path, "wb") as dep_file: dep_file.write(resp.content) minio_utils.file_to_minio( filename=local_path, filename_prefix_override=( f"{service_request_map_layers_to_minio.SERVICE_REQUEST_MAP_PREFIX}" f"{DEP_DIR}/"), minio_bucket=MINIO_BUCKET, minio_key=minio_access, minio_secret=minio_secret, data_classification=MINIO_CLASSIFICATION, ) new_path = (f"{DEP_DIR}/{filename}") return new_path
def write_to_minio(html, minio_filename, minio_access, minio_secret): with tempfile.TemporaryDirectory() as tempdir: local_path = os.path.join(tempdir, minio_filename) logging.debug(f"Writing out HTML to '{local_path}'") with open(local_path, "w") as line_plot_file: line_plot_file.write(html) logging.debug(f"Uploading '{local_path}' to Minio") result = minio_utils.file_to_minio( filename=local_path, filename_prefix_override=WIDGETS_RESTRICTED_PREFIX, minio_bucket=MINIO_BUCKET, minio_key=minio_access, minio_secret=minio_secret, data_classification=MINIO_CLASSIFICATION, ) assert result
def write_map_to_minio(map, directorate_file_prefix, time_period_prefix, map_suffix, tempdir, minio_access, minio_secret, js_libs, css_libs): map_filename = f"{time_period_prefix}_{directorate_file_prefix}_{map_suffix}" local_path = os.path.join(tempdir, map_filename) folium.folium._default_js = js_libs folium.folium._default_css = css_libs map.save(local_path) result = minio_utils.file_to_minio( filename=local_path, filename_prefix_override=service_request_map_layers_to_minio. SERVICE_REQUEST_MAP_PREFIX, minio_bucket=MINIO_BUCKET, minio_key=minio_access, minio_secret=minio_secret, data_classification=MINIO_CLASSIFICATION, ) assert result
def write_html_to_minio(plotly_fig, outfile, prefix, bucket, secret_access, secret_secret): with tempfile.TemporaryDirectory() as tempdir: local_path = os.path.join(tempdir, outfile) logging.debug(f"Push[ing] '{local_path}' to Minio") with open(local_path, "w") as plot_file: plotly_fig.write_html( plot_file, include_plotlyjs='directory', default_height='100%', default_width='100%' ) result = minio_utils.file_to_minio( filename=local_path, filename_prefix_override=prefix, minio_bucket=bucket, minio_key=secret_access, minio_secret=secret_secret, data_classification=EDGE_CLASSIFICATION, ) assert result logging.debug(f"Push[ed] '{local_path}' to Minio") return result
def plot_risk(plot_df_willing, label_name, out_name): label_name_cln = label_name.replace("_", " ").capitalize() formatter = mdates.DateFormatter('%y/%m/%d') locator = mdates.WeekdayLocator(interval=1) fig, (ax1, ax2) = plt.subplots(ncols=2, nrows=1, sharex=True, figsize=(20, 8)) sns.lineplot(data=plot_df_willing, x=VAX_DATE, y=VACCINATED_CUMSUM, hue=AGG_LEVEL_NAME, ax=ax1) sns.lineplot(data=plot_df_willing, x=VAX_DATE, y=VACCINATED_PERCENT, hue=AGG_LEVEL_NAME, ax=ax2) legend_list = [] for ax in [ax1, ax2]: ax.set_xlabel("") ax.xaxis.set_major_formatter(formatter) ax.xaxis.set_major_locator(locator) ax.tick_params(axis='x', rotation=45) if label_name != TOP_LEVEL: for leg_labels in ax.get_legend_handles_labels(): legend_list.append(leg_labels) new_leg_list = [] for label in legend_list[1]: if "," in label: new_lab = (label.strip("\(\)").split(",")[1]).strip() else: new_lab = label new_leg_list.append(new_lab) ax.legend(handles=legend_list[0], labels=new_leg_list, loc=2, title=label_name_cln) else: ax.legend(loc=2) ax1.set_ylabel(WILLNIG_COUNT_LAB) ax2.set_ylabel(WILLNIG_PERC_LAB) ax2.set_ylim(0, 100) plt.suptitle(f"{TITLE_PREFIX} {label_name_cln}") with tempfile.TemporaryDirectory() as tempdir: outfile_name = pathlib.Path(tempdir, f"{out_name}.png") plt.savefig(str(outfile_name), dpi=96, format='png', facecolor='white', bbox_inches='tight') logging.debug(f"writing {outfile_name} to minio") result = minio_utils.file_to_minio( filename=outfile_name, minio_bucket=COVID_BUCKET, filename_prefix_override=VACC_PLOT_PREFIX, data_classification=EDGE_CLASSIFICATION, ) return result
for logger in exchangelib_loggers: logger.setLevel(logging.INFO) SUBJECT_FILTER = 'MCSH-Billing_Invoicing' filtered_items = exchange_utils.filter_account(account, SUBJECT_FILTER) CURRENT_FILENAME = "MCSH-Billing_Invoicing.xlsx" BUCKET = 'covid' BUCKET_PREFIX = "data/staging/" for attachment_path in exchange_utils.get_latest_attachment_file( filtered_items): logging.debug("Uploading '{}' to minio://covid/{}".format( attachment_path, BUCKET_PREFIX)) print(attachment_path) new_path = os.path.join(os.path.dirname(attachment_path), CURRENT_FILENAME) print(new_path) if attachment_path.endswith(".xlsx"): os.rename(attachment_path, new_path) minio_utils.file_to_minio( filename=new_path, filename_prefix_override=BUCKET_PREFIX, minio_bucket=BUCKET, minio_key=secrets["minio"]["edge"]["access"], minio_secret=secrets["minio"]["edge"]["secret"], data_classification=minio_utils.DataClassification.EDGE, ) # Delete filtered items because we feel sorry for our poor inbox filtered_items.delete()
temp_path = pathlib.Path(tempfile.TemporaryDirectory().name) temp_path.mkdir() directorate_file_prefix = directorate_file_prefix.replace("_", "-") out_points_geojson = pathlib.Path( temp_path, f"{directorate_file_prefix}-{POINT_OUTPUT_FILENAME_SUFFIX}") out_hex_geojson = pathlib.Path( temp_path, f"{directorate_file_prefix}-{HEX_OUTPUT_FILENAME_SUFFIX}") if out_points_geojson.exists(): os.unlink(out_points_geojson) if out_hex_geojson.exists(): os.unlink(out_hex_geojson) latest_work_status_counts_point.to_file(out_points_geojson, driver="GeoJSON") latest_work_status_counts_hex.to_file(out_hex_geojson, driver="GeoJSON") file_tuples = [(out_hex_geojson, latest_work_status_counts_hex), (out_points_geojson, latest_work_status_counts_point)] for (file_name, df) in file_tuples: minio_utils.file_to_minio( filename=file_name, minio_bucket=MINIO_BUCKET, filename_prefix_override=DATA_RESTRICTED_PREFIX, minio_key=secrets["minio"]["edge"]["access"], minio_secret=secrets["minio"]["edge"]["secret"], data_classification=MINIO_CLASSIFICATION, ) logging.debug(f"Done")
password=receiver_dict["password"], dashboard_link="ds3.capetown.gov.za/covid-dash/ct-covid-dash-city.html", request_id=message_uuid, iso8601_timestamp=iso8601_timestamp) message_dict = dict( subject=subject, body=HTMLBody(body), to_recipients=[receiver_dict["email_address"]], ) message = Message(account=account, **message_dict) message.attach(city_logo_attachment) logging.info(f"Saving {message_uuid}.json to Minio") with tempfile.TemporaryDirectory() as tempdir: local_path = os.path.join(tempdir, f"{message_uuid}.json") with open(local_path, "w") as message_file: json.dump(message_dict, message_file) minio_utils.file_to_minio( filename=local_path, minio_bucket="covid-19-dash-user-cred-emails", minio_key=secrets["minio"]["edge"]["access"], minio_secret=secrets["minio"]["edge"]["secret"], data_classification=minio_utils.DataClassification.EDGE, ) if not dry_run: logging.info(f"Sending email to {receiver_dict['email_address']}") message.send(save_copy=False)
def send_email(account, email_message_dict, attachments, message_uuid, dry_run): logging.debug(f"Saving {message_uuid}.json to Minio") with tempfile.TemporaryDirectory() as tempdir: local_path = os.path.join(tempdir, f"{message_uuid}.json") with open(local_path, "w") as message_file: json.dump(email_message_dict, message_file) minio_utils.file_to_minio( filename=local_path, minio_bucket="covid-19-dash-hr-bp-emails", minio_key=secrets["minio"]["edge"]["access"], minio_secret=secrets["minio"]["edge"]["secret"], data_classification=minio_utils.DataClassification.EDGE, ) message = Message(account=account, **email_message_dict) logging.debug("Attaching logo") logo_path = os.path.join(RESOURCES_PATH, CITY_LOGO_FILENAME) with open(logo_path, "rb") as logo_file: message.attach( FileAttachment(name=CITY_LOGO_FILENAME, content=logo_file.read())) logging.debug(f"Attaching data files") for attachment_name, attachment_file in attachments: logging.debug(f"attachment_name='{attachment_name}'") attachment_content = attachment_file.read() logging.debug("Backing up attachment file to Minio...") with tempfile.TemporaryDirectory() as tempdir: local_path = os.path.join(tempdir, attachment_name) with open(local_path, "wb") as attachment_temp_file: attachment_temp_file.write(attachment_content) minio_utils.file_to_minio( filename=local_path, filename_prefix_override=message_uuid + "/", minio_bucket="covid-19-dash-hr-bp-emails", minio_key=secrets["minio"]["edge"]["access"], minio_secret=secrets["minio"]["edge"]["secret"], data_classification=minio_utils.DataClassification.EDGE, ) logging.debug("Attaching the content") message.attach( FileAttachment(name=attachment_name, content=attachment_content)) logging.debug(f"Sending {message_uuid} email") # Turning down various exchange loggers for the send - they're a bit noisy exchangelib_loggers = [ logging.getLogger(name) for name in logging.root.manager.loggerDict if name.startswith("exchangelib") ] for logger in exchangelib_loggers: logger.setLevel(logging.INFO) if not dry_run: message.send(save_copy=True) else: logging.warning( "**--not-a-drill flag not set, hence not sending emails...**") return True