def non_active_pairs(ctx: callable, config: callable, statsd: callable,
logger: callable, run_id: int, conn: callable,
metadata_conn: callable, command: str,
metrics_root: callable, metrics_run_root: callable,
output_dir: str, period: int) -> None:
"""Generate list of Non-Active pairs over specified period.
Arguments:
ctx: click context object
config: DIRBS config object
statsd: DIRBS statsd connection object
logger: DIRBS custom logger object
run_id: run id of the current job
conn: DIRBS PostgreSQL connection object
metadata_conn: DIRBS PostgreSQL metadata connection object
command: name of the command
metrics_root: root object for the statsd metrics
metrics_run_root: root object for the statsd run metrics
output_dir: output directory path
period: period in days for a pair being count as not active (not active for these many days)
Returns:
None
"""
metadata.add_optional_job_metadata(
metadata_conn,
command,
run_id,
report_schema_version=report_schema_version,
output_dir=os.path.abspath(str(output_dir)))
current_date = datetime.date.today()
last_seen_date = datetime.date(
current_date.year, current_date.month,
current_date.day) - datetime.timedelta(period)
logger.info(
'List of None-Active Pairs with last_seen less than {0} will be generated'
.format(last_seen_date))
report_dir = make_report_directory(ctx, output_dir, run_id, conn, config)
with utils.CodeProfiler() as cp:
report_metadata = write_non_active_pairs(conn, logger, report_dir,
last_seen_date)
statsd.gauge(
'{0}runtime.per_report.non_active_pairs'.format(metrics_run_root),
cp.duration)
metadata.add_optional_job_metadata(metadata_conn,
command,
run_id,
report_outputs=report_metadata)
def stolen_violations(ctx: callable, config: callable, statsd: callable,
logger: callable, run_id: int, conn: callable,
metadata_conn: callable, command: str,
metrics_root: callable, metrics_run_root: callable,
output_dir: str, newer_than: str,
filter_by_conditions: list) -> None:
"""Generate per-MNO list of IMEIs seen on the network after they were reported stolen.
Arguments:
ctx: click context object
config: DIRBS config object
statsd: DIRBS statsd connection object
logger: DIRBS custom logger object
run_id: run id of the current job
conn: DIRBS PostgreSQL connection object
metadata_conn: DIRBS PostgreSQL metadata connection object
command: name of the command
metrics_root: root object for the statsd metrics
metrics_run_root: root object for the statsd run metrics
output_dir: output directory path
newer_than: violation newer then this date
filter_by_conditions: list of condition to filter by
Returns:
None
"""
operators_configured_check(config, logger)
metadata.add_optional_job_metadata(
metadata_conn,
command,
run_id,
report_schema_version=report_schema_version,
output_dir=os.path.abspath(str(output_dir)))
report_dir = make_report_directory(ctx, output_dir, run_id, conn, config)
with utils.CodeProfiler() as cp:
report_metadata = write_stolen_violations(config, logger, report_dir,
conn, filter_by_conditions,
newer_than)
statsd.gauge(
'{0}runtime.per_report.blacklist_violations_stolen'.format(
metrics_run_root), cp.duration)
# Store metadata about the report data ID and classification run ID
metadata.add_optional_job_metadata(metadata_conn,
command,
run_id,
report_outputs=report_metadata)
def unregistered_subscribers(ctx: callable, config: callable, statsd: callable,
logger: callable, run_id: int, conn: callable,
metadata_conn: callable, command: str,
metrics_root: callable,
metrics_run_root: callable, output_dir: str,
newer_than: str):
"""Generate per-MNO list of IMSIs that are not registered in subscribers list.
Arguments:
ctx: click context object
config: DIRBS config object
statsd: DIRBS statsd connection object
logger: DIRBS custom logger object
run_id: run id of the current job
conn: DIRBS PostgreSQL connection object
metadata_conn: DIRBS PostgreSQL metadata connection object
command: name of the command
metrics_root: root object for the statsd metrics
metrics_run_root: root object for the statsd run metrics
output_dir: output directory path
newer_than: violation newer then this date
Returns:
None
"""
operators_configured_check(config, logger)
metadata.add_optional_job_metadata(
metadata_conn,
command,
run_id,
report_schema_version=report_schema_version,
output_dir=os.path.abspath(str(output_dir)))
report_dir = make_report_directory(ctx, output_dir, run_id, conn, config)
with utils.CodeProfiler() as cp:
report_metadata = write_un_registered_subscribers(
logger, config, report_dir, conn, newer_than)
statsd.gauge(
'{0}runtime.per_report.unregistered_subscribers'.format(
metrics_run_root), cp.duration)
# store metadata
metadata.add_optional_job_metadata(metadata_conn,
command,
run_id,
report_outputs=report_metadata)
def blacklist_violations(ctx: callable, config: callable, statsd: callable,
logger: callable, run_id: int, conn: callable,
metadata_conn: callable, command: str,
metrics_root: callable, metrics_run_root: callable,
output_dir: str, month: int, year: int) -> None:
"""Generate per-operator blacklist violations.
Arguments:
ctx: click context object
config: DIRBS config object
statsd: DIRBS statsd connection object
logger: DIRBS custom logger object
run_id: run id of the current job
conn: DIRBS PostgreSQL connection object
metadata_conn: DIRBS PostgreSQL metadata connection object
command: name of the command
metrics_root: root object for the statsd metrics
metrics_run_root: root object for the statsd run metrics
output_dir: output directory path
month: reporting month
year: reporting year
Returns:
None
"""
metadata.add_optional_job_metadata(
metadata_conn,
command,
run_id,
report_schema_version=report_schema_version,
output_dir=os.path.abspath(str(output_dir)))
report_dir = make_report_directory(ctx, output_dir, run_id, conn, config)
with utils.CodeProfiler() as cp:
report_metadata = write_blacklist_violations(logger, config,
report_dir, conn, month,
year)
statsd.gauge(
'{0}runtime.per_report.blacklist_violation'.format(metrics_run_root),
cp.duration)
metadata.add_optional_job_metadata(metadata_conn,
command,
run_id,
report_outputs=report_metadata)
def classified_triplets(ctx: callable, config: callable, statsd: callable,
logger: callable, run_id: int, conn: callable,
metadata_conn: callable, command: str,
metrics_root: callable, metrics_run_root: callable,
output_dir: str, conditions: list) -> None:
"""Generate per-condition classified triplets list.
Arguments:
ctx: click context object
config: DIRBS config object
statsd: DIRBS statsd connection object
logger: DIRBS custom logger object
run_id: run id of the current job
conn: DIRBS PostgreSQL connection object
metadata_conn: DIRBS PostgreSQL metadata connection object
command: name of the command
metrics_root: root object for the statsd metrics
metrics_run_root: root object for the statsd run metrics
output_dir: output directory path
conditions: list of conditions for classified triplets
Returns:
None
"""
metadata.add_optional_job_metadata(
metadata_conn,
command,
run_id,
report_schema_version=report_schema_version,
output_dir=os.path.abspath(str(output_dir)))
report_dir = make_report_directory(ctx, output_dir, run_id, conn, config)
with utils.CodeProfiler() as cp:
report_metadata = write_classified_triplets(logger, conditions,
report_dir, conn)
statsd.gauge(
'{0}runtime.per_report.classified_triplets'.format(metrics_run_root),
cp.duration)
metadata.add_optional_job_metadata(metadata_conn,
command,
run_id,
report_outputs=report_metadata)
def transient_msisdns(ctx: callable, config: callable, statsd: callable,
logger: callable, run_id: int, conn: callable,
metadata_conn: callable, command: str,
metrics_root: callable, metrics_run_root: callable,
output_dir: str, period: int, num_of_imeis: int,
current_date: str) -> None:
"""Generate list of MSISDNS used with possible transient IMEIs.
Required Arguments:
period: Analysis period in days (positive integer)
num_of_imeis: Number of IMEIs a MSISDN must be seen with for analysis
"""
metadata.add_optional_job_metadata(
metadata_conn,
command,
run_id,
report_schema_version=report_schema_version,
output_dir=os.path.abspath(str(output_dir)))
report_dir = make_report_directory(ctx, output_dir, run_id, conn, config)
with utils.CodeProfiler() as cp:
report_metadata = write_transient_msisdns(logger,
period,
report_dir,
conn,
config,
num_of_imeis,
current_date=current_date)
statsd.gauge(
'{0}runtime.per_report.transient_msisdns'.format(metrics_run_root),
cp.duration)
metadata.add_optional_job_metadata(metadata_conn,
command,
run_id,
report_outputs=report_metadata)
def condition_imei_overlaps(ctx: callable, config: callable, statsd: callable,
logger: callable, run_id: int, conn: callable,
metadata_conn: callable, command: str,
metrics_root: callable, metrics_run_root: callable,
force_refresh: bool, disable_retention_check: bool,
disable_data_check: bool,
debug_query_performance: bool, month: int,
year: int, output_dir: str):
"""Generate per-condition reports showing matched IMEIs seen on more than one MNO network.
Arguments:
ctx: click context object
config: DIRBS config object
statsd: DIRBS statsd connection object
logger: DIRBS custom logger object
run_id: run id of the current job
conn: DIRBS PostgreSQL connection object
metadata_conn: DIRBS PostgreSQL metadata connection object
command: name of the command
metrics_root: root object for the statsd metrics
metrics_run_root: root object for the statsd run metrics
force_refresh: bool to force writing/generating reports from scratch
disable_retention_check: bool to disable data retention check
disable_data_check: bool to disable data check
debug_query_performance: bool to debug query performance
month: reporting month
year: reporting year
output_dir: output directory path
Returns:
None
"""
reports_validation_checks(disable_retention_check, year, month, logger,
config, conn, disable_data_check)
metadata.add_optional_job_metadata(
metadata_conn,
command,
run_id,
refreshed_data=force_refresh,
month=month,
year=year,
report_schema_version=report_schema_version,
output_dir=os.path.abspath(str(output_dir)))
report_dir = make_report_directory(ctx,
output_dir,
run_id,
conn,
config,
year=year,
month=month)
report_metadata = []
with utils.CodeProfiler() as cp:
country_name = config.region_config.name
logger.info(
'Generating country per-condition IMEI overlap reports (classified IMEIs seen on more than '
"one MNO\'s network this month...")
cond_names = [c.label for c in config.conditions]
report_metadata.extend(
write_condition_imei_overlaps(conn, config, month, year,
country_name, report_dir,
cond_names))
statsd.gauge(
'{0}runtime.per_report.condition_imei_overlaps'.format(
metrics_run_root), cp.duration)
# Store metadata about the report data ID and classification run ID
metadata.add_optional_job_metadata(metadata_conn,
command,
run_id,
report_outputs=report_metadata)
def top_duplicates(ctx: callable, config: callable, statsd: callable,
logger: callable, run_id: int, conn: callable,
metadata_conn: callable, command: str,
metrics_root: callable, metrics_run_root: callable,
force_refresh: bool, disable_retention_check: bool,
disable_data_check: bool, debug_query_performance: bool,
month: int, year: int, output_dir: str) -> None:
"""Generate report listing IMEIs seen with more than 5 IMSIs in a given month and year.
Arguments:
ctx: click context object
config: DIRBS config object
statsd: DIRBS statsd connection object
logger: DIRBS custom logger object
run_id: run id of the current job
conn: DIRBS PostgreSQL connection object
metadata_conn: DIRBS PostgreSQL metadata connection object
command: name of the command
metrics_root: root object for the statsd metrics
metrics_run_root: root object for the statsd run metrics
force_refresh: bool to force writing/generating reports from scratch
disable_retention_check: bool to disable data retention check
disable_data_check: bool to disable data check
debug_query_performance: bool to debug query performance
month: reporting month
year: reporting year
output_dir: output directory path
Returns:
None
"""
reports_validation_checks(disable_retention_check, year, month, logger,
config, conn, disable_data_check)
metadata.add_optional_job_metadata(
metadata_conn,
command,
run_id,
refreshed_data=force_refresh,
month=month,
year=year,
report_schema_version=report_schema_version,
output_dir=os.path.abspath(str(output_dir)))
report_metadata = []
report_dir = make_report_directory(ctx,
output_dir,
run_id,
conn,
config,
year=year,
month=month)
with utils.CodeProfiler() as cp:
imsi_min_limit = 5
country_name = config.region_config.name
logger.info(
'Generating country duplicate IMEI report (IMEIs seen with more than {0:d} IMSIs this '
'reporting month)...'.format(imsi_min_limit))
report_metadata.extend(
write_country_duplicates_report(conn,
config,
month,
year,
country_name,
report_dir,
imsi_min_limit=imsi_min_limit))
statsd.gauge(
'{0}runtime.per_report.top_duplicates'.format(metrics_run_root),
cp.duration)
# Store metadata about the report data ID and classification run ID
metadata.add_optional_job_metadata(metadata_conn,
command,
run_id,
report_outputs=report_metadata)
def standard(ctx: callable, config: callable, statsd: callable,
logger: callable, run_id: int, conn: callable,
metadata_conn: callable, command: str, metrics_root: callable,
metrics_run_root: callable, force_refresh: bool,
disable_retention_check: bool, disable_data_check: bool,
debug_query_performance: bool, month: int, year: int,
output_dir: str) -> None:
"""Generate standard monthly operator and country-level reports.
Arguments:
ctx: click context object
config: DIRBS config object
statsd: DIRBS statsd connection object
logger: DIRBS custom logger object
run_id: run id of the current job
conn: DIRBS PostgreSQL connection object
metadata_conn: DIRBS PostgreSQL metadata connection object
command: name of the command
metrics_root: root object for the statsd metrics
metrics_run_root: root object for the statsd run metrics
force_refresh: bool to force writing/generating reports from scratch
disable_retention_check: bool to disable data retention check
disable_data_check: bool to disable data check
debug_query_performance: bool to debug query performance
month: reporting month
year: reporting year
output_dir: output directory path
Returns:
None
"""
# Store metadata
metadata.add_optional_job_metadata(
metadata_conn,
command,
run_id,
refreshed_data=force_refresh,
month=month,
year=year,
report_schema_version=report_schema_version,
output_dir=os.path.abspath(str(output_dir)))
reports_validation_checks(disable_retention_check, year, month, logger,
config, conn, disable_data_check)
# Next, generate all the report data so that report generation can happen very quickly
data_id, class_run_id, per_tac_compliance_data = generate_monthly_report_stats(
config, conn, month, year, statsd, metrics_run_root, run_id,
force_refresh, debug_query_performance)
# Store metadata about the report data ID and classification run ID
metadata.add_optional_job_metadata(metadata_conn,
command,
run_id,
data_id=data_id,
classification_run_id=class_run_id)
report_dir = make_report_directory(ctx,
output_dir,
run_id,
conn,
config,
class_run_id=class_run_id,
year=year,
month=month,
data_id=data_id)
# First, copy all the report JS/CSS files into the output directory in
# cachebusted form and get the cachebusted filenames
asset_map = {}
report_assets = ['js/report.js', 'css/report.css']
for fn in report_assets:
logger.info('Copying required asset "%s" to report folder', fn)
asset = pkgutil.get_data('dirbs', fn)
name, ext = fn.split('/')[-1].split('.')
filename = '{0}_{1}.{2}'.format(
name, utils.cachebusted_filename_from_contents(asset), ext)
asset_map[fn] = filename
with open(os.path.join(report_dir, filename), 'wb') as of:
of.write(asset)
js_filename = asset_map['js/report.js']
css_filename = asset_map['css/report.css']
# Next, generate the country level report
report_metadata = []
with utils.CodeProfiler() as cp:
logger.info('Generating country report...')
country_name = config.region_config.name
country_per_tac_compliance_data = None
if per_tac_compliance_data is not None:
country_per_tac_compliance_data = per_tac_compliance_data[
OperatorConfig.COUNTRY_OPERATOR_NAME]
report = CountryReport(
conn,
data_id,
config,
month,
year,
country_name,
has_compliance_data=country_per_tac_compliance_data is not None)
report_metadata.extend(
write_report(report, month, year, report_dir, country_name,
css_filename, js_filename,
country_per_tac_compliance_data))
statsd.gauge('{0}runtime.per_report.country'.format(metrics_run_root),
cp.duration)
operators = config.region_config.operators
# Finally, generate the operator reports
for op in operators:
with utils.CodeProfiler() as cp:
logger.info('Generating operator report for operator ID %s...',
op.id)
operator_per_tac_compliance_data = None
if per_tac_compliance_data is not None:
operator_per_tac_compliance_data = per_tac_compliance_data.get(
op.id)
report = OperatorReport(
conn,
data_id,
config,
month,
year,
op,
has_compliance_data=operator_per_tac_compliance_data
is not None)
report_prefix = '{0}_{1}'.format(country_name, op.id)
report_metadata.extend(
write_report(report, month, year, report_dir, report_prefix,
css_filename, js_filename,
operator_per_tac_compliance_data))
statsd.gauge(
'{0}runtime.per_report.operators.{1}'.format(
metrics_run_root, op.id), cp.duration)
# Store per-report job metadata
metadata.add_optional_job_metadata(metadata_conn,
command,
run_id,
report_outputs=report_metadata)
