def test_cdets_import_from_excel():
#---------------------------------------------------------
excel_df = dataimport.import_from_excel(CDETSconfig, 'CDETS', 'PSIRT')
assert isinstance(excel_df, pd.DataFrame)
assert excel_df.columns.values.tolist() == [
'Identifier', 'Status', 'SIR', 'Product', 'OPENED', 'CLOSED'
]
def test_get_swdl_grouped_data_by_date():
#---------------------------------------------------------
excel_df = dataimport.import_from_excel(SWDLconfig, 'SWDL', 'SWDL')
assert isinstance(excel_df, pd.DataFrame)
if isinstance(excel_df, pd.DataFrame):
filtered_df = swdlprep.filter_swdl(excel_df)
# check for 'CMA' 18M
cma = dataprep.get_product_data(filtered_df, 'CMA', SWDLconfig, 'SWDL')
cma_grp = swdlprep.group_data_by_date(cma, '18M', 'CMA')
# check column names for CMA releases
for c in cma_grp.columns.values.tolist():
assert 'CMA' in c
# check min/max dates are 18M apart (incl.)
min_mth = cma_grp.index.values.tolist()[0]
max_mth = cma_grp.index.values.tolist()[-1]
start_dt = datetime.strptime("01-" + min_mth, "%d-%b-%Y")
dt = util.get_next_date(
datetime(start_dt.year, start_dt.month, start_dt.day), 17, 0)
assert dt.strftime("%b-%Y") == max_mth
else:
pytest.fail()
def test_jira_import_from_excel():
#---------------------------------------------------------
excel_df = dataimport.import_from_excel(JIRAconfig, 'JIRA', 'CPFD')
assert isinstance(excel_df, pd.DataFrame)
assert excel_df.columns.values.tolist() == [
'Issue id', 'Status', 'Priority', 'Issue key', 'Created', 'Resolved'
]
def test_swdl_import_from_excel():
#---------------------------------------------------------
excel_df = dataimport.import_from_excel(SWDLconfig, 'SWDL', 'SWDL')
assert isinstance(excel_df, pd.DataFrame)
assert 'Download Date and Time' in excel_df.columns.values.tolist()
assert 'Full File Name' in excel_df.columns.values.tolist()
assert 'Access Level Name' in excel_df.columns.values.tolist()
def test_get_swdl_product():
#---------------------------------------------------------
excel_df = dataimport.import_from_excel(SWDLconfig, 'SWDL', 'SWDL')
assert isinstance(excel_df, pd.DataFrame)
if isinstance(excel_df, pd.DataFrame):
filtered_df = swdlprep.filter_swdl(excel_df)
# filter for 'CMM'
cmm1 = filtered_df[filtered_df.Product == "CMM"]
cmm2 = dataprep.get_product_data(filtered_df, 'CMM', SWDLconfig,
'SWDL')
assert len(cmm1) == len(cmm2)
else:
pytest.fail()
def test_cms_swdl_kpi_chart():
#---------------------------------------------------------
excel_df = dataimport.import_from_excel(SWDLconfig, 'SWDL', 'SWDL')
assert isinstance(excel_df, pd.DataFrame)
if isinstance(excel_df, pd.DataFrame):
filtered_df = swdlprep.filter_swdl(excel_df)
CMS = dataprep.get_product_data(filtered_df, 'CMS', SWDLconfig, 'SWDL')
# plot data
df_plot = swdlprep.group_data_by_date(CMS, 'allW', 'CMS')
kpi_chart = plotkpi.plot_swdl_chart(df_plot, 'CMS', 'allW', True)
assert 'png' in kpi_chart # assert kpi_chart is a picture file
assert os.path.exists(kpi_chart)
else:
pytest.fail()
def test_filter_swdl():
#---------------------------------------------------------
excel_df = dataimport.import_from_excel(SWDLconfig, 'SWDL', 'SWDL')
assert isinstance(excel_df, pd.DataFrame)
if isinstance(excel_df, pd.DataFrame):
filtered_df = swdlprep.filter_swdl(excel_df)
# check additional columns added during filter function
assert 'DownloadFile' in filtered_df.columns
assert 'Product' in filtered_df.columns
assert 'DownloadMonth' in filtered_df.columns
# check export file created
exportfile = os.path.join(os.getcwd(), config.autokpi["savedir"],
"exportswdl.csv")
assert os.path.exists(exportfile)
else:
pytest.fail()
def main(kpi_dict, importfromxl):
kpilog.debug("Input Parms: {}".format(kpi_dict))
clear_kpi_output()
# Setup dates for filters and for plotting
months_to_plot_df = dataprep.get_plot_months(None,
None) # use default dates
fyq_start, fyq_end = util.get_kpi_fyq_start_end(None,
None) # use default dates
months_fyq_df = dataprep.get_plot_months(fyq_start, fyq_end)
# set var to determine if at end of current fyq
end_fyq = False
if util.is_fyq_start(fyq_end):
end_fyq = True
# Process each tool/kpi combination
for tool, kpis in kpi_dict.items():
import_df = None
toolcfg = config.autokpi["tools"][tool]
kpilog.info("Processing KPI for: {}".format(tool))
for kpi in kpis:
kpilog.debug("Selected - {}".format(kpi))
plot_fyq = True
# call separate process for ATC
if kpi == 'ATC':
process_atc_schedules(toolcfg, tool, kpi)
continue
if importfromxl: # import data from (saved) excel workbook
import_df = dataimport.import_from_excel(toolcfg, tool, kpi)
elif kpi == 'BEMS': # import from Oracle DB
import_df = dataimport.import_from_db(toolcfg, kpi)
elif kpi == 'SWDL':
import_df = dataimport.import_from_excel(toolcfg, tool, kpi)
else: # import data using tool api
import_df = dataimport.import_from_api(toolcfg, tool, kpi)
if import_df is None:
kpilog.warning("Could not import data for {0}: {1}".format(
tool, kpi))
continue
# call separate process for SWDL (Software Downloads)
if kpi == 'SWDL':
process_swdl_data(import_df, toolcfg)
continue
if kpi == 'PSIRT': plot_fyq = False
# reformat dates
df_reformat = dataprep.reformat_df_dates(import_df, toolcfg,
importfromxl)
# Set kpi chart text
kpi_title = toolcfg["kpi"][kpi]["kpi_title"]
# call separate process for BEMS (Escalations)
if kpi == 'BEMS':
process_bems_data(df_reformat, toolcfg, months_fyq_df,
months_to_plot_df, kpi_title, end_fyq)
continue
# process data by project
all_products = []
df_all_by_month = None
df_all_by_fyq = None
for product in toolcfg["products"]:
# get project code
if product in ['CLIENT', 'meeting_apps']:
product_code = 'CMA'
elif product in ['SERVER', 'meetingserver']:
product_code = 'CMS'
else: # management
product_code = 'CMM'
if kpi == 'CFPD' and product_code == 'CMM': # ignore CMM for CFPD ??
continue
df_product = dataprep.get_product_data(df_reformat, product,
toolcfg, kpi)
if df_product is None:
kpilog.warning("No data for {0}: {1}".format(
kpi, product_code))
continue
kpilog.info("Processing {0} data for: {1}".format(
kpi, product_code))
# get mttr days
kpilog.debug("1. Calculate MTTR days....")
mttr_df = dataprep.get_mttr_days(df_product, months_fyq_df,
toolcfg)
# get open/closed counts
kpilog.debug("2. Get open/closed counts....")
df_open_grp, df_closed_grp = dataprep.get_product_counts(
df_product)
# merge open/closed counts
kpilog.debug("3. Merge open/closed/mttr data....")
df_plot_data = dataprep.get_plot_data(df_open_grp,
df_closed_grp, mttr_df,
months_fyq_df)
# perform mttr calc
kpilog.debug("4. Perform MTTR calcs....")
mttr_calcs = dataprep.get_mttr_calcs(df_plot_data,
product_code)
#***********************
# Monthly Product chart
#***********************
xaxis = "Months"
kpilog.debug("5. Prepare monthly data and chart....")
df_product_by_month = dataprep.group_counts_by_month(
df_plot_data, mttr_calcs, months_to_plot_df)
chart_title = str.join(
' ', [kpi_title.replace('XXX', product_code), 'Month\n'])
kpi_chart = plotkpi.plot_kpi_chart(df_product_by_month,
product_code, chart_title,
kpi, xaxis)
if kpi_chart:
kpilog.info("Monthly chart created: {}".format(kpi_chart))
#**********************
# FYQ Product chart
#**********************
if plot_fyq:
xaxis = 'FYQ'
kpilog.debug("6. Prepare FYQ data and chart....")
df_product_by_fyq = dataprep.group_counts_by_fyq(
df_plot_data, mttr_calcs, end_fyq)
chart_title = chart_title.replace('Month',
'Financial Quarter')
kpi_chart = plotkpi.plot_kpi_chart(df_product_by_fyq,
product_code,
chart_title, kpi, xaxis)
if kpi_chart:
kpilog.info("FYQ chart created: {}".format(kpi_chart))
# store totals
if all_products == []:
df_all_mttr = mttr_df
df_all_months = df_plot_data
else:
columnlist = ["OpenCnt", "ClosedCnt", "OpenDefects"]
df_all_months = sum_df_columns(df_all_months, df_plot_data,
columnlist)
df_all_mttr.MTTR += mttr_df.MTTR
all_products.append(product_code)
#***********************
# Monthly & FYQ Totals
#***********************
if not all_products == []:
# perform mttr calc for al months
kpilog.debug("7. Perform All MTTR calcs....")
df_all_months = dataprep.merge_mttr(df_all_months, df_all_mttr,
True)
mttr_calcs = dataprep.get_mttr_calcs(df_all_months)
product_str = str.join(', ', all_products)
#****************************
# All Products Monthly chart
#****************************
kpilog.debug("8. Group all by month....")
df_all_by_month = dataprep.group_counts_by_month(
df_all_months, mttr_calcs, months_to_plot_df)
kpilog.debug("9. Chart All monthly....")
chart_title = str.join(
' ', [kpi_title.replace('XXX', product_str), 'Month\n'])
kpi_chart = plotkpi.plot_kpi_chart(df_all_by_month,
product_str, chart_title,
kpi, 'AllMonths')
if kpi_chart:
kpilog.info(
"All Months chart created: {}".format(kpi_chart))
#****************************
# All Products FYQ chart
#****************************
if plot_fyq:
kpilog.debug("10. Group all by FYQ....")
df_all_by_fyq = dataprep.group_counts_by_fyq(
df_all_months, mttr_calcs, end_fyq)
kpilog.debug("11. Chart All FYQ....")
chart_title = chart_title.replace('Month',
'Financial Quarter')
kpi_chart = plotkpi.plot_kpi_chart(df_all_by_fyq,
product_str,
chart_title, kpi,
'AllFYQ')
if kpi_chart:
kpilog.info(
"All FYQ chart created: {}".format(kpi_chart))
return None