def create_or_load_item_monthly_sales_profit_mat(client_id):
# now = datetime.datetime.now()
# year = now.year
year = get_current_or_max_year(client_id)
cur_sale_name = f'{year}_sales_mat'
prev_sale_name = f'{year-1}_sales_mat'
cur_profit_name = f'{year}_profit_mat'
prev_profit_name = f'{year-1}_profit_mat'
if (not does_collection_exist(cur_sale_name, client_id)
or not does_collection_exist(prev_sale_name, client_id)):
df = get_item_wise_sales_data(client_id, year)
print("In item_analyis load data complete")
item2idx, idx2item, code2name = load_item_dict(client_id)
curr_df, prev_df = df[df['year'] == year], df[df['year'] == year - 1]
create_collection_item_sales(cur_sale_name, curr_df, item2idx,
client_id)
create_collection_item_sales(prev_sale_name, prev_df, item2idx,
client_id)
print(f'{prev_sale_name}')
create_collection_item_profit(cur_profit_name, curr_df, item2idx,
client_id)
create_collection_item_profit(prev_profit_name, prev_df, item2idx,
client_id)
print(f'{prev_profit_name}')
else:
item2idx, idx2item, code2name = load_item_dict(client_id)
return item2idx, idx2item, code2name
def make_forecast(client_id, periods):
df = get_daily_sales_data(client_id)
pkl_path = PATH + f'{client_id}_propeht_model.plk'
#for prophet requirement date column name ds and data column name y
daily_ts = df[['date', 'Total_sales']].rename(columns={
'date': 'ds',
'Total_sales': 'y'
})
#print(daily_ts.head(4))
m = Prophet(daily_seasonality=False)
m.add_country_holidays(country_name='US')
m.fit(daily_ts)
future = m.make_future_dataframe(periods=periods)
forecast_df = m.predict(future)
forecast_df = forecast_df[['ds', 'yhat', 'yhat_lower', 'yhat_upper']]
#print(forecast.head())
data_name = 'forecast_data'
if does_collection_exist(data_name, client_id):
drop_collection(data_name, client_id)
create_collection_and_insert_datafrme(data_name, forecast_df, client_id)
with open(pkl_path, "wb") as f:
pickle.dump(m, f)
logging.info(
f'timeseries sales prediction model and data Mongo collection for {client_id} in sales_forcast created'
)
def create_collection_item_sales(data_name, df, item2idx, client_id):
if does_collection_exist(data_name, client_id):
drop_collection(data_name, client_id)
rows = len(item2idx)
cols = 13
X = np.zeros((rows, cols), np.float32)
for item, month, sales in zip(df['Item_code'], df['month'],
df['Total_sales']):
if len(item) > 2:
#print(item2idx[item])
X[item2idx[item], month] += sales
doc_list = []
for i in range(rows):
doc = {
'idx': i,
'total': float(X[i, :].sum()),
'row': X[i, :].tolist()
}
doc_list.append(doc)
create_collection_and_insert_many(data_name=data_name,
doc_list=doc_list,
client_id=client_id)
logging.info(
f'item_wise_sales Mongo collection for {client_id} in item_analysis created'
)
def process_item_analysis(client_id):
try:
create_or_load_item_monthly_sales_profit_mat(client_id)
except Exception as e:
logging.error(
f"while creating itemwise_sales_profit_mat for {client_id} in item_analysis{e}"
)
try:
_, _, _ = load_item_dict(client_id)
except Exception as e:
logging.error(
f'while creating pickle code2name, idx2code for {client_id}in item_analysis {e}'
)
logging.info(
f'pickle for code2name, idx2code for {client_id} in item_analysis exists'
)
# now = datetime.datetime.now()
# year = now.year
year = get_current_or_max_year(client_id)
cur_sale_name = f'{year}_sales_mat'
prev_sale_name = f'{year-1}_sales_mat'
cur_profit_name = f'{year}_profit_mat'
prev_profit_name = f'{year-1}_profit_mat'
data_names = [
cur_sale_name, prev_sale_name, cur_profit_name, prev_profit_name
]
collection_exist = True
for data_name in data_names:
if does_collection_exist(data_name, client_id):
collection_exist = collection_exist and True
return collection_exist
def make_forecast(client_id, periods=52):
#weekly_balance = weekly_cash_blance_data(client_id)
weekly_balance = weekly_agg_cash_blance(client_id)
pkl_path = PATH + f'{client_id}_cash_flow_propeht_model.plk'
#for prophet requirement date column name ds and data column name y
weekly_ts = weekly_balance.rename(columns={
'date': 'ds',
'cash_balance': 'y'
})
#print(daily_ts.head(4))
m = Prophet(daily_seasonality=False,
weekly_seasonality=True,
yearly_seasonality=True)
m.add_country_holidays(country_name='IN')
m.fit(weekly_ts)
future = m.make_future_dataframe(periods=periods, freq='W')
forecast_df = m.predict(future)
forecast_df = forecast_df[['ds', 'yhat', 'yhat_lower', 'yhat_upper']]
#print(forecast.head())
data_name = 'cash_flow_weekly_forecast_data'
if does_collection_exist(data_name, client_id):
drop_collection(data_name, client_id)
create_collection_and_insert_datafrme(data_name, forecast_df, client_id)
with open(pkl_path, "wb") as f:
pickle.dump(m, f)
logging.info(
f'timeseries sales prediction model and data Mongo collection for \
{client_id} in sales_forcast created')
def create_coocc_mat_mango_collecion(client_id):
data_name = 'coocc_mat'
if not does_collection_exist(data_name, client_id):
vitemcodes_list = create_itemcodes_list(client_id)
#print(vitemcodes_list[:10])
item2idx, idx2item, code2name = load_item_dict(client_id)
#print(code2name)
itemcodes = [*code2name]
mat_sz = len(itemcodes)
X = np.zeros((mat_sz, mat_sz), dtype=np.int32)
#print(X.shape)
for vitems in vitemcodes_list:
n = len(vitems)
if n == 1: continue
for i in range(n):
vi = vitems[i]
for j in range(i + 1, n):
vj = vitems[j]
#print(item2idx[vi], item2idx[vj])
X[item2idx[vi], item2idx[vj]] += 1
X[item2idx[vj], item2idx[vi]] += 1
# insering data into mongo collecions row by row
doc_list = []
for i in range(X.shape[0]):
arr = np.nonzero(X[i, :] >= 1)[0].tolist()
nonzero = {}
sorted_x = []
if arr:
for idx in arr:
nonzero[idx] = int(X[i, :][idx])
sorted_x = sorted(nonzero.items(),
key=lambda x: x[1],
reverse=True)
sorted_x = [list(t) for t in sorted_x]
doc = {
'idx': i,
'freq': int(X[i, :].max()),
'bskts': len(arr),
'nonzero': sorted_x
}
doc_list.append(doc)
create_collection_and_insert_many(data_name, doc_list, client_id)
logging.info(
f'created coocc_matrix Mongo collection for {client_id}in Market Basket analysis'
)
else:
#print(f'{client_id}_{data_name} collection exists')
logging.info(
f'coocc_matrix Mongo collection for {client_id} in Market Basket analysis exists'
)
item2idx, idx2item, code2name = load_item_dict(client_id)
return item2idx, idx2item, code2name
def process_mba(client_id):
data_name = 'coocc_mat'
try:
create_coocc_mat_mango_collecion(client_id)
except Exception as e:
logging.error(
f"while creating coocc_matirx for {client_id} in market basket analysi{e}"
)
return does_collection_exist(
data_name,
client_id) and os.path.exists(PATH + f'{client_id}_item2idx.pkl')
def load_propet_forecast_data_n_model(client_id):
pkl_path = PATH + f'{client_id}_propeht_model.plk'
data_name = 'forecast_data'
if not os.path.exists(pkl_path) or not does_collection_exist(
data_name, client_id):
make_forecast(client_id, periods=360)
forecast_df = get_dataframe_collection(data_name, client_id)
with open(pkl_path, 'rb') as f:
model = pickle.load(f)
return forecast_df, model
def process_abc_analysis(client_id):
try:
get_saleswise_abc_data(client_id)
except Exception as e:
logging.error(
f"while creating sales_abc_df for {client_id} in abc_analysis{e}")
try:
get_profitwise_abc_data(client_id)
except Exception as e:
logging.error(
f"while creating profit_abc for {client_id} in abc_analysis{e}")
try:
_, _, _ = load_item_dict(client_id)
except Exception as e:
logging.error(
f'while creating pickle code2name, idx2code for {client_id}in abc_analysis {e}'
)
return (does_collection_exist('sales_abc', client_id)
and does_collection_exist('profit_abc', client_id))
def process_forecast(client_id):
try:
load_propet_forecast_data_n_model(client_id)
except Exception as e:
logging.error(
f"while creating timeseries sales prediction model for {client_id} in sales_forcast{e}"
)
logging.info(
f'timeseries sales prediction model and data Mongo collection for {client_id} in sales_forcast exists'
)
pkl_path = PATH + f'{client_id}_propeht_model.plk'
data_name = 'forecast_data'
return (os.path.exists(pkl_path)
and does_collection_exist(data_name, client_id))
def get_saleswise_abc_data(client_id):
data_name = 'sales_abc'
if not does_collection_exist(data_name, client_id):
#df, code2name = load_n_process_salesdetail_data(client_id)
df = load_n_process_salesdetail_data(client_id)
item2idx, idx2item, code2name = load_item_dict(client_id)
itemwise_sales = df.groupby(['Item_code'], as_index=False). \
agg({'Unitsale_price':np.average, 'Sale_quantity': np.sum})
itemwise_sales['totalsales'] = itemwise_sales[
'Unitsale_price'] * itemwise_sales['Sale_quantity']
itemwise_sales = itemwise_sales.sort_values(by='totalsales',
ascending=False)
itemwise_sales['cumulativesales'] = itemwise_sales[
'totalsales'].cumsum()
itemwise_sales['%sales'] = (itemwise_sales['cumulativesales'] /
itemwise_sales['totalsales'].sum())
itemwise_sales['ABC_category'] = itemwise_sales['%sales']. \
apply(lambda x: 'A' if x < 0.71
else 'B' if (x >= 0.71 and x < 0.91)
else 'C')
itemwise_sales.insert(
1, 'Item_name',
itemwise_sales['Item_code'].apply(lambda x: code2name[x]))
create_collection_and_insert_datafrme(data_name, itemwise_sales,
client_id)
logging.info(
f'created sales_abc Mongo collection for {client_id}in abc analysis'
)
else:
itemwise_sales = get_dataframe_collection(data_name, client_id)
logging.info(
f'sales_abc Mongo collection for {client_id} in abc_analysis exists'
)
return itemwise_sales