# ###################################################################################################################
print("Adding forecaster.zip to system path")
sys.path.insert(0, "forecaster.zip")
# ###################################################################################################################
from support_func import get_current_date, get_sample_customer_list, date_check, \
get_sample_customer_list_new_addition, get_previous_sundays
from properties import MODEL_BUILDING
import properties as p
from run_weekly import run_weekly
from run_monthly import run_monthly
####################################################################################################################
# Getting Current Date Time for AppName
appName = "_".join([MODEL_BUILDING, get_current_date()])
####################################################################################################################
# conf = SparkConf()
spark = SparkSession \
.builder \
.appName(appName) \
.enableHiveSupport() \
.getOrCreate()
sc = spark.sparkContext
sqlContext = spark
print("Setting LOG LEVEL as ERROR")
sc.setLogLevel("ERROR")
start_time = time.time()
if 'backlog' in kwargs.keys() and kwargs.get('backlog'):
print("run_weekly() - Backlog -- Running MA")
build_prediction_weekly(sc=sc, sqlContext=sqlContext, _model_bld_date_string=_model_bld_date_string,
backlog_run=True, _bottlers=_bottlers)
else:
build_prediction_weekly(sc=sc, sqlContext=sqlContext, _model_bld_date_string=_model_bld_date_string,
_bottlers=_bottlers)
print("Time taken for running WEEKLY MODELS:\t\t--- %s seconds ---" % (time.time() - start_time))
if __name__ == "__main__":
####################################################################################################################
# Getting Current Date Time for AppName
appName = "_".join([MODEL_BUILDING, "WEEKLY", get_current_date()])
####################################################################################################################
# conf = SparkConf()
spark = SparkSession \
.builder \
.appName(appName) \
.enableHiveSupport() \
.getOrCreate()
sc = spark.sparkContext
sqlContext = spark
print ("Setting LOG LEVEL as ERROR")
sc.setLogLevel("ERROR")
# from run_distributed_arima import _run_dist_arima
# from run_distributed_prophet import _run_dist_prophet
# from run_moving_average import _run_moving_average_weekly
# from support_func import assign_category, get_current_date
# from transform_data.spark_dataframe_func import final_select_dataset
# from properties import MODEL_BUILDING, weekly_pdt_cat_123_location, weekly_pdt_cat_7_location
# from pyspark.sql.functions import *
# from transform_data.data_transform import string_to_gregorian
# from support_func import get_current_date, get_sample_customer_list
# import properties as p
###################################################################################################################
# Getting Current Date Time for AppName
appName_Weekly = "_".join([MODEL_BUILDING, "W", get_current_date()])
####################################################################################################################
# conf = SparkConf().setAppName(appName)
#
# sc = SparkContext(conf=conf)
# sqlContext = HiveContext(sparkContext=sc)
spark = SparkSession \
.builder \
.config("spark.sql.warehouse.dir",
"wasb://[email protected]/user/sshuser/spark-warehouse") \
.appName(appName_Weekly) \
.enableHiveSupport() \
.getOrCreate()
"""
author: rajarshi
"""
from pyspark import SparkContext, SparkConf
from pyspark.sql import HiveContext
from pyspark.sql.functions import *
from support_func import _generate_invoice, get_order_dates_between
from transform.support_func import _get_visit_list_from_invoice
from support_func import get_current_date
from properties import *
appName = "_".join(["CSO_INVOICE_GEN_", get_current_date()])
conf = SparkConf() \
.setAppName(appName) \
.set("spark.io.compression.codec", "snappy")
sc = SparkContext(conf=conf)
sqlContext = HiveContext(sparkContext=sc)
print "Setting LOG LEVEL as ERROR"
sc.setLogLevel("ERROR")
print "Add cso.zip to system path"
import sys
sys.path.insert(0, "cso.zip")
# Obtain visit list from invoice data. TODO : This will be replaced.
_visit_list = _get_visit_list_from_invoice(sqlContext=sqlContext,
start_date=START_DATE_INVOICE,
end_date=END_DATE_INVOICE)
from support_func import get_current_date, get_sample_customer_list, date_check, \
get_sample_customer_list_new_addition, get_previous_sundays
from properties import MODEL_BUILDING
import properties as p
from run_weekly import run_weekly
from run_monthly import run_monthly
####################################################################################################################
mdl_bld_date_string = ["".join(sys.argv[1])]
_model_bld_date_string_stg = mdl_bld_date_string[0]
# Getting Current Date Time for AppName
appName = "_".join([
MODEL_BUILDING, "SIMULATION_RUN::Dated::", _model_bld_date_string_stg,
get_current_date()
])
####################################################################################################################
# conf = SparkConf()
spark = SparkSession \
.builder \
.appName(appName) \
.enableHiveSupport() \
.getOrCreate()
sc = spark.sparkContext
sqlContext = spark
print("Setting LOG LEVEL as ERROR")
print("Adding forecaster.zip to system path")
sys.path.insert(0, "forecaster.zip")
####################################################################################################################
from support_func import get_current_date, get_sample_customer_list, date_check
from properties import MODEL_BUILDING
import properties as p
# from run_weekly import run_weekly
# from run_monthly import run_monthly
from run_weekly_baseline import run_weekly_baseline
from run_monthly_baseline import run_monthly_baseline
####################################################################################################################
# Getting Current Date Time for AppName
appName = "_".join([MODEL_BUILDING, get_current_date()])
####################################################################################################################
# conf = SparkConf()
spark = SparkSession \
.builder \
.appName(appName) \
.enableHiveSupport() \
.getOrCreate()
sc = spark.sparkContext
sqlContext = spark
print ("Setting LOG LEVEL as ERROR")
sc.setLogLevel("ERROR")
###################################################################################################################
# from run_distributed_prophet_monthly import _run_dist_prophet_monthly
# from run_moving_average import _run_moving_average_monthly
# from support_func import assign_category, get_current_date, _get_last_day_of_previous_month
# from properties import MODEL_BUILDING, monthly_pdt_cat_456_location, monthly_pdt_cat_8910_location
# from pyspark.sql.functions import *
# from transform_data.data_transform import string_to_gregorian
# from support_func import get_current_date, get_sample_customer_list
# import properties as p
###################################################################################################################
# Getting Current Date Time for AppName
appName_Monthly = "_".join([MODEL_BUILDING, "M", get_current_date()])
####################################################################################################################
# conf = SparkConf().setAppName(appName)
#
# sc = SparkContext(conf=conf)
# sqlContext = HiveContext(sparkContext=sc)
spark = SparkSession \
.builder \
.config("spark.sql.warehouse.dir",
"wasb://[email protected]/user/sshuser/spark-warehouse") \
.appName(appName_Monthly) \
.enableHiveSupport() \
.getOrCreate()