def __init__(self):
self.all_conf = config.Config("../conf/all.conf")
self.tuner_conf = config.TunerConfig("../conf/tuner.yaml")
self.cases_conf = config.BenchmarkConfig()
self.required_lists = self.list_required_config()
for request_type in self.required_lists:
self.get_group(request_type)
def main():
spark = create_spark_session()
conf = config.Config()
city_list = conf.get_city_list()
today = datetime.today()
year_to_process = today.year
month_to_process = today.month
# mortgage data is updated annually
if month_to_process == 1:
print("Processing Mortgage Data")
df = process_mortgage.read_mortgage_data(spark, conf, year_to_process)
for city in city_list:
process_mortgage.process_mortgage_data(spark, conf, city, year_to_process, df)
# mortgage data is updated monthly
print("Processing Crime Data")
for city in city_list:
process_crime.process_crime_data(spark, conf, city, year_to_process, month_to_process)
# get latest school data
print("Processing School Data")
for city in city_list:
process_school.process_school_data(spark, conf, city)
return
#!/usr/bin/env python # -*- coding:utf-8 -*- # @Datetime : 2017/12/8 上午10:40 # @Author : Alfred Xue # @E-Mail : [email protected] # @GitHub : https://github.com/Alfred-Xue # @Blog : http://www.cnblogs.com/alfred0311/ import os baseDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) from conf import config config = config.Config() # 服务器地址 ipAddr = config.ipAddr # WebLogic 域相关配置信息 WLS_HOME = config.WLS_HOME DOMAIN_HOME = config.DOMAIN_HOME domainName = config.domainName domainConfig = ''.join([DOMAIN_HOME, 'config/', 'config.xml']) # 管理服务器配置 adminUser = config.adminUser adminPwd = config.adminPwd adminPort = config.adminPort adminURL = ''.join(['t3://', ipAddr, ':', str(adminPort)]) startAS = ''.join( ['java weblogic.WLST -i ', baseDir, '/scripts/', 'startAdminServer.py'])
def __init__(self, config_file_path):
self.cfg = config.Config(config_file_path)