now = datetime.datetime.now().strftime("%Y%m%d%H%M%S") QUEUE_DOWNLOAD_MAX_SIZE = 3 QUEUE_LOAD_MAX_SIZE = 2 #-- fetch DB info db_dict = get_yaml(DB_YML) DB_NAME = db_dict["DB"] DB_HOST = db_dict["Host"] DB_PORT = db_dict["Port"] DB_UNAME = db_dict["Username"] DB_PWD = db_dict["Password"] #-- open db connection conn = get_conn(db_dict["DB"], db_dict["Username"], db_dict["Password"], db_dict["Host"], db_dict["Port"]) recent_working_day = recent_working_day(is_skip_holiday=True, conn=conn) #-- opts parser = OptionParser() parser.add_option("--mode", "-m", dest="mode", action="store", default='downloadAndLoad', help="download|load|downloadAndLoad") parser.add_option("--start_date", "-s", dest="start_date", action="store", default=recent_working_day, help="The default value is " + recent_working_day + ", the format is YYYYMMDD") parser.add_option("--end_date", "-e", dest="end_date", action="store", default=recent_working_day, help="The default value is " + recent_working_day + ", the format is YYYYMMDD") parser.add_option("--stock_id", "-i", dest="stock_id", action="store", help="--stock_id|-i is optional") parser.add_option("--obj_selection", "-o", dest="obj_selection", action="store", help="--obj_selection|-o can indicate which object interface you want to use, T:Tengxun, S:Sina, N:Netease, e.g. -o \"T|S\"") parser.add_option("--merge_before_copy", "-g", dest="merge_before_copy", action="store_true", default=True, help="Merge all the files in the data directory into one file and load it into table") #parser.add_option("--enable_copy", "-c", dest="enable_copy", action="store_true", default=True if platform.system() == 'Linux' else False, help="Enable postgres copy when loading data into table") parser.add_option("--enable_copy", "-c", dest="enable_copy", action="store_true", default=True, help="Enable postgres copy when loading data into table") (options, args) = parser.parse_args() #-- function
return_new_name_for_existing_file, ) from Sys_paths import Sys_paths from tooling.psql import get_conn, get_cur from tooling.db_func import insert_into_table from object_impl.Sina_stock import Sina_stock from object_impl.Tengxun_stock import Tengxun_stock from object_impl.Yahoo_stock import Yahoo_stock # -- sys var SEP = os.path.sep FILE_PATH = sys.path[0] FILE_BASE_NAME = __file__ FILE_NAME = FILE_PATH + SEP + FILE_BASE_NAME recent_working_day = recent_working_day() data_dir = Sys_paths.DATA_STOCK_DAILY YML_DIR = Sys_paths.YML_DIR LOG_DIR = Sys_paths.LOG_DIR DB_YML = YML_DIR + SEP + "db.yml" STOCK_YML = YML_DIR + SEP + "table" + SEP + "dw.stock.yml" now = datetime.datetime.now().strftime("%Y%m%d%H%M%S") # -- opts parser = OptionParser() parser.add_option( "--object_class", "-o", dest="object_class", action="store", default="tengxun",
from urllib2 import HTTPError from tooling.common_tool import replace_vars, print_log, error_log, warn_log, get_date, recent_working_day, get_yaml, return_new_name_for_existing_file from Sys_paths import Sys_paths from tooling.psql import get_conn, get_cur from tooling.db_func import insert_into_table from object_impl.Sina_stock import Sina_stock from object_impl.Tengxun_stock import Tengxun_stock from object_impl.Yahoo_stock import Yahoo_stock #-- sys var SEP = os.path.sep FILE_PATH = sys.path[0] FILE_BASE_NAME = __file__ FILE_NAME = FILE_PATH + SEP + FILE_BASE_NAME recent_working_day = recent_working_day() data_dir = Sys_paths.DATA_STOCK_DAILY YML_DIR = Sys_paths.YML_DIR LOG_DIR = Sys_paths.LOG_DIR DB_YML = YML_DIR + SEP + "db.yml" STOCK_YML = YML_DIR + SEP + "table" + SEP + "dw.stock.yml" now = datetime.datetime.now().strftime("%Y%m%d%H%M%S") #-- opts parser = OptionParser() parser.add_option( "--object_class", "-o", dest="object_class", action="store", default='tengxun',
} #-- file load seq, based on load_seq in table_mapping, put table names into load_seq_tables in a sorted order load_seq_mapping = {} for i in table_mapping: load_seq_mapping[table_mapping[i]["load_seq"]] = i load_seq_tables = [load_seq_mapping[i] for i in load_seq_mapping] #-- fetch DB info db_dict = get_yaml(DB_YML) #-- open db connection conn = get_conn(db_dict["DB"], db_dict["Username"], db_dict["Password"], db_dict["Host"], db_dict["Port"]) today = get_date("today") yesterday = get_date("yesterday") recent_working_day = recent_working_day(today, is_skip_holiday=True, conn=conn) start_date = "" end_date = "" files_to_load = {} #-- opts parser = OptionParser() parser.add_option("--mode", "-m", dest="mode", action="store", type="string", default="downloadAndLoad", help="download:Just download data from internet|load:load downloaded data into db|downloadAndLoad:do download and load in order") parser.add_option("--start_date", "-s", dest="start_date", action="store", type="string", default=recent_working_day, help="Start date of the date range, e.g. 20150101") parser.add_option("--end_date", "-e", dest="end_date", action="store", type="string", default=recent_working_day, help="End date of the date range, e.g. 20150101") parser.add_option("--table", "-t", dest="table", action="store", type="string", help="dim_bankuai|dim_stock|dim_stock_bankuai|bankuai") parser.add_option("--in_file", "-f", dest="in_file", action="store", type="string", help="To load a specific file, $DATE would be replaced from --start_date and --end_date") (options, args) = parser.parse_args() #-- var assignment