now = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
QUEUE_DOWNLOAD_MAX_SIZE = 3
QUEUE_LOAD_MAX_SIZE = 2
#-- fetch DB info
db_dict = get_yaml(DB_YML)
DB_NAME = db_dict["DB"]
DB_HOST = db_dict["Host"]
DB_PORT = db_dict["Port"]
DB_UNAME = db_dict["Username"]
DB_PWD = db_dict["Password"]
#-- open db connection
conn = get_conn(db_dict["DB"], db_dict["Username"], db_dict["Password"], db_dict["Host"], db_dict["Port"])
recent_working_day = recent_working_day(is_skip_holiday=True, conn=conn)
#-- opts
parser = OptionParser()
parser.add_option("--mode", "-m", dest="mode", action="store", default='downloadAndLoad', help="download|load|downloadAndLoad")
parser.add_option("--start_date", "-s", dest="start_date", action="store", default=recent_working_day, help="The default value is " + recent_working_day + ", the format is YYYYMMDD")
parser.add_option("--end_date", "-e", dest="end_date", action="store", default=recent_working_day, help="The default value is " + recent_working_day + ", the format is YYYYMMDD")
parser.add_option("--stock_id", "-i", dest="stock_id", action="store", help="--stock_id|-i is optional")
parser.add_option("--obj_selection", "-o", dest="obj_selection", action="store", help="--obj_selection|-o can indicate which object interface you want to use, T:Tengxun, S:Sina, N:Netease, e.g. -o \"T|S\"")
parser.add_option("--merge_before_copy", "-g", dest="merge_before_copy", action="store_true", default=True, help="Merge all the files in the data directory into one file and load it into table")
#parser.add_option("--enable_copy", "-c", dest="enable_copy", action="store_true", default=True if platform.system() == 'Linux' else False, help="Enable postgres copy when loading data into table")
parser.add_option("--enable_copy", "-c", dest="enable_copy", action="store_true", default=True, help="Enable postgres copy when loading data into table")
(options, args) = parser.parse_args()
#-- function
return_new_name_for_existing_file,
)
from Sys_paths import Sys_paths
from tooling.psql import get_conn, get_cur
from tooling.db_func import insert_into_table
from object_impl.Sina_stock import Sina_stock
from object_impl.Tengxun_stock import Tengxun_stock
from object_impl.Yahoo_stock import Yahoo_stock
# -- sys var
SEP = os.path.sep
FILE_PATH = sys.path[0]
FILE_BASE_NAME = __file__
FILE_NAME = FILE_PATH + SEP + FILE_BASE_NAME
recent_working_day = recent_working_day()
data_dir = Sys_paths.DATA_STOCK_DAILY
YML_DIR = Sys_paths.YML_DIR
LOG_DIR = Sys_paths.LOG_DIR
DB_YML = YML_DIR + SEP + "db.yml"
STOCK_YML = YML_DIR + SEP + "table" + SEP + "dw.stock.yml"
now = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
# -- opts
parser = OptionParser()
parser.add_option(
"--object_class",
"-o",
dest="object_class",
action="store",
default="tengxun",
from urllib2 import HTTPError
from tooling.common_tool import replace_vars, print_log, error_log, warn_log, get_date, recent_working_day, get_yaml, return_new_name_for_existing_file
from Sys_paths import Sys_paths
from tooling.psql import get_conn, get_cur
from tooling.db_func import insert_into_table
from object_impl.Sina_stock import Sina_stock
from object_impl.Tengxun_stock import Tengxun_stock
from object_impl.Yahoo_stock import Yahoo_stock
#-- sys var
SEP = os.path.sep
FILE_PATH = sys.path[0]
FILE_BASE_NAME = __file__
FILE_NAME = FILE_PATH + SEP + FILE_BASE_NAME
recent_working_day = recent_working_day()
data_dir = Sys_paths.DATA_STOCK_DAILY
YML_DIR = Sys_paths.YML_DIR
LOG_DIR = Sys_paths.LOG_DIR
DB_YML = YML_DIR + SEP + "db.yml"
STOCK_YML = YML_DIR + SEP + "table" + SEP + "dw.stock.yml"
now = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
#-- opts
parser = OptionParser()
parser.add_option(
"--object_class",
"-o",
dest="object_class",
action="store",
default='tengxun',
}
#-- file load seq, based on load_seq in table_mapping, put table names into load_seq_tables in a sorted order
load_seq_mapping = {}
for i in table_mapping:
load_seq_mapping[table_mapping[i]["load_seq"]] = i
load_seq_tables = [load_seq_mapping[i] for i in load_seq_mapping]
#-- fetch DB info
db_dict = get_yaml(DB_YML)
#-- open db connection
conn = get_conn(db_dict["DB"], db_dict["Username"], db_dict["Password"], db_dict["Host"], db_dict["Port"])
today = get_date("today")
yesterday = get_date("yesterday")
recent_working_day = recent_working_day(today, is_skip_holiday=True, conn=conn)
start_date = ""
end_date = ""
files_to_load = {}
#-- opts
parser = OptionParser()
parser.add_option("--mode", "-m", dest="mode", action="store", type="string", default="downloadAndLoad", help="download:Just download data from internet|load:load downloaded data into db|downloadAndLoad:do download and load in order")
parser.add_option("--start_date", "-s", dest="start_date", action="store", type="string", default=recent_working_day, help="Start date of the date range, e.g. 20150101")
parser.add_option("--end_date", "-e", dest="end_date", action="store", type="string", default=recent_working_day, help="End date of the date range, e.g. 20150101")
parser.add_option("--table", "-t", dest="table", action="store", type="string", help="dim_bankuai|dim_stock|dim_stock_bankuai|bankuai")
parser.add_option("--in_file", "-f", dest="in_file", action="store", type="string", help="To load a specific file, $DATE would be replaced from --start_date and --end_date")
(options, args) = parser.parse_args()
#-- var assignment
