import os import sys reload(sys) sys.setdefaultencoding('utf8') sys.path.append("{0}".format(os.environ["DIDP_HOME"])) from uuid import uuid1 import datetime from archive.archive_enum import PartitionKey, AddColumn from archive.hive_field_info import HiveFieldInfo from archive.didp_db_operator import DbOperator from utils.didp_logger import Logger from utils.didp_tools import get_db_login_info LOG = Logger() HIVE_CLASS = "org.apache.hive.jdbc.HiveDriver" DRIVER_PATH = "drivers\\jdbc\\inceptor-driver-6.0.0.jar" def get_uuid(): return str(uuid1()).replace("-", "") class BizException(Exception): """ 自定义异常 """ def __init__(self, *args): self.args = args
import ftplib from datetime import datetime sys.path.append("{0}".format(os.environ["DIDP_HOME"])) from utils.didp_logger import Logger from utils.didp_tools import stat_file_record from utils.didp_db_operator import DbOperator from utils.didp_ddl_operator import DdlOperator from utils.didp_log_recorder import LogRecorder from utils.didp_ddlfile_parser import DDLFileParser from utils.didp_ctlfile_parser import CtlFileParser from utils.didp_tools import get_db_login_info, generate_common_ddl_type # 全局变量 LOG = Logger() JDBC_DRIVER_PATH = os.getenv("DIDP_JDBC_DRIVER_PATH") # 配置库用户 DIDP_CFG_DB_USER = os.environ["DIDP_CFG_DB_USER"] # 配置库用户密码 DIDP_CFG_DB_PWD = os.environ["DIDP_CFG_DB_PWD"] # JDBC信息python DIDP_CFG_DB_JDBC_CLASS = os.environ["DIDP_CFG_DB_JDBC_CLASS"] DIDP_CFG_DB_JDBC_URL = os.environ["DIDP_CFG_DB_JDBC_URL"] class FileCollector: """ 文件采集类 Attributes: __args : 参数
import traceback import importlib import jaydebeapi reload(sys) sys.setdefaultencoding('utf8') sys.path.append("{0}".format(os.environ["DIDP_HOME"])) from datetime import datetime from utils.didp_logger import Logger from utils.didp_tools import get_db_login_info from utils.didp_log_recorder import LogRecorder # 全局变量 LOG = Logger() class ExportData(object): """ 数据卸载类 Attributes: __args : 参数 """ def __init__(self, args): self.__args = args # argparse接收到的参数没有转义,后续使用的时候需要将其替换成对应的值 self.__args.rcdelim = re.sub(r"\\n", "\n", self.__args.rcdelim) self.__args.rcdelim = re.sub(r"\\r", "\r", self.__args.rcdelim)
import jaydebeapi reload(sys) sys.setdefaultencoding('utf8') from datetime import datetime sys.path.append("{0}".format(os.environ["DIDP_HOME"])) from utils.didp_logger import Logger from utils.didp_tools import get_db_login_info, stat_file_record, stat_table_record from utils.didp_ddl_operator import DdlOperator from utils.didp_ddlfile_parser import DDLFileParser from utils.didp_log_recorder import LogRecorder # 全局变量 LOG = Logger() # 配置库用户 DIDP_CFG_DB_USER = os.environ["DIDP_CFG_DB_USER"] # 配置库用户密码 DIDP_CFG_DB_PWD = os.environ["DIDP_CFG_DB_PWD"] # JDBC信息python DIDP_CFG_DB_JDBC_CLASS = os.environ["DIDP_CFG_DB_JDBC_CLASS"] DIDP_CFG_DB_JDBC_URL = os.environ["DIDP_CFG_DB_JDBC_URL"] class DataLoader: """ 数据加载类 Attributes: __args : 参数