def get_spec_dir(key): ''' ''' global s_save_path unzip_save_dir = os.path.join(s_save_path, key) log('tmp_unzip_save_dir:', unzip_save_dir) if os.path.exists(unzip_save_dir) == False: os.makedirs(unzip_save_dir) return unzip_save_dir
def module_init(): ''' brief:主要是模块初始化 ''' for module in modules_name: try: eval(module).init() except Exception, err: log("init module %s failed" % module) return False
def untar(file, path, num=3): ''' ''' cmd = "tar xvf " + file + " --strip-components " + str( num) + " -C " + path + " >>/dev/null" try: os.system(cmd) except Exception, err: log("file:[" + file + "] failed,err:", sys.exc_info()) return False
def module_run(argv): ''' breief:主要是控制模块的运行 ''' for module in modules_name: try: eval(module).run() except Exception, err: log("run module %s failed,err %s" % (module, err)) return False
def get_ini_sections(filename): ''' brief:获取ini配置文件中的段 ''' try: cf = ConfigParser.ConfigParser() cf.read(filename) section = cf.sections() except Exception, err: log('get ini[' + filename + '] section failed,err:', sys.exc_info) return False, False
def get_need_process_sections(filename): ''' brief:对配置文件中的段进行处理 ''' cf,section = get_ini_sections(filename) if not section: return False section = get_use_sec(section) log('sections:',section) return cf,section
def unzip(file, path): ''' ''' try: fz = zipfile.ZipFile(file, mode='r') for file in fz.namelist(): fz.extract(file, path) fz.close() except Exception, err: log("file:" + file + " unzip failed,err:", sys.exc_info()) return False
def init(): ''' Brief:初始化一些配置 ''' log("download init====") global s_ftp_info s_ftp_info['username'] = FTP_INFO['username'] s_ftp_info['password'] = FTP_INFO['password'] s_ftp_info['host'] = FTP_INFO['host'] s_ftp_info['port'] = FTP_INFO['port'] s_ftp_info['timeout'] = FTP_INFO['timeout'] s_ftp_info['localdir'] = FTP_INFO['localdir'] s_ftp_info['remote_root_dir'] = FTP_INFO['remote_root_dir']
def run(): ''' breief:主要是控制模块的运行 ''' if int(get_status()) != 2: return True s_date_list = get_last_date() for date in s_date_list: if not plugin_run(date): log("date %s parse failed,err ", sys.exc_info()) return False return True
def ftp_login(host, username, password, port=21, timeout=300): ''' brief:ftp登录 ''' global s_conn try: # s_conn.set_debuglevel(2) s_conn.connect(host, port, timeout) s_conn.login(username, password) except Exception, err: log("ftp server login failed:", sys.exc_info()) return False
def plugin_run(date): ''' breief:主要是控制模块的运行 ''' if int(get_status()) != 2: return True s_path = os.path.join(ROOT_PATH,'/saas/data') for module in modules_name: try: eval(module).run() except Exception,err: log("run module %s failed,err %s" %(module,err)) return False
def unrar(file_list): ''' ''' print 's_root_path:', s_root_path for filename in file_list: if os.path.exists(filename) == False: continue try: tar = tarfile.open(filename) for names in tar.getnames(): tar.extract(names, path=s_root_path) tar.close() except Exception, err: log("untar file failed,err:", sys.exc_info()) return False
def run(): ''' ''' if int(get_status()) != 1: return True uncompress_file_list = get_uncompress_file_list() if not uncompress_file_list: log("uncompress file list is not exists ,module exit") return True log("current need uncompress file list is", uncompress_file_list) ret = unrar(uncompress_file_list) if ret == False: return False set_status(2) return True
def run(path): ''' ''' log("plugin [%s] start running" %key) file_path_list = get_need_file_list(path , key) save_dir = get_spec_dir(key) preproces_filename = os.path.join(save_dir,s_process_file) for file in file_path_list: if os.path.exists(preproces_filename) == True: os.unlink(preproces_filename) ret = uncompress(file,key,save_dir) if not ret: return False if not process(preproces_filename): return False return True
def get_need_file_list(path, key): ''' 根据根路径和key值获取解析的文件列表 ''' dir_info = get_dir_by_key(path, key) if not dir_info: log("current upload info is not exist info [%s]" % key) return True log("dir_info:", dir_info) file_info = get_filepath_by_dir(dir_info) if not file_info: log("current upload [%s] is empty file list" % key) return True #log("file_info:",file_info) return file_info
def run(): ''' 主要是执行FTP下载任务 ''' if not is_can_download_check(): return True download_file_list = get_need_filenames(get_last_date()) if not download_file_list: log("have no file to download ,exit") return True log("need download file list:", download_file_list) if download(download_file_list) == False: close() return False log("==============download success====") #下载完成之后更新下载状态 set_status(1) return True
def ftp_login(host, username, password, port=21, timeout=300): ''' brief:ftp登录 ''' global s_conn try: # s_conn.set_debuglevel(2) s_conn.connect(host, port, timeout) s_conn.login(username, password) except Exception, err: log("ftp server login failed:", sys.exc_info()) return False log(s_conn.welcome) return True def get_file(ftp_path, local_path='.'): '''''' global s_conn ftp_login(s_ftp_info['host'], s_ftp_info['username'],\ s_ftp_info['password'],s_ftp_info['port'],\ s_ftp_info['timeout']) ftp_path = ftp_path.rstrip('/') file_name = os.path.basename(ftp_path) local_file = os.path.join(local_path, file_name) ftp_path = os.path.join(s_ftp_info['remote_root_dir'], ftp_path)