def get_spec_dir(key):
'''
'''
global s_save_path
unzip_save_dir = os.path.join(s_save_path, key)
log('tmp_unzip_save_dir:', unzip_save_dir)
if os.path.exists(unzip_save_dir) == False:
os.makedirs(unzip_save_dir)
return unzip_save_dir
def module_init():
'''
brief:主要是模块初始化
'''
for module in modules_name:
try:
eval(module).init()
except Exception, err:
log("init module %s failed" % module)
return False
def untar(file, path, num=3):
'''
'''
cmd = "tar xvf " + file + " --strip-components " + str(
num) + " -C " + path + " >>/dev/null"
try:
os.system(cmd)
except Exception, err:
log("file:[" + file + "] failed,err:", sys.exc_info())
return False
def module_run(argv):
'''
breief:主要是控制模块的运行
'''
for module in modules_name:
try:
eval(module).run()
except Exception, err:
log("run module %s failed,err %s" % (module, err))
return False
def get_ini_sections(filename):
'''
brief:获取ini配置文件中的段
'''
try:
cf = ConfigParser.ConfigParser()
cf.read(filename)
section = cf.sections()
except Exception, err:
log('get ini[' + filename + '] section failed,err:', sys.exc_info)
return False, False
def get_need_process_sections(filename):
'''
brief:对配置文件中的段进行处理
'''
cf,section = get_ini_sections(filename)
if not section:
return False
section = get_use_sec(section)
log('sections:',section)
return cf,section
def unzip(file, path):
'''
'''
try:
fz = zipfile.ZipFile(file, mode='r')
for file in fz.namelist():
fz.extract(file, path)
fz.close()
except Exception, err:
log("file:" + file + " unzip failed,err:", sys.exc_info())
return False
def init():
'''
Brief:初始化一些配置
'''
log("download init====")
global s_ftp_info
s_ftp_info['username'] = FTP_INFO['username']
s_ftp_info['password'] = FTP_INFO['password']
s_ftp_info['host'] = FTP_INFO['host']
s_ftp_info['port'] = FTP_INFO['port']
s_ftp_info['timeout'] = FTP_INFO['timeout']
s_ftp_info['localdir'] = FTP_INFO['localdir']
s_ftp_info['remote_root_dir'] = FTP_INFO['remote_root_dir']
def run():
'''
breief:主要是控制模块的运行
'''
if int(get_status()) != 2:
return True
s_date_list = get_last_date()
for date in s_date_list:
if not plugin_run(date):
log("date %s parse failed,err ", sys.exc_info())
return False
return True
def ftp_login(host, username, password, port=21, timeout=300):
'''
brief:ftp登录
'''
global s_conn
try:
# s_conn.set_debuglevel(2)
s_conn.connect(host, port, timeout)
s_conn.login(username, password)
except Exception, err:
log("ftp server login failed:", sys.exc_info())
return False
def plugin_run(date):
'''
breief:主要是控制模块的运行
'''
if int(get_status()) != 2:
return True
s_path = os.path.join(ROOT_PATH,'/saas/data')
for module in modules_name:
try:
eval(module).run()
except Exception,err:
log("run module %s failed,err %s" %(module,err))
return False
def unrar(file_list):
'''
'''
print 's_root_path:', s_root_path
for filename in file_list:
if os.path.exists(filename) == False:
continue
try:
tar = tarfile.open(filename)
for names in tar.getnames():
tar.extract(names, path=s_root_path)
tar.close()
except Exception, err:
log("untar file failed,err:", sys.exc_info())
return False
def run():
'''
'''
if int(get_status()) != 1:
return True
uncompress_file_list = get_uncompress_file_list()
if not uncompress_file_list:
log("uncompress file list is not exists ,module exit")
return True
log("current need uncompress file list is", uncompress_file_list)
ret = unrar(uncompress_file_list)
if ret == False:
return False
set_status(2)
return True
def run(path):
'''
'''
log("plugin [%s] start running" %key)
file_path_list = get_need_file_list(path , key)
save_dir = get_spec_dir(key)
preproces_filename = os.path.join(save_dir,s_process_file)
for file in file_path_list:
if os.path.exists(preproces_filename) == True:
os.unlink(preproces_filename)
ret = uncompress(file,key,save_dir)
if not ret:
return False
if not process(preproces_filename):
return False
return True
def get_need_file_list(path, key):
'''
根据根路径和key值获取解析的文件列表
'''
dir_info = get_dir_by_key(path, key)
if not dir_info:
log("current upload info is not exist info [%s]" % key)
return True
log("dir_info:", dir_info)
file_info = get_filepath_by_dir(dir_info)
if not file_info:
log("current upload [%s] is empty file list" % key)
return True
#log("file_info:",file_info)
return file_info
def run():
'''
主要是执行FTP下载任务
'''
if not is_can_download_check():
return True
download_file_list = get_need_filenames(get_last_date())
if not download_file_list:
log("have no file to download ,exit")
return True
log("need download file list:", download_file_list)
if download(download_file_list) == False:
close()
return False
log("==============download success====")
#下载完成之后更新下载状态
set_status(1)
return True
def ftp_login(host, username, password, port=21, timeout=300):
'''
brief:ftp登录
'''
global s_conn
try:
# s_conn.set_debuglevel(2)
s_conn.connect(host, port, timeout)
s_conn.login(username, password)
except Exception, err:
log("ftp server login failed:", sys.exc_info())
return False
log(s_conn.welcome)
return True
def get_file(ftp_path, local_path='.'):
''''''
global s_conn
ftp_login(s_ftp_info['host'], s_ftp_info['username'],\
s_ftp_info['password'],s_ftp_info['port'],\
s_ftp_info['timeout'])
ftp_path = ftp_path.rstrip('/')
file_name = os.path.basename(ftp_path)
local_file = os.path.join(local_path, file_name)
ftp_path = os.path.join(s_ftp_info['remote_root_dir'], ftp_path)
