def __login(self):
socket.setdefaulttimeout(120)
ftp = FTP()
ftp.set_debuglevel(2)
ftp.set_pasv(False)
ftp.connect(self.__ftp_ip, self.__ftp_port)
ftp.login(self.__ftp_userName, self.__ftp_passWord)
from cfg import custom_log
custom_log.log_info_cmd('Login Success!')
return ftp
def upload_file(self, source_file_path, target_file_path):
from cfg import custom_log
upload_flag = False
ftp_server = self.__login()
try:
ftp_server.upload_file(source_file_path, target_file_path)
upload_flag = True
custom_log.log_info_cmd('Upload File:%s To:%s Success.' %
(source_file_path, self.name))
except IOError:
custom_log.log_error_cmd('[Error]Upload File:%s To:%s Fail!' %
(source_file_path, self.name))
return upload_flag
def spider_data(self):
data_list = pd.read_csv(self.save_path)
check_day = data_list[data_list['date'] ==
self.last_trading_day]['date'].values
if len(check_day) != 0:
custom_log.log_info_cmd('trading_day:[%s] data Already exist.' %
check_day[0])
return
headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) \
Chrome/70.0.3538.77 Safari/537.36'
}
for url in self.url_list:
try:
xml = requests.get(url, headers=headers)
except BaseException:
continue
text = xml.text
soup = BeautifulSoup(text, 'lxml')
if soup.find('tradingday') is not None:
trading_day = soup.find('tradingday').string
trade_list = list(
set([i['text'] for i in soup.find_all('data')]))
datatype_id_list = sorted(
set([i['value'] for i in soup.find_all('data')]))
for trade in trade_list:
csv_list = [trading_day, trade.strip()]
for datatype_id in datatype_id_list:
data_list = soup.find_all(attrs={
'text': trade,
'value': datatype_id
})
volume_sum = 0
volume_change_sum = 0
for data in data_list:
volume = re.search(r'<volume>(\d*)</volume>',
str(data)).group(1)
volume_change = re.search(
r'<varvolume>(.*)</varvolume>',
str(data)).group(1)
volume_sum += int(volume)
volume_change_sum += int(volume_change)
csv_list.extend([volume_sum, volume_change_sum])
self.writer_csv(csv_list)
def upload_file(self, source_file_path, target_file_path):
from cfg import custom_log
upload_flag = False
t, sftp = self.__login()
try:
sftp.put(source_file_path, target_file_path)
upload_flag = True
custom_log.log_info_cmd('Upload File:%s To:%s Success.' %
(source_file_path, self.name))
except IOError:
custom_log.log_error_cmd('[Error]Upload File:%s To:%s Fail!' %
(source_file_path, self.name))
finally:
if t is not None:
t.close()
return upload_flag
def run_cmd_str2(self, cmd_str):
from cfg import custom_log
custom_log.log_info_cmd('Server[%s],Cmd[%s] Start.' %
(self.name, cmd_str))
cmd_result_list = []
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(self.ip, self.port, self.user, self.pwd, timeout=30)
stdin, stdout, stderr = ssh.exec_command(cmd_str)
error_message_list = []
cmd_result = stderr.readlines()
for item in cmd_result:
error_message_list.append(item)
if len(error_message_list) > 0:
error_message_str = '\n'.join(error_message_list)
custom_log.log_debug_cmd(
'-----------[Server:%s,stderr]-----------' % self.name +
'\n' + error_message_str)
if 'Traceback' in error_message_str:
error_message = 'Server[%s],Cmd[%s] Error2!Error_message:%s' % \
(self.name, cmd_str, error_message_str)
custom_log.log_error_cmd(error_message)
raise Exception(error_message)
cmd_result = stdout.read()
for item in cmd_result.splitlines():
cmd_result_list.append(item)
if len(cmd_result_list) > 0:
custom_log.log_debug_cmd(
'-----------[Server:%s,stdout]-----------' % self.name +
'\n' + '\n'.join(cmd_result_list))
ssh.close()
except Exception:
error_msg = traceback.format_exc()
error_message = 'Server[%s],Cmd[%s] Exception2!Error_msg:%s' % (
self.name, cmd_str, error_msg)
custom_log.log_error_cmd(error_message)
raise Exception(error_message)
custom_log.log_info_cmd('Server[%s],Cmd[%s] Stop!' %
(self.name, cmd_str))
return cmd_result_list
def download_file(self, source_file_path, target_file_path):
from cfg import custom_log
download_flag = False
ftp = self.__login()
try:
with open(target_file_path, 'wb') as file_handler:
ftp.retrbinary("RETR %s" % source_file_path,
file_handler.write, self.__buff_size)
custom_log.log_info_cmd('Download File:%s From:%s Success.' %
(source_file_path, self.name))
download_flag = True
except error_perm:
custom_log.log_error_cmd(error_perm)
custom_log.log_error_cmd('[Error]Download File:%s From:%s Fail!' %
(source_file_path, self.name))
finally:
ftp.set_debuglevel(0)
ftp.quit()
return download_flag
def upload_file(self, source_file_path, target_file_path):
from cfg import custom_log
upload_flag = False
ftp = self.__login()
try:
with open(source_file_path, 'rb') as file_handler:
ftp.storbinary('STOR ' + target_file_path, file_handler,
self.__buff_size)
custom_log.log_info_cmd('Upload File:%s To:%s Success.' %
(source_file_path, self.name))
upload_flag = True
except error_perm:
custom_log.log_error_cmd(error_perm)
custom_log.log_error_cmd('[Error]Upload File:%s To:%s Fail!' %
(source_file_path, self.name))
finally:
ftp.set_debuglevel(0)
ftp.quit()
return upload_flag
def check_connect(self):
"""
检查VPN是否可连接
"""
check_flag = False
from cfg import custom_log
sk = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sk.settimeout(1)
port = 8088
try:
sk.connect((self.ip, port))
custom_log.log_info_cmd('Server %s:%d Connect OK!' %
(self.ip, port))
check_flag = True
except Exception:
from cfg import custom_log
custom_log.log_info_cmd('[Error]Server %s:%d Connect Fail!' %
(self.ip, port))
finally:
sk.close()
return check_flag
def __exit__(self, exc_type, exc_val, exc_tb):
custom_log.log_info_cmd('spider finish')
self.csv_file.close()
