def check_valid_tips_ip():
ips = connectable_ips()
code = ts.get_stock_basics().index.values.tolist()[0]
with MultiTasks(32) as mt:
res = mt.run_list_tasks(pip_checker,
var_args=ips,
fix_args={'code': code, 'timeout': sets.PROXY_TIMEOUT},
en_bar=True, desc='CheckIP')
valid_ips = list(filter(None, res))
log.info('{} valid IPs.'.format(len(valid_ips)))
if not os.path.exists(sets.PROXY_IP_LIB):
with open(sets.PROXY_IP_LIB, 'w') as f:
f.write('\n'.join(valid_ips))
return
else:
with open(sets.PROXY_IP_LIB, 'r') as f:
pre_ips = f.read().split('\n')
valid_ips.extend(pre_ips)
valid_ips = list(filter(None, valid_ips))
valid_ips = rm_dupl(valid_ips)
with open(sets.PROXY_IP_LIB, 'w') as f:
f.write('\n'.join(valid_ips))
def backup_t0002(src_path, en_compress=True):
dst_folder_name = 't0002_backup'
dst_path = os.path.join(os.getcwd(), dst_folder_name)
log.info('Backup TDX customer data...')
if not os.path.exists(dst_path):
os.mkdir(dst_path)
else:
shutil.rmtree(dst_path)
os.mkdir(dst_path)
if not os.path.exists(sets.T0002_LIST):
log.error(r'Can not find {}'.format(sets.T0002_LIST))
exit(0)
src_list = glob(os.path.join(src_path, '*'))
with open(sets.T0002_LIST, 'r') as f:
t0002_list = filter(None, f.read().split('\n'))
for i in t0002_list:
if '*' in i:
key = i.split('.')[-1]
key_list = list(filter(lambda x: key in x, src_list))
for k in key_list:
shutil.copy(os.path.join(src_path, k), dst_path)
else:
copy_func = shutil.copy if '.' in i else shutil.copytree
dst_tmp = dst_path if '.' in i else os.path.join(dst_path, i)
copy_func(os.path.join(src_path, i), dst_tmp)
if en_compress:
cmd = '{} a -t7z {} {}\*'.format(sets.EXE_7Z, dst_folder_name, os.path.join(dst_path, ))
run_cmd(cmd)
shutil.rmtree(dst_path)
zip_full_path = os.path.realpath(dst_folder_name+'.7z')
subj = 'stool backup' + time_str()
py_send(subj=subj, atta_files=[zip_full_path])
os.remove(zip_full_path)
def terminal_socket(ws, namespace, pod, container):
log.info('Try create socket connection')
ssl_ca_cert, key_file, cert_file = K8SClient.gen_ca()
kub = K8SClient(api_host='kubernetes_api_url',
ssl_ca_cert=ssl_ca_cert,
key_file=key_file,
cert_file=cert_file)
try:
container_stream = kub.terminal_start(namespace, pod, container)
except Exception as err:
log.error('Connect container error: {}'.format(err))
ws.close()
return
kub_stream = K8SStreamThread(ws, container_stream)
kub_stream.start()
log.info('Start terminal')
try:
while not ws.closed:
message = ws.receive()
if message is not None:
if message != '__ping__':
container_stream.write_stdin(message)
container_stream.write_stdin('exit\r')
except Exception as err:
log.error('Connect container error: {}'.format(err))
finally:
container_stream.close()
ws.close()
def make_list_of_t0002(template_path):
p = template_path + ('*' if template_path[-1] == '\\' else '\\*')
os.rename(sets.T0002_LIST, sets.T0002_LIST + '.bak')
with open(sets.T0002_LIST, 'w') as f:
for i in glob(p):
print(i.split('\\')[-1], file=f)
log.info('Make th0002 list done.')
def terminal_socket(ws, namespace, pod, container, token):
log.info('Try create socket connection')
kub = K8SClient(
api_server=os.environ["API_SERVER"].rstrip("/"), id_token=token)
try:
container_stream = kub.terminal_start(namespace, pod, container)
except Exception as err:
log.error('Connect container error: {}'.format(err))
ws.close()
return
kub_stream = K8SStreamThread(ws, container_stream)
kub_stream.start()
log.info('Start terminal')
try:
while not ws.closed:
message = ws.receive()
if message is not None:
if message != '__ping__':
container_stream.write_stdin(message)
container_stream.write_stdin('exit\r')
except Exception as err:
log.error('Connect container error: {}'.format(err))
finally:
container_stream.close()
ws.close()
def get_codes(self):
if not os.path.exists(sets.INFO_FILE):
log.info('No local info file: {}, get all codes online.'.format(
sets.INFO_FILE))
ret = ts.get_stock_basics().index
else:
ret = pd.read_csv(sets.INFO_FILE, dtype={
'code': str
}).loc[:, 'code']
return ret
def save2wiz(offset=0):
if not os.path.exists(sets.MRZTBFP_PAPER_URLS):
assert (0)
with open(sets.MRZTBFP_PAPER_URLS, 'r') as f:
urls = f.read().split('\n')[::-1]
for url in tqdm(urls, ascii=True):
idx = urls.index(url)
if idx < offset:
continue
try:
req_i.url2wiz(url)
except Exception as err:
log.info('Save fail: {}'.format(url))
print(err)
sleep(6)
sleep(2)
def down_tips_singleprocess(incre=True):
if incre is True and os.path.exists(sets.ZT_REC):
with open(sets.ZT_REC, 'r') as f:
cont = list(filter(None, f.read().split('\n')))
codes = cont[-1].split(',')[-1].split(' ')
else:
codes = ts.get_stock_basics().index.values.tolist()
with open(sets.PROXY_IP_LIB, 'r') as f:
proxies_ori = f.read().split('\n')
res = []
proxies = proxies_ori
proxy_expire_flag = -1
p = proxies.pop(0)
for code in tqdm(codes, ascii=True, desc='DownTips'):
if codes.index(code) == int(len(codes) * 2 / 3):
proxies = proxies_ori
while True:
ret = get_one_tip(code, proxy_ip=p, timeout=sets.PROXY_TIMEOUT)
if ret is not None:
break
elif len(proxies) > 0:
p = proxies.pop(0)
continue
elif p is None and ret is None:
proxy_expire_flag = codes.index(code)
break
else:
p = None
continue
if proxy_expire_flag != -1 and ret is None:
break
res.append(ret)
if len(res) > 0:
print('{} proxies remaining.'.format(len(proxies)))
var = list(map(lambda x: x[0], res))
var = '\n'.join(var)
with open(sets.CONCEPT_FILE, 'w') as f:
f.write(var)
var = list(map(lambda x: x[1], res))
var = '\n'.join(var)
with open(sets.ZT_REASON_FILE, 'w') as f:
f.write(var)
if proxy_expire_flag != -1:
log.info('Not all tips downed, updated {}'.format(proxy_expire_flag))
def check_ip_batch(ip_list: list, mt=None, timeout=1, drop=True):
if mt is not None:
ret_ips = mt.run_list_tasks(func=check_ip_valid,
var_args=ip_list,
fix_args={'timeout': sets.PROXY_TIMEOUT},
en_bar=1,
desc='CheckRawIPs')
res = np.c_[np.array(ip_list), np.array(ret_ips)].tolist()
else:
res = []
for ip in ip_list:
print('Checking IP: {}:{}'.format(*ip))
res.append([*ip, check_ip_valid(*ip, timeout)])
res0 = list(filter(lambda x: x[2] is not None, res))
log.info('{} valid raw IPs.'.format(len(res0)))
if drop:
return res0
else:
return res
def run(self):
while not self.ws.closed:
if not self.stream.is_open():
log.info('container stream closed')
self.ws.close()
try:
if self.stream.peek_stdout():
stdout = self.stream.read_stdout()
self.ws.send(stdout)
if self.stream.peek_stderr():
stderr = self.stream.read_stderr()
self.ws.send(stderr)
except Exception as err:
log.error('container stream err: {}'.format(err))
self.ws.close()
break
def run(self):
while not self.ws.closed:
if not self.stream.is_open():
log.info('container stream closed')
self.ws.close()
try:
if self.stream.peek_stdout():
stdout = self.stream.read_stdout()
self.ws.send(stdout)
if self.stream.peek_stderr():
stderr = self.stream.read_stderr()
self.ws.send(stderr)
except Exception as err:
log.error('container stream err: {}'.format(err))
self.ws.close()
break
def import_tdx_list():
blk_dir = os.path.join(sets.TDX_ROOT, 'T0002', 'blocknew')
blk_files = glob(os.path.join(blk_dir, '*.blk'))
choose_range = ['Z1ZT.blk', 'Z2ZT.blk', 'Z3ZT.blk', 'Z4ZT.blk', 'Z5ZT.blk',
'TEMPTTA6030.blk', 'TEMPTTA120DAY.blk', 'BBB.blk', 'ZZB.blk']
var_list = []
for file in blk_files:
if os.path.split(file)[-1] not in choose_range:
continue
else:
with open(file, 'r') as f:
c = f.read().split('\n')
c = list(map(lambda x: x[1:], c))
var_list.extend(c)
var_list = list(filter(None, var_list))
var_list = list(set(var_list))
with open(sets.TDX_IMPORT_LIST, 'w') as f:
f.write('\n'.join(var_list))
log.info('Successfully import tdx list')
def raw_ip_parse():
if not os.path.exists(sets.RAW_IP_FILE):
log.error(
'Must manual make and prepare {} for proxy ip parsing job.'.format(
sets.RAW_IP_FILE))
exit(0)
f = open(sets.RAW_IP_FILE, 'r')
content = f.read()
f.close()
ip_list = []
pat0 = re.compile(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s+(\d{1,5}).*?',
re.S)
pat1 = re.compile(
r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s*:\s*(\d{1,5}).*?', re.S)
ip_list.extend(pat0.findall(content))
ip_list.extend(pat1.findall(content))
ip_list = rm_dupl(ip_list)
log.info('Raw ips: {}'.format(len(ip_list)))
return ip_list
def py_send(subj, body=None, atta_files: list = None):
mail = MIMEMultipart()
mail['from'] = 'neilwang'
mail['to'] = ADDR['o']
mail['subject'] = subj
mail.attach(MIMEText(body if body is not None else '', 'plain', 'utf-8'))
if atta_files is not None:
for file in atta_files:
atta = py_attach(file)
mail.attach(atta)
try:
server = smtplib.SMTP(SRV['o'], 25)
server.ehlo()
server.starttls()
server.login(ADDR['o'], '0q164b13')
log.info('Login email server with user {}'.format(ADDR['o']))
server.sendmail(ADDR['o'], ADDR['o'], mail.as_string())
log.info('Sending mail...')
server.quit()
log.info('Success!')
except smtplib.SMTPException as err:
print(err)