def print_progress(ind, last=False):
if last or (ind > 0 and ind % K_PRINT_ITER == 0):
ZLog.info('Iteration %d/%d\n' % (ind + 1, iter_n))
ZLog.debug(' content loss: %g\n' % org_loss.eval())
ZLog.debug(' style loss: %g\n' % style_loss.eval())
ZLog.debug(' tv loss: %g\n' % tv_loss.eval())
ZLog.debug(' total loss: %g\n' % loss.eval())
def down_load_img(self, url_dict, thread_lock):
img_url = url_dict['url']
img_url_thumb = url_dict['url_thumb']
file_name = self.img_dir + Md5Helper.mkmd5frombinary(img_url) + '.jpg'
if ZCommonUtil.file_exist(file_name):
ZLog.debug('{} has already exist'.format(img_url))
"""
也还是应该算成功收集了
"""
with thread_lock:
self.collect_cnt += 1
return
def begin_req(a_proxy):
headers = {"User-Agent": K_UA}
if self.proxy['type'] == 'HTTP':
proxy_dict = dict(http='http://{}'.format(a_proxy['proxy']),
https='http://{}'.format(a_proxy['proxy']))
else:
proxy_dict = dict(http='socks5://{}'.format(a_proxy['proxy']),
https='socks5://{}'.format(a_proxy['proxy']))
try:
if g_enable_stream:
return self._down_load_img_stream(
img_url,
file_name,
headers,
proxy_dict,
thread_lock,
img_url_thumb=img_url_thumb)
else:
return self._down_load_img_direct(
img_url,
file_name,
headers,
proxy_dict,
thread_lock,
img_url_thumb=img_url_thumb)
except Exception:
# ZLog.exception(e)
return False
bps = self.back_proxys[:]
random.shuffle(bps)
for proxy in bps:
if begin_req(proxy):
break
def _down_load_img_direct(self,
img_url,
file_name,
headers,
proxy_dict,
thread_lock,
img_url_thumb=None):
response = requests.get(img_url,
headers=headers,
proxies=proxy_dict,
timeout=(15, 360))
if response.status_code == 200:
try:
with open(file_name, 'wb') as f:
f.write(response.content)
f.flush()
"""
检测ok了，才可以，暂时就先算了，
需要添加重试次数限制
"""
check_img = PIL.Image.open(file_name)
check_img.verify()
check_img.close()
except Exception:
os.remove(file_name)
return True
with thread_lock:
self.collect_cnt += 1
if img_url_thumb is None:
ZLog.debug('down_load_img img_url_thumb ok!')
return True
else:
if img_url_thumb is None:
ZLog.debug('down_load_img fail url={} code={}'.format(
img_url, response.status_code))
return False
"""
原始地址下载失败，改为下载thumb，要是再失败了就失败了
"""
return self._down_load_img_stream(img_url_thumb,
file_name,
headers,
proxy_dict,
thread_lock,
img_url_thumb=None)
def stats_max_winth(order):
try:
bd = order['buy Date']
kp = order['keep_days']
'''
n_folds=3多来一个，避免由于cap对齐造成的缺失
'''
kl_pd = SymbolPd.make_kfold_pd(order.Symbol, cap=cap)
b_key = kl_pd[kl_pd['date'] == bd].key.values[0]
'''
bKey + 1
第一天的return不计算，factor赞按照mean买入，可以认为
符合正态分布，均值是0
'''
maxth = kl_pd[b_key + 1: b_key + kp + 1].netChangeRatio.cumsum().max()
maxth95 = maxth if kp >= keep95 or (b_key + keep95 + 1) > kl_pd.shape[0] \
else kl_pd[b_key + 1:b_key + keep95 + 1].netChangeRatio.cumsum().max()
except Exception, e:
# import pdb
# pdb.set_trace()
ShowMsg.show_msg('Exception', e.__str__)
ZLog.debug(e.__str__)
ZLog.debug(bd)
ZLog.debug(kl_pd)
return 0, 0
def _down_load_img_stream(self,
img_url,
file_name,
headers,
proxy_dict,
thread_lock,
img_url_thumb=None):
response = requests.get(img_url,
headers=headers,
proxies=proxy_dict,
timeout=(15, 90),
stream=True)
if response.status_code == 200:
try:
with open(file_name, 'wb') as f:
start_time = time.time() # 文件下载开始时间
for chunk in response.iter_content(chunk_size=1024):
now_time = time.time()
if now_time - start_time > 90:
ZLog.debug('now_time - start_time > 90')
raise RuntimeError("now_time - start_time > 90")
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
"""
检测ok了，才可以，暂时就先算了，
需要添加重试次数限制
"""
check_img = PIL.Image.open(file_name)
check_img.close()
except Exception:
os.remove(file_name)
return True
# with open(file_name, 'wb') as out_file:
# shutil.copyfileobj(response.raw, out_file)
with thread_lock:
self.collect_cnt += 1
if img_url_thumb is None:
ZLog.debug('down_load_img img_url_thumb ok!')
return True
else:
if img_url_thumb is None:
ZLog.debug('down_load_img fail url={} code={}'.format(
img_url, response.status_code))
return False
return self._down_load_img_stream(img_url_thumb,
file_name,
headers,
proxy_dict,
thread_lock,
img_url_thumb=None)