def adjust_learning_rate(init_lr, optimizer, epoch):
lr = init_lr * (0.1 ** (epoch//100))
toprint = True
for param_group in optimizer.param_groups:
if param_group["lr"]!=lr:
param_group["lr"] = lr
if toprint:
logging.info("Switching to learning rate %f" % lr)
toprint = False
def do_something():
request.params = get_request_args()
url = str(request.url)
# get ip
ip = request.headers.get('X-Real-IP')
if ip is None or ip == '':
ip = "127.0.0.1"
logging.info('ip: %2s request url is: [%s], input is: [%s]', ip, url,
request.params)
api_version, real_path = parse_path(request.path, web['url_pre'],
web['api_version'])
request.api_version = api_version
validate_json(real_path)
def response_handler(response):
# 关闭session
try:
g.session.close()
except:
pass
response.headers['Access-Control-Allow-Origin'] = '*'
response.headers[
'Access-Control-Allow-Headers'] = 'x-requested-with,content-type,authorization,Captcha-ETag'
response.headers[
'Access-Control-Expose-Headers'] = 'authorization, Captcha-ETag'
if not (response.headers['Content-Type'] == "image/png"):
logging.info("response is [{}]".format(response.data))
return response
def send_sms_dy(phone_number, template_param):
req = top.api.AlibabaAliqinFcSmsNumSendRequest()
req.set_app_info(
top.appinfo(R_SMS['ACCESS_KEY_ID'], R_SMS['ACCESS_KEY_SECRET']))
req.extend = uuid.uuid1()
req.sms_type = "normal"
req.sms_free_sign_name = R_SMS['sign_name']
req.sms_param = template_param
req.rec_num = phone_number
req.sms_template_code = R_SMS['template_code']
try:
resp = req.getResponse()
print resp
logging.info('fun_out: [sms_response: %2s]', resp)
if resp['alibaba_aliqin_fc_sms_num_send_response']['result'][
'err_code'] == "0":
return True
except Exception, e:
logging.info('fun_out: [sms_response: %2s]', e)
def sendmail(toaddrs, subject=email["subject"], msg=None):
"""
:param toaddrs: 收件人地址可多个["",""]
:param subject: 邮件主题
:param msg: 邮件内容
:return:
"""
mail_msg = MIMEMultipart()
if not isinstance(subject, unicode):
subject = unicode(subject, 'utf-8')
mail_msg['Subject'] = subject
mail_msg['From'] = email["from_email"]
mail_msg['To'] = ','.join(toaddrs)
mail_msg.attach(MIMEText(msg, 'html', 'utf-8'))
try:
s = smtplib.SMTP_SSL()
s.connect(email["smtp_addr"])
s.login(email["from_email"], email["from_email_pwd"])
s.sendmail(email["from_email"], toaddrs, mail_msg.as_string())
s.quit()
logging.info("email send sucess. %s", ','.join(toaddrs))
except IOError:
logging.error("Error unable to send email. %s", traceback.format_exc())
def send_sms_al(phone, param_string):
url = 'https://sms.aliyuncs.com/'
ts = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
once = str(uuid.uuid4())
data = {
'Action': 'SingleSendSms',
'SignName': sign_name,
'TemplateCode': template_code,
'RecNum': phone,
'ParamString': param_string,
'Format': 'JSON',
'Version': '2016-09-27',
'AccessKeyId': access_key_id,
'SignatureMethod': 'HMAC-SHA1',
'Timestamp': ts,
'SignatureVersion': '1.0',
'SignatureNonce': once,
}
def __percent_encode(s):
import urllib
s = str(s)
s = urllib.quote(s.decode('utf8').encode('utf8'), '')
s = s.replace('+', '%20')
s = s.replace('*', '%2A')
s = s.replace('%7E', '~')
return s
def __gen_signature(data, req_method, secret):
import hashlib
import hmac
import base64
sorted_data = sorted(data, key=lambda v: v[0])
vals = []
for k, v in sorted_data:
vals.append(__percent_encode(k) + '=' + __percent_encode(v))
params = '&'.join(vals)
string_to_sign = req_method + '&%2F&' + __percent_encode(params)
key = secret + '&'
signature = base64.encodestring(
hmac.new(key, string_to_sign, hashlib.sha1).digest()).strip()
return signature
try:
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
}
signature = __gen_signature(data.items(), 'POST', access_key_secret)
data['Signature'] = signature
r = requests.post(url, data=data, headers=headers)
resp = r.json()
logging.info('fun_out: [sms_response: %2s]', json.dumps(resp))
print resp
if resp.get('Model'):
return True
except Exception, e:
logging.error('sms_err: [sms_err: %2s]', e)
def fit(self, trainloader, validloader, lr=0.001, batch_size=128, num_epochs=10, corrupt=0.3,
loss_type="mse"):
"""
data_x: FloatTensor
valid_x: FloatTensor
"""
use_cuda = torch.cuda.is_available()
if use_cuda:
self.cuda()
logging.info("=====Denoising Autoencoding layer=======")
# optimizer = optim.Adam(filter(lambda p: p.requires_grad, self.parameters()), lr=lr)
optimizer = optim.SGD(filter(lambda p: p.requires_grad, self.parameters()), lr=lr, momentum=0.9)
if loss_type=="mse":
criterion = MSELoss()
elif loss_type=="cross-entropy":
criterion = BCELoss()
# validate
total_loss = 0.0
total_num = 0
for batch_idx, (inputs, _) in enumerate(validloader):
# inputs = inputs.view(inputs.size(0), -1).float()
# if use_cuda:
# inputs = inputs.cuda()
inputs = Variable(inputs)
hidden = self.encode(inputs)
if loss_type=="cross-entropy":
outputs = self.decode(hidden, binary=True)
else:
outputs = self.decode(hidden)
valid_recon_loss = criterion(outputs, inputs)
total_loss += valid_recon_loss.data * len(inputs)
total_num += inputs.size()[0]
valid_loss = total_loss / total_num
logging.info("#Epoch 0: Valid Reconstruct Loss: %.4f" % (valid_loss))
self.train()
for epoch in range(num_epochs):
# train 1 epoch
train_loss = 0.0
adjust_learning_rate(lr, optimizer, epoch)
for batch_idx, (inputs, _) in enumerate(trainloader):
# inputs = inputs.view(inputs.size(0), -1).float()
inputs_corr = masking_noise(inputs, corrupt)
# if use_cuda:
# inputs = inputs.cuda()
# inputs_corr = inputs_corr.cuda()
optimizer.zero_grad()
inputs = Variable(inputs)
inputs_corr = Variable(inputs_corr)
hidden = self.encode(inputs_corr)
if loss_type=="cross-entropy":
outputs = self.decode(hidden, binary=True)
else:
outputs = self.decode(hidden)
recon_loss = criterion(outputs, inputs)
train_loss += recon_loss.data*len(inputs)
recon_loss.backward()
optimizer.step()
# validate
valid_loss = 0.0
for batch_idx, (inputs, _) in enumerate(validloader):
# inputs = inputs.view(inputs.size(0), -1).float()
# if use_cuda:
# inputs = inputs.cuda()
inputs = Variable(inputs)
hidden = self.encode(inputs, train=False)
if loss_type=="cross-entropy":
outputs = self.decode(hidden, binary=True)
else:
outputs = self.decode(hidden)
valid_recon_loss = criterion(outputs, inputs)
valid_loss += valid_recon_loss.data * len(inputs)
logging.info("#Epoch %3d: Reconstruct Loss: %.4f, Valid Reconstruct Loss: %.4f" % (
epoch+1, train_loss / len(trainloader.dataset), valid_loss / len(validloader.dataset)))