def catch_station_map(self):
Logging.debug('<---------- 获取站名信息 ---------->')
try:
response = self.session.get(INITURL, verify=False)
CLeftTicketUrl = re.search(r"var\sCLeftTicketUrl\s=\s'(.*?)';",
response.text).group(1)
tag = re.search(
r'<script type="text/javascript" src="(.*?station_name.*?)".*</script>',
response.text).group(1)
map_url = stringByAppendingPathComponent(DOMAIN, tag)
js_rep = self.session.get(map_url, verify=False)
js_data = re.search(r"'(.*)'", js_rep.text).group(1).split('|')
key = u''
country_ls_map = {}
country_sl_map = {}
for index in range(len(js_data)):
if index % 5 == 1:
key = js_data[index]
elif index % 5 == 2:
country_ls_map[key] = js_data[index]
country_sl_map[js_data[index]] = key
return country_ls_map, country_sl_map, CLeftTicketUrl
except Exception as e:
Logging.warning(e)
return None, None, None
def forward(self, X):
# Perform matrix multiplication and activation twice (one for each layer).
# (hint: add a bias term before multiplication)
# input layer to 1st hidden layer
Logging.debug("X: {}, W1: {}, b_0:{}".format(X.shape, self.W_1.shape,
self.b_0.shape))
self.a_1 = (
np.matmul(self.W_1, X.T) + self.b_0
).T # W1^T * X + B_0 | W1=num_nodes_hidden x 2(input nodes), X = BTx2, a1 = BT x num_nodes_hidden
Logging.debug("a_1: %s" % repr(self.a_1.shape))
# 1st hidden layer - activation
self.h_1 = self.activate(self.a_1) # h1 = BT x num_nodes_hidden
Logging.debug("h_1: %s" % repr(self.h_1.shape))
# output layer
self.a_2 = np.matmul(
self.h_1, self.W_2
) + self.b_1.T # a2 = BT x 1, h1= BT x num_nodes_hidden, W2 = num_nodes_hidden x 1, b1 = BTx1
#TODO: np.repeat(self.b_1, self.batch_size, axis=1)
Logging.debug("a_2: %s" % repr(self.a_2.shape))
if self.task == "regression":
final_activation = relu
else:
final_activation = softmax
self.y_hat = final_activation(self.a_2) # = BT x K
Logging.debug("y_hat: %s" % repr(self.y_hat.shape))
return self.y_hat
def request_ticket_info(self):
Logging.debug('<---------- 请求车票信息 ---------->')
train_infos = []
for url in self.request_ticket_urls:
proxies = []
for ip in IPProxyTool().getIPs():
proxies.append(dict(http='http://' + ip, https='http://' + ip))
print url
if len(proxies) != 0:
response = self.session.get(url,
verify=False,
proxies=choice(proxies))
else:
response = self.session.get(url, verify=False)
try:
if response.status_code == 200:
# Logging.debug(requests.utils.dict_from_cookiejar(self.session.cookies))
self.session.cookies.save(ignore_discard=True)
info_json = response.json()
result = info_json['data']['result']
train_infos.append(result)
else:
Logging.warning(response.text.encode('u8'))
continue
except Exception as e:
Logging.debug(response.text)
Logging.warning(e.message)
continue
return self.parse_result(train_infos)
def ticket_schedule(self):
Logging.debug('<---------- 开始定时任务:获取车票信息 --------->')
scheduler = BlockingScheduler()
scheduler.add_job(self.request_ticket_info,
'interval',
seconds=5,
max_instances=5)
scheduler.start()
def proxy_schedule(self):
Logging.debug('<---------- 开始定时任务:获取代理 --------->')
scheduler = BlockingScheduler()
scheduler.add_job(self.request_proxy,
'interval',
seconds=300,
max_instances=5)
scheduler.start()
def parse_captcha(self, image_name):
Logging.debug('<---------- 解析验证码 ---------->')
if platform.system() == "Linux":
Logging.info("Command: xdg-open %s &" % image_name)
os.system("xdg-open %s &" % image_name)
elif platform.system() == "Darwin":
Logging.info("Command: open %s &" % image_name)
os.system("open %s &" % image_name)
elif platform.system() in ("SunOS", "FreeBSD", "Unix", "OpenBSD",
"NetBSD"):
os.system("open %s &" % image_name)
elif platform.system() == "Windows":
os.system("%s" % image_name)
else:
Logging.info("无法获取当前操作系统，请自行打开验证码 %s 文件，并输入验证码。" %
os.path.join(os.getcwd(), image_name))
pic_index_str = raw_input("请输入验证答案(1-8,以','分割)：")
pic_indexs = pic_index_str.split(',')
offsetX = 0
offsetY = 0
answer = ''
for pic_index in pic_indexs:
if pic_index == '1':
offsetX = '45'
offsetY = '47'
elif pic_index == '2':
offsetX = '110'
offsetY = '47'
elif pic_index == '3':
offsetX = '187'
offsetY = '44'
elif pic_index == '4':
offsetX = '253'
offsetY = '44'
elif pic_index == '5':
offsetX = '46'
offsetY = '114'
elif pic_index == '6':
offsetX = '110'
offsetY = '114'
elif pic_index == '7':
offsetX = '187'
offsetY = '114'
elif pic_index == '8':
offsetX = '253'
offsetY = '114'
else:
Logging.warning('输入的答案无效')
answer = answer + offsetX + ',' + offsetY + ','
return answer[:-1]
def start(self, min_ip_count=5):
Logging.debug('<---------- 设置代理定时任务 ---------->')
proxy_thread = threading.Thread(target=self.proxy_schedule)
proxy_thread.setDaemon(True)
Logging.debug('<---------- 获取代理 ---------->')
self.request_proxy(min_ip_count=min_ip_count)
# 设置获取票信息定时任务
ticket_thread = threading.Thread(target=self.ticket_schedule)
ticket_thread.setDaemon(True)
ticket_thread.start()
proxy_thread.start()
def isLogin(self):
Logging.debug('<---------- 判断登录状态 ---------->')
try:
self.session.cookies.load(ignore_discard=True)
tk = requests.utils.dict_from_cookiejar(self.session.cookies)['tk']
res = self.session.post(self.check_user_url,
params={'_json_att': ''},
verify=False)
res_json = json.loads(res.text)
return res_json['data']['flag']
except Exception as e:
Logging.debug(e.message)
Logging.warning("cookies 加载失败 !!! 请重新运行 'login12306.py'验证登录 !!!")
return False
def umount(self):
if os.path.exists(self.mountPoint) and os.path.isdir(self.mountPoint):
Logging.debug("Unmounting %s." % self.mountPoint)
mountProcess = subprocess.Popen(["/bin/umount", self.mountPoint])
mountProcess.wait()
if mountProcess.returncode == 0 and len(os.listdir(self.mountPoint)) == 0:
if self.createdMountPoint:
Logging.debug("Deleting mounting point: %s" % self.mountPoint)
os.rmdir(self.mountPoint)
self.createdMountPoint = False
return True
return False
def config_task_urls(self, CLeftTicketUrl):
Logging.debug('<---------- 配置请求url ---------->')
request_ticket_urls = []
save_query_urls = []
for date in self.dates:
request_ticket_urls.append(
'%s?leftTicketDTO.train_date=%s&leftTicketDTO.from_station=%s&leftTicketDTO.to_station=%s'
'&purpose_codes=%s' %
(CLeftTicketUrl, date, self.start_place_ab, self.to_place_ab,
'ADULT'))
save_query_urls.append(
'%s?leftTicketDTO.train_date=%s&leftTicketDTO.from_station=%s&leftTicketDTO.to_station=%s'
'&purpose_codes=%s' %
(save_query_log_url, date, self.start_place_ab,
self.to_place_ab, 'ADULT'))
return request_ticket_urls, save_query_urls
def install(self, handler=DefaultHandler):
self.setValues()
destroy = None
servers = self.startHttpd()
master, slave = pty.openpty()
stdin = os.fdopen(master, "w")
logPath = self.log % self.values
Logging.debug("Trying write install log of VM '%s' to %s." %
(self.guest.getHostName(), logPath))
stdout = open(logPath, "w")
try:
self.cmdInstall = self.cmdInstall.replace("\n", " ")
Logging.info("Trying install guest '%s'..." %
self.guest.getHostName())
Logging.debug(self.cmdInstall % self.values)
process = subprocess.Popen(self.cmdInstall % self.values,
shell=True,
stdin=slave,
stdout=stdout,
stderr=stdout,
close_fds=True)
analyzator = handler(stdin, stdout, process)
analyzator.handle()
except InstallationError as e:
destroy = subprocess.Popen(self.cmdDestroy % self.values,
shell=True)
Logging.info("Check installation log for details: %s" % logPath)
raise e
finally:
if servers[0]: servers[0].kill()
if servers[1]: servers[1].kill()
if destroy:
return not destroy.wait()
Logging.info("Guest '%s' installed." % self.guest.getHostName())
return
def getNetwork(self):
self.stdin.write("\n")
stdoutRead = open(self.stdout.name)
stdoutRead.seek(0, os.SEEK_END)
Logging.info("Getting network settings...")
command = self.config.get("commands", "get")
self.stdin.write("%s\necho ==GET-NETWORK-DONE==\n\n" % command)
addresses = ""
i = 0
while not "\n==GET-NETWORK-DONE==" in addresses:
addresses += stdoutRead.read()
if i > 50:
raise NetworkSetterError(
"Error occured while getting network settings from guest '%s'."
% self.guest.getHostName())
i += 1
time.sleep(1)
Logging.debug("Raw network settings: %s" % str(addresses))
inets = re.findall("inet6?\s+([a-f0-9:./]+)", addresses)
ipAddresses = map(lambda inet: inet.split("/"), inets)
ipAddresses = filter(lambda ip: not ip[0].startswith("fe80"),
ipAddresses)
Logging.debug("Processed network settings: %s" % str(ipAddresses))
ipDict = {}
for ip in ipAddresses:
ipDict[ip[0]] = self.guest.getHostName()
time.sleep(2)
stdoutRead.close()
return ipDict
def download(self):
fileName = self.path.rsplit("/", 1)
filePath = "%s/%i-%s" % (self.tempStorage, hash(self), fileName[1])
if os.path.exists(filePath):
self.path = filePath
self.downloaded = True
return self.downloaded
Logging.debug("Trying download %s to %s." % (self.path, filePath))
try:
httpRequest = urllib2.urlopen(self.path)
except urllib2.HTTPError as e:
return None
if (not os.path.exists(self.tempStorage)
or not os.access(self.tempStorage, os.W_OK)
or System.getFreeSpace(self.tempStorage) < int(httpRequest.info().get("Content-Length"))):
return None
try:
iso = file(filePath, "w")
while 1:
buf = httpRequest.read(16*1024)
if not buf:
break
iso.write(buf)
iso.close()
except IOError as e:
return None
self.path = filePath
self.downloaded = os.path.exists(filePath)
return self.downloaded
def insertDNS(self, dnsDict):
conn = libvirt.open(DC.get("virsh", "connect"))
xmlDesc = self.network.XMLDesc(0)
Logging.debug("Inserting DNS entries: %s" % str(dnsDict))
localDns = self.makeDNS(dnsDict)
networkXml = xml.parseString(xmlDesc)
dns = networkXml.getElementsByTagName("dns")
if dns:
dns = dns[0]
for entry in localDns.getElementsByTagName("host"):
dns.appendChild(entry)
else:
networkXml.documentElement.appendChild(localDns)
self.network.destroy()
self.network.undefine()
self.network = conn.networkDefineXML(
networkXml.documentElement.toxml())
self.network.setAutostart(1)
self.network.create()
values = {
'addr': self.getIPv6Address(),
'prefix': self.getIPv6Prefix(),
'interface': self.getInterface()
}
if (DC.has_option("network", "manual-route")
and DC.get("network", "manual-route", True)):
cmd = DC.get("network", "manual-route", False, values)
cmd = cmd.replace("\n", " ")
subprocess.Popen(cmd, shell=True)
else:
Logging.warning("Manual route not set. Skipping.")
return
def getNetwork(self):
self.stdin.write("\n")
stdoutRead = open(self.stdout.name)
stdoutRead.seek(0, os.SEEK_END)
Logging.info("Getting network settings...")
command = self.config.get("commands", "get")
self.stdin.write("%s\necho ==GET-NETWORK-DONE==\n\n" % command)
addresses = ""
i = 0
while not "\n==GET-NETWORK-DONE==" in addresses:
addresses += stdoutRead.read()
if i > 50:
raise NetworkSetterError("Error occured while getting network settings from guest '%s'." % self.guest.getHostName())
i += 1
time.sleep(1)
Logging.debug("Raw network settings: %s" % str(addresses))
inets = re.findall("inet6?\s+([a-f0-9:./]+)", addresses)
ipAddresses = map(lambda inet: inet.split("/"), inets)
ipAddresses = filter(lambda ip: not ip[0].startswith("fe80"), ipAddresses)
Logging.debug("Processed network settings: %s" % str(ipAddresses))
ipDict = {}
for ip in ipAddresses:
ipDict[ip[0]] = self.guest.getHostName()
time.sleep(2)
stdoutRead.close()
return ipDict
def install(self, handler=DefaultHandler):
self.setValues()
destroy = None
servers = self.startHttpd()
master, slave = pty.openpty()
stdin = os.fdopen(master, "w")
logPath = self.log % self.values
Logging.debug("Trying write install log of VM '%s' to %s."
% (self.guest.getHostName(), logPath))
stdout = open(logPath, "w")
try:
self.cmdInstall = self.cmdInstall.replace("\n", " ")
Logging.info("Trying install guest '%s'..." % self.guest.getHostName())
Logging.debug(self.cmdInstall % self.values)
process = subprocess.Popen(self.cmdInstall % self.values, shell=True,
stdin=slave, stdout=stdout, stderr=stdout,
close_fds=True)
analyzator = handler(stdin, stdout, process)
analyzator.handle()
except InstallationError as e:
destroy = subprocess.Popen(self.cmdDestroy % self.values, shell=True)
Logging.info("Check installation log for details: %s" % logPath)
raise e
finally:
if servers[0]: servers[0].kill()
if servers[1]: servers[1].kill()
if destroy:
return not destroy.wait()
Logging.info("Guest '%s' installed." % self.guest.getHostName())
return
def insertDNS(self, dnsDict):
conn = libvirt.open(DC.get("virsh", "connect"))
xmlDesc = self.network.XMLDesc(0)
Logging.debug("Inserting DNS entries: %s" % str(dnsDict))
localDns = self.makeDNS(dnsDict)
networkXml = xml.parseString(xmlDesc)
dns = networkXml.getElementsByTagName("dns")
if dns:
dns = dns[0]
for entry in localDns.getElementsByTagName("host"):
dns.appendChild(entry)
else:
networkXml.documentElement.appendChild(localDns)
self.network.destroy()
self.network.undefine()
self.network = conn.networkDefineXML(networkXml.documentElement.toxml())
self.network.setAutostart(1)
self.network.create()
values = {'addr': self.getIPv6Address(), 'prefix': self.getIPv6Prefix(),
'interface': self.getInterface()}
if (DC.has_option("network", "manual-route")
and DC.get("network", "manual-route", True)):
cmd = DC.get("network", "manual-route", False, values)
cmd = cmd.replace("\n", " ")
subprocess.Popen(cmd, shell=True)
else:
Logging.warning("Manual route not set. Skipping.")
return
def initialize_param(self):
self.W_1 = np.random.uniform(-1, 1, (self.NNodes, self.input_dim))
self.W_2 = np.random.uniform(-1, 1, (self.NNodes, self.output_dim))
self.b_0 = np.random.uniform(-1, 1, (self.NNodes, 1))
self.b_1 = np.random.uniform(-1, 1, (self.output_dim, 1))
# self.W_1 = np.ones((self.NNodes, self.input_dim))
# self.W_2 = np.ones((self.NNodes, self.output_dim))
# self.b_0 = 0.5 * np.ones((self.NNodes, 1))
# self.b_1 = 0.5 * np.ones((self.output_dim, 1))
Logging.debug("W1: %s" % repr(self.W_1.shape))
Logging.debug("W2: %s" % repr(self.W_2.shape))
Logging.debug("b_0: %s " % repr(self.b_0.shape))
Logging.debug("b_1: %s " % repr(self.b_1.shape))
def setNetwork(self):
self.stdin.write("\n")
stdoutRead = open(self.stdout.name)
stdoutRead.seek(0, os.SEEK_END)
Logging.info("Trying set network...")
settings = self.prepareData()
commands = self.config.get("commands", "set", False,
{"settings": settings})
Logging.debug("Commands to write:\n%s" % str(commands))
singleCommands = commands.split("\n")
for cmd in singleCommands:
self.stdin.write("%s\n" % cmd)
time.sleep(0.1)
self.stdin.write("\necho ==SET-NETWORK-DONE==\n\n")
time.sleep(1)
output = ""
i = 0
while not "\n==SET-NETWORK-DONE==" in output:
output += stdoutRead.read()
if i > 50:
raise NetworkSetterError(
"Error occured while setting network settings from guest '%s'."
% self.guest.getHostName())
i += 1
time.sleep(1)
time.sleep(1)
stdoutRead.close()
def parse_result(self, train_infos):
Logging.debug('<---------- 解析车票信息 ---------->')
valid_train_infos = []
for train_info in train_infos:
results = train_info
for result in results:
try:
search_result = re.search(ur'\|?(.*?)(\|预订\|.*)', result)
train_data_list = search_result.group(2).split('|')
if len(search_result.group(1)) != 0:
train_data_list.insert(0, search_result.group(1))
else:
train_data_list.insert(0, "")
train_data_list.reverse()
if self.target_trains is None or len(
self.target_trains) == 0:
# 返回全部有效的车次信息
valid_train_info = self.structure_result(
train_data_list)
valid_train_infos.append(
valid_train_info
) if valid_train_info is not None else None
else:
# 返回指定查询的有效的车次信息
for target_train in self.target_trains:
for train_data in train_data_list:
if target_train in train_data:
valid_train_info = self.structure_result(
train_data_list)
valid_train_infos.append(
valid_train_info
) if valid_train_info is not None else None
except Exception as e:
Logging.warning(e.message)
# print valid_train_infos
# print json.dumps(valid_train_infos, encoding='UTF-8', ensure_ascii=False)
return valid_train_infos
def getCost(self, YTrue, YPredict):
# Compute loss / cost in terms of crossentropy.
# (hint: your regularization term should appear here)
# flatten all params
thetas = np.concatenate([
self.W_2.ravel(),
self.b_1.ravel(),
self.W_1.ravel(),
self.b_0.ravel()
]).ravel()
Logging.debug("Theta shape: {}".format(thetas.shape))
regular_term = self.reg_lambda * np.linalg.norm(
thetas) / YTrue.shape[0]
if self.task == "regression":
return np.mean(np.linalg.norm(YTrue - YPredict,
axis=1)) + regular_term
else:
# print(np.sum(YPredict, axis=0))
return np.mean(-np.sum(YTrue *
np.log(YPredict), axis=1)) + regular_term
def mount(self):
if not os.path.exists(self.mountPoint) or not os.path.isdir(self.mountPoint):
Logging.debug("Creating mounting point: %s" % self.mountPoint)
os.makedirs(self.mountPoint)
self.createdMountPoint = True
if not os.path.exists(self.getPath()):
raise IOError("Path '%s' does not exists." % self.getPath())
Logging.debug("Mounting %s to %s." % (self.getPath(), self.mountPoint))
mountProcess = subprocess.Popen(["/bin/mount", '-o', 'loop', self.getPath(), self.mountPoint])
mountProcess.wait()
if mountProcess.returncode == 0 and len(os.listdir(self.mountPoint)) > 0:
treeinfo = "%s/.treeinfo" % self.mountPoint
if not os.path.exists(treeinfo):
Logging.warn("The image doesn't contain .treeinfo file.")
else:
cp = ConfigParser.ConfigParser()
cp.read(treeinfo)
if cp.has_section("general") and cp.has_option("general", "arch"):
arch = cp.get("general", "arch")
imagesArch = "images-%s" % arch
if cp.has_section(imagesArch):
if (not cp.has_option(imagesArch, "kernel")
or not cp.has_option(imagesArch, "initrd")):
raise IOError("There's no kernel or initrd option"
" in '%s' section in .treeinfo file."
% imagesArch)
return True
return False
def get_plot_ROC_2(model, XTest, YTest):
"""
Plots the ROC curve.
Parameters
----------
model : NeuralNetwork object
This should be a trained NN model.
XTest : numpy matrix
The matrix containing samples features (not indices) for testing.
YTest : numpy matrix
This array contains the ground truth.
Returns
plt : matplot lib object
The ROC plot
"""
# for each threshold
thresholds = np.arange(0.01, 1.01, 0.01)
fprs = []
tprs = []
# get TPR, FPR
for threshold in thresholds:
YPred = model.predict(XTest, threshold)
tpr, fpr = get_TPR_FPR(YTest, YPred)
Logging.debug(
"ROC test: threshold {}, tpr {} ({}), fpr {} ({})".format(
threshold, tpr[1], tpr.shape, fpr[1], fpr.shape))
tprs.append(tpr[1])
fprs.append(fpr[1])
# add to X,Y
# plt.plot([f[1] for f in fprs], [t[1] for t in tprs])
plt.figure()
plt.plot(fprs, tprs)
plt.title("ROC Curve")
plt.xlabel("FPR")
plt.ylabel("TPR")
plt.xlim(-0.05, 1.05)
plt.ylim(-0.05, 1.05)
def setNetwork(self):
self.stdin.write("\n")
stdoutRead = open(self.stdout.name)
stdoutRead.seek(0, os.SEEK_END)
Logging.info("Trying set network...")
settings = self.prepareData()
commands = self.config.get("commands", "set", False, { "settings": settings })
Logging.debug("Commands to write:\n%s" % str(commands))
singleCommands = commands.split("\n")
for cmd in singleCommands:
self.stdin.write("%s\n" % cmd)
time.sleep(0.1)
self.stdin.write("\necho ==SET-NETWORK-DONE==\n\n")
time.sleep(1)
output = ""
i = 0
while not "\n==SET-NETWORK-DONE==" in output:
output += stdoutRead.read()
if i > 50:
raise NetworkSetterError("Error occured while setting network settings from guest '%s'." % self.guest.getHostName())
i += 1
time.sleep(1)
time.sleep(1)
stdoutRead.close()
def fit(self, X, Y, learningRate, epochs, regLambda, batchSize=5):
"""
This function is used to train the model.
Parameters
----------
X : numpy matrix
The matrix containing sample features for training.
Y : numpy array
The array containing sample labels for training.
Returns
-------
None
"""
# Initialize your weight matrices first.
# (hint: check the sizes of your weight matrices first!)
self.batch_size = batchSize
self.reg_lambda = regLambda
self.lr = learningRate
self.input_dim = X.shape[1]
self.output_dim = Y.shape[1]
self.initialize_param()
self.current_epoch = 0
self.current_step = 0
# For each epoch, do
while self.current_epoch < epochs:
self.current_epoch += 1
Logging.debug("Starting %i epoch" %
(self.current_epoch)) # DEBUG, INFO, WARNING
# For each training sample (X[i], Y[i]), do
for x_batch, y_batch in data_iterator(X, Y, self.batch_size):
self.current_step += 1
Logging.debug("Start forward step!")
# 1. Forward propagate once. Use the function "forward" here!
self.forward(x_batch)
Logging.debug("Back propagating")
# 2. Backward progate once. Use the function "backpropagate" here!
self.backpropagate(x_batch, y_batch)
self.count = 0
def get_uam(self):
Logging.debug('<---------- 获取时效信息 ---------->')
res = self.session.post(self.passport_authuam_url,
params=dict(appid=self.passport_appId),
verify=False)
try:
res_json = json.loads(res.text)
Logging.debug('<---------- 获取时效信息: %s ---------->' %
res_json["result_message"].encode('u8'))
if res_json['result_code'] == 0:
if res_json["apptk"]:
return res_json["apptk"]
elif len(res_json["newapptk"]) is not 0:
return res_json["newapptk"]
else:
return None
except Exception as e:
Logging.debug(res.text.decode('u8'))
Logging.warning(e.message)
def get_captcha(self):
captcha_param = dict(login_site='E', module='login', rand='sjrand')
Logging.debug('验证码请求参数:%s' % captcha_param)
Logging.debug('<---------- 获取验证码 ---------->')
res = self.session.get(self.passport_captcha_url,
params=captcha_param,
verify=False)
if res.status_code == 200:
captcha_name = 'captcha-image.png'
with open(captcha_name, 'wb') as f:
for data in res.iter_content(chunk_size=1024):
if data:
f.write(data)
f.flush()
f.close()
Logging.debug('<---------- 获取验证码成功 ---------->')
return captcha_name
else:
Logging.error('<---------- 获取验证码失败 ---------->')
return None
# -*- coding: utf-8 -*-
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
from Logging import Logging
if __name__ == '__main__':
log = Logging('BBDSpider').get_logging()
log.debug('this is debug message')
log.info('this is info message')
log.error('this is error message')
def __del__(self):
if self.downloaded and os.path.exists(self.path):
Logging.debug("Trying unlink file %s." % self.path)
os.unlink(self.path)
def login(self):
Logging.debug('<---------- 正在登录 ---------->')
captcha_name = self.get_captcha()
if captcha_name == None or len(captcha_name) == 0:
return False
# self.get_uam()
answer = self.parse_captcha(captcha_name)
captcha_check_param = dict(answer=answer,
login_site='E',
rand='sjrand')
# Logging.debug('验证码请求验证参数:%s' % captcha_check_param)
res = self.session.get(self.passport_captcha_check_url,
params=captcha_check_param,
verify=False)
if res.status_code == 200:
result = json.loads(res.text)
if result['result_code'] == '4':
Logging.debug('<---------- 验证成功 ---------->')
else:
Logging.debug('<---------- %s ---------->' %
result['result_message'])
return False
else:
Logging.error('<---------- 验证失败 ---------->')
return False
login_params = dict(username=self.username,
password=self.password,
appid=self.passport_appId)
res = self.session.post(self.passport_login, params=login_params)
try:
if json.loads(res.text)['result_code'] == 0:
Logging.debug('<---------- 登录成功 ---------->')
Logging.info(res.text)
# Logging.info(requests.utils.dict_from_cookiejar(self.session.cookies))
user_login_res = self.session.post(self.user_login_url,
verify=False,
allow_redirects=False)
Logging.debug(user_login_res.headers)
try:
Logging.debug('<---------- 获取登录重定向url ---------->')
redirect_login_url = user_login_res.headers['Location']
Logging.debug('redirect_login_url: %s' %
redirect_login_url)
Logging.debug(user_login_res.headers)
Logging.debug('<---------- 重定向获取 JSESSIONID ---------->')
redirect_res = self.session.get(redirect_login_url,
verify=False)
Logging.debug(redirect_res.headers)
# Logging.info(requests.utils.dict_from_cookiejar(self.session.cookies))
Logging.debug('<---------- 获取token ---------->')
token = self.get_uam()
if token is None or len(token) == 0:
return False
else:
Logging.debug(
'<---------- 将token保存进cookies ---------->')
res = self.session.post(self.uamauthclient,
params={'tk': token},
verify=False)
res_json = json.loads(res.text)
Logging.debug('<---------- %s ---------->' %
res_json['result_message'].encode('u8'))
if res_json['result_code'] != 0:
return False
self.session.cookies.save(ignore_discard=True)
return True
except Exception as e:
Logging.warning(e)
return False
else:
Logging.debug('<---------- 登录失败 ---------->')
Logging.debug(res.text)
return False
except Exception as e:
Logging.warning(e)
return False
import sys
from datetime import datetime
from Logging import Logging
from SetupEnvironment import SetupEnvironment
""" Getting Basic Logging Options """
logger = Logging().get_logger("setup")
if logger != None:
# Checking Python Version
python_major_version = sys.version_info.major
if python_major_version < 3:
logger.critical(
"Setup Module : Version Issue : Please install Python Version 3 or greater to execute this script"
)
else:
logger.debug("Setup Module : Correct Python Version Found")
logger.debug("Setup Module : Preparing Script Environment")
# Staring Environment Setup
setupEnvironment = SetupEnvironment(logger)
setupEnvironment.installAndUnpgradeLibraries()
logger.debug("Setup Module : Environment Setup Completed")
else:
print("Setup Module : Critical : Logging Setup Could Not Be Completed")
print("Setup Module : Critical : Process will Exit")
print("Setup Module : Critical : Contact Administrator For Resolution")
sys.exit()
def __init__(self, username=None, password=None):
self.cookies = cookielib.LWPCookieJar('cookies')
self.session = requests.Session()
self.session.cookies = self.cookies
Logging.debug('<---------- 正在初始化 ---------->')
self.username = username
self.password = password
Logging.debug('<---------- 登录用户名:%s ---------->' %
self.username.encode('u8'))
Logging.debug('<---------- 登录密码:%s ---------->' %
self.password.encode('u8'))
res = self.session.get(self.init_url, verify=False)
if res.status_code == 200:
try:
self.passport_appId = re.search(
r"var\spassport_appId\s=\s'(.*)'", res.text).group(1)
Logging.debug('获取passport_appId:%s' %
self.passport_appId.encode(res.encoding))
self.passport_captcha_url = re.search(
r"var\spassport_captcha\s=\s'(http.*)'", res.text).group(1)
Logging.debug('获取验证码url:%s' %
self.passport_captcha_url.encode(res.encoding))
self.passport_captcha_check_url = re.search(
r"var\spassport_captcha_check\s=\s'(http.*)'",
res.text).group(1)
Logging.debug(
'获取验证码验证url:%s' %
self.passport_captcha_check_url.encode(res.encoding))
self.passport_authuam_url = re.search(
r"var\spassport_authuam\s=\s'(http.*)'", res.text).group(1)
Logging.debug('获取时效验证url:%s' %
self.passport_authuam_url.encode(res.encoding))
self.passport_login = re.search(
r"var\spassport_login\s=\s'(http.*)'", res.text).group(1)
Logging.debug('获取登录url:%s' %
self.passport_login.encode(res.encoding))
self.uamauthclient = "https://kyfw.12306.cn/otn/uamauthclient"
except Exception as e:
Logging.error('%s' % e)
def check_user(self):
Logging.debug('<---------- 检查用户登录状态 ---------->')
res = self.session.post(check_user_url, verify=False)
Logging.debug(res.text)
def __init__(self,
start,
to,
trains=None,
dates=None,
seat_types=None,
delay=5):
cookies = cookielib.LWPCookieJar('cookies')
self.session = requests.session()
self.session.cookies = cookies
headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36',
'Content-Type': 'application/json;charset=UTF-8',
'cache-control': 'no-cache',
'Accept-Encoding': 'gzip, deflate, br'
}
self.session.headers = headers
try:
self.session.cookies.load(ignore_discard=True)
except Exception as e:
Logging.debug(e.message)
Logging.warning(
"cookies 加载失败 !!! 如需订票，请重新运行 'login12306.py'验证登录 !!!")
self.session.cookies = requests.utils.add_dict_to_cookiejar(
self.session.cookies, {
"current_captcha_type":
"Z",
"fp_ver":
"4.5.1",
"RAIL_EXPIRATION":
str(int(round(time.time() * 1000))),
"RAIL_DEVICEID":
"JCTFFU_Ut9rAYiNh49CcBhxCWjxxzfRWpEH9MVv78I-EnTFvtJxZopvJjdKInEm2k0gtSfg06x_xnH2FiRak7_uzab62y0QgnW-GmiRG8GCwa3cxSUeuJeqxG9s_mCn-aQ92yA3h8KElnUty4HAOmm6IYxeiXtm7"
})
self.dates = dates
# 获取站名及其对应缩写
self.country_ls_map, self.country_sl_map, CLeftTicketUrl = self.catch_station_map(
)
if self.country_ls_map is None:
Logging.error('country_ls_map is None')
return
if self.country_sl_map is None:
Logging.error('country_sl_map is None')
return
if CLeftTicketUrl is None:
Logging.error('CLeftTicketUrl is None')
return
self.start_place_ab = self.country_ls_map.get(start)
self.to_place_ab = self.country_ls_map.get(to)
if len(self.start_place_ab) == 0:
Logging.error("param 'start' is invalid")
return
if len(self.to_place_ab) == 0:
Logging.error("param 'to' is invalid")
return
self.target_trains = trains
self.start_place = start
self.to_place = to
self.seat_types = seat_types
self.delay = delay
self.request_ticket_urls, self.save_query_urls = self.config_task_urls(
stringByAppendingPathComponent(DOMAIN, 'otn/' + CLeftTicketUrl))
if len(self.request_ticket_urls) == 0:
Logging.error('request_ticket_urls is None')
return
if len(self.save_query_urls) == 0:
Logging.error('save_query_urls is None')
return
# 检查用户登录状态
self.check_user()
def save_query_log(self):
Logging.debug('<---------- 保存查询log ---------->')
for url in self.save_query_urls:
Logging.debug(url)
res = self.session.get(url, verify=False)
