def success_parse(self, response):
utils.log('name:%s success_parse proxy:%s meta:%s' %
(self.name, str(
response.meta.get('proxy_info')), str(response.meta)))
filename = datetime.datetime.now().strftime('%Y-%m-%d %H_%M_%S_%f')
self.save_page(filename, response.body)
if response.body.find(self.success_mark) or self.success_mark is '':
proxy = response.meta.get('proxy_info')
speed = time.time() - response.meta.get('cur_time')
table = response.meta.get('table')
id = response.meta.get('id')
utils.log('speed:%s table:%s id:%s' % (speed, table, id))
if table == self.name:
if speed > self.timeout:
command = utils.get_delete_data_command(table, id)
self.sql.execute(command)
else:
command = utils.get_update_data_command(table, id, speed)
self.sql.execute(command)
else:
if speed < self.timeout:
command = utils.get_insert_data_command(self.name)
msg = (None, proxy.get('ip'), proxy.get('port'),
proxy.get('country'), proxy.get('anonymity'),
proxy.get('https'), speed, proxy.get('source'),
None)
self.sql.insert_data(command, msg)
def success_parse(self, response):
utils.log('success_parse proxy:%s meta:%s' % (str(response.meta.get('proxy_info')), response.meta))
proxy = response.meta.get('proxy_info')
table = response.meta.get('table')
id = response.meta.get('id')
ip = proxy.get('ip')
self.save_page(ip, response.body)
if self.success_mark in response.body or self.success_mark is '':
speed = time.time() - response.meta.get('cur_time')
utils.log('speed:%s table:%s id:%s' % (speed, table, id))
if table == self.name:
if speed > self.timeout:
command = utils.get_delete_data_command(table, id)
self.sql.execute(command)
else:
vali_count = response.meta.get('vali_count', 0) + 1
command = utils.get_update_data_command(table, id, speed, vali_count)
self.sql.execute(command)
else:
if speed < self.timeout:
command = utils.get_insert_data_command(self.name)
msg = (None, proxy.get('ip'), proxy.get('port'), proxy.get('country'), proxy.get('anonymity'),
proxy.get('https'), speed, proxy.get('source'), None, 1)
self.sql.insert_data(command, msg, commit = True)
else:
# 如果没有找到成功标示，说明这里返回信息有误，需要删除当前库的 ip
if table == self.name:
command = utils.get_delete_data_command(table, id)
self.sql.execute(command)
def success_parse(self, response):
utils.log('name:%s success_parse meta:%s' % (self.name, response.meta))
https = response.meta.get('https')
filename = '%s_%s' % (response.meta.get('proxy_info').get('ip'), https)
self.save_page(filename, response.body)
if response.body.find(self.success_mark) or self.success_mark is '':
proxy = response.meta.get('proxy_info')
speed = time.time() - response.meta.get('cur_time')
table = response.meta.get('table')
id = response.meta.get('id')
utils.log('speed:%s table:%s id:%s https:%s' %
(speed, table, id, https))
if https == 'no':
data = json.loads(response.body)
origin = data.get('origin')
headers = data.get('headers')
x_forwarded_for = headers.get('X-Forwarded-For', None)
x_real_ip = headers.get('X-Real-Ip', None)
via = headers.get('Via', None)
anonymity = 3
if self.origin_ip in origin:
anonymity = 3
elif via is not None:
anonymity = 2
elif x_forwarded_for is not None and x_real_ip is not None:
anonymity = 1
if table == self.name:
if speed > self.timeout:
command = utils.get_delete_data_command(table, id)
self.sql.execute(command)
else:
# command = utils.get_update_data_command(table, id, speed)
# self.sql.execute(command)
command = "UPDATE {0} SET speed={1}, https={2}, anonymity={3} WHERE id={4}".format(
self.name, speed, https, anonymity, id)
self.sql.execute(command)
else:
if speed < self.timeout:
command = utils.get_insert_data_command(self.name)
msg = (None, proxy.get('ip'), proxy.get('port'),
proxy.get('country'), anonymity, https, speed,
proxy.get('source'), None)
self.sql.insert_data(command, msg)
else:
command = "UPDATE {0} SET https=\'{1}\' WHERE ip=\'{2}\'".format(
self.name, https, proxy.get('ip'))
self.sql.execute(command)
def success_parse(self, response):
utils.log('success_parse proxy:%s meta:%s' %
(str(response.meta.get('proxy_info')), response.meta))
proxy = response.meta.get('proxy_info')
table = response.meta.get('table')
id = response.meta.get('id')
ip = proxy.get('ip')
https = response.meta.get('https')
self.save_page(ip, response.body)
if response.body.find(self.success_mark) or self.success_mark is '':
speed = time.time() - response.meta.get('cur_time')
utils.log('speed:%s table:%s id:%s https:%s' %
(speed, table, id, https))
if https == 'no':
data = json.loads(response.body)
origin = data.get('origin')
headers = data.get('headers')
x_forwarded_for = headers.get('X-Forwarded-For', None)
x_real_ip = headers.get('X-Real-Ip', None)
via = headers.get('Via', None)
anonymity = 3
if self.origin_ip in origin:
anonymity = 3
elif via is not None:
anonymity = 2
elif x_forwarded_for is not None and x_real_ip is not None:
anonymity = 1
if table == self.name:
if speed > self.timeout:
command = utils.get_delete_data_command(table, id)
self.sql.execute(command)
else:
# command = utils.get_update_data_command(table, id, speed)
# self.sql.execute(command)
vali_count = response.meta.get('vali_count', 0) + 1
command = "UPDATE {name} SET speed={speed}, https='{https}', vali_count={vali_count}, " \
"anonymity={anonymity}, save_time={save_time} WHERE id={id}". \
format(name = self.name, speed = speed, https = https, vali_count = vali_count,
anonymity = anonymity, id = id, save_time = 'NOW()')
self.sql.execute(command)
else:
if speed < self.timeout:
command = utils.get_insert_data_command(self.name)
msg = (None, proxy.get('ip'), proxy.get('port'),
proxy.get('country'), anonymity, https, speed,
proxy.get('source'), None, 1)
self.sql.insert_data(command, msg, commit=True)
elif https == 'yes':
command = "UPDATE {name} SET https=\'{https}\' WHERE ip=\'{ip}\'". \
format(name = self.name, https = https, ip = ip)
self.sql.execute(command)
else:
pass