def _do_insert(self, conn, item, spider):
try:
conn.execute("select * from base_location where lac=%s and cid=%s",
(item['lac'], item['cid']))
ret = conn.fetchone()
if ret:
logging.info(LogUtil.get_time_now() +
"do db update, item ==>" + str(item))
conn.execute(
"update base_location set errcode=%s, lat=%s, lon=%s, address=%s, type=%s, update_time=now() where lac=%s and cid=%s",
(item['errcode'], item['lat'], item['lon'],
item['address'],
self._get_base_type(
item['lac'], item['cid']), item['lac'], item['cid']))
else:
logging.info(LogUtil.get_time_now() +
"do db insert, item ==>" + str(item))
conn.execute(
"insert into base_location (errcode, lac, cid, lat, lon, address, type, update_time) values (%s, %s, %s, %s, %s, %s, %s, now())",
(item['errcode'], item['lac'], item['cid'], item['lat'],
item['lon'], item['address'],
self._get_base_type(item['lac'], item['cid'])))
except:
logging.info(LogUtil.get_time_now() + traceback.format_exc())
def disable_ip(self, ip, port):
conn = mysql.connector.connect(host=settings.MYSQL_HOST, user=settings.MYSQL_USER, password=settings.MYSQL_PASSWD, db=settings.MYSQL_PROXY_DBNAME)
cursor = conn.cursor()
sqlStr = "update ip_proxy_info i set i.isvalid=0 where i.ip='%s' and i.port='%s'" % (ip, port)
try:
cursor.execute(sqlStr)
logging.info(LogUtil.get_time_now() + 'Disableing ip proxy %s' % ip)
except:
print LogUtil.get_time_now(), traceback.print_exc()
conn.commit()
cursor.close()
conn.close()
def _do_insert(self, conn, item, spider):
try:
conn.execute("select * from base_location where lac=%s and cid=%s", (item['lac'], item['cid']))
ret = conn.fetchone()
if ret:
logging.info(LogUtil.get_time_now() + "do db update, item ==>" + str(item))
conn.execute("update base_location set errcode=%s, lat=%s, lon=%s, address=%s, type=%s, update_time=now() where lac=%s and cid=%s",
(item['errcode'], item['lat'], item['lon'], item['address'], self._get_base_type(item['lac'], item['cid']), item['lac'], item['cid']))
else:
logging.info(LogUtil.get_time_now() + "do db insert, item ==>" + str(item))
conn.execute("insert into base_location (errcode, lac, cid, lat, lon, address, type, update_time) values (%s, %s, %s, %s, %s, %s, %s, now())",
(item['errcode'], item['lac'], item['cid'], item['lat'], item['lon'], item['address'], self._get_base_type(item['lac'], item['cid'])))
except:
logging.info(LogUtil.get_time_now() + traceback.format_exc())
def process_request(self, request, spider):
print request.meta
if settings.PROXY:
if request.meta.has_key('tried'):
request.meta['tried'] = int(request.meta['tried']) + 1
else:
proxy = random.choice(settings.PROXY)
request.meta['proxy'] = "%s://%s:%s" % (proxy['protocol'], proxy['ip'], proxy['port'])
request.meta['ip'] = proxy['ip']
request.meta['port'] = proxy['port']
request.meta['tried'] = 1
logging.info(LogUtil.get_time_now() + "use ip: %s proxy: %s, try for %s times" % (request.url, request.meta['proxy'], request.meta['tried']))
else:
logging.info('NO PROXY')
def _handle_error(self, failure, item, spider):
logging.info(LogUtil.get_time_now(), "Error ==>" + failure)
def _handle_error(self, failure, item, spider):
logging.info(LogUtil.get_time_now(), "Error ==>" + failure)
