def typeValuetoStr(s): #对数据库入库字段值格式化
if not s:
return s
decode_s, dump_s = '', ''
if isinstance(s, str): #检测str编码并转成unicode
try:
json.loads(s)
except:
decode_s = strtodecode(s)
else:
decode_s = s #json的结果直接存数据库
elif isinstance(s, unicode):
decode_s = strtodecode(s)
else: #非str类型的先转成json格式
dump_s = json.dumps(s)
if decode_s: #转unicode成功
try:
rs = decode_s.encode(config.mysql_charset)
except:
return decode_s.encode('utf8').replace(
'\\', '\\\\') #对数据库指定编码转码失败返回默认utf8编码
else:
return rs.replace('\\', '\\\\')
elif dump_s: #转unicode失败
return dump_s.replace('\\', '\\\\')
else:
return False
def typeValuetoStr(s): #对数据库入库字段值格式化
if not s:
return s
decode_s,dump_s = '',''
if isinstance(s, str): #检测str编码并转成unicode
try:
json.loads(s)
except:
decode_s = strtodecode(s)
else:
decode_s = s #json的结果直接存数据库
elif isinstance(s, unicode):
decode_s = strtodecode(s)
else: #非str类型的先转成json格式
dump_s = json.dumps(s)
if decode_s: #转unicode成功
try:
rs = decode_s.encode(config.mysql_charset)
except:
return decode_s.encode('utf8').replace('\\','\\\\') #对数据库指定编码转码失败返回默认utf8编码
else:
return rs.replace('\\','\\\\')
elif dump_s: #转unicode失败
return dump_s.replace('\\','\\\\')
else:
return False
def setTable(self, table_name='', table_struc=''): if not table_name: return False if not table_struc: return False try: sql = "create table if not exists `%s` %s" % (strtodecode(table_name), strtodecode(table_struc)) self.sql = sql tag = self.cur.execute(sql) except: print "sql error:%s" % sql else: return tag
def setTable(self, table_name='', table_struc=''):
if not table_name:
return False
if not table_struc:
return False
try:
sql = "create table if not exists `%s` %s" % (
strtodecode(table_name), strtodecode(table_struc))
self.sql = sql
tag = self.cur.execute(sql)
except:
print "sql error:%s" % sql
else:
return tag
def getNumPage(segnum='', num='', url=''): #获取号码页详细数据
if not segnum:
return False
if not num:
return False
if not url:
return False
gevent.sleep(random.randint(10,22)*0.81) #从此处协程并发
db_file = './data/mobile_area.db'
html = catchPage(url)
if not html:
print "catch %s num page error!" % num
print "url:%s\n" % (url)
with open("./logs/errornum.txt", "a") as f:
f.write(segnum+','+num+','+url+"\n")
return False
#json数据
try:
page_temp_dict = json.loads(unquote(html))
except:
print segnum+','+num+','+url+",result error convert to dict\n"
with open('./logs/errorlog.txt','a') as f:
f.write(segnum+','+num+','+url+",result error convert to dict\n")
return False
else:
try:
cx = sqlite3.connect(db_file)
cu = cx.cursor()
except:
print "can not find sqlite db file\n"
with open('./logs/errorlog.txt','a') as f:
f.write("can not find sqlite db file '%s'\n" % str(db_file))
return False
insdata = {}
#mobile_num
if page_temp_dict.get('Mobile', False):
insdata['mobile_num'] = int(page_temp_dict['Mobile'])
else:
with open('./logs/errorlog.txt','a') as f:
f.write(segnum+','+num+','+url+",No matching data\n")
return False #无号码
#mobile_area
if page_temp_dict.get('Province', False):
if page_temp_dict['Province'] == u'未知':
with open('./logs/errorlog.txt','a') as f:
f.write(segnum+','+num+','+url+",province is weizhi\n")
return False #无地区
if page_temp_dict.get('City', False):
insdata['mobile_area'] = strtodecode(page_temp_dict['Province']+' '+page_temp_dict['City'])
else:
insdata['mobile_area'] = strtodecode(page_temp_dict['Province']+' '+page_temp_dict['Province'])
else:
with open('./logs/errorlog.txt','a') as f:
f.write(segnum+','+num+','+url+",No matching province\n")
return False #无地区
#mobile_type
if page_temp_dict.get('Corp', False):
if page_temp_dict.get('Card', False):
insdata['mobile_type'] = strtodecode(page_temp_dict['Corp']+' '+page_temp_dict['Card'])
else:
insdata['mobile_type'] = strtodecode(page_temp_dict['Corp'])
#area_code
if page_temp_dict.get('AreaCode', False):
insdata['area_code'] = strtodecode(page_temp_dict['AreaCode'])
#post_code
if page_temp_dict.get('PostCode', False):
insdata['post_code'] = strtodecode(page_temp_dict['PostCode'])
if insdata:
sql = "insert into mobile_area values (?,?,?,?,?,?)"
cu.execute(sql, (None,insdata['mobile_num'],insdata['mobile_area'],insdata['mobile_type'],insdata['area_code'],insdata['post_code']))
try:
cx.commit() #执行insert
except:
with open('./logs/errorlog.txt','a') as f:
f.write(segnum+','+num+','+url+",insert sqlitdb faild\n")
return False
else:
print "%d write db ok" % insdata['mobile_num']
return True
