def __init__(self):
db_type = config.DB_config.get('db_type', 'mysql')
db_config = config.DB_config.get(db_type)
if db_type == 'mysql':
from mysql import MySql
self.sql = MySql(**db_config)
elif db_type == 'redis':
pass
elif db_type == 'sqlite':
pass
else: # default mysql
from mysql import MySql
self.sql = MySql(**config.DB_config.get('db_type'))
def main():
data_warehosue = soufangwang_data_extract_in_multiprocess()
d = MySql(host='121.42.188.55',
port=3306,
username='******',
password='******',
charset="utf8")
d.connect_to_database_server()
d.change_database('SouFangWang')
# sql = "INSERT INTO SouFangWang_New_House(loupan_name, loupan_name_detail, address, price) VALUES(%s, %s, %s, %s);"
# sql = "INSERT INTO SouFangWang_Second_House(loupan_name, loupan_name_detail, area, allocation, address, per_price, " \
# "price) VALUES(%s, %s, %s, %s, %s, %s, %s);"
sql = "INSERT INTO SouFangWang_Renting_House(loupan_name, loupan_name_detail, allocation, renting_type, floor, " \
"address, price) VALUES(%s, %s, %s, %s, %s, %s, %s);"
d.commit_to_database(sql, data_warehosue)
d.disconnect_from_database_server()
def __init__(self, book_num):
self.douban_url = "https://api.douban.com/v2/book/"
self.start_num = book_num[0]
self.end_num = book_num[1]
self.header = {'User-Agent': 'Mozilla/5.0'}
self.sql = MySql("Your MySQL setting")
self.data = []
def main():
data_warehosue = anjuke_data_extract_in_multiprocess()
d = MySql(host='121.42.188.55', port=3306,
username='******', password='******',
charset="utf8")
d.connect_to_database_server()
d.change_database('AnJuKe')
# sql = "INSERT INTO AnJuKe_New_House(loupan_name, loupan_name_detail, address, price) VALUES(%s, %s, %s, %s);"
# sql = "INSERT INTO AnJuKe_Second_House(loupan_name, loupan_name_detail, area, allocation, address, per_price, " \
# "price) VALUES(%s, %s, %s, %s, %s, %s, %s);"
sql = "INSERT INTO AnJuKe_Renting_House(loupan_name, loupan_name_detail, allocation, renting_type, finish_type, " \
"floor, address, price) VALUES(%s, %s, %s, %s, %s, %s, %s, %s);"
d.commit_to_database(sql, data_warehosue)
d.disconnect_from_database_server()
def __init__(self, start_num, count_num):
# Init function:
# Add request Header
# Set start&count number
# Connect MySQL
self.movie_url = "http://api.douban.com/v2/movie/top250?"
self.header = {'User-Agent': 'Mozilla/5.0'}
self.start = "start=" + str(start_num)
self.count = "count=" + str(count_num)
self.sql = MySql("Your MySQL setting")
self.data = []
def factory(self, dbtype):
dbtype = dbtype.lower()
if dbtype == 'oracle':
return Oracle(self._params)
elif dbtype == 'mssql':
return MsSql(self._params)
elif dbtype == 'mysql':
return MySql(self._params)
#elif dbtype == 'db2':
# return DB2(self._params)
#elif dbtype == 'sybase':
# return SaiBase(self._params)
elif dbtype == 'postgresql':
return Postgres(self._params)
else:
return None
def main():
d = MySql(host='121.42.188.55', port=3306,
username='******', password='******',
charset="utf8")
d.connect_to_database_server()
d.change_database('AnJuKe')
sql = "SELECT * FROM AnJuKe_New_House WHERE loupan_name_detail LIKE 'http://sz.fang.anjuke.com/loupan/%.html' AND " \
"price != '售价待定' AND price LIKE '均价%';"
results = d.query_from_database(sql)
d.disconnect_from_database_server()
excel = xlwt.Workbook()
sheet = excel.add_sheet('DataSheet', cell_overwrite_ok=True)
for (j, result) in enumerate(results):
sheet.write(j, 0, result[1])
sheet.write(j, 1, result[3])
price = re.findall(r'[0-9]+', result[4])
sheet.write(j, 2, int(price[0]))
excel.save("anjuke_new_house_data/ShenZhen_New_House.xls")
class SqlManager(object):
def __init__(self):
db_type = config.DB_config.get('db_type', 'mysql')
db_config = config.DB_config.get(db_type)
if db_type == 'mysql':
from mysql import MySql
self.sql = MySql(**db_config)
elif db_type == 'redis':
pass
elif db_type == 'sqlite':
pass
else: # default mysql
from mysql import MySql
self.sql = MySql(**config.DB_config.get('db_type'))
def init_database(self, database_name):
pass
def init_proxy_table(self, table_name):
return self.sql.init_proxy_table(table_name)
def insert_proxy(self, table_name, proxy):
return self.sql.insert_proxy(table_name, proxy)
def select_proxy(self, table_name, **kwargs):
return self.sql.select_proxy(table_name, **kwargs)
def update_proxy(self, table_name, proxy):
return self.sql.update_proxy(table_name, proxy)
def delete_proxy(self, table_name, proxy):
return self.sql.delete_proxy(table_name, proxy)
def delete_old(self, table_name, day):
return self.sql.delete_old(table_name, day)
def get_proxy_count(self, table_name):
return self.sql.get_proxy_count(table_name=table_name)
def get_proxy_ids(self, table_name):
return self.sql.get_proxy_ids(table_name=table_name)
def get_proxy_with_id(self, table_name, id):
return self.sql.get_proxy_with_id(table_name=table_name, id=id)
def del_proxy_with_id(self, table_name, id):
return self.sql.del_proxy_with_id(table_name=table_name, id=id)
def del_proxy_with_ip(self, table_name, ip):
return self.sql.del_proxy_with_ip(table_name=table_name, ip=ip)
def commit(self):
return self.sql.commit()
def init_paremeter():
global mysql_db_mta, http, logging
http = HttpClient("logger")
mysql_db_mta = MySql('master_mysql', 'logger')
logging = config.init_logger("constellation_crawler", False)
def connectionMade(self):
logger.info("New connection from {!s}".format(self.getId()))
self.mysql=MySql()
class Chat(Int32StringReceiver):
def __init__(self):
self.login=False
self.guid = None
self.groups=None
self.mysql=None
self.msgHandle={
'REGISTER' : self.register,
'UNREGISTER' : self.unregister,
'GET_CUSTOM_GROUPS' : self.getGroups,
'CREATE_CUSTOM_GROUP' : self.addGroup,
'DELETE_CUSTOM_GROUP' : self.delGroup,
'CHAT' : self.chat,
'ADDUSER_CUSTOM_GROUP' : self.addUser,
'CUSTOM_USER_LOGIN' : self.customLogin,
'UPDATE_CUSTOM_GROUP' : self.updateGroupName,
'DELETEUSER_CUSTOM_GROUP' : self.delUser,
'FILE_TRANSFER_REQUEST' : self.sendFile,
'FILE_TRANSFER_CANCELLED' : self.sendFileCancel,
'FILE_TRANSFER_ACCEPT' : self.sendFileAccept,
'FILE_TRANSFER_REFUSED' : self.sendFileRefused,
'SHAKE_WINDOW' : self.shakeWindow,
'GET_USER_STATUS' : self.getUserStatus,
'CHANGE_PHOTO' : self.updatePhoto,
'PERSONALSIGN' : self.updateSignature,
'QUERY_USER_BASIC' : self.getUserBasicInfo,
'CHANGE_FONTSEETINGS' : self.changeFontSetting,
}
def connectionMade(self):
logger.info("New connection from {!s}".format(self.getId()))
self.mysql=MySql()
def stringReceived(self,msg):
logger.debug("Received msg : {!s}".format(msg))
msgType=self.msg.getValue(msg,'Type')
msgHandler=self.msgHandle.get(msgType,self.errHandle)
msgHandler(msg)
def register(self,msg):
self.guid=self.msg.getValue(msg,'OwnGuid')
''' register user '''
logger.debug("Register user {!s}...".format(self.guid))
self.factory.addClient(self.guid,self)
logger.debug("Done")
if isinstance(self.mysql,MySql):
''' send online msg '''
self.factory.sendNotify(self.guid,self.mysql,'ONLINE_NOTIFY')
''' check offline msg '''
try:
msgs=self.mysql.getOffLineMsg(self.guid)
except _mysql_exceptions.ProgrammingError as error:
logger.error(error)
logger.error("{}".format(traceback.format_exc()))
return
if not msgs:
logger.debug("User {!s} has no offline msgs".format(self.guid))
else:
logger.debug(msgs)
for _msg in msgs:
arg=('OFFLINE_MSG',_msg[1],_msg[2],_msg[3],_msg[4])
message=self.msg.packMsg(*arg)
logger.debug("Sending offline msgs : {!s}".format(message))
self.transport.write(message)
logger.debug("Insert offline msgs to online msg...")
arg=(_msg[0],_msg[1],_msg[2],_msg[3],_msg[4],self.mysql)
try:
setOnLineMsg(*arg)
except _mysql_exceptions.ProgrammingError:
logger.error("Insert error")
except UnicodeDecodeError as error:
logger.error(error)
self.mysql.delOffLineMsg(_msg[5])
''' register in database '''
self.mysql.setOnLineUser(self.guid)
''' send online msg '''
else:
''' database init failed '''
pass
def unregister(self,msg):
OwnGuid=self.msg.getValue(msg,'OwnGuid')
logger.debug("Unregister user {!s}...".format(self.guid))
self.factory.delClient(self.guid)
logger.debug("Done")
self.factory.sendNotify(self.guid,self.mysql,'OFFLINE_NOTIFY')
self.mysql.delOnLineUser(self.guid)
def getGroups(self,msg):
def sendGroup():
self.groups=self.mysql.getGroups(self.guid)
if self.groups :
for group in self.groups :
arg=('CUSTOM_GROUPS',group['gid'],group['gname'],group['users'],group['created'],group['weight'])
groupMsg=self.msg.packMsg(*arg)
logger.debug("Sending group msg : {!s}".format(groupMsg))
self.transport.write(groupMsg)
else:
logger.debug("{!s} has no custom groups,created them!".format(self.guid))
try:
self.mysql.setDefaultGroups(self.guid)
except Exception :#as error:
logger.error("Set default groups failed!!!")
#logger.error("{}".format(error))
#logger.error("{}".format(error.__doc__))
logger.error("{}".format(traceback.format_exc()))
sendGroup()
return
logger.debug("Sending group msgs begin...")
sendGroup()
logger.debug("Sending group msgs end")
return
def getUserStatus(self,msg):
Type=self.msg.getValue(msg,'Type')
#UserGuid=self.msg.getValue(msg,'UserGuid')
GroupID=self.msg.getValue(msg,'GroupID')
Users=self.mysql.getOnlineUser(GroupID)
arg=(Type,GroupID,Users)
msg=self.msg.packMsg(*arg)
logger.debug("Sending user status msg...")
logger.debug(msg)
self.transport.write(msg)
def delGroup(self,msg):
#gid=self.msg.getValue(msg,'GroupID')
#uid=self.msg.getValue(msg,'OwnGuid')
GroupID=self.msg.getValue(msg,'GroupID')
OwnGuid=self.msg.getValue(msg,'OwnGuid')
if OwnGuid != self.guid:
return
logger.debug("Delete group {!s}...".format(GroupID))
try:
self.mysql.delGroup(GroupID,OwnGuid)
except Exception:
logger.error("Delete group failed!!!")
logger.error("{}".format(traceback.format_exc()))
logger.debug("Done")
arg=('DELETE_CUSTOM_GROUP',GroupID,OwnGuid)
msg=self.msg.packMsg(*arg)
logger.debug("Send deleted group msg : {!s}".format(msg))
self.transport.write(msg)
def addGroup(self,msg):
try:
OwnGuid=self.msg.getValue(msg,'OwnGuid')
except KeyError:
logger.error("add group msg has no OwnGuid")
return
try:
GroupName=self.msg.getValue(msg,'GroupName')
except KeyError:
logger.error("add group msg has no GroupName")
return
try:
Weight=self.msg.getValue(msg,'Weight')
except KeyError:
logger.error("add group msg has no Weight")
return
try:
GroupName=GroupName.encode('utf8')
logger.debug("Add group {!s}...".format(GroupName))
pass
except UnicodeDecodeError:
pass
try:
self.mysql.addGroup(OwnGuid,GroupName,Weight)
logger.debug("Done")
except _mysql_exceptions.IntegrityError as err:
logger.error("Failed")
logger.error("{!s}".format(err[1]))
group=self.mysql.getGroups(OwnGuid,GroupName)[0]
arg=('CREATE_CUSTOM_GROUP',group)
groupMsg=self.msg.packMsg(*arg)
logger.debug("Send group msg : {!s}".format(groupMsg))
self.transport.write(groupMsg)
def updateGroupName(self,msg):
GroupID=self.msg.getValue(msg,'GroupID')
GroupName=self.msg.getValue(msg,'GroupName')
OwnGuid=self.msg.getValue(msg,'OwnGuid')
if OwnGuid != self.guid:
return
try:
self.mysql.renameGroup(GroupID,GroupName)
except :
logger.error("Rename group failed!!!\n")
arg=('UPDATE_CUSTOM_GROUP',GroupID,GroupName,OwnGuid)
msg=self.msg.packMsg(*arg)
logger.debug("Sending rename group msg : {!s}".format(msg))
self.transport.write(msg)
def addUser(self,msg):
GroupID=self.msg.getValue(msg,'GroupID')
UserGuid=self.msg.getValue(msg,'UserGuid')
Created=int(time.time())
self.mysql.addUser(GroupID,UserGuid)
arg=('ADDUSER_CUSTOM_GROUP',GroupID,UserGuid,Created)
msg=self.msg.packMsg(*arg)
logger.debug("Sending add user msg : {!s}".format(msg))
self.transport.write(msg)
def delUser(self,msg):
GroupID=self.msg.getValue(msg,'GroupID')
OwnGuid=self.msg.getValue(msg,'OwnGuid')
UserGuid=self.msg.getValue(msg,'UserGuid')
if OwnGuid != self.guid:
return
self.mysql.delUser(GroupID,UserGuid)
logger.debug("Delete user {!s} of group {!s}".format(UserGuid,GroupID))
arg=('DELETEUSER_CUSTOM_GROUP',GroupID,OwnGuid,UserGuid)
msg=self.msg.packMsg(*arg)
logger.debug("Sending delete user msg : {!s}".format(msg))
self.transport.write(msg)
def chat(self,msg):
#if self.login :
msgBody=self.msg.getValue(msg,'Body')
msgCreated=int(time.time())
msgRecvGuid=self.msg.getValue(msg,'RecvGuid')
msgSendGuid=self.msg.getValue(msg,'SendGuid')
blacklist=self.mysql.getBlackList(msgRecvGuid)
if msgSendGuid not in blacklist:
arg=('CHAT',msgBody,msgCreated,msgSendGuid,msgRecvGuid)
msg=self.msg.packMsg(*arg)
receiver=self.factory.getClient(msgRecvGuid)
if not receiver:
logger.debug("User {!s} if offline,write the msg to mysql".format(msgRecvGuid))
arg=('CHAT',msgBody,msgCreated,msgSendGuid,msgRecvGuid,self.mysql)
setOffLineMsg(*arg)
#T=setOffLineMsg(*arg)
#T.setDaemon(True)
#T.start()
return
logger.debug("Sending msgs to user {!s}".format(msgRecvGuid))
receiver.transport.write(msg)
else:
logger.debug("you are in receiver's blacklist,so sorry...")
logger.debug("Writing msgs to database...")
arg=('CHAT',msgBody,msgCreated,msgSendGuid,msgRecvGuid,self.mysql)
try:
setOnLineMsg(*arg)
except _mysql_exceptions.ProgrammingError:
logger.error("Insert error")
#T=setOnLineMsg(*arg)
#T.setDaemon(True)
#T.start()
#else:
# stdout.write("[{!s}] you haven't login,sorry!!!\n".format(time.ctime()))
def shakeWindow(self,msg):
Type=self.msg.getValue(msg,'Type')
RecvGuid=self.msg.getValue(msg,'RecvGuid')
SendGuid=self.msg.getValue(msg,'SendGuid')
blacklist=self.mysql.getBlackList(RecvGuid)
if SendGuid not in blacklist:
msg=self.msg.packMsg(Type,RecvGuid,SendGuid)
logger.debug("Sending shake-window msg ...")
logger.debug(msg)
self.factory.clients[RecvGuid].transport.write(msg)
else:
logger.debug("you are in receiver's blacklist,so sorry...")
def sendFile(self,msg):
RecvGuid=self.msg.getValue(msg,'RecvGuid')
SendGuid=self.msg.getValue(msg,'SendGuid')
Type=self.msg.getValue(msg,'Type')
Created=self.msg.getValue(msg,'Created')
DisplayName=self.msg.getValue(msg,'DisplayName')
Length=self.msg.getValue(msg,'Length')
NodeID=self.msg.getValue(msg,'NodeID')
IP=self.getId()
blacklist=self.mysql.getBlackList(RecvGuid)
if SendGuid not in blacklist:
if RecvGuid in self.factory.clients:
logger.debug("User {!s} is online...".format(RecvGuid))
arg=(Type,Created,DisplayName,RecvGuid,SendGuid,Length,NodeID,IP)
msg=self.msg.packMsg(*arg)
logger.debug("Send send-file-msg to {!s}...".format(RecvGuid))
logger.debug("{!s}".format(msg))
self.factory.clients[RecvGuid].transport.write(msg)
logger.debug("Write send-file-msg to database...")
arg=(Type,Created,DisplayName,RecvGuid,SendGuid,Length,NodeID,IP,self.mysql)
setSendFileMsg(*arg)
logger.debug("Done")
else:
logger.debug("User {!s} is offline...".format(RecvGuid))
logger.debug("Give it to me...")
else:
logger.debug("you are in receiver's blacklist,so sorry...")
def sendFileCancel(self,msg):
'''
{"Type":"FILE_TRANSFER_CANCELLED","NodeID":0,"RecvGuid":"custom-00001"} '''
Type=self.msg.getValue(msg,'Type')
NodeID=self.msg.getValue(msg,'NodeID')
RecvGuid=self.msg.getValue(msg,'RecvGuid')
ActionCode=self.msg.getValue(msg,'ActionCode')
arg=(Type,NodeID,ActionCode)
msg=self.msg.packMsg(*arg)
logger.debug("Send send-file-cancel msg...")
logger.debug(msg)
if RecvGuid in self.factory.clients:
self.factory.clients[RecvGuid].transport.write(msg)
else:
logger.debug("User {} is offline!".format(RecvGuid))
def sendFileAccept(self,msg):
Type=self.msg.getValue(msg,'Type')
NodeID=self.msg.getValue(msg,'NodeID')
RecvGuid=self.msg.getValue(msg,'RecvGuid')
arg=(Type,NodeID)
msg=self.msg.packMsg(*arg)
logger.debug("Send send-file-accept msg...")
logger.debug(msg)
if RecvGuid in self.factory.clients:
self.factory.clients[RecvGuid].transport.write(msg)
else:
logger.debug("User {} is offline!".format(RecvGuid))
def sendFileRefused(self,msg):
pass
def updatePhoto(self,msg):
OwnGuid=self.msg.getValue(msg,'OwnGuid')
Photo=self.msg.getValue(msg,'Photo')
self.mysql.updatePhoto(OwnGuid,Photo)
def updateSignature(self,msg):
Guid=self.msg.getValue(msg,'OwnGuid')
Sign=self.msg.getValue(msg,'SignContent')
logger.debug("Update user signature...")
self.mysql.updateSignature(Guid,Sign)
logger.debug("Done")
def getUserBasicInfo(self,msg):
OwnGuid=self.msg.getValue(msg,'OwnGuid')
UserGuids=self.msg.getValue(msg,'UserGuids')
user_basic_info=self.mysql.getUserBasicInfo(UserGuids)
Type='USER_BASICS'
Users=user_basic_info
msg=self.msg.packMsg(Type,Users)
logger.debug("Sending get-user-basic-info msg...")
self.transport.write(msg)
logger.debug("Done")
def changeFontSetting(self,msg):
OwnGuid=self.msg.getValue(msg,'OwnGuid')
Settings=self.msg.getValue(msg,'Settings')
logger.debug("Changing user font-setting ...")
self.mysql.changeFontSetting(OwnGuid,Settings)
logger.debug("Done")
def customLogin(self,msg):
logger.debug("Recv login msg : {!s}".format(msg))
UserName=self.msg.getValue(msg,'UserName')
PassWord=self.msg.getValue(msg,'PassWord')
SerialNumber=self.msg.getValue(msg,'SerialNumber')
try:
UserInfo=self.mysql.getUserInfo(UserName)
except _mysql_exceptions.OperationalError as err:
logger.error(err)
if not UserInfo :
UserID=-1 # no uid
ErrorCode=1 # no such user
DisplayName=''
arg=('CUSTOM_USER_LOGIN',UserID,ErrorCode,SerialNumber,DisplayName)
msg=self.msg.packMsg(*arg)
logger.warning("No such user,check please!!!")
logger.debug("Sending login msg : {!s}".format(msg))
self.transport.write(msg)
return
__password=UserInfo[1]
if PassWord != __password :
UserID=-1 # can not get uid
ErrorCode=2 # password does not match
DisplayName=''
arg=('CUSTOM_USER_LOGIN',UserID,ErrorCode,SerialNumber,DisplayName)
msg=self.msg.packMsg(*arg)
logger.warning("Password does not match!!!")
logger.debug("Sending login msg : {!s}".format(msg))
self.transport.write(msg)
return
UserID=UserInfo[0] # real user id
ErrorCode=0 # 0 means check pass
DisplayName=UserInfo[2]
arg=('CUSTOM_USER_LOGIN',UserID,ErrorCode,SerialNumber,DisplayName)
msg=self.msg.packMsg(*arg)
logger.debug("Login succeed,congratulations!")
logger.debug("Sending login msg : {!s}".format(msg))
self.transport.write(msg)
self.login=True
def errHandle(self,msg):
pass
def connectionLost(self, reason):
logger.info("Connection lost from {!s}".format(self.getId()))
logger.debug(reason)
self.factory.delClient(self.guid)
self.factory.sendNotify(self.guid,self.mysql,'OFFLINE_NOTIFY')
self.mysql.delOnLineUser(self.guid)
logger.info("Shutdown database connection")
self.mysql.close()
logger.info("Done")
def getId(self):
return str(self.transport.getPeer().host)
class CrawlTopMovie:
def __init__(self, start_num, count_num):
# Init function:
# Add request Header
# Set start&count number
# Connect MySQL
self.movie_url = "http://api.douban.com/v2/movie/top250?"
self.header = {'User-Agent': 'Mozilla/5.0'}
self.start = "start=" + str(start_num)
self.count = "count=" + str(count_num)
self.sql = MySql("Your MySQL setting")
self.data = []
def getDataFromUrl(self, url):
# Get data
request = urllib2.Request(url, None, self.header)
data = urllib2.urlopen(request).read()
return json.loads(data)
def getSubjects(self):
url = self.movie_url + self.start + "&" + self.count
print url
return self.getDataFromUrl(url)
def crawlData(self):
subject_data = self.getSubjects()
if subject_data:
for subject in subject_data["subjects"]:
self.data = subject
self.importToMySQL("movie")
self.sql.quit()
else:
print "no subject"
def importToMySQL(self, table):
id = self.sql.getMaxID(table)
self.sql.insert(table,
(id,
self.getTitle(),
self.getMovieID(),
self.getDoubanURL(),
self.getRating(),
self.getDirector(),
self.getGenres(),
self.getImage(),
self.getDate()
))
def getRating(self):
return float(self.data["rating"]["average"])
def getDoubanURL(self):
return self.data["alt"]
def getDirector(self):
return self.data["directors"][0]["name"]
def getGenres(self):
return str(self.data["genres"])
def getMovieID(self):
return int(self.data["id"])
def getImage(self):
return self.data["images"]["large"]
def getTitle(self):
return self.data["title"]
def getDate(self):
return self.data["year"]
class BookCrawl:
"""Crawl book data douban.com by API"""
def __init__(self, book_num):
self.douban_url = "https://api.douban.com/v2/book/"
self.start_num = book_num[0]
self.end_num = book_num[1]
self.header = {'User-Agent': 'Mozilla/5.0'}
self.sql = MySql("Your MySQL setting")
self.data = []
def __iter__(self):
return self
def next(self):
if self.start_num < self.end_num:
try:
self.crawlData(self.douban_url + str(self.start_num))
except:
print self.start_num
self.start_num += 1
return self.start_num
else:
self.sql.quit()
raise StopIteration()
def test(self, url):
self.data = self.getDataFromUrl(url)
print self.getPrice()
def crawlData(self, url):
self.data = self.getDataFromUrl(url)
self.insertBookInfo("bookinfo")
self.insertBookTag("booktag")
def getDataFromUrl(self, url):
request = urllib2.Request(url, None, self.header)
data = urllib2.urlopen(request).read()
return eval(data)
def getRating(self):
return float(self.data["rating"]["average"])
def getAuthor(self):
if self.data["author"] == []:
return ""
else:
return self.data["author"][0]
def getPublishDate(self):
date_str = self.data["pubdate"].split("-")
date_format = ["%y", "%m", "%d"]
format_type = "-".join(date_format[:len(date_str)])
date = datetime.datetime.strptime(self.data["pubdate"][2:],
format_type).date()
return date
def getImage(self):
return self.data["image"]
def getBookID(self):
return int(self.data["id"])
def getPublisher(self):
return self.data["publisher"]
def getTitle(self):
return self.data["title"]
def getUrl(self):
return self.data["alt"]
def getSummary(self):
return self.data["summary"]
def getPrice(self):
price = self.data["price"]
return float(re.search(r'\d+\.?\d*', price).group(0))
def getBookTag(self):
tags = self.data["tags"]
if tags:
return [item["name"] for item in tags]
else:
return []
def insertBookInfo(self, table):
id = self.sql.getMaxID(table)
self.sql.insert(table,
(id,
self.getBookID(),
self.getTitle(),
self.getAuthor(),
self.getPublisher(),
self.getPublishDate(),
self.getPrice(),
self.getRating(),
1,
self.getSummary(),
self.getUrl(),
self.getImage())
)
def insertBookTag(self, table):
if self.getBookTag():
for item in self.getBookTag():
self.sql.insert(table,
(item,
self.getBookID())
)
def init_fund():
bms = MySql('192.168.0.32', 3306, 'ceshi', 'ceshi', 'gmf_bms')
sql = 'delete from product_group where org_id="23"'
bms.excute_sql(sql)
bms.close()
