def countRoomsForRecentDays(self, query, city, days):
"""
get the number of available individual rooms for recent days
:param query: the query string
:param city: the city name specified in config.json, this will take effect only when query is None
:param days: the number of days before now
:return:
"""
ret = 0
cursor = self._conn.cursor()
try:
if query:
cursor.execute(
"SELECT DISTINCT(room_id) FROM room_info WHERE update_time>? AND query_str=?",
(getDeltaTimeStamp(datetime.datetime.now(), -days), query))
elif city:
cursor.execute(
"SELECT DISTINCT(room_id) FROM room_info WHERE update_time>? AND city=?",
(getDeltaTimeStamp(datetime.datetime.now(), -days), city))
else:
Log.w(StorageService._TAG, "query and city all empty!!")
return 0
ret = len(cursor.fetchall())
except BaseException as e:
Log.w(StorageService._TAG, "countRoomsForRecentDays() failed", e)
finally:
cursor.close()
self._conn.commit()
return ret
def write_pid_file(pid_file, pid):
import fcntl
import stat
try:
fd = os.open(pid_file, os.O_RDWR | os.O_CREAT,
stat.S_IRUSR | stat.S_IWUSR)
except OSError as e:
Log.w(MAIN_TAG, "exception for write_pid_file()", e)
return -1
flags = fcntl.fcntl(fd, fcntl.F_GETFD)
assert flags != -1
flags |= fcntl.FD_CLOEXEC
r = fcntl.fcntl(fd, fcntl.F_SETFD, flags)
assert r != -1
# There is no platform independent way to implement fcntl(fd, F_SETLK, &fl)
# via fcntl.fcntl. So use lockf instead
try:
fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB, 0, 0, os.SEEK_SET)
except IOError:
r = os.read(fd, 32)
if r:
Log.e(MAIN_TAG, 'already started at pid %s' % Utils.to_str(r))
else:
Log.e(MAIN_TAG, 'already started')
os.close(fd)
return -1
os.ftruncate(fd, 0)
os.write(fd, Utils.to_bytes(str(pid)))
return 0
def doQuery(self, checkin_date, checkout_dates, time_title):
"""
the entry for build and execute a home api query
:param checkin_date: the date when u are going to check in.
:param checkout_dates: multiple checkout dates for enumerating the home info as completable as possible
:param time_title:
:return:
"""
Log.d(SearchService.TAG, "===> querying on " + getDateStr())
storageService = StorageService(self._config.localStoragePath)
try:
for (cityName, cityList) in self._config.cityList.items():
retry_time = 0
while retry_time < 3:
homeInfoCollection = dict()
for city in cityList:
for checkout_date in checkout_dates:
for adults in [1, 2]:
self.tryGetHomeInfo(checkin_date, checkout_date, homeInfoCollection, adults, city)
if len(homeInfoCollection) > 0:
roomInfos = [x for (_, x) in homeInfoCollection.items()]
storageService.saveOrUpdateRoomBatch(roomInfos)
analyze, excelRet = self.performAnalyze(roomInfos, cityName, storageService, time_title)
self.analyzeCollection.append(analyze)
self.forExcel.append(excelRet)
break
else:
retry_time += 1
Log.w(SearchService.TAG, "no data for query={0}, will retry for {1}th time".format(cityName, retry_time))
time.sleep(self._queryIntervalForFailInSec) # take your time to do it
except BaseException as e:
Log.w(SearchService.TAG, "doQuery() failed, ", str(e) + "\r\nexception detail:" + traceback.format_exc())
finally:
storageService.close()
def tryGetHomeInfo(self, checkin_date, checkout_date, homeInfoCollection, adults, city):
"""
the kernel logic for getting home information through cgi
:param checkin_date:
:param checkout_date:
:param homeInfoCollection:
:param adults:
:param city:
:return:
"""
api = Api(randomize=False)
query = getQueryStr(city)
hasNextPage = True
startOffset = 0
sectionOffset = 0
try:
while hasNextPage:
homes = api.get_homes(query=query, checkin=checkin_date, checkout=checkout_date,
offset=startOffset, sectionOffset=sectionOffset, items_per_grid=50, adults=adults)
pagination = self.retrieveHomeData(query, homes, homeInfoCollection)
if not pagination:
hasNextPage = False
else:
print("paging for next...")
hasNextPage = pagination["hasNextPage"]
startOffset = pagination.get("itemsOffset")
sectionOffset = pagination.get("sectionOffset")
time.sleep(self._queryIntervalInSec) # do not query too fast
except BaseException as e:
Log.w(SearchService.TAG, "get_homes() failed, ",
str(e) + "\r\nexception detail:" + traceback.format_exc())
def isAvailable(self, roomId, dateStr):
"""
query whether a room specified by roomId is available on given date
:param roomId:
:param dateStr: like 2019-05-30, be sure to use '{:%Y-%m-%d}'.format(datetime)
:return: True for available that day, False otherwise
"""
cursor = self._conn.cursor()
ret = False
try:
cursor.execute(
"SELECT * FROM reservation_info WHERE room_id=? AND date=?",
(roomId, dateStr))
ret = len(cursor.fetchall()) > 0
except BaseException as e:
Log.w(StorageService._TAG, "hasReservation() failed", e)
finally:
cursor.close()
self._conn.commit()
return ret
def saveReservationInfo(self, roomId):
"""
mark a room as available for today
:param roomId:
:return:
"""
cursor = self._conn.cursor()
success = False
try:
now_date = getDateStr()
cursor.execute(
"INSERT INTO reservation_info (room_id, date) VALUES(?, ?)",
(roomId, now_date))
success = cursor.rowcount > 0
except BaseException as e:
Log.w(StorageService._TAG, "saveReservationInfo() failed", e)
finally:
cursor.close()
self._conn.commit()
return success
def start(self):
print("airbnb service start")
while True:
while True:
now = datetime.datetime.now()
if now.hour == self._config.startHour and now.minute == self._config.startMinute:
dates = self.prepareDate()
self.analyzeCollection.clear()
self.forExcel.clear()
try:
for item in dates:
self.doQuery(item[0], item[1], item[2])
except BaseException as e:
Log.e(SearchService.TAG, "doQuery() failed, ", e)
if len(self.analyzeCollection) > 0:
self.reportForAnalyzeResult(self.analyzeCollection, self.forExcel)
else:
Log.w(SearchService.TAG, "no analyze result generated")
time.sleep(70)
else:
time.sleep(10) # in second
def saveOrUpdateRoomBatch(self, roomInfos):
"""
save or update information for a given room
:param roomInfos: a list of RoomInfo
:return:
"""
cursor = self._conn.cursor()
success = True
try:
for roomInfo in roomInfos:
cursor.execute("SELECT room_id FROM room_info WHERE room_id=?",
(roomInfo.roomId, ))
if len(cursor.fetchall()) > 0:
# exist
cursor.execute(
"UPDATE room_info SET person_capacity=?, city=?, beds=?,\
localized_neighborhood=?, price=?, pic=?, update_time=?, query_str=? WHERE room_id=?",
(roomInfo.personCapacity, roomInfo.city, roomInfo.beds,
roomInfo.neighbourhood, int(
roomInfo.price), roomInfo.pic, getNowTimeStamp(),
roomInfo.query, roomInfo.roomId))
else:
# new
cursor.execute(
"INSERT INTO room_info (room_id, person_capacity, city, beds, localized_neighborhood, \
price, pic, update_time, query_str) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(roomInfo.roomId, roomInfo.personCapacity,
roomInfo.city, roomInfo.beds,
roomInfo.neighbourhood, roomInfo.price, roomInfo.pic,
getNowTimeStamp(), roomInfo.query))
success = cursor.rowcount > 0 and success
except BaseException as e:
success = False
Log.w(StorageService._TAG, "saveOrUpdateRoomBatch() failed", e)
finally:
cursor.close()
self._conn.commit()
return success
def getRoomById(self, roomId):
"""
query full information for a given roomId
:param roomId:
:return: a RoomInfo with given roomId if found, None if not
"""
cursor = self._conn.cursor()
ret = None
try:
cursor.execute(
"SELECT room_id, person_capacity, city, beds, localized_neighborhood, \
price, pic, query_str FROM room_info WHERE room_id=?",
(roomId, ))
values = cursor.fetchall()
if len(values) > 0:
ret = RoomInfo(values[0][0], values[0][1], values[0][2],
values[0][3], values[0][4], values[0][5],
values[0][6], values[0][7])
except BaseException as e:
Log.w(StorageService._TAG, "getRoomById() failed", e)
finally:
cursor.close()
self._conn.commit()
return ret
def retrieveHomeData(self, query, originReturn, homeInfoCollection):
'''
pagenationMeta is like
"paginationMetadata":{
"__typename":"DoraExploreV3PaginationMetadata",
"hasNextPage":true,
"itemsOffset":40,
"sectionOffset":3,
"hasPreviousPage":true,
"previousPageSectionOffset":0,
"previousPageItemsOffset":0,
"searchSessionId":"d91045aa-0d95-49ba-ad44-0d7bcf8b2813",
"pageLimit":20,
"totalCount":"124"
}
'''
Log.d(SearchService.TAG, "analyzing " + query)
sections = originReturn["data"]["dora"]["exploreV3"]["filters"]["sections"]
pagination = originReturn["data"]["dora"]["exploreV3"]["metadata"]["paginationMetadata"]
if sections and len(sections) > 0:
# if pagination:
# print("has_next_page = " + str(pagination["has_next_page"]))
# print("items_offset = " + str(pagination.get("items_offset")))
# print("section_offset = " + str(pagination.get("section_offset")))
for item in sections:
if item and item["__typename"] == "DoraExploreV3ListingsSection":
real_homes = item["items"]
Log.d(SearchService.TAG, str(len(real_homes)) + " rooms for " + query)
for roomItem in real_homes:
roomInfo = RoomInfo.parseFromDict(roomItem, query)
if roomInfo:
homeInfoCollection[roomInfo.roomId] = roomInfo
else:
Log.w(SearchService.TAG, "explore_tabs is empty")
return pagination
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# 测试日志辅助类
import sys
sys.path.append("../")
from LogHelper import Log
TAG = "Test"
Log.d(TAG, "Hello")
Log.e(TAG, "World!")
Log.i(TAG, "Hehe")
Log.w(TAG, "XXXX")
def parseConfigFile(path):
'''
example for a valid config file, should be in json format
{
"start_hour":14,
"start_minute":0,
"counting_days":30,
"room_limit":0,
"smtp_host":"smtp.exmail.qq.com",
"smtp_port":465,
"email_receiver":"*****@*****.**",
"email_sender":"*****@*****.**",
"sender_passwd":"***",
"query":[
"上海,徐汇区,中国",
"上海,虹口区,中国",
"上海,浦东新区,中国",
"上海,普陀区,中国",
"上海,长宁区,中国",
"上海,闸北区,中国",
"上海,杨浦区,中国",
"上海,黄浦区,中国",
"上海,卢湾区,中国",
"上海,静安区,中国",
"上海,宝山区,中国",
"上海,闵行区,中国",
"上海,嘉定区,中国",
"上海,金山区,中国",
"上海,松江区,中国",
"上海,青浦区,中国",
"上海,南汇区,中国",
"上海,奉贤区,中国"
]
}
:param path:
:return:
'''
try:
f = open(path, "r", encoding='utf-8')
jsonStr = f.read()
configDict = json.loads(jsonStr.replace("\\n", ""))
hour = configDict.get("start_hour")
minute = configDict.get("start_minute")
counting_days = configDict.get("counting_days")
room_limit = configDict.get("room_limit")
email_receiver = configDict.get("email_receiver")
email_sender = configDict.get("email_sender")
sender_passwd = configDict.get("sender_passwd")
query = configDict.get("query")
smtp_host = configDict.get("smtp_host")
smtp_port = configDict.get("smtp_port")
send_type = configDict.get("send_type")
cities = dict()
for item in query:
splited = item.split(",")
if len(splited) < 2:
continue
content = cities.get(splited[0])
if not content:
content = list()
cities[splited[0]] = content
content.append(splited)
return Config(hour,
minute,
cities,
email_receiver,
email_sender,
sender_passwd,
room_limit,
countingDays=counting_days,
smtpHost=smtp_host,
smtpPort=smtp_port,
sendType=send_type)
except json.decoder.JSONDecodeError as e:
Log.w(MAIN_TAG, "parseConfigFile() fail, ", e)
return None
