def unregChat(self, chat_id):
'''
chat_id of user
'''
db = Database()
db.connection()
return db.delUser(chat_id=chat_id)
def unregChat(self, chat_id):
db = Database()
db.connection()
##Check if user registered to db##
if db.delUser(chat_id=chat_id):
return
else:
return False
def listChat(self, chat_id):
'''
chat_id of owner
'''
db = Database()
db.connection()
hasil = messageformat.parsingList(db.getUser(chat_id=chat_id))
return hasil
def listToken(cls):
'''
Return hasilparsing = (markup, teks)
'''
db = Database()
db.connection()
hasilquery = db.getOwner()
print(f'ini hasil query {hasilquery}\n masuk messageformat')
hasilparsing = messageformat.parsingGetOwner(hasilquery)
print(hasilparsing)
return hasilparsing
def __init__(self, cityName, area=None):
self.cityName = cityName
self.area = area
self.city = City(cityName,
searchDB=Database(MongoDB),
commentsDB=Database(MongoDB))
self.city.get()
self.category_list = []
self.coa_category = []
self.fin_category = []
self.process_category(self.city.category, self.category_list)
self.coarsness_category(self.category_list)
self.fine_grained_category(self.category_list)
def setUp(self):
self.keyspace = 'Claimspace'
self.host = 'localhost'
self.port = '9160'
self.pool = ConnectionPool(self.keyspace,
[self.host + ':' + self.port])
self.cf = pycassa.ColumnFamily(self.pool, Database.providerCFname)
self.db = Database(self.keyspace)
self.providerDict = {
'username': '******',
'realname': 'Testy User',
'providerid': '8675309'
}
self.providerObj = Provider(**self.providerDict)
def notifUpdate():
parser.read("config/message.ini")
msg = parser.get("bot", "notifUpdate")
sticker = parser.get("bot", "notifUpdateSticker")
parser.clear()
db = Database()
db.connection()
listchat_id = db.getAllUserOwner()
#send message
#send message
for i in listchat_id:
dispatcher.bot.send_message(chat_id=i, text=msg)
dispatcher.bot.send_sticker(chat_id=i, sticker=sticker)
def main():
last_update_id = None
USERDB = Database(my_file = 'recorded_respond.csv')
INDEX = {}
while True:
updates = get_updates(last_update_id)
if len(updates["result"]) > 0:
last_update_id = get_last_update_id(updates) + 1
INDEX = response(updates, INDEX)
time.sleep(1)
def notifMaintenance():
parser.read("config/message.ini")
msg = parser.get("bot", "notifMaintenance")
#msg.replace('\\n','\n')
sticker = parser.get("bot", "notifMaintenanceSticker")
parser.clear()
db = Database()
db.connection()
print(msg)
listchat_id = db.getAllUserOwner()
print(listchat_id)
#send message
for i in listchat_id:
dispatcher.bot.send_message(chat_id=i,
text=msg,
parse_mode=ParseMode.HTML)
dispatcher.bot.send_sticker(chat_id=i, sticker=sticker)
def setUp(self):
self.keyspace = 'Claimspace'
self.host = 'localhost'
self.port = '9160'
self.pool = ConnectionPool(self.keyspace,
[self.host + ':' + self.port])
self.proCF = pycassa.ColumnFamily(self.pool, Database.providerCFname)
self.patCF = pycassa.ColumnFamily(self.pool, Database.patientCFname)
self.patMapCF = pycassa.ColumnFamily(self.pool,
Database.patientMapCFname)
self.db = Database(self.keyspace)
self.providerDict = {
'username': '******',
'realname': 'Testy User',
'providerid': '8675309'
}
self.providerObj = Provider(**self.providerDict)
self.patientDict = {'pname': 'John Doe', 'psex': 'm', 'pid': '88888'}
self.patientObj = Patient(**self.providerDict)
self.patientDict['hashkey'] = self.patientObj.hashkey
def get_category(self, save=False):
"""
获取该城市所有的店铺分类结果
:param save: 是否将城市所有店铺分类保存到数据库
"""
logger.info(f'获取 “{self.cityName}” 所有店铺分类成功.')
if save:
categoryDB = init_category_db(Database(MongoDB))
categoryDB.save({'category': self.category_list},
tname=self.cityName)
logger.info(f'已将 “{self.cityName}” 所有店铺分类信息保存到数据库中.')
return self.city.category
def unregToken(self, chat_id: int):
'''
chat_id of owner
'''
db = Database()
db.connection()
db.delTeamOrOwner(chat_id=chat_id)
def registerChat(self, chat_id: int, chat_name: str, owner_id: str):
'''
:param token: chat_id of owner
'''
db = Database()
db.connection()
db.insertUser(chat_id, chat_name, owner_id)
def reqPasscode(self, tokenid, nexttoken=None):
'''
Return passcode->str
:param tokenid: chat_id of current user
'''
db = Database()
db.connection()
passcode = None
button = []
fetchtoken = db.getUser(tokenid=tokenid)
if fetchtoken:
print("masuk if fetchtoken")
username = messageformat.parsingPasscode(fetchtoken)
print(f"ini username {username}")
#condition check callback data button
if not nexttoken:
cmd = os.popen(f"stoken --rcfile=rcfile/.{username}")
passcode = cmd.read()
cmd.close
print(passcode)
return passcode, username
else:
cmd = os.popen(
f"stoken --rcfile=rcfile/.{username} --use-time=+{nexttoken}"
)
passcode = cmd.read()
cmd.close()
waktu = (datetime.now() +
timedelta(seconds=nexttoken)).strftime("%H:%M:%S")
return passcode, waktu, username
else:
print("masuk else fetchtoken")
return None
def init_cache_db():
"""
:return: Database
"""
db: Database
db = Database(MongoDB)
if db and not db.connected:
db.connect()
db.use_db(MongoDB['cacheDB'])
return db
def registerToken(self,
chat_id: int,
chat_name: str,
username: str,
setpin: int,
team_name,
token=None,
sdtid=None):
db = Database()
db.connection()
#check if this user already have token imported
if db.getOwner(chat_id=chat_id):
return False
self.importToken(username, setpin, sdtid, token)
db.insertOwner(chat_id, chat_name, team_name, username)
return
def get_area(self, save=False):
"""
获取该城市所有的区
:param save: 是否将城市所有区保存到数据库
"""
area_list = []
for item in self.city.locations:
try:
if (item['text'] == '全部地区'):
continue
area_list.append(item['text'])
except:
logger.debug(f'获取城市分区失败:[城市:{self.cityName}]')
logger.info(f'获取 “{self.cityName}” 所有区成功.')
if save:
areaDB = init_area_db(Database(MongoDB))
areaDB.save({'area': area_list}, self.cityName)
logger.info(f'已将 “{self.cityName}” 所有区信息保存到数据库中.')
return area_list
def __init__(self, chat_id):
db = Database()
db.connection()
isadmin = db.getAdmin(chat_id)
isowner = db.getOwner(chat_id=chat_id)
if isadmin:
self.currentRole = Admin()
print("jadi admin")
elif isowner:
self.currentRole = Owner()
print("jadi owner")
else:
self.currentRole = User()
print("jadi user")
from config import *
from dbhelper import Database
from fangline import FangLine
mongo = Database(MONGO)
mongo.connect()
'''
1. 获取杭州所有 出售 房源
2. 将每一个房源和其详情存储进数据库
tips:
1.传入的cookie必须是登陆了 房蚁浙江站 的cookie，选择地区为 杭州
'''
def GetHouses(cookie, cate='sell', page=1):
"""
:param cookie: 用户cookie
:param cate: 四个类别：sell,rent,wantSell,wantRent
"""
fl = FangLine(cookie)
types = {
'sell': fl.SearchSellHouses,
'rent': fl.SearchRentHouses,
'wantSell': fl.SearchWantSellHouses,
'wantRent': fl.SearchWantRentHouses,
}
tip = {
'sell': '出售',
'rent': '出租',
'wantSell': '求售',
from city import City
from dbhelper import Database
from config import MongoDB
if __name__ == '__main__':
db = Database(MongoDB)
beijing = City('南宁', searchDB=db)
beijing.get()
results = beijing.search(keyword='朝阳广场地铁站',
category='美食',
save=True,
details=False)
def removeToken(self, chat_id: int):
db = Database()
db.connection()
db.updateOwner(chat_id=chat_id, token=None)
def __init__(self):
self.tokens = set()
self.db = Database(MongoDB)
class Waiter:
def __init__(self):
self.tokens = set()
self.db = Database(MongoDB)
@check
def generate_tokens(self,
proxy_api=PROXY_API,
count=MAX_TOKEN_GETS,
toDB=False,
tname=MongoDB['tokens'],
path=None):
_tokens = set()
lower = min(MAX_TOKEN_THREADS, count)
_got = 0
step = min(lower, count)
while len(_tokens) != count:
if not proxy_api:
logger.error(f'No proxy api for requesting tokens.Set one!')
return
threads = []
for i in range(step):
threads.append(CrawlThread(request_token, args=(proxy_api, )))
for i in threads:
i.start()
for i in threads:
i.join()
res = i.get_result()
if res:
_tokens.add(i.get_result())
_got += 1
if _got >= count:
logger.info(f'Got {_got} tokens.')
self.tokens.update(_tokens)
break
else:
step = count - _got
if toDB:
for i in _tokens:
_ = {'token': i, "generate_at": time_to_date(time.time())}
self.db.save(_, tname=tname)
logger.info(f'Tokens saved into table "{tname}" successfully.')
if path:
with open(path, 'a') as f:
for i in _tokens:
f.write(i + '\n')
logger.info(f'Tokens saved into file "{path}" successfully.')
def _check_token_files(self, file_path=TOKEN_FILE):
if os.path.isfile(file_path):
with open(file_path) as f:
lines = f.readlines()
for i in lines:
self.tokens.add(i.strip('\n'))
logger.info(
f'Loaded tokens from "{file_path}",amount:{len(self.tokens)}'
)
else:
logger.warning(f'Token files not found.')
def _check_token_table(self):
if not self.db.connected:
self.db.connect()
tokens = self.db.all(MongoDB['tokens'])
if tokens:
_ = 0
for i in tokens:
self.tokens.add(i['token'])
_ += 1
logger.info(
f'Loaded tokens from table "{MongoDB["tokens"]}",amount:{_}')
else:
logger.warning(f'Token data table not found.')
def tesaja(self):
db = Database()
db.connection()
hasilquery = db.tesquery()
return hasilquery
class PatientTests(unittest.TestCase):
def setUp(self):
self.keyspace = 'Claimspace'
self.host = 'localhost'
self.port = '9160'
self.pool = ConnectionPool(self.keyspace,
[self.host + ':' + self.port])
self.proCF = pycassa.ColumnFamily(self.pool, Database.providerCFname)
self.patCF = pycassa.ColumnFamily(self.pool, Database.patientCFname)
self.patMapCF = pycassa.ColumnFamily(self.pool,
Database.patientMapCFname)
self.db = Database(self.keyspace)
self.providerDict = {
'username': '******',
'realname': 'Testy User',
'providerid': '8675309'
}
self.providerObj = Provider(**self.providerDict)
self.patientDict = {'pname': 'John Doe', 'psex': 'm', 'pid': '88888'}
self.patientObj = Patient(**self.providerDict)
self.patientDict['hashkey'] = self.patientObj.hashkey
def tearDown(self):
pass
def test_create_patient(self):
try:
self.patCF.get(self.patientObj.hashkey)
except NotFoundException:
self.db.create_patient(self.patientObj)
else:
self.patCF.remove(self.username)
self.db.create_patient(self.patientObj)
resultDict = self.patCF.get(self.patientObj.hashkey)
self.assertEqual(self.patientObj.get_fields(), resultDict)
def test_update_patient(self):
self.patCF.insert(self.patientObj.hashkey,
self.patientObj.get_fields())
self.patientObj.pname = 'John Deer'
self.db.update_patient(self.patientObj)
resultDict = self.patCF.get(self.patientObj.hashkey)
resultObj = Patient(**resultDict)
self.assertEqual(self.patientObj.get_fields(), resultDict)
def test_get_patient(self):
self.patCF.insert(self.patientObj.hashkey,
self.patientObj.get_fields())
resultObj = self.db.get_patient(self.patientObj.hashkey)
self.assertEqual(self.patientObj, resultObj)
def test_get_not_found(self):
self.assertRaises(PatientDoesNotExistError, self.db.get_patient,
self.patientObj.hashkey)
def test_delete_patient(self):
self.patCF.insert(self.patientObj.hashkey,
self.patientObj.get_fields())
self.db.delete_patient(self.patientObj.hashkey)
self.assertRaises(NotFoundException, self.patCF.get,
self.patientObj.hashkey)
def test_assign_patient(self):
self.db.assign_patient(self.patientObj, self.providerObj)
resultDict = self.patMapCF.get(self.providerObj.username)
self.assertTrue(self.patientObj.hashkey in resultDict)
def test_unassign_patient(self):
self.patMapCF.insert(self.providerObj.username,
{self.patientObj.hashkey: ''})
self.db.unassign_patient(self.patientObj, self.providerObj)
resultDict = self.patMapCF.get(self.providerObj.username)
self.assertTrue(self.patientObj.hashkey not in resultDict)
class ProviderTests(unittest.TestCase):
def setUp(self):
self.keyspace = 'Claimspace'
self.host = 'localhost'
self.port = '9160'
self.pool = ConnectionPool(self.keyspace,
[self.host + ':' + self.port])
self.cf = pycassa.ColumnFamily(self.pool, Database.providerCFname)
self.db = Database(self.keyspace)
self.providerDict = {
'username': '******',
'realname': 'Testy User',
'providerid': '8675309'
}
self.providerObj = Provider(**self.providerDict)
def tearDown(self):
if hasattr(self.providerObj, 'username'):
self.cf.remove(self.providerObj.username)
def test_create_provider(self):
try:
self.cf.get(self.providerObj.username)
except NotFoundException:
self.db.create_provider(self.providerObj)
else:
self.cf.remove(self.providerObj.username)
self.db.create_provider(self.providerObj)
resultDict = self.cf.get(self.providerObj.username)
self.assertEqual(self.providerObj.get_fields(), resultDict)
def test_create_provider_no_id(self):
del self.providerObj.username
self.assertRaises(ProviderHasNoUsernameError, self.db.create_provider,
self.providerObj)
def test_create_provider_already_exists(self):
self.cf.insert(self.providerObj.username,
self.providerObj.get_fields())
self.assertRaises(ProviderAlreadyExistsError, self.db.create_provider,
self.providerObj)
def test_delete_provider(self):
self.cf.insert(self.providerObj.username,
self.providerObj.get_fields())
self.db.delete_provider(self.providerObj.username)
self.assertRaises(NotFoundException, self.cf.get,
self.providerObj.username)
def test_get_provider(self):
self.cf.insert(self.providerObj.username,
self.providerObj.get_fields())
resultObj = self.db.get_provider(self.providerObj.username)
self.assertEqual(self.providerObj, resultObj)
def test_get_not_found(self):
self.assertRaises(ProviderDoesNotExistError, self.db.get_provider,
self.providerObj.username)
def test_update_provider(self):
self.cf.insert(self.providerObj.username,
self.providerObj.get_fields())
self.providerObj.realname = 'John Doe'
self.db.update_provider(self.providerObj)
resultDict = self.cf.get(self.providerObj.username)
resultObj = Provider(**resultDict)
self.assertEqual(self.providerObj.get_fields(), resultDict)
def test_update_no_id(self):
del self.providerObj.username
self.assertRaises(ProviderHasNoUsernameError, self.db.update_provider,
self.providerObj)
logger = getLogger(__name__)
#
# while True:
# try:
# citySpider = CitySpider('上海')
# if citySpider.save_shop_comments():
# break
# except:
# print('重新执行保存评论程序')
# logger.debug(f'重新执行保存评论程序')
# cityList = ['上海']
# for cityName in cityList:
# citySpider = CitySpider(cityName)
# citySpider.get_area(save=True)
# citySpider.get_category(save=True)
commentsDB = init_comments_db(Database(MongoDB))
shopId_list = [
"jP3YpIoLqaljyubk", "k4Iu2JQ91Wk41qjI", "H40Qg42dzbVjKDFX",
"G1Dk3HxdesOijK1M", "k98zHWVG0LvxOps7", "H9frHtAYhwMus16B",
"H4XmSOeHDBiYu1IZ", "k3KYmMzbXlaRK0FJ", "l1JpNElJ2kXKi4JN",
"l3o0OLgdStIUzRfB", "H6kjfusig9Q0cK0K", "k30sKl77PHb7IFDQ",
"l57CJvGEtSFx8wNI", "l7xindmRaEdyY5Tn", "l1HU0qrWDul57k35",
"G8zYWpKOwdbE1VrP", "l5LIFnuGRuftPihV", "k5K4HpwmBXBjWp8v",
"G1AoCT767WTT0YMy", "k4aMta58i7RZZi4k", "H98rq56ZNQ1cls4T",
"k5Gl4g3RZrkUc1r2", "l1KvwxcdqviRVhZo", "G3CUJMKe2GyLqsud",
"k5PqMUXNGHylZ34d", "G4qRxdPNlOPZoTmh", "lalvI86XRED7aHc5",
"kahSWHudTeP4lt6g", "k9Dj8EMtZ9eVedT0", "G6AoUF1nPd7266km",
"H6keUnEwp3iX39Pv", "k5UAEDdidLmCGkaP", "jEY6s3Gp5yhLckxu",
"k4R7WOFHGPGleArH", "H9SltKJ36sA5fMNG", "k6PqwGSzDY4kHRWQ",
"jCw9VwkfMYRbrODc", "H5Rvj6P2h75xL9P7", "l5iYpgOqk7ezKOuj",
"k9Ksy8vFduLw26hF", "k7mo5Ahi9GEiPrzL", "G9jysntot0XZ37CT",