class FeedPuller(object):
def __init__(self, ident, secret, port, host, feeds):
self.ident = ident
self.secret = secret
self.port = port
self.host = host
self.feeds = feeds
self.last_received = datetime.now()
self.hpc = None
self.enabled = True
def handle_url(self,url):
print "Time: %s -- URL: %s" % (self.last_received, url)
def start_listening(self):
gevent.spawn_later(15, self._activity_checker)
while self.enabled:
try:
self.hpc = hpfeeds.new(self.host, self.port, self.ident, self.secret)
def on_error(payload):
print 'Error message from broker: {0}'.format(payload)
self.hpc.stop()
def on_message(ident, chan, payload):
self.last_received = datetime.now()
data = json.loads(str(payload))
site_id = data['id']
url = data['url'].encode('unicode-escape')
self.handler = UrlHandler(url)
self.handler.process()
#self.handle_url(url)
#print "Time: %s --- Site: %s - URL: %s" % (self.last_received, site_id, url)
self.hpc.subscribe(self.feeds)
self.hpc.run(on_message, on_error)
except Exception as ex:
print ex
self.hpc.stop()
gevent.sleep(5)
def stop(self):
self.hpc.stop()
self.enabled = False
def _activity_checker(self):
while self.enabled:
if self.hpc is not None and self.hpc.connected:
difference = datetime.now() - self.last_received
if difference.seconds > 15:
print "No activity for 15 seconds, forcing reconnect"
self.hpc.stop()
gevent.sleep(15)
def on_message(ident, chan, payload):
self.last_received = datetime.now()
data = json.loads(str(payload))
site_id = data['id']
url = data['url'].encode('unicode-escape')
self.handler = UrlHandler(url)
self.handler.process()
def main():
"""get updates continuosly and manage instructions"""
last_update_id = None
chat_bot = ChatBot.chat_bot_start()
url_handler = UrlHandler()
bot_manager = BotManager()
while True:
print("Updates")
updates = url_handler.get_updates(last_update_id)
if updates["result"]:
last_update_id = url_handler.get_last_update_id(updates) + 1
bot_manager.handle_updates(updates, chat_bot)
time.sleep(0.5)
class Spider():
table_pro_city_init = UrlHandler().web_init
thread_1_put_zufangUrl = ParseUrl()
thread_2_put_AreaUrl_info = ParseCity_MultiThreading()
parse_rent_info_consumer = ParseRent()
parse_rent_info_producer = Proudcer()
def __init__(self):
self.task_1 = Spider.table_pro_city_init
self.task_2_1 = Spider.thread_1_put_zufangUrl
self.task_2_2 = Spider.thread_2_put_AreaUrl_info
self.task_3_2 = Spider.parse_rent_info_consumer
self.task_3_1 = Spider.parse_rent_info_producer
def task1(self):
'''
Initialization. Parse tables of Province, City and insert the information into the DB.
'''
logger.info("Task1 Running...")
self.task_1()
logger.info("Task1 Finish!")
def task2(self):
'''
1. Using City_url from DB to get the url containing a string of 'zufang', then put into the global Queue.
2. Geting url form Queue and Parse the Area information into the DB.
'''
logger.info("Task2 Running...")
self.task_2_1.start(
) # Thread1: Stop when raise StopInteration form generator
self.task_2_2.start() # Thread2: Stop when T_FLAG changes by Thread1
self.task_2_2.join()
self.task_2_1.join()
logger.info("Task2 Finish!")
def task3(self):
'''
1. City_Url(in parseRent.py when it ends then) Put into Queue for iter every rent information of City
2. Insert the Rent information into the DB.
'''
logger.info("Task3 Running...")
self.task_3_1.start() # Thread1: Stop when CITY_URL_QUEUE is empty
self.task_3_2.start() # Thread2: Stop when T_FLAG changes by Thread1
self.task_3_2.join()
self.task_3_1.join()
logger.info("Task3 Finish!")
def spider(self):
logger.info('-' * 60)
#self.task1()
logger.info('-' * 60)
#self.task2()
logger.info('-' * 60)
self.task3()
def __init__(self):
self.url_handler = UrlHandler()
class IssueManager:
def __init__(self):
self.url_handler = UrlHandler()
def split_message(self, msg):
"""split a message"""
text = ''
if msg != '':
if len(msg.split(' ', 1)) > 1:
text = msg.split(' ', 1)[1]
msg = msg.split(' ', 1)[0]
return msg, text
def new_issue(self, name, chat):
"""Create an Issue"""
payload = "{\n \"title\": \""+name+"\",\n \"labels\": [\n \"telegram\"\n ]\n}"
print(payload)
headers = {
'Content-Type': "application/json",
'Authorization': "Basic WWVzdGVyZGF5WW91U2FpZFRvbW9ycm93Qm90Olllc3RlcmRheVlvdVNhaWRUb21vcnJvdw==",
'Cache-Control': "no-cache",
'Postman-Token': "49817fa1-698d-496d-b6e3-252e81bc792f"
}
response = requests.request("POST", constants.URL_GITHUB, data=payload, headers=headers)
print(response.text)
return self.url_handler.send_message("New Issue created {}".format(name), chat)
def rename_issue(self, msg, chat):
"""rename a task by id"""
msg, text = self.split_message(msg)
if text == '':
self.url_handler.send_message("""
You want to modify the issue {},
but you didn't provide any new text
""".format(msg), chat)
return
payload = "{\n \"title\": \""+text+"\",\n \"labels\": [\n \"telegram\"\n ]\n}"
print(payload)
headers = {
'Content-Type': "application/json",
'Authorization': "Basic WWVzdGVyZGF5WW91U2FpZFRvbW9ycm93Qm90Olllc3RlcmRheVlvdVNhaWRUb21vcnJvdw==",
'Cache-Control': "no-cache",
'Postman-Token': "49817fa1-698d-496d-b6e3-252e81bc792f"
}
result = requests.request("GET", constants.URL_GITHUB + '/' + msg)
result_json = result.json()
print(result_json)
try:
result_json['state']
except:
return self.url_handler.send_message("Issue does not exist", chat)
requests.request("POST", constants.URL_GITHUB+'/'+msg, data=payload, headers=headers)
return self.url_handler.send_message("Issue renamed {}".format(text), chat)
def list_issues(self, chat):
"""lists all the issues active in the T--yesterday_you_said_tomorrow repo"""
issues = self.url_handler.get_json_from_url(constants.URL_GITHUB)
msg = ''
msg += '\U0001F4CB Issues List\n\n'
for aux in issues:
msg += "[[{}]] - {}\n\n".format(str(aux['number']), aux['title'])
self.url_handler.send_message(msg, chat)
class TaskManager:
def __init__(self):
self.url_handler = UrlHandler()
def deps_text(self, task, chat, preceed=''):
"""list tasks in a tree view"""
text = ''
for i in range(len(task.dependencies.split(',')[:-1])):
line = preceed
query = (db.SESSION
.query(Task)
.filter_by(id=int(task
.dependencies
.split(',')[:-1][i]), chat=chat))
dep = query.one()
icon = constants.NEW_ICON
if dep.status == 'DOING':
icon = constants.DOING_ICON
elif dep.status == 'DONE':
icon = constants.DONE_ICON
if i + 1 == len(task.dependencies.split(',')[:-1]):
line += '└── [[{}]] {} {}\n'.format(dep.id, icon, dep.name)
line += self.deps_text(dep, chat, preceed + ' ')
else:
line += '├── [[{}]] {} {}\n'.format(dep.id, icon, dep.name)
line += self.deps_text(dep, chat, preceed + '│ ')
text += line
return text
def get_task(self, msg, chat):
"""send message acusing missing id in the command"""
if not msg.isdigit():
self.url_handler.send_message("You must inform the task id {}".format(constants.THINKING_EMOJI), chat)
raise MessageException('id not provided')
task_id = int(msg)
query = db.SESSION.query(Task).filter_by(id=task_id, chat=chat)
try:
task = query.one()
except sqlalchemy.orm.exc.NoResultFound:
self.url_handler.send_message("Sorry, I didn't find task {} {}.".format(task_id, constants.CONFUSED_EMOJI), chat)
raise MessageException('task not found')
return task
def new_task(self, name, chat):
"""Create and returns a new task named by the user"""
task = Task(chat=chat,
name=name,
status='TODO',
dependencies='',
parents='',
priority='2')
db.SESSION.add(task)
db.SESSION.commit()
self.url_handler.send_message("Okay, I'll write this {}.\nThe ID is [[{}]]\n{}".format(constants.WRITING_EMOJI,
task.id,
task.name),
chat)
return task
def rename_task(self, msg, chat):
"""rename a task by id"""
msg, text = self.split_message(msg)
try:
task = self.get_task(msg, chat)
except MessageException:
return
if self.validate_rename_task(task, chat, text) is True:
old_text = task.name
task.name = text
db.SESSION.commit()
self.url_handler.send_message("""
Task {} redefined from {} to {}
""".format(task.id, old_text, text), chat)
else:
# NOTHING TO DO
pass
def validate_rename_task(self, task, chat, text):
"""validate if it is possible to rename a task"""
if text == '':
self.url_handler.send_message("""
You want to modify task {},
but you didn't provide any new text
""".format(task.id), chat)
return False
return True
def duplicate_task(self, msg, chat):
"""copy a task by id"""
for id in msg.split():
try:
task = self.get_task(id, chat)
except MessageException:
continue
dtask = self.new_task(task.name, chat)
for item in task.dependencies.split(',')[:-1]:
querry = db.SESSION.query(Task).filter_by(id=int(item), chat=chat)
item = querry.one()
item.parents += '{},'.format(dtask.id)
def delete_task(self, msg, chat):
"""delete a task by id"""
for id in msg.split():
try:
task = self.get_task(id, chat)
except MessageException:
continue
dependencies = []
for item in task.dependencies.split(',')[:-1]:
dependencies.append(item)
for item in task.parents.split(',')[:-1]:
querry = db.SESSION.query(Task).filter_by(id=int(item), chat=chat)
item = querry.one()
item.dependencies = item.dependencies.replace('{},'.format(task.id), '')
for item in dependencies:
querry = db.SESSION.query(Task).filter_by(id=int(item), chat=chat)
item = querry.one()
item.parents = item.parents.replace('{},'.format(task.id), '')
db.SESSION.delete(task)
db.SESSION.commit()
self.url_handler.send_message("Task [[{}]] deleted".format(task.id), chat)
def set_task_status(self, msg, chat, status):
"""set status of task to TODO"""
for id in msg.split():
try:
task = self.get_task(id, chat)
except MessageException:
continue
task.status = status
db.SESSION.commit()
self.url_handler.send_message("*{}* task [[{}]] {}".format(status, task.id, task.name), chat)
def list_tasks(self, chat, order):
"""lists all the tasks"""
msg = ''
msg += '\U0001F4CB Task List\n'
query = (db.SESSION
.query(Task)
.filter_by(parents='', chat=chat)
.order_by(Task.id))
for task in query.all():
icon = constants.NEW_ICON
if task.status == 'DOING':
icon = constants.DOING_ICON
elif task.status == 'DONE':
icon = constants.DONE_ICON
msg += '[[{}]] {} {}\n'.format(task.id, icon, task.name)
msg += self.deps_text(task, chat)
self.url_handler.send_message(msg, chat)
msg = ''
msg += constants.STATUS_ICON + '_Status_\n'
for status, status_icon in zip(['TODO', 'DOING', 'DONE'],
[constants.NEW_ICON, constants.DOING_ICON, constants.DONE_ICON]):
query_result = self.query(status, chat, order)
msg += '\n'+ status_icon + '*' + status + '*\n'
for task in query_result.all():
duedate = ''
if task.duedate != None:
if datetime.datetime.strptime(task.duedate, '%Y-%m-%d') < datetime.datetime.now():
duedate = task.duedate + ' ' + constants.END_TIME_EMOJI
else:
duedate = task.duedate + ' ' + constants.TIME_RUN_EMOJI
msg += '[[{}]] {} {} {}\n'.format(task.id,
constants.PRIORITY[self.dict_priority(task.priority)],
task.name,
duedate)
self.url_handler.send_message(msg, chat)
def query(self, status, chat, order):
"""makes a query with those parameters"""
query = (db.SESSION
.query(Task)
.filter_by(status=status, chat=chat)
.order_by(order))
return query
def circular_dependency(self, task_id, depid, chat):
"""checks if link the task with a circular dependency
will cause some circular dependency deadlock"""
try:
task = self.get_task(str(task_id), chat)
except MessageException:
return True
if str(depid) in task.parents.split(',')[:-1]:
return True
for i in task.parents.split(',')[:-1]:
if self.circular_dependency(i, depid, chat):
return True
return False
def depend_on_task(self, msg, chat):
"""set dependencies of the task"""
msg, text = self.split_message(msg)
try:
task = self.get_task(msg, chat)
except MessageException:
return
if text == '':
for i in task.dependencies.split(',')[:-1]:
i = int(i)
querry = db.SESSION.query(Task).filter_by(id=i, chat=chat)
item = querry.one()
item.parents = item.parents.replace('{},'.format(task.id), '')
task.dependencies = ''
self.url_handler.send_message("Task {} doesn't have any dependencies anymore".format(task.id),
chat)
else:
for depid in text.split(' '):
if task.id == int(depid):
self.url_handler.send_message("Invalid task: {}".format(depid), chat)
elif self.circular_dependency(task.id, depid, chat):
self.url_handler.send_message("Circular dependency, task {} depends on a task {}"
.format(depid, task.id), chat)
continue
else:
try:
taskdep = self.get_task(depid, chat)
except MessageException:
continue
aux_parents = taskdep.parents.split(',')
if str(task.id) not in aux_parents:
taskdep.parents += str(task.id) + ','
deplist = task.dependencies.split(',')
if str(depid) not in deplist:
task.dependencies += str(depid) + ','
db.SESSION.commit()
self.url_handler.send_message("Task {} dependencies up to date".format(task.id), chat)
def split_message(self, msg):
"""split a message"""
text = ''
if msg != '':
if len(msg.split(' ', 1)) > 1:
text = msg.split(' ', 1)[1]
msg = msg.split(' ', 1)[0]
return msg, text
def split_list(self, msg):
"""split a list of parameters and a comand as
first parameter"""
ids = []
if len(msg.split()) > 1:
ids = msg.split()[1:]
text = msg.split()[0]
return text, ids
def prioritize_task(self, msg, chat):
"""set the priority of given task"""
text, ids = self.split_list(msg)
for id in ids:
try:
task = self.get_task(id, chat)
except MessageException:
continue
if text == '':
task.priority = ''
self.url_handler.send_message("_Cleared_ all priorities from task {}"
.format(task.id), chat)
else:
if text.lower() not in ['high', 'medium', 'low']:
self.url_handler.send_message("""
I'm not so smart, sorry. {}
Please, tell me 'high', 'medium', or 'low'
""".format(constants.ZANY_EMOJI), chat)
else:
task.priority = self.dict_priority(text.lower())
self.url_handler.send_message("*Task {}* priority has priority *{}*"
.format(task.id, text.lower()), chat)
db.SESSION.commit()
def duedate_task(self, msg, chat):
"""set the priority of given task"""
text, ids = self.split_list(msg)
for id in ids:
try:
task = self.get_task(id, chat)
except MessageException:
continue
if text == '':
task.duedate = ''
self.url_handler.send_message("_Cleared_ duedate from task {}"
.format(task.name), chat)
else:
if self.validate_date(text, chat) is True:
task.duedate = text
self.url_handler.send_message("*Task {}* duedate is *{}*"
.format(task.id, text), chat)
db.SESSION.commit()
def validate_date(self, text, chat):
"""cheks if input is a valid date"""
try:
datetime.datetime.strptime(text, '%Y-%m-%d')
except ValueError:
self.url_handler.send_message("""
Incorrect data format, should be YYYY-MM-DD
""", chat)
return
if datetime.datetime.strptime(text, '%Y-%m-%d') < datetime.datetime.now():
self.url_handler.send_message("""
You can't travel to the past {}
If you can please tell us how :)""".format(constants.MONOCLE_EMOJI), chat)
return False
return True
def dict_priority(self, priority):
"""translate priority by the following dictionary"""
return {
'high': '1',
'medium': '2',
'low': '3',
'1': 'high',
'2': 'medium',
'3': 'low',
}[priority]
def __init__(self):
self.task_manager = TaskManager()
self.issue_manager = IssueManager()
self.url_handler = UrlHandler()
class BotManager:
def __init__(self):
self.task_manager = TaskManager()
self.issue_manager = IssueManager()
self.url_handler = UrlHandler()
def handle_updates(self, updates, chat_bot):
"""read the user command and calls the property methods"""
for update in updates["result"]:
try:
message = self.url_handler.get_message(update)
command = message["text"].split(" ", 1)[0]
except:
return
msg = ''
if len(message["text"].split(" ", 1)) > 1:
msg = message["text"].split(" ", 1)[1].strip()
chat = message["chat"]["id"]
print('\n\n\n')
print(command, msg, chat)
print('\n\n\n')
if command == '/new' or command == '/n':
self.task_manager.new_task(msg, chat)
elif command == '/newIssue' or command == '/ni':
self.issue_manager.new_issue(msg, chat)
elif command == '/renameIssue' or command == '/ri':
self.issue_manager.rename_issue(msg, chat)
elif command == '/rename' or command == '/r':
self.task_manager.rename_task(msg, chat)
elif command == '/duplicate' or command == '/dc':
self.task_manager.duplicate_task(msg, chat)
elif command == '/delete' or command == '/d':
self.task_manager.delete_task(msg, chat)
elif command == '/todo':
self.task_manager.set_task_status(msg, chat, constants.TODO)
elif command == '/doing':
self.task_manager.set_task_status(msg, chat, constants.DOING)
elif command == '/done':
self.task_manager.set_task_status(msg, chat, constants.DONE)
elif command == '/listP' or command == '/lp':
order = Task.priority
self.task_manager.list_tasks(chat, order)
elif command == '/list' or command == '/l':
order = Task.id
self.task_manager.list_tasks(chat, order)
elif command == '/listIssues' or command == '/li':
self.issue_manager.list_issues(chat)
elif command == '/dependson' or command == '/dp':
self.task_manager.depend_on_task(msg, chat)
elif command == '/priority' or command == '/p':
self.task_manager.prioritize_task(msg, chat)
elif command == '/duedate' or command == '/dd':
self.task_manager.duedate_task(msg, chat)
elif command == '/start':
self.url_handler.send_message(
"Welcome! Here is a list of things you can do.", chat)
self.url_handler.send_message(constants.HELP, chat)
elif command == '/help' or command == '/h':
self.url_handler.send_message(
"Here is a list of things you can do.", chat)
self.url_handler.send_message(constants.HELP, chat)
else:
response = chat_bot.predict([message['text']])
print(response)
print(message['text'])
response = str(response)
print(response)
self.url_handler.send_message(response[2:-2], chat)