def writeall(self):
''' writeall is called for writing all 'real' outmessage objects; but not for enveloping.
writeall is call from transform.translate()
'''
self.nrmessagewritten = 0
if not self.root.children:
raise botslib.OutMessageError(
_(u'No outgoing message')) #then there is nothing to write...
for message in self.root.getloop({'BOTSID': 'STX'}, {'BOTSID': 'MHD'}):
self.outmessagegrammarread(
self.ta_info['editype'],
message.get({
'BOTSID': 'MHD',
'TYPE.01': None
}) + message.get({
'BOTSID': 'MHD',
'TYPE.02': None
}))
if not self.nrmessagewritten:
self._initwrite()
self.normalisetree(message)
self._write(message)
self.nrmessagewritten += 1
self._closewrite()
self.ta_info['nrmessages'] = self.nrmessagewritten
def valid(self):
# check argument size
if len(self._args) != 1:
StdOut.display(msg=message.get(message.INVALID_ARGS))
return False
drive_name = self._args[0]
# reject invalid drive
if not ConfUtil.valid_drive(drive_name):
StdOut.display(msg=message.get(message.INVALID_DRIVE_NAME, drive=drive_name))
return False
return True
def execute(self, **kwargs):
# find all the drive config
for name, drive in Conf.drives.items():
mark = message.NO_MARK
if ConfUtil.exists_drive(name):
mark = message.MARK
StdOut.display(msg=message.get(message.STATUS, mark=mark, drive=name))
def __insert_folder(self, src=None, meta=None):
folder_name = fsutil.filename(src)
try:
children = self.__drive.children().list(folderId=meta.get('id')).execute()
folder_meta = None
for child in children.get('items'):
child_meta = self.__drive.files().get(fileId=child.get('id')).execute()
if child_meta.get('title') == folder_name and child_meta.get('mimeType') == self.__FOLDER_MIME:
folder_meta = child_meta
self.__drive.files().untrash(fileId=child_meta.get('id')).execute()
break
if folder_meta is None:
body = {
'title': folder_name,
'mimeType': self.__FOLDER_MIME,
'parents': [{'id': str(meta.get('id'))}]
}
folder_meta = self.__drive.files().insert(body=body).execute()
for path in fsutil.list_folder(src):
self.__insert(path, folder_meta)
except HttpError, error:
StdOut.display(
msg=message.get(message.ERROR_REMOTE_OPERATION, file=src, drive=self.name(), error=error)
)
def valid(self):
if len(self._args) == 0:
StdOut.display(msg=message.get(message.INVALID_ARGS))
return False
# make sure drive has been checked out
checkout = ConfUtil.get_checkout()
if not checkout:
StdOut.display(msg=message.get(message.MISSING_CHECKOUT))
return False
# make sure drive is tracked
if not ConfUtil.exists_drive(checkout):
StdOut.display(msg=message.get(message.MISSING_REMOTE, drive=checkout))
return False
return True
def __write_file(self, meta, dst):
url = meta.get('downloadUrl', None)
# todo google document types not supported at this time
content = self.__get_content(url)
if content is None:
StdOut.display(
msg=message.get(message.ERROR_UNSUPPORTED_DOWNLOAD, file=meta.get('title'), drive=self.name()))
else:
fsutil.write(dst, content, True)
def execute(self, **kwargs):
proceed = True
if ConfUtil.exists_home():
if not StdIn.prompt_yes(message.get(message.HOME_EXISTS)):
proceed = False
# create home and files
if proceed:
ConfUtil.remove_home()
ConfUtil.create_home()
def delete(self, **kwargs):
src = kwargs.get('src', None)
# find file
meta = self.__find_meta(src)
if meta is None:
StdOut.display(msg=message.get(message.MISSING_FILE, location=self.name()))
return
if StdIn.prompt_yes(message.get(message.PROMPT_TRASH, file=src, drive=self.name())):
# delete the file
try:
resp = self.__drive.files().trash(fileId=meta.get('id')).execute()
if resp:
StdOut.display(message.get(message.TRASH, file=src, drive=self.name()))
except HttpError, error:
StdOut.display(
msg=message.get(message.ERROR_REMOTE_OPERATION, file=src, drive=self.name(), error=error)
)
def make_row(self, message, cid, parent_user_id):
"""
create a Row dictionary for insertion into DynamoDB
"""
ts = message['ts']
date = utils.make_day(ts)
try:
user_id = message.get("user") or message.get("bot_id")
except BaseException:
print("How the hell no user? {}".format(
json.dumps(message, indent=4)))
return None
text = message.get("text", "")
if text is None:
text = ""
word_count = len(text.split())
mentions = utils.find_user_mentions(text)
mentions = [x for x in mentions if x != user_id]
(reaction_count, reactions) = self.get_reactions(message)
(reply_count, replies) = self.get_replies(message)
files = json.dumps(message.get("files", None))
thread_ts = message.get("thread_ts")
is_threadhead = thread_ts == ts
is_threaded = 'thread_ts' in message
cid_ts = "{}_{}".format(cid, ts)
if files == 'null':
files = None
Row = {
"cid_ts": cid_ts,
"date": date,
"is_threaded": is_threaded,
"is_thread_head": is_threadhead,
"ts": ts,
"thread_ts": thread_ts,
"slack_cid": cid,
"user_id": user_id,
"word_count": word_count,
"reaction_count": reaction_count,
"reactions": reactions,
"replies": replies,
"reply_count": reply_count,
"files": files
}
Row['subtype'] = message.get("subtype")
if parent_user_id:
Row['parent_user_id'] = parent_user_id
if mentions:
Row['mentions'] = ",".join(mentions)
else: # if it's a thread head, we want to capture that
if message.get("thread_ts") == message.get("ts"):
Row['parent_user_id'] = user_id
Row = utils.prune_empty(Row)
return Row
def execute(self, **kwargs):
# read checkout file
checkout = ConfUtil.get_checkout()
# read branches and compare with checkout
for name in Conf.drives.keys():
mark = message.NO_MARK
if name == checkout:
mark = message.MARK
StdOut.display(msg=message.get(message.STATUS, mark=mark, drive=name))
def valid(self):
if len(self._args) == 0:
return True
elif len(self._args) == 1:
parser = CommandParser()
cmd = parser.parse(self._args)
if type(cmd) is not Unknown:
return True
StdOut.display(msg=message.get(message.INVALID_ARGS))
return False
def download(self, **kwargs):
src = kwargs.get('src', None)
dst = kwargs.get('dst', None)
# find and persist
meta = self.__find_meta(src)
if meta is not None:
self.__write(meta, dst)
else:
# file not found
StdOut.display(msg=message.get(message.MISSING_FILE, location=self.name()))
def __get_credentials():
storage = Storage(ConfUtil.get_drive_prop(Conf.GOOGLE, Conf.CREDENTIALS))
credentials = storage.get()
# if no credentials
if credentials is None or credentials.invalid:
# validate credentials
client_id = StdIn.prompt(message.get(message.GOOGLE_ID))
client_secret = StdIn.prompt(message.get(message.GOOGLE_SECRET))
if not client_id or not client_secret:
StdOut.display(msg=message.get(message.INVALID_CREDENTIALS))
return None
StdOut.display(msg=message.GOOGLE_CONSENT)
# run flow and store credentials
flow = OAuth2WebServerFlow(client_id, client_secret, 'https://www.googleapis.com/auth/drive')
credentials = run(flow, storage)
return credentials
def stats(bot, update):
if update.message.from_user.id != ADMIN_ID:
return
global db
sent_message = bot.send_message(chat_id=update.message.chat.id,
text=message.get(db),
parse_mode="Markdown",
disable_web_page_preview=True)
db.tasks.insert_one({
"chat_id": sent_message.chat.id,
"message_id": sent_message.message_id
})
def poll(username):
"""
Pops messages from the user's queue.
"""
msg_lst = []
while True:
msg_id = r.lpop("user:%s:queue" % username)
if not msg_id: # no more messages left
break
msg_lst.append(message.get(msg_id))
message.recieved(msg_id)
return msg_lst
def valid(self):
if len(self._args) != 2:
StdOut.display(msg=message.get(message.INVALID_ARGS))
return False
# make sure drive has been checked out
checkout = ConfUtil.get_checkout()
if not checkout:
StdOut.display(msg=message.get(message.MISSING_CHECKOUT))
return False
# make sure drive is tracked
if not ConfUtil.exists_drive(checkout):
StdOut.display(msg=message.get(message.MISSING_REMOTE, drive=checkout))
return False
if not fsutil.is_folder(self._args[1]):
StdOut.display(msg=message.get(message.INVALID_DST, file=self._args[1], location='local'))
return False
return True
def poll(username):
"""
Pops messages from the user's queue.
"""
msg_lst = []
while True:
msg_id = r.lpop("user:%s:queue" % username)
if not msg_id: # no more messages left
break
msg_lst.append(message.get(msg_id))
message.recieved(msg_id)
return msg_lst
def update_messages():
global db
message_text = message.get(db)
for task in db.tasks.find():
bot.edit_message_text(
chat_id=task["chat_id"],
message_id=task["message_id"],
text=message_text,
parse_mode="Markdown",
disable_web_page_preview=True)
def __find_meta(self, path=''):
# preconditions
if not path:
return None
# split file and cleanup
titles = fsutil.split(path)
titles.insert(0, 'My Drive')
size = len(titles) - 1
root = False
# build query
query = []
params = {}
if len(titles) > 1:
query = ['title', ' = \'', titles[len(titles) - 1], '\'']
else:
query = ["'root' in parents"]
params['maxResults'] = 1
root = True
size = 1
query.append(' and trashed = False')
params['q'] = ''.join(query)
req = self.__drive.files().list(**params)
try:
while req:
resp = req.execute()
metas = resp.get('items')
for meta in metas:
parents = meta.get('parents')
index = size
while True:
if len(parents) == 0 and index == 0:
if root:
return anc
else:
return meta
elif len(parents) > 0 and index > 0:
index -= 1
anc = self.__drive.files().get(fileId=parents[0].get('id')).execute()
if anc.get('title') == titles[index]:
parents = anc.get('parents')
continue
break
# paginate
req = self.__drive.files().list_next(req, resp)
except HttpError, error:
StdOut.display(
msg=message.get(message.ERROR_REMOTE_OPERATION, file=path, drive=self.name(), error=error)
)
def invoke(self, command):
if command is None:
return
# if in new state, can execute only init, help and invalid
if not ConfUtil.exists_home():
if not isinstance(command, Help) and not isinstance(command, Init) and not isinstance(command, Unknown):
StdOut.display(msg=message.get(message.MISSING_HOME))
return
if command.valid():
# if in ready state, all commands can be executed
command.execute()
def __insert_file(self, src=None, parent_meta=None):
file_name = fsutil.filename(src)
mime = mimetypes.guess_type(src)[0]
if not mime:
mime = 'text/plain'
media_body = MediaFileUpload(src, mimetype=mime, resumable=True)
parent_id = str(parent_meta.get('id'))
folder = str(parent_meta.get('title'))
try:
# search for file under folder,
query = ['\'', parent_id, '\' in parents', ' and title = ', '\'', file_name, '\'',
' and trashed = False']
params = {'q': ''.join(query), 'maxResults': 1}
resp = self.__drive.files().list(**params).execute()
child_metas = resp.get('items')
if child_metas:
# if it exists, prompt and update
child_meta = child_metas[0]
if StdIn.prompt_yes(msg=message.get(message.PROMPT_OVERWRITE, file=file_name)):
self.__drive.files().untrash(fileId=child_meta.get('id')).execute()
self.__drive.files().update(fileId=child_meta.get('id'), media_body=media_body).execute()
StdOut.display(msg=message.get(message.LOADED, file=file_name, load='uploaded', folder=folder,
drive=self.name()))
return
else:
# else insert it as a child to the folder
body = {
'title': file_name,
'parents': [{'id': parent_id}],
'mimeType': mime
}
self.__drive.files().insert(body=body, media_body=media_body).execute()
StdOut.display(msg=message.get(message.LOADED, file=file_name, load='uploaded', folder=folder,
drive=self.name()))
except HttpError, error:
StdOut.display(
msg=message.get(message.ERROR_REMOTE_OPERATION, file=src, drive=self.name(), error=error)
)
def upload(self, **kwargs):
src = kwargs.get('src', None)
dst = kwargs.get('dst', None)
# if not folder, print error and return
if not fsutil.exists(src):
StdOut.display(msg=message.get(message.MISSING_FILE, location='local'))
return
# get meta data
meta = self.__find_meta(dst)
# if remote does not exist, print and return
if meta is None:
StdOut.display(msg=message.get(message.MISSING_FILE, location=self.name()))
return
if meta.get('mimeType') != self.__FOLDER_MIME:
StdOut.display(msg=message.get(message.INVALID_DST, location=self.name()))
return
self.__insert(src, meta)
def insert_new_message(conn, message):
cur = conn.cursor()
print(type(message))
m = json.dumps(message)
print(type(m))
try:
cur.execute("insert into messages_tbl values (?, ?)",
[message.get('message_id'), m])
except sqlite3.OperationalError as msg:
print(msg)
return ' it is not succeeded to insert the new message!!!!'
conn.commit()
return cur.lastrowid
def execute(self, **kwargs):
drive_name = self._args[0]
drive = device.get_drive(drive_name)
if drive is not None:
# create configs home
ConfUtil.create_drive(drive_name)
# create remote
if drive.open(**kwargs):
drive.close(**kwargs)
else:
# don't create home if auth fails
ConfUtil.remove_drive(drive_name)
else:
StdOut.display(msg=message.get(message.MISSING_IMPLEMENTATION, drive=drive_name))
def writeall(self):
''' writeall is called for writing all 'real' outmessage objects; but not for enveloping.
writeall is call from transform.translate()
'''
self.nrmessagewritten = 0
if not self.root.children:
raise botslib.OutMessageError(_(u'No outgoing message')) #then there is nothing to write...
for message in self.root.getloop({'BOTSID':'STX'},{'BOTSID':'MHD'}):
self.outmessagegrammarread(self.ta_info['editype'],message.get({'BOTSID':'MHD','TYPE.01':None}) + message.get({'BOTSID':'MHD','TYPE.02':None}))
if not self.nrmessagewritten:
self._initwrite()
self.normalisetree(message)
self._write(message)
self.nrmessagewritten += 1
self._closewrite()
self.ta_info['nrmessages'] = self.nrmessagewritten
def send_messages(devices):
while open("master alive", 'r').read() == "T":
res = None
if mess.exist():
message_to_send = mess.get()
message_to_send.update_master(devices[0])
for i in range(1):
print "Try to send..."
res = send_message(message_to_send)
if res is not None:
mess.remove_from_queue()
print "Message arrived"
break
if res is None:
print "Message not arrived"
open("master alive", 'w').write("F")
def get_replies(self, message):
"""
Given a message, return (reply_count, replies)
where reply_count is the number of replies and
replies is str of the form "UID:TS,UID:TS"
If no replies, returns (0,"")
"""
reply_count = 0
replies = message.get("replies", [])
replies.sort(key=lambda x: int(float(x['ts'])))
reply_count = len(replies)
replies = ["{}:{}".format(x['user'], x['ts']) for x in replies]
if replies:
replies = ",".join(replies)
else:
replies = None
return (reply_count, replies)
def __write_folder(self, meta, dst):
# create folder if it does not exist
fsutil.create_dir(dst)
# list all its children,
try:
children = self.__drive.children().list(folderId=meta.get('id')).execute()
for child in children.get('items'):
child_meta = self.__drive.files().get(fileId=child.get('id')).execute()
child_dst = fsutil.join_paths(dst, child_meta.get('title'))
# if child is folder, recurs
if child_meta.get('mimeType') == self.__FOLDER_MIME:
self.__write_folder(child_meta, child_dst)
# if child is file, write it to folder
else:
self.__write_file(child_meta, child_dst)
except HttpError, error:
StdOut.display(
msg=message.get(message.ERROR_REMOTE_OPERATION, file=dst, drive=self.name(), error=error)
)
def get_reactions(self, message):
"""
given a message, return (COUNT_OF_REACTIONS, REACTIONS)
where COUNT_OF_REACTIONS is the total number of reactions and
REACTIONS is of the form REACTION_TYPE:UID:UID,REACTION_TYPE:UID:UID, etc
if no reactions, return (0, "")
"""
count = 0
reactions = []
for reaction in message.get("reactions", []):
reaction_name = reaction['name']
users = ";".join(reaction['users'])
count += len(reaction['users'])
reaction_text = "{};{}".format(reaction_name, users)
reactions.append(reaction_text)
if reactions:
reactions = ",".join(reactions)
else:
reactions = None
return (count, reactions)
def info(self, **kwargs):
# preconditions
src = kwargs.get('src', None)
# get meta information
meta = self.__find_meta(src)
# print data
if meta is not None:
# pprint version for utf8 conversions. Will remote the u prefixes.
class Utf8PrettyPrinter(pprint.PrettyPrinter):
def format(self, object, context, maxlevels, level):
if isinstance(object, unicode):
return object.encode('utf8'), True, False
return pprint.PrettyPrinter.format(self, object, context, maxlevels, level)
Utf8PrettyPrinter().pprint(meta)
else:
StdOut.display(msg=message.get(message.MISSING_FILE, location=self.name()))
def write(self, list_of_messages, cid, parent_user_id=None):
"""
Given the list of message JSONs, write them to DynamoDB
cid is the Slack channel ID
if parent_user_id is provided, this is the UID of the originator of the thread
of which this message is a part
"""
# Map message table name to actual message table objects
message_tables = {}
# Map message table names to list of messages
messages = {}
self.seen = {}
table = self.Message.table
with table.batch_writer() as batch:
ctr = 0
for message in list_of_messages:
ctr += 1
if ctr > 0:
if ctr % 2000 == 0:
sys.stdout.write(str(ctr))
sys.stdout.flush()
elif ctr % 200 == 0:
sys.stdout.write(".")
sys.stdout.flush()
if message.get("type") != "message":
continue
ts = message['ts']
Row = self.make_row(message, cid, parent_user_id)
if not Row:
continue
if Row['cid_ts'] in self.seen:
continue
self.seen[Row['cid_ts']] = 1
if not self.fake:
batch.put_item(Row)
else:
pass
def execute(self, **kwargs):
StdOut.display(msg=message.get(message.INVALID_COMMAND, command=self._args[0]))
def delete(path=None, prompt=False):
if exists(path):
if prompt and not StdIn.prompt_yes(message.get(msg=message.PROMPT_OVERWRITE, file=path)):
return False
os.remove(abs_path(path))
return True
def valid(self):
if len(self._args) != 0:
StdOut.display(msg=message.get(message.INVALID_ARGS))
return False
return True