def __init__(self, wttQueue, groups):
super(MediaWorker, self).__init__()
self.daemon = True
self.wttQ = wttQueue
self.groups = groups
self._jobs = Queue()
self._media_cipher = MediaCipher()
class MediaCipherTest(unittest.TestCase):
IMAGE = (
b'EOnnZIBu1vTSI51IeJvaKR+8W1FqBETATI2Ikl6nVQ8=',
b'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8M'
b'CgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQ'
b'EBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAABAAEDAREAAhEBAxEB/8QAHwAAAQUBAQEB'
b'AQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKB'
b'kaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1'
b'dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl'
b'5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcF'
b'BAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5'
b'OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0'
b'tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD9U6AP/9kL'
b'CwsLCwsLCwsLCw==',
b'dT9YPFSz4dprkH6uiXPEVERGZVIZbGYHzwue21WoLP1RCE2wmu11M8n6ysfROPtI39DCRFQhBBEVGFCT/nfV'
b'pt+fouIENBSXY44mR2en4HGvRR//dlM5OBLz2WuEOf01iKPazGtfacy6lnV0X5JagL4r1mKeyuSXJEV81kxj'
b'Vd3OArpLKt13XM36PcnTd/U6DHOV6Vf982Wc1UjR7kMb5JT+HlWrvz9CCGMTX5mqBYWEr3InCFyrmaZu8DXC'
b'60YUZCPLTHJP0hFQA1ooKQks4f3F39tzVL3dbX3io7XPQiSgHN6nuPCD0PF7dWINep+amk+ODjeQd/o2guqx'
b'O/AngNIxfFWq3915jMQWAXeARUFw7x+9Qx93UWC/sfQ72nTqdQaH5W4vsMUKaocZcAJ7YWudKo5Y15uo3ulP'
b'744Cyo54tvxUSV0zLC90LYCe8fJefREjreO73RlVqoynTyFHuVS7/YMnesO5lCHZJ2NpHuzpGKCU08RgCuSr'
b'K/wh4SLXZ1nhZEYX/xY4cDO7swrA3Y3Qt0gjFzNBfUT9aABQaU+WHY8pBS/oVfJsuj2iod2fsW8UoplImoOk'
b'bosYlMkdZSIfwBQ5kt3On2d+HPUNt7HVZWRECFj0C4IHhZCZIJw0wFmPMOiIHyzittXGb45uJA2Sd1gnRbw0'
b'1XFuoIyvS7z0MtW0QUiD6kaJFBkpTo7hxN/HyN8xQwOkeBcKORdpeSp62iPBuRvel9I2p07it7UyYhzas+Jv'
b'tl6i+hIz6Z5nzQEZ+zPAYxDmoy4h9GQuXUivTzU9I0/Sd4QvM3rfewGeXsNSjWI1CLVZG+4wL1TPCbQGaHEB'
b'NF8zmpTMYV+NvCzv/2DwYuYoiUI=')
def setUp(self):
self._cipher = MediaCipher() # type: MediaCipher
def test_decrypt_image(self):
media_key, media_plaintext, media_ciphertext = map(
base64.b64decode, self.IMAGE)
self.assertEqual(
media_plaintext,
self._cipher.decrypt_image(media_ciphertext, media_key))
def test_encrypt_image(self):
media_key, media_plaintext, media_ciphertext = map(
base64.b64decode, self.IMAGE)
encrypted = self._cipher.encrypt(media_plaintext, media_key,
MediaCipher.INFO_IMAGE)
self.assertEqual(media_ciphertext, encrypted)
class MediaCipherTest(unittest.TestCase):
IMAGE = (
b'EOnnZIBu1vTSI51IeJvaKR+8W1FqBETATI2Ikl6nVQ8=',
b'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8M'
b'CgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQ'
b'EBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAABAAEDAREAAhEBAxEB/8QAHwAAAQUBAQEB'
b'AQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKB'
b'kaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1'
b'dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl'
b'5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcF'
b'BAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5'
b'OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0'
b'tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD9U6AP/9k'
b'==',
b'dT9YPFSz4dprkH6uiXPEVERGZVIZbGYHzwue21WoLP1RCE2wmu11M8n6ysfROPtI39DCRFQhBBEVGFCT/nfV'
b'pt+fouIENBSXY44mR2en4HGvRR//dlM5OBLz2WuEOf01iKPazGtfacy6lnV0X5JagL4r1mKeyuSXJEV81kxj'
b'Vd3OArpLKt13XM36PcnTd/U6DHOV6Vf982Wc1UjR7kMb5JT+HlWrvz9CCGMTX5mqBYWEr3InCFyrmaZu8DXC'
b'60YUZCPLTHJP0hFQA1ooKQks4f3F39tzVL3dbX3io7XPQiSgHN6nuPCD0PF7dWINep+amk+ODjeQd/o2guqx'
b'O/AngNIxfFWq3915jMQWAXeARUFw7x+9Qx93UWC/sfQ72nTqdQaH5W4vsMUKaocZcAJ7YWudKo5Y15uo3ulP'
b'744Cyo54tvxUSV0zLC90LYCe8fJefREjreO73RlVqoynTyFHuVS7/YMnesO5lCHZJ2NpHuzpGKCU08RgCuSr'
b'K/wh4SLXZ1nhZEYX/xY4cDO7swrA3Y3Qt0gjFzNBfUT9aABQaU+WHY8pBS/oVfJsuj2iod2fsW8UoplImoOk'
b'bosYlMkdZSIfwBQ5kt3On2d+HPUNt7HVZWRECFj0C4IHhZCZIJw0wFmPMOiIHyzittXGb45uJA2Sd1gnRbw0'
b'1XFuoIyvS7z0MtW0QUiD6kaJFBkpTo7hxN/HyN8xQwOkeBcKORdpeSp62iPBuRvel9I2p07it7UyYhzas+Jv'
b'tl6i+hIz6Z5nzQEZ+zPAYxDmoy4h9GQuXUivTzU9I0/Sd4QvM3rfewGeXsNSjWI1CLVZG+4wL1TPCbQGaHEB'
b'NF8zmpTMYV+NvCzv/2DwYuYoiUI='
)
def setUp(self):
self._cipher = MediaCipher() # type: MediaCipher
def test_decrypt_image(self):
media_key, media_plaintext, media_ciphertext = map(base64.b64decode, self.IMAGE)
self.assertEqual(media_plaintext, self._cipher.decrypt_image(media_ciphertext, media_key))
def test_encrypt_image(self):
media_key, media_plaintext, media_ciphertext = map(base64.b64decode, self.IMAGE)
encrypted = self._cipher.encrypt(media_plaintext, media_key, MediaCipher.INFO_IMAGE)
self.assertEqual(media_ciphertext, encrypted)
def __init__(self, storage_dir):
super(SinkWorker, self).__init__()
self.daemon = True
self._storage_dir = storage_dir
self._jobs = Queue()
self._media_cipher = MediaCipher()
class SinkWorker(threading.Thread):
def __init__(self, storage_dir):
super(SinkWorker, self).__init__()
self.daemon = True
self._storage_dir = storage_dir
self._jobs = Queue()
self._media_cipher = MediaCipher()
def enqueue(self, media_message_protocolentity):
self._jobs.put(media_message_protocolentity)
def _create_progress_iterator(self, iterable, niterations, desc):
return tqdm(iterable,
total=niterations,
unit='KB',
dynamic_ncols=True,
unit_scale=True,
leave=True,
desc=desc,
ascii=True)
def _download(self, url):
response = requests.get(url, stream=True)
total_size = int(response.headers.get('content-length', 0))
logger.debug("%s total size is %s, downloading" % (url, total_size))
block_size = 1024
wrote = 0
enc_data = b""
for data in self._create_progress_iterator(
response.iter_content(block_size),
math.ceil(total_size // block_size) + 1,
"Download ",
):
wrote = wrote + len(data)
enc_data = enc_data + data
if total_size != 0 and wrote != total_size:
logger.error("Something went wrong")
return None
return enc_data
def _decrypt(self, ciphertext, ref_key, media_info):
length_kb = int(math.ceil(len(ciphertext) / 1024))
progress = self._create_progress_iterator(range(length_kb), length_kb,
"Decrypt ")
try:
plaintext = self._media_cipher.decrypt(ciphertext, ref_key,
media_info)
progress.update(length_kb)
return plaintext
except Exception as e:
progress.set_description("Decrypt Error ")
logger.error(e)
return None
def _write(self, data, filename):
length_kb = int(math.ceil(len(data) / 1024))
progress = self._create_progress_iterator(range(length_kb), length_kb,
"Write ")
try:
with open(filename, 'wb') as f:
f.write(data)
progress.update(length_kb)
return filename
except Exception as e:
progress.set_description("Write error ")
logger.error(e)
return None
def _create_unique_filepath(self, filepath):
file_dir = os.path.dirname(filepath)
filename = os.path.basename(filepath)
result_filename = filename
dissected = os.path.splitext(filename)
count = 0
while os.path.exists(os.path.join(file_dir, result_filename)):
count += 1
result_filename = "%s_%d%s" % (dissected[0], count, dissected[1])
return os.path.join(file_dir, result_filename)
def run(self):
logger.debug("SinkWorker started, storage_dir=%s" % self._storage_dir)
while True:
media_message_protocolentity = self._jobs.get()
if media_message_protocolentity is None:
sys.exit(0)
if isinstance(media_message_protocolentity,
DownloadableMediaMessageProtocolEntity):
logger.info(
"Processing [url=%s, media_key=%s]" %
(media_message_protocolentity.url,
base64.b64encode(media_message_protocolentity.media_key)))
else:
logger.info("Processing %s" %
media_message_protocolentity.media_type)
filedata = None
fileext = None
if isinstance(media_message_protocolentity,
ImageDownloadableMediaMessageProtocolEntity):
media_info = MediaCipher.INFO_IMAGE
filename = "image"
elif isinstance(media_message_protocolentity,
AudioDownloadableMediaMessageProtocolEntity):
media_info = MediaCipher.INFO_AUDIO
filename = "ptt" if media_message_protocolentity.ptt else "audio"
elif isinstance(media_message_protocolentity,
VideoDownloadableMediaMessageProtocolEntity):
media_info = MediaCipher.INFO_VIDEO
filename = "video"
elif isinstance(media_message_protocolentity,
DocumentDownloadableMediaMessageProtocolEntity):
media_info = MediaCipher.INFO_DOCUM
filename = media_message_protocolentity.file_name
elif isinstance(media_message_protocolentity,
ContactMediaMessageProtocolEntity):
filename = media_message_protocolentity.display_name
filedata = media_message_protocolentity.vcard
fileext = "vcard"
else:
logger.error("Unsupported Media type: %s" %
media_message_protocolentity.__class__)
sys.exit(1)
if filedata is None:
enc_data = self._download(media_message_protocolentity.url)
if enc_data is None:
logger.error("Download failed")
sys.exit(1)
filedata = self._decrypt(
enc_data, media_message_protocolentity.media_key,
media_info)
if filedata is None:
logger.error("Decrypt failed")
sys.exit(1)
if not isinstance(media_message_protocolentity,
DocumentDownloadableMediaMessageProtocolEntity):
if fileext is None:
fileext = media_message_protocolentity.mimetype.split(
'/')[1].split(';')[0]
filename_full = "%s.%s" % (filename, fileext)
else:
filename_full = filename
filepath = self._create_unique_filepath(
os.path.join(self._storage_dir, filename_full))
if self._write(filedata, filepath):
logger.info("Wrote %s" % filepath)
else:
sys.exit(1)
def __init__(self, storage_dir):
super(WADocumentDownloader, self).__init__()
self._storage_dir = storage_dir
self._media_cipher = MediaCipher()
class MediaWorker(threading.Thread):
def __init__(self, wttQueue, groups):
super(MediaWorker, self).__init__()
self.daemon = True
self.wttQ = wttQueue
self.groups = groups
self._jobs = Queue()
self._media_cipher = MediaCipher()
def enqueue(self, media_message_protocolentity):
self._jobs.put(media_message_protocolentity)
def _create_progress_iterator(self, iterable, niterations, desc):
return tqdm(iterable,
total=niterations,
unit='KB',
dynamic_ncols=True,
unit_scale=True,
leave=True,
desc=desc,
ascii=True)
def _download(self, url):
response = requests.get(url, stream=True)
total_size = int(response.headers.get('content-length', 0))
logger.debug("%s total size is %s, downloading" % (url, total_size))
block_size = 1024
wrote = 0
enc_data = b""
for data in self._create_progress_iterator(
response.iter_content(block_size),
math.ceil(total_size // block_size) + 1,
"Download ",
):
wrote = wrote + len(data)
enc_data = enc_data + data
if total_size != 0 and wrote != total_size:
logger.error("Something went wrong")
return None
return enc_data
def _decrypt(self, ciphertext, ref_key, media_info):
length_kb = int(math.ceil(len(ciphertext) / 1024))
progress = self._create_progress_iterator(range(length_kb), length_kb,
"Decrypt ")
try:
plaintext = self._media_cipher.decrypt(ciphertext, ref_key,
media_info)
progress.update(length_kb)
return plaintext
except Exception as e:
progress.set_description("Decrypt Error ")
logger.error(e)
return None
def _write(self, media_message_protocolentity, data, filename):
msg = WTTMessage(
media_message_protocolentity.media_type,
media_message_protocolentity.getNotify().encode(
'latin-1').decode(),
data,
waID=media_message_protocolentity.getFrom(),
title=(self.groupIdToSubject(
media_message_protocolentity.getFrom())
if media_message_protocolentity.isGroupMessage() else None),
isGroup=media_message_protocolentity.isGroupMessage(),
filename=filename)
self.wttQ.put(msg)
return None
def run(self):
logger.debug("MediaWorker started")
while True:
if self._jobs.empty():
time.sleep(1)
continue
media_message_protocolentity = self._jobs.get()
if media_message_protocolentity is None:
logger.error("MediaMessageEntity is none")
continue
if isinstance(media_message_protocolentity,
DownloadableMediaMessageProtocolEntity):
logger.info(
"Processing [url=%s, media_key=%s]" %
(media_message_protocolentity.url,
base64.b64encode(media_message_protocolentity.media_key)))
else:
logger.info("Processing %s" %
media_message_protocolentity.media_type)
filedata = None
fileext = None
if isinstance(media_message_protocolentity,
ImageDownloadableMediaMessageProtocolEntity):
media_info = MediaCipher.INFO_IMAGE
filename = "image"
elif isinstance(media_message_protocolentity,
AudioDownloadableMediaMessageProtocolEntity):
media_info = MediaCipher.INFO_AUDIO
filename = "ptt" if media_message_protocolentity.ptt else "audio"
elif isinstance(media_message_protocolentity,
VideoDownloadableMediaMessageProtocolEntity):
media_info = MediaCipher.INFO_VIDEO
filename = "video"
elif isinstance(media_message_protocolentity,
DocumentDownloadableMediaMessageProtocolEntity):
media_info = MediaCipher.INFO_DOCUM
filename = media_message_protocolentity.file_name
elif isinstance(media_message_protocolentity,
ContactMediaMessageProtocolEntity):
filename = media_message_protocolentity.display_name
filedata = media_message_protocolentity.vcard
fileext = "vcard"
# elif isinstance(media_message_protocolentity, StickerDownloadableMediaMessageProtocolEntity):
# media_info = MediaCipher.INFO_IMAGE
# filename = "sticker"
else:
logger.error("Unsupported Media type: %s" %
media_message_protocolentity.__class__)
continue
if filedata is None:
enc_data = self._download(media_message_protocolentity.url)
if enc_data is None:
logger.error("Download failed")
continue
filedata = self._decrypt(
enc_data, media_message_protocolentity.media_key,
media_info)
if filedata is None:
logger.error("Decrypt failed")
continue
if not isinstance(media_message_protocolentity,
DocumentDownloadableMediaMessageProtocolEntity):
if fileext is None:
fileext = media_message_protocolentity.mimetype.split(
'/')[1].split(';')[0]
filename_full = "%s.%s" % (filename, fileext)
else:
filename_full = filename
if self._write(media_message_protocolentity, filedata,
filename_full):
logger.info("Pushing processed media...")
else:
continue
def groupIdToSubject(self, groupId):
rest = groupId.split('@', 1)[0]
for group in self.groups:
if group["groupId"] == rest:
return group["subject"]
def __init__(self, storage_dir):
super(SinkWorker, self).__init__()
self.daemon = True
self._storage_dir = storage_dir
self._jobs = Queue()
self._media_cipher = MediaCipher()
class SinkWorker(threading.Thread):
def __init__(self, storage_dir):
super(SinkWorker, self).__init__()
self.daemon = True
self._storage_dir = storage_dir
self._jobs = Queue()
self._media_cipher = MediaCipher()
def enqueue(self, media_message_protocolentity):
self._jobs.put(media_message_protocolentity)
def _create_progress_iterator(self, iterable, niterations, desc):
return tqdm(
iterable, total=niterations, unit='KB', dynamic_ncols=True,
unit_scale=True, leave=True, desc=desc, ascii=True)
def _download(self, url):
response = requests.get(url, stream=True)
total_size = int(response.headers.get('content-length', 0))
logger.debug("%s total size is %s, downloading" % (url, total_size))
block_size = 1024
wrote = 0
enc_data = b""
for data in self._create_progress_iterator(
response.iter_content(block_size), math.ceil(total_size // block_size) + 1, "Download ",
):
wrote = wrote + len(data)
enc_data = enc_data + data
if total_size != 0 and wrote != total_size:
logger.error("Something went wrong")
return None
return enc_data
def _decrypt(self, ciphertext, ref_key, media_info):
length_kb = int(math.ceil(len(ciphertext) / 1024))
progress = self._create_progress_iterator(range(length_kb), length_kb, "Decrypt ")
try:
plaintext = self._media_cipher.decrypt(ciphertext, ref_key, media_info)
progress.update(length_kb)
return plaintext
except Exception as e:
progress.set_description("Decrypt Error ")
logger.error(e)
return None
def _write(self, data, filename):
length_kb = int(math.ceil(len(data) / 1024))
progress = self._create_progress_iterator(range(length_kb), length_kb, "Write ")
try:
with open(filename, 'wb') as f:
f.write(data)
progress.update(length_kb)
return filename
except Exception as e:
progress.set_description("Write error ")
logger.error(e)
return None
def _create_unique_filepath(self, filepath):
file_dir = os.path.dirname(filepath)
filename = os.path.basename(filepath)
result_filename = filename
dissected = os.path.splitext(filename)
count = 0
while os.path.exists(os.path.join(file_dir, result_filename)):
count += 1
result_filename = "%s_%d%s" % (dissected[0], count, dissected[1])
return os.path.join(file_dir, result_filename)
def run(self):
logger.debug(
"SinkWorker started, storage_dir=%s" % self._storage_dir
)
while True:
media_message_protocolentity = self._jobs.get()
if media_message_protocolentity is None:
sys.exit(0)
if isinstance(media_message_protocolentity, DownloadableMediaMessageProtocolEntity):
logger.info(
"Processing [url=%s, media_key=%s]" %
(media_message_protocolentity.url, base64.b64encode(media_message_protocolentity.media_key))
)
else:
logger.info("Processing %s" % media_message_protocolentity.media_type)
filedata = None
fileext = None
if isinstance(media_message_protocolentity, ImageDownloadableMediaMessageProtocolEntity):
media_info = MediaCipher.INFO_IMAGE
filename = "image"
elif isinstance(media_message_protocolentity, AudioDownloadableMediaMessageProtocolEntity):
media_info = MediaCipher.INFO_AUDIO
filename = "ptt" if media_message_protocolentity.ptt else "audio"
elif isinstance(media_message_protocolentity, VideoDownloadableMediaMessageProtocolEntity):
media_info = MediaCipher.INFO_VIDEO
filename = "video"
elif isinstance(media_message_protocolentity, DocumentDownloadableMediaMessageProtocolEntity):
media_info = MediaCipher.INFO_DOCUM
filename = media_message_protocolentity.file_name
elif isinstance(media_message_protocolentity, ContactMediaMessageProtocolEntity):
filename = media_message_protocolentity.display_name
filedata = media_message_protocolentity.vcard
fileext = "vcard"
elif isinstance(media_message_protocolentity, StickerDownloadableMediaMessageProtocolEntity):
media_info = MediaCipher.INFO_IMAGE
filename = "sticker"
else:
logger.error("Unsupported Media type: %s" % media_message_protocolentity.__class__)
sys.exit(1)
if filedata is None:
enc_data = self._download(media_message_protocolentity.url)
if enc_data is None:
logger.error("Download failed")
sys.exit(1)
filedata = self._decrypt(enc_data, media_message_protocolentity.media_key, media_info)
if filedata is None:
logger.error("Decrypt failed")
sys.exit(1)
if not isinstance(media_message_protocolentity, DocumentDownloadableMediaMessageProtocolEntity):
if fileext is None:
fileext = media_message_protocolentity.mimetype.split('/')[1].split(';')[0]
filename_full = "%s.%s" % (filename, fileext)
else:
filename_full = filename
filepath = self._create_unique_filepath(os.path.join(self._storage_dir, filename_full))
if self._write(filedata, filepath):
logger.info("Wrote %s" % filepath)
else:
sys.exit(1)
def setUp(self):
self._cipher = MediaCipher() # type: MediaCipher
def setUp(self):
self._cipher = MediaCipher() # type: MediaCipher