def delayed_delete_message(msg_id=None, process_id=None, retries=0): process = CleanUserProcess.get_by_id(process_id) criteria = process.search_criteria msg_process = CleanMessageProcess.query(ndb.AND( CleanMessageProcess.msg_id == msg_id, CleanMessageProcess.clean_process_id == process_id) ).get() if msg_process.status != constants.MIGRATED: if retries < constants.MAX_RETRIES: deferred.defer(delayed_delete_message, msg_id=msg_id, process_id=process_id, retries=retries+1, _countdown=60*2**retries, _queue="elimination") else: logging.error("Couldn't delete msg %s for user %s" % (msg_id, process.source_email)) return imap = IMAPHelper() imap.login(process.source_email, process.source_password) imap.select() # Look for the migrated email, if it doesn't exist yet # retry later try: subject = imap.get_subject(msg_id=msg_id) except Exception as e: if retries < constants.MAX_RETRIES: deferred.defer(delayed_delete_message, msg_id=msg_id, process_id=process_id, retries=retries+1, _countdown=60*2**retries) else: logging.error("Couldn't delete msg %s for user %s, error %s" % (msg_id, process.source_email, e.message)) return messages = imap.list_messages(criteria="subject:(%s) label:Migrated-Migrados" % subject) if len(messages) < 1: if retries < constants.MAX_RETRIES: deferred.defer(delayed_delete_message, msg_id=msg_id, process_id=process_id, retries=retries+1, _countdown=60*2**retries, _queue="elimination") else: logging.error("Couldn't delete msg %s for user %s" % (msg_id, process.source_email)) return imap.delete_message(msg_id=msg_id, criteria=criteria) imap.close() msg_process.status = constants.FINISHED msg_process.put() all_done = True all_cleaning_messages = CleanMessageProcess.query( CleanMessageProcess.clean_process_id == process_id ).fetch() progress = 0 for message in all_cleaning_messages: if not message.status == constants.FINISHED: all_done = False else: progress += 1 if all_done: process.status = constants.FINISHED utc_now = datetime.datetime.utcnow() local_tz = pytz.timezone('America/Bogota') tz_offset = local_tz.utcoffset(utc_now) now = utc_now + tz_offset process.progress = progress process.latest_activity = "%s" % now process.put()
def clean_message(msg_id='', imap=None, drive=None, migration=None, folder_id=None, user_email=None, process_id=None): logging.info("Trying to clean message %s for user %s" % (msg_id, user_email)) process = CleanUserProcess.get_by_id(process_id) criteria = process.search_criteria msg_process = CleanMessageProcess.get_or_create(msg_id, process_id) if msg_process.status == constants.FINISHED: return True result, message = imap.get_message(msg_id=msg_id) if result != 'OK': raise Exception("Couldn't read message") result, label_data = imap.get_message_labels(msg_id=msg_id) labels = [] if label_data and label_data[0]: labels = (((label_data[0].split('('))[2].split(')'))[0]).split() mail = email.message_from_string(message[0][1]) attachments = [] number_of_attachments = 0 if mail.get_content_maintype() == 'multipart': for part in mail.walk(): if part.get_content_maintype() == 'multipart': continue if part.get('Content-Disposition') is None: continue # Is attachment attached = False number_of_attachments += 1 attachment_process = CleanAttachmentProcess.get_or_create( msg_id, msg_process.key.id(), number_of_attachments ) file_id = '' if (attachment_process.status == constants.FINISHED and attachment_process.url and attachment_process.filename and attachment_process.file_id ): attached = True attachments.append( (attachment_process.url, attachment_process.filename)) file_id = attachment_process.file_id if not attached: attachment = part.get_payload(decode=True) mime_type = part.get_content_type() filename = part.get_filename() text, encoding = email.Header.decode_header(filename)[0] if encoding: filename = text.decode(encoding) insert_result = drive.insert_file(filename=filename, mime_type=mime_type, content=attachment, parent_id=folder_id) if not insert_result: attachment_process.error_description = ( "Error inserting file" ) attachment_process.put() raise Exception("Insert file error") drive_url = insert_result['webContentLink'] file_id = insert_result['id'] attachment_process.url = drive_url attachment_process.file_id = file_id attachment_process.status = constants.FINISHED attachment_process.filename = filename attachment_process.put() attachments.append((drive_url, filename)) permission_result = drive.insert_permission(file_id=file_id, value=user_email, type='user', role='writer') if not permission_result: attachment_process.error_description = ( "Permission error" ) attachment_process.put() raise Exception("Permission error") part.set_payload("") for header in part.keys(): part.__delitem__(header) msg_process.status = constants.DUPLICATED msg_process.put() for url, filename in attachments: body_suffix = '<a href="%s">%s</a>' % (url, filename) new_payload = email.MIMEText.MIMEText(body_suffix.encode('utf-8'), 'html', 'utf-8') mail.attach(new_payload) # Send new mail time.sleep(1.2) migration_result = migration.migrate_mail(user_email=user_email, msg=mail, labels=labels) if not migration_result: msg_process.error_description = "Migration error" msg_process.put() raise Exception("Migration error") else: msg_process.status = constants.MIGRATED msg_process.put() # Then delete previous email logging.info("Delaying delete of msg %s for user %s" % (msg_id, user_email)) deferred.defer(delayed_delete_message, msg_id=msg_id, process_id=process_id, _countdown=30, _queue="elimination") return True