def on_savemessage(accountname, username, message_file, flags, label):
print 'ON_SAVEMESSAGE: account=%s message file=%s flags=%s' % (username, message_file, flags)
MODULE_NAME = 'mail_parser' # os.path.splitext(os.path.basename(__file__))[0]
DEFAULT_INPUTS = [ AssetClass.UPLOAD ]
DEFAULT_OUTPUTS = [ AssetClass.MESSAGE_PART ]
DEFAULT_ACCEPTED_MIME_TYPES = [ MimeType.MAIL ]
owner = User.objects.get(username=username)
processor = operations.initialize_processor(
MODULE_NAME,
DEFAULT_INPUTS,
DEFAULT_OUTPUTS,
DEFAULT_ACCEPTED_MIME_TYPES ) [0]
new_item = operations.create_asset_from_file(
file_name = message_file,
owner = owner,
producer = processor,
asset_class = AssetClass.UPLOAD,
child_number = 0,
mime_type = MimeType.MAIL )
new_item.orig_file_name += ',ACCOUNT=%s' % accountname
new_item.orig_file_name += ',LABEL=%s' % label
new_item.orig_file_name += ',FLAGS='
for f in flags:
new_item.orig_file_name += ',' + f
new_item.save()
operations.publish_work_item(new_item)
return
def on_savemessage(accountname, username, message_file, flags, label):
print 'ON_SAVEMESSAGE: account=%s message file=%s flags=%s' % (
username, message_file, flags)
MODULE_NAME = 'mail_parser' # os.path.splitext(os.path.basename(__file__))[0]
DEFAULT_INPUTS = [AssetClass.UPLOAD]
DEFAULT_OUTPUTS = [AssetClass.MESSAGE_PART]
DEFAULT_ACCEPTED_MIME_TYPES = [MimeType.MAIL]
owner = User.objects.get(username=username)
processor = operations.initialize_processor(MODULE_NAME, DEFAULT_INPUTS,
DEFAULT_OUTPUTS,
DEFAULT_ACCEPTED_MIME_TYPES)[0]
new_item = operations.create_asset_from_file(file_name=message_file,
owner=owner,
producer=processor,
asset_class=AssetClass.UPLOAD,
child_number=0,
mime_type=MimeType.MAIL)
new_item.orig_file_name += ',ACCOUNT=%s' % accountname
new_item.orig_file_name += ',LABEL=%s' % label
new_item.orig_file_name += ',FLAGS='
for f in flags:
new_item.orig_file_name += ',' + f
new_item.save()
operations.publish_work_item(new_item)
return
def upload_file(processor, user, local_path):
""" Transactionally upload a new work item """
operations.publish_work_item(
operations.create_asset_from_file(
file_name=local_path,
owner=user,
producer=processor,
child_number=0,
asset_class=models.AssetClass.UPLOAD))
def upload_file( processor, user, local_path ):
""" Transactionally upload a new work item """
operations.publish_work_item(
operations.create_asset_from_file(
file_name = local_path,
owner = user,
producer = processor,
child_number = 0,
asset_class = models.AssetClass.UPLOAD ))
def _enqueue_message(self, message_file, accountname, label, flags):
new_item = operations.create_asset_from_file(
file_name=message_file,
owner=self.user,
producer=self.processor,
asset_class=AssetClass.UPLOAD,
child_number=0,
mime_type=MimeType.MAIL)
new_item.orig_file_name += ',ACCOUNT=%s' % accountname
new_item.orig_file_name += ',LABEL=%s' % label
new_item.orig_file_name += ',FLAGS='
for f in flags:
new_item.orig_file_name += ',' + f
new_item.save()
operations.publish_work_item(new_item)
def test_create_asset_from_stream(self):
asset = operations.create_asset_from_stream(
owner = self.user,
producer = self.producer,
asset_class = 'test_data',
data_stream = StringIO(TEST_DATA),
file_name = 'create_asset_from_string.txt',
child_number = 0,
mime_type = 'text/plain' )
self.assert_( asset is not None )
operations.publish_work_item(asset)
self._validate_consumer(asset, TEST_DATA)
def _enqueue_message(self, message_file, accountname, label, flags):
new_item = operations.create_asset_from_file(
file_name = message_file,
owner = self.user,
producer = self.processor,
asset_class = AssetClass.UPLOAD,
child_number = 0,
mime_type = MimeType.MAIL )
new_item.orig_file_name += ',ACCOUNT=%s' % accountname
new_item.orig_file_name += ',LABEL=%s' % label
new_item.orig_file_name += ',FLAGS='
for f in flags:
new_item.orig_file_name += ',' + f
new_item.save()
operations.publish_work_item(new_item)
def main():
""" Program to import mail messages from files on disk """
import optparse
parser = optparse.OptionParser()
parser.add_option(
'-d',
'--delete',
action ='store_true',
default = False,
help = 'Delete each input file after successfully queing it' )
parser.add_option(
'-u',
'--user')
options, file_name_list = parser.parse_args()
logging.info(options)
owner = User.objects.get(username=options.user)
processor = operations.initialize_processor(
MODULE_NAME,
DEFAULT_INPUTS,
DEFAULT_OUTPUTS,
DEFAULT_ACCEPTED_MIME_TYPES ) [0]
# pylint: disable-msg=W0702
# -> no exception type given
for file_name in file_name_list:
try:
new_items = process_mail(owner, processor, file_name)
operations.publish_work_item(*new_items)
except:
logging.exception("Failed to process %s", file_name )
else:
if options.delete:
os.remove(file_name)
def test_create_asset_from_file(self):
fd, temp_file_name = tempfile.mkstemp()
try:
os.write(fd, TEST_DATA)
os.close(fd)
fd = None
asset = operations.create_asset_from_file(
owner = self.user,
producer = self.producer,
asset_class = 'test_data',
file_name = temp_file_name,
child_number = 0,
mime_type = 'text/plain' )
self.assert_( asset is not None )
finally:
if fd is not None:
os.close(fd)
os.remove(temp_file_name)
operations.publish_work_item(asset)
self._validate_consumer(asset, TEST_DATA)
processor = driver.init_processor(module)
try:
work_item.update(operations.instantiate_asset(asset_id))
except models.Asset.DoesNotExist:
logging.error('Asset no longer exists: %s' % asset_id)
except S3ResponseError, error:
if error.status == 404:
logging.error('Could not find asset in S3: %s' % asset_id)
else:
logging.exception('Unexpected error!')
raise
else:
new_items = handle_work_item(module, processor, work_item)
operations.publish_work_item( *new_items )
except NotReadyException, e:
logging.info(e)
sys.exit(1)
except:
logging.exception('Failed to run processor')
sys.exit(1)
finally:
local_path = work_item.get('Local-Path')
if local_path and os.path.exists(local_path):
shutil.rmtree(os.path.dirname(local_path))
if __name__ == '__main__':
