def f(bytstr):
rand_name = _random_alphanum_string(num_random_bits, chars=rename_from_set)
tries = 1
while rand_name in seen_rand_names:
rand_name = _random_alphanum_string(num_random_bits, chars=rename_from_set)
tries += 1
if tries > 20:
raise EZArchiveError('20 random filename selections collided: perhaps you need to increase num_rand_bits?')
seen_rand_names.append(rand_name)
return [rand_name, bytstr]
def _verify_filename(name, seen_filenames, filename_collision, num_random_bits,
rename_from_set):
# name could be a filename or directory.
if name in seen_filenames:
seen_filenames[name] += 1
times = seen_filenames[name]
(dirname, basename) = os.path.split(name)
if filename_collision == 'throw_error':
raise EZArchiveError('filename collision: %s' % (name))
elif filename_collision == 'rename_similar':
# Just in case archive contains a list of
# filenames that follow the same pattern as this
# incrementing, we need to check the increment
# doesn't collide as well:
incremented_basename = str(times) + '.' + basename
while os.path.join(dirname,
incremented_basename) in seen_filenames:
times += 1
incremented_basename = str(times) + '.' + basename
# Make a note of how many increments we've had to
# do:
seen_filenames[name] = times
name = os.path.join(dirname, incremented_basename)
elif filename_collision == 'rename_random':
# Just in case of random collision, we introduce the while loop.
randbasename = _random_alphanum_string(num_random_bits,
chars=rename_from_set)
tries = 1
while os.path.join(dirname, randbasename) in seen_filenames:
randbasename = _random_alphanum_string(num_random_bits,
chars=rename_from_set)
# If user gives small set rename_from_set and low number of bits,
# then it is possible we will exhaust all posibile combinations:
tries += 1
if tries > 20:
raise EZArchiveError(
'20 random filename selections collided: perhaps you need to increase num_rand_bits?'
)
seen_filenames[os.path.join(dirname, randbasename)] = 0
name = os.path.join(dirname, randbasename)
elif filename_collision == 'overwrite':
pass
elif filename_collision == 'skip':
return ['skip']
else:
seen_filenames[name] = 0
return name
def f(bytstr):
rand_name = _random_alphanum_string(num_random_bits,
chars=rename_from_set)
tries = 1
while rand_name in seen_rand_names:
rand_name = _random_alphanum_string(
num_random_bits, chars=rename_from_set)
tries += 1
if tries > 20:
raise EZArchiveError(
'20 random filename selections collided: perhaps you need to increase num_rand_bits?'
)
seen_rand_names.append(rand_name)
return [rand_name, bytstr]
def _verify_filename(name, seen_filenames, filename_collision, num_random_bits, rename_from_set):
# name could be a filename or directory.
if name in seen_filenames:
seen_filenames[name] += 1
times = seen_filenames[name]
(dirname, basename) = os.path.split(name)
if filename_collision == 'throw_error':
raise EZArchiveError('filename collision: %s' % (name))
elif filename_collision == 'rename_similar':
# Just in case archive contains a list of
# filenames that follow the same pattern as this
# incrementing, we need to check the increment
# doesn't collide as well:
incremented_basename = str(times) + '.' + basename
while os.path.join(dirname, incremented_basename) in seen_filenames:
times += 1
incremented_basename = str(times) + '.' + basename
# Make a note of how many increments we've had to
# do:
seen_filenames[name] = times
name = os.path.join(dirname, incremented_basename)
elif filename_collision == 'rename_random':
# Just in case of random collision, we introduce the while loop.
randbasename = _random_alphanum_string(num_random_bits, chars=rename_from_set)
tries = 1
while os.path.join(dirname, randbasename) in seen_filenames:
randbasename = _random_alphanum_string(num_random_bits, chars=rename_from_set)
# If user gives small set rename_from_set and low number of bits,
# then it is possible we will exhaust all posibile combinations:
tries += 1
if tries > 20:
raise EZArchiveError('20 random filename selections collided: perhaps you need to increase num_rand_bits?')
seen_filenames[os.path.join(dirname, randbasename)] = 0
name = os.path.join(dirname, randbasename)
elif filename_collision == 'overwrite':
pass
elif filename_collision == 'skip':
return ['skip']
else:
seen_filenames[name] = 0
return name
def generate_filename(filename=None,
file=None,
content_type=None,
no_rand_chars=8,
prefix='',
postfix=''):
name_stub = _random_alphanum_string(no_rand_chars)
name_ext = calculate_filename_extension(filename, file, content_type)
return prefix + name_stub + postfix + '.' + name_ext
def generate_filename(filename=None, file=None, content_type=None, no_rand_chars=8, prefix='', postfix=''):
name_stub = _random_alphanum_string(no_rand_chars)
name_ext = calculate_filename_extension(filename, file, content_type)
return prefix + name_stub + postfix + '.' + name_ext
def process_email(email_string):
""" main entry point of the module, handles whole processing of the email
"""
# See if we can parse the email:
try:
e = elmsubmit_EZEmail.ParseMessage(email_string)
except elmsubmit_EZEmail.EZEmailParseError as err:
try:
if err.basic_email_info['from'] is None:
raise ValueError
response = elmsubmit_EZEmail.CreateMessage(to=err.basic_email_info['from'],
_from=elmsubmit_config.CFG_ELMSUBMIT_PEOPLE['admin'],
message=elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['bad_email'],
subject="Re: " + (err.basic_email_info.get('Subject', '') or ''),
references=[err.basic_email_info.get('message-id', '') or ''],
wrap_message=False)
_send_smtp(_from=elmsubmit_config.CFG_ELMSUBMIT_PEOPLE['admin'], to=err.basic_email_info['from'], msg=response)
raise elmsubmitError("Email could not be parsed. Reported to sender.")
except ValueError:
raise elmsubmitError("From: field of submission email could not be parsed. Could not report to sender.")
# See if we can parse the submission fields in the email:
try:
# Note that this returns a dictionary loaded with utf8 byte strings:
submission_dict = elmsubmit_submission_parser.parse_submission(e.primary_message.encode('utf8'))
# Add the submitter's email:
submission_dict['SuE'] = e.from_email.encode('utf8')
except elmsubmit_submission_parser.SubmissionParserError:
_notify(msg=e, response=elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['bad_submission'])
raise elmsubmitSubmissionError("Could not parse submission.")
# Check we have been given the required fields:
available_fields = submission_dict.keys()
if not len(filter(lambda x: x in available_fields, elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS)) == len(elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS):
response = elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['missing_fields_1'] + elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['missing_fields_2'] + "\n\n" + repr(elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS)
_notify(msg=e, response=response)
raise elmsubmitSubmissionError("Submission does not contain the required fields %s." % (elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS))
# Check that the fields we have been given validate OK:
map(lambda field: validate_submission_field(e, submission_dict, field, submission_dict[field]), elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS)
# Get a submission directory:
folder_name = 'elmsubmit_' + _random_alphanum_string(15)
storage_dir = os.path.join(CFG_TMPDIR, folder_name)
try:
os.makedirs(storage_dir)
except EnvironmentError:
_notify(msg=e, response=elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['temp_problem'])
_notify_admin(response="Could not create directory: %s" % (storage_dir))
raise elmsubmitError("Could not create directory: %s" % (storage_dir))
# Process the files list:
process_files(e, submission_dict, storage_dir)
#generate the appropriate Marc_XML for the submission
marc_xml = elmsubmit_generate_marc.generate_marc(submission_dict)
# Write the Marc to a file in CFG_TMPDIR
file_name = folder_name + '.xml'
fullpath = os.path.join(CFG_TMPDIR, file_name)
try:
open(fullpath, 'wb').write(marc_xml)
except EnvironmentError:
response_email = elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['temp_problem']
admin_response_email = "There was a problem writing data to directory %s." % (storage_dir)
error = elmsubmitError("There was a problem writing data to directory %s." % (storage_dir))
return (response_email, admin_response_email, error)
# print marc_xml
return marc_xml
def process_files(msg, submission_dict, storage_dir):
""" extract the files out of the email and include them in the submission dict
"""
files = map(lambda filename: filename.decode('utf8'), submission_dict['files'])
# Check for the special filename 'all': if we find it, add all of
# the files attached to the email to the list of files to submit:
if 'all' in files:
f = lambda attachment: attachment['filename'] is not None
g = lambda attachment: attachment['filename'].lower()
attached_filenames = map(g, filter(f, msg.attachments))
files.extend(attached_filenames)
files = filter(lambda name: name != 'all', files)
# Filter out duplicate filenames:
_temp = {}
map(lambda filename: _temp.update({ filename : 1}), files)
files = _temp.keys()
# Get the files out of the mail message:
# file dictionary with file content needed for saving the file to proper directory
file_dict = {}
# file list needed to be included in submission_dict
file_list = []
for filename in files:
# See if we have special keyword self (which uses the mail message itself as the file):
if filename == 'self':
file_attachment = msg.original_message
filename = _random_alphanum_string(8) + '_' + msg.date_sent_utc.replace(' ', '_').replace(':', '-') + '.msg'
else:
nominal_attachments = filter(lambda attachment: attachment['filename'].lower() == filename, msg.attachments)
try:
file_attachment = nominal_attachments[0]['file']
except IndexError:
_notify(msg=msg, response=elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['missing_attachment'] + ' ' + filename)
raise elmsubmitSubmissionError("Submission is missing attached file: %s" % (filename))
file_dict[filename.encode('utf8')] = file_attachment
#merge the file name and the storage dir in the submission_dict
full_file_name = os.path.join(storage_dir, filename.encode('utf8'))
file_list.append(full_file_name)
submission_dict['files'] = file_list
def create_files((path, dictionary_or_data)):
"""
Take any dictionary, eg.:
{ 'title' : 'The loveliest title.',
'name' : 'Pete the dog.',
'info' : 'pdf file content'
}
and create a set of files in the given directory:
directory/title
directory/name
directory/info
so that each filename is a dictionary key, and the contents of
each file is the value that the key pointed to.
"""
fullpath = os.path.join(storage_dir, path)
try:
dictionary_or_data.has_key
except AttributeError:
open(fullpath, 'wb').write(dictionary_or_data)
try:
map(create_files, file_dict.items())
except EnvironmentError:
response_email = elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['temp_problem']
admin_response_email = "There was a problem writing data to directory %s." % (storage_dir)
error = elmsubmitError("There was a problem writing data to directory %s." % (storage_dir))
return (response_email, admin_response_email, error)
return None
def process_email(email_string):
""" main entry point of the module, handles whole processing of the email
"""
# See if we can parse the email:
try:
e = elmsubmit_EZEmail.ParseMessage(email_string)
except elmsubmit_EZEmail.EZEmailParseError as err:
try:
if err.basic_email_info['from'] is None:
raise ValueError
response = elmsubmit_EZEmail.CreateMessage(
to=err.basic_email_info['from'],
_from=elmsubmit_config.CFG_ELMSUBMIT_PEOPLE['admin'],
message=elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['bad_email'],
subject="Re: " +
(err.basic_email_info.get('Subject', '') or ''),
references=[err.basic_email_info.get('message-id', '') or ''],
wrap_message=False)
_send_smtp(_from=elmsubmit_config.CFG_ELMSUBMIT_PEOPLE['admin'],
to=err.basic_email_info['from'],
msg=response)
raise elmsubmitError(
"Email could not be parsed. Reported to sender.")
except ValueError:
raise elmsubmitError(
"From: field of submission email could not be parsed. Could not report to sender."
)
# See if we can parse the submission fields in the email:
try:
# Note that this returns a dictionary loaded with utf8 byte strings:
submission_dict = elmsubmit_submission_parser.parse_submission(
e.primary_message.encode('utf8'))
# Add the submitter's email:
submission_dict['SuE'] = e.from_email.encode('utf8')
except elmsubmit_submission_parser.SubmissionParserError:
_notify(msg=e,
response=elmsubmit_config.
CFG_ELMSUBMIT_NOLANGMSGS['bad_submission'])
raise elmsubmitSubmissionError("Could not parse submission.")
# Check we have been given the required fields:
available_fields = submission_dict.keys()
if not len(
filter(lambda x: x in available_fields,
elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS)) == len(
elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS):
response = elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS[
'missing_fields_1'] + elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS[
'missing_fields_2'] + "\n\n" + repr(
elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS)
_notify(msg=e, response=response)
raise elmsubmitSubmissionError(
"Submission does not contain the required fields %s." %
(elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS))
# Check that the fields we have been given validate OK:
map(
lambda field: validate_submission_field(e, submission_dict, field,
submission_dict[field]),
elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS)
# Get a submission directory:
folder_name = 'elmsubmit_' + _random_alphanum_string(15)
storage_dir = os.path.join(CFG_TMPDIR, folder_name)
try:
os.makedirs(storage_dir)
except EnvironmentError:
_notify(
msg=e,
response=elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['temp_problem'])
_notify_admin(response="Could not create directory: %s" %
(storage_dir))
raise elmsubmitError("Could not create directory: %s" % (storage_dir))
# Process the files list:
process_files(e, submission_dict, storage_dir)
#generate the appropriate Marc_XML for the submission
marc_xml = elmsubmit_generate_marc.generate_marc(submission_dict)
# Write the Marc to a file in CFG_TMPDIR
file_name = folder_name + '.xml'
fullpath = os.path.join(CFG_TMPDIR, file_name)
try:
open(fullpath, 'wb').write(marc_xml)
except EnvironmentError:
response_email = elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS[
'temp_problem']
admin_response_email = "There was a problem writing data to directory %s." % (
storage_dir)
error = elmsubmitError(
"There was a problem writing data to directory %s." %
(storage_dir))
return (response_email, admin_response_email, error)
# print marc_xml
return marc_xml
def process_files(msg, submission_dict, storage_dir):
""" extract the files out of the email and include them in the submission dict
"""
files = map(lambda filename: filename.decode('utf8'),
submission_dict['files'])
# Check for the special filename 'all': if we find it, add all of
# the files attached to the email to the list of files to submit:
if 'all' in files:
f = lambda attachment: attachment['filename'] is not None
g = lambda attachment: attachment['filename'].lower()
attached_filenames = map(g, filter(f, msg.attachments))
files.extend(attached_filenames)
files = filter(lambda name: name != 'all', files)
# Filter out duplicate filenames:
_temp = {}
map(lambda filename: _temp.update({filename: 1}), files)
files = _temp.keys()
# Get the files out of the mail message:
# file dictionary with file content needed for saving the file to proper directory
file_dict = {}
# file list needed to be included in submission_dict
file_list = []
for filename in files:
# See if we have special keyword self (which uses the mail message itself as the file):
if filename == 'self':
file_attachment = msg.original_message
filename = _random_alphanum_string(
8) + '_' + msg.date_sent_utc.replace(' ', '_').replace(
':', '-') + '.msg'
else:
nominal_attachments = filter(
lambda attachment: attachment['filename'].lower() == filename,
msg.attachments)
try:
file_attachment = nominal_attachments[0]['file']
except IndexError:
_notify(msg=msg,
response=elmsubmit_config.
CFG_ELMSUBMIT_NOLANGMSGS['missing_attachment'] + ' ' +
filename)
raise elmsubmitSubmissionError(
"Submission is missing attached file: %s" % (filename))
file_dict[filename.encode('utf8')] = file_attachment
#merge the file name and the storage dir in the submission_dict
full_file_name = os.path.join(storage_dir, filename.encode('utf8'))
file_list.append(full_file_name)
submission_dict['files'] = file_list
def create_files((path, dictionary_or_data)):
"""
Take any dictionary, eg.:
{ 'title' : 'The loveliest title.',
'name' : 'Pete the dog.',
'info' : 'pdf file content'
}
and create a set of files in the given directory:
directory/title
directory/name
directory/info
so that each filename is a dictionary key, and the contents of
each file is the value that the key pointed to.
"""
fullpath = os.path.join(storage_dir, path)
try:
dictionary_or_data.has_key
except AttributeError:
open(fullpath, 'wb').write(dictionary_or_data)
try:
map(create_files, file_dict.items())
except EnvironmentError:
response_email = elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS[
'temp_problem']
admin_response_email = "There was a problem writing data to directory %s." % (
storage_dir)
error = elmsubmitError(
"There was a problem writing data to directory %s." %
(storage_dir))
return (response_email, admin_response_email, error)
return None
