def f(bytstr): rand_name = _random_alphanum_string(num_random_bits, chars=rename_from_set) tries = 1 while rand_name in seen_rand_names: rand_name = _random_alphanum_string(num_random_bits, chars=rename_from_set) tries += 1 if tries > 20: raise EZArchiveError('20 random filename selections collided: perhaps you need to increase num_rand_bits?') seen_rand_names.append(rand_name) return [rand_name, bytstr]
def _verify_filename(name, seen_filenames, filename_collision, num_random_bits, rename_from_set): # name could be a filename or directory. if name in seen_filenames: seen_filenames[name] += 1 times = seen_filenames[name] (dirname, basename) = os.path.split(name) if filename_collision == 'throw_error': raise EZArchiveError('filename collision: %s' % (name)) elif filename_collision == 'rename_similar': # Just in case archive contains a list of # filenames that follow the same pattern as this # incrementing, we need to check the increment # doesn't collide as well: incremented_basename = str(times) + '.' + basename while os.path.join(dirname, incremented_basename) in seen_filenames: times += 1 incremented_basename = str(times) + '.' + basename # Make a note of how many increments we've had to # do: seen_filenames[name] = times name = os.path.join(dirname, incremented_basename) elif filename_collision == 'rename_random': # Just in case of random collision, we introduce the while loop. randbasename = _random_alphanum_string(num_random_bits, chars=rename_from_set) tries = 1 while os.path.join(dirname, randbasename) in seen_filenames: randbasename = _random_alphanum_string(num_random_bits, chars=rename_from_set) # If user gives small set rename_from_set and low number of bits, # then it is possible we will exhaust all posibile combinations: tries += 1 if tries > 20: raise EZArchiveError( '20 random filename selections collided: perhaps you need to increase num_rand_bits?' ) seen_filenames[os.path.join(dirname, randbasename)] = 0 name = os.path.join(dirname, randbasename) elif filename_collision == 'overwrite': pass elif filename_collision == 'skip': return ['skip'] else: seen_filenames[name] = 0 return name
def f(bytstr): rand_name = _random_alphanum_string(num_random_bits, chars=rename_from_set) tries = 1 while rand_name in seen_rand_names: rand_name = _random_alphanum_string( num_random_bits, chars=rename_from_set) tries += 1 if tries > 20: raise EZArchiveError( '20 random filename selections collided: perhaps you need to increase num_rand_bits?' ) seen_rand_names.append(rand_name) return [rand_name, bytstr]
def _verify_filename(name, seen_filenames, filename_collision, num_random_bits, rename_from_set): # name could be a filename or directory. if name in seen_filenames: seen_filenames[name] += 1 times = seen_filenames[name] (dirname, basename) = os.path.split(name) if filename_collision == 'throw_error': raise EZArchiveError('filename collision: %s' % (name)) elif filename_collision == 'rename_similar': # Just in case archive contains a list of # filenames that follow the same pattern as this # incrementing, we need to check the increment # doesn't collide as well: incremented_basename = str(times) + '.' + basename while os.path.join(dirname, incremented_basename) in seen_filenames: times += 1 incremented_basename = str(times) + '.' + basename # Make a note of how many increments we've had to # do: seen_filenames[name] = times name = os.path.join(dirname, incremented_basename) elif filename_collision == 'rename_random': # Just in case of random collision, we introduce the while loop. randbasename = _random_alphanum_string(num_random_bits, chars=rename_from_set) tries = 1 while os.path.join(dirname, randbasename) in seen_filenames: randbasename = _random_alphanum_string(num_random_bits, chars=rename_from_set) # If user gives small set rename_from_set and low number of bits, # then it is possible we will exhaust all posibile combinations: tries += 1 if tries > 20: raise EZArchiveError('20 random filename selections collided: perhaps you need to increase num_rand_bits?') seen_filenames[os.path.join(dirname, randbasename)] = 0 name = os.path.join(dirname, randbasename) elif filename_collision == 'overwrite': pass elif filename_collision == 'skip': return ['skip'] else: seen_filenames[name] = 0 return name
def generate_filename(filename=None, file=None, content_type=None, no_rand_chars=8, prefix='', postfix=''): name_stub = _random_alphanum_string(no_rand_chars) name_ext = calculate_filename_extension(filename, file, content_type) return prefix + name_stub + postfix + '.' + name_ext
def process_email(email_string): """ main entry point of the module, handles whole processing of the email """ # See if we can parse the email: try: e = elmsubmit_EZEmail.ParseMessage(email_string) except elmsubmit_EZEmail.EZEmailParseError as err: try: if err.basic_email_info['from'] is None: raise ValueError response = elmsubmit_EZEmail.CreateMessage(to=err.basic_email_info['from'], _from=elmsubmit_config.CFG_ELMSUBMIT_PEOPLE['admin'], message=elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['bad_email'], subject="Re: " + (err.basic_email_info.get('Subject', '') or ''), references=[err.basic_email_info.get('message-id', '') or ''], wrap_message=False) _send_smtp(_from=elmsubmit_config.CFG_ELMSUBMIT_PEOPLE['admin'], to=err.basic_email_info['from'], msg=response) raise elmsubmitError("Email could not be parsed. Reported to sender.") except ValueError: raise elmsubmitError("From: field of submission email could not be parsed. Could not report to sender.") # See if we can parse the submission fields in the email: try: # Note that this returns a dictionary loaded with utf8 byte strings: submission_dict = elmsubmit_submission_parser.parse_submission(e.primary_message.encode('utf8')) # Add the submitter's email: submission_dict['SuE'] = e.from_email.encode('utf8') except elmsubmit_submission_parser.SubmissionParserError: _notify(msg=e, response=elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['bad_submission']) raise elmsubmitSubmissionError("Could not parse submission.") # Check we have been given the required fields: available_fields = submission_dict.keys() if not len(filter(lambda x: x in available_fields, elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS)) == len(elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS): response = elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['missing_fields_1'] + elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['missing_fields_2'] + "\n\n" + repr(elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS) _notify(msg=e, response=response) raise elmsubmitSubmissionError("Submission does not contain the required fields %s." % (elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS)) # Check that the fields we have been given validate OK: map(lambda field: validate_submission_field(e, submission_dict, field, submission_dict[field]), elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS) # Get a submission directory: folder_name = 'elmsubmit_' + _random_alphanum_string(15) storage_dir = os.path.join(CFG_TMPDIR, folder_name) try: os.makedirs(storage_dir) except EnvironmentError: _notify(msg=e, response=elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['temp_problem']) _notify_admin(response="Could not create directory: %s" % (storage_dir)) raise elmsubmitError("Could not create directory: %s" % (storage_dir)) # Process the files list: process_files(e, submission_dict, storage_dir) #generate the appropriate Marc_XML for the submission marc_xml = elmsubmit_generate_marc.generate_marc(submission_dict) # Write the Marc to a file in CFG_TMPDIR file_name = folder_name + '.xml' fullpath = os.path.join(CFG_TMPDIR, file_name) try: open(fullpath, 'wb').write(marc_xml) except EnvironmentError: response_email = elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['temp_problem'] admin_response_email = "There was a problem writing data to directory %s." % (storage_dir) error = elmsubmitError("There was a problem writing data to directory %s." % (storage_dir)) return (response_email, admin_response_email, error) # print marc_xml return marc_xml
def process_files(msg, submission_dict, storage_dir): """ extract the files out of the email and include them in the submission dict """ files = map(lambda filename: filename.decode('utf8'), submission_dict['files']) # Check for the special filename 'all': if we find it, add all of # the files attached to the email to the list of files to submit: if 'all' in files: f = lambda attachment: attachment['filename'] is not None g = lambda attachment: attachment['filename'].lower() attached_filenames = map(g, filter(f, msg.attachments)) files.extend(attached_filenames) files = filter(lambda name: name != 'all', files) # Filter out duplicate filenames: _temp = {} map(lambda filename: _temp.update({ filename : 1}), files) files = _temp.keys() # Get the files out of the mail message: # file dictionary with file content needed for saving the file to proper directory file_dict = {} # file list needed to be included in submission_dict file_list = [] for filename in files: # See if we have special keyword self (which uses the mail message itself as the file): if filename == 'self': file_attachment = msg.original_message filename = _random_alphanum_string(8) + '_' + msg.date_sent_utc.replace(' ', '_').replace(':', '-') + '.msg' else: nominal_attachments = filter(lambda attachment: attachment['filename'].lower() == filename, msg.attachments) try: file_attachment = nominal_attachments[0]['file'] except IndexError: _notify(msg=msg, response=elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['missing_attachment'] + ' ' + filename) raise elmsubmitSubmissionError("Submission is missing attached file: %s" % (filename)) file_dict[filename.encode('utf8')] = file_attachment #merge the file name and the storage dir in the submission_dict full_file_name = os.path.join(storage_dir, filename.encode('utf8')) file_list.append(full_file_name) submission_dict['files'] = file_list def create_files((path, dictionary_or_data)): """ Take any dictionary, eg.: { 'title' : 'The loveliest title.', 'name' : 'Pete the dog.', 'info' : 'pdf file content' } and create a set of files in the given directory: directory/title directory/name directory/info so that each filename is a dictionary key, and the contents of each file is the value that the key pointed to. """ fullpath = os.path.join(storage_dir, path) try: dictionary_or_data.has_key except AttributeError: open(fullpath, 'wb').write(dictionary_or_data) try: map(create_files, file_dict.items()) except EnvironmentError: response_email = elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['temp_problem'] admin_response_email = "There was a problem writing data to directory %s." % (storage_dir) error = elmsubmitError("There was a problem writing data to directory %s." % (storage_dir)) return (response_email, admin_response_email, error) return None
def process_email(email_string): """ main entry point of the module, handles whole processing of the email """ # See if we can parse the email: try: e = elmsubmit_EZEmail.ParseMessage(email_string) except elmsubmit_EZEmail.EZEmailParseError as err: try: if err.basic_email_info['from'] is None: raise ValueError response = elmsubmit_EZEmail.CreateMessage( to=err.basic_email_info['from'], _from=elmsubmit_config.CFG_ELMSUBMIT_PEOPLE['admin'], message=elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['bad_email'], subject="Re: " + (err.basic_email_info.get('Subject', '') or ''), references=[err.basic_email_info.get('message-id', '') or ''], wrap_message=False) _send_smtp(_from=elmsubmit_config.CFG_ELMSUBMIT_PEOPLE['admin'], to=err.basic_email_info['from'], msg=response) raise elmsubmitError( "Email could not be parsed. Reported to sender.") except ValueError: raise elmsubmitError( "From: field of submission email could not be parsed. Could not report to sender." ) # See if we can parse the submission fields in the email: try: # Note that this returns a dictionary loaded with utf8 byte strings: submission_dict = elmsubmit_submission_parser.parse_submission( e.primary_message.encode('utf8')) # Add the submitter's email: submission_dict['SuE'] = e.from_email.encode('utf8') except elmsubmit_submission_parser.SubmissionParserError: _notify(msg=e, response=elmsubmit_config. CFG_ELMSUBMIT_NOLANGMSGS['bad_submission']) raise elmsubmitSubmissionError("Could not parse submission.") # Check we have been given the required fields: available_fields = submission_dict.keys() if not len( filter(lambda x: x in available_fields, elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS)) == len( elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS): response = elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS[ 'missing_fields_1'] + elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS[ 'missing_fields_2'] + "\n\n" + repr( elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS) _notify(msg=e, response=response) raise elmsubmitSubmissionError( "Submission does not contain the required fields %s." % (elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS)) # Check that the fields we have been given validate OK: map( lambda field: validate_submission_field(e, submission_dict, field, submission_dict[field]), elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS) # Get a submission directory: folder_name = 'elmsubmit_' + _random_alphanum_string(15) storage_dir = os.path.join(CFG_TMPDIR, folder_name) try: os.makedirs(storage_dir) except EnvironmentError: _notify( msg=e, response=elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['temp_problem']) _notify_admin(response="Could not create directory: %s" % (storage_dir)) raise elmsubmitError("Could not create directory: %s" % (storage_dir)) # Process the files list: process_files(e, submission_dict, storage_dir) #generate the appropriate Marc_XML for the submission marc_xml = elmsubmit_generate_marc.generate_marc(submission_dict) # Write the Marc to a file in CFG_TMPDIR file_name = folder_name + '.xml' fullpath = os.path.join(CFG_TMPDIR, file_name) try: open(fullpath, 'wb').write(marc_xml) except EnvironmentError: response_email = elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS[ 'temp_problem'] admin_response_email = "There was a problem writing data to directory %s." % ( storage_dir) error = elmsubmitError( "There was a problem writing data to directory %s." % (storage_dir)) return (response_email, admin_response_email, error) # print marc_xml return marc_xml
def process_files(msg, submission_dict, storage_dir): """ extract the files out of the email and include them in the submission dict """ files = map(lambda filename: filename.decode('utf8'), submission_dict['files']) # Check for the special filename 'all': if we find it, add all of # the files attached to the email to the list of files to submit: if 'all' in files: f = lambda attachment: attachment['filename'] is not None g = lambda attachment: attachment['filename'].lower() attached_filenames = map(g, filter(f, msg.attachments)) files.extend(attached_filenames) files = filter(lambda name: name != 'all', files) # Filter out duplicate filenames: _temp = {} map(lambda filename: _temp.update({filename: 1}), files) files = _temp.keys() # Get the files out of the mail message: # file dictionary with file content needed for saving the file to proper directory file_dict = {} # file list needed to be included in submission_dict file_list = [] for filename in files: # See if we have special keyword self (which uses the mail message itself as the file): if filename == 'self': file_attachment = msg.original_message filename = _random_alphanum_string( 8) + '_' + msg.date_sent_utc.replace(' ', '_').replace( ':', '-') + '.msg' else: nominal_attachments = filter( lambda attachment: attachment['filename'].lower() == filename, msg.attachments) try: file_attachment = nominal_attachments[0]['file'] except IndexError: _notify(msg=msg, response=elmsubmit_config. CFG_ELMSUBMIT_NOLANGMSGS['missing_attachment'] + ' ' + filename) raise elmsubmitSubmissionError( "Submission is missing attached file: %s" % (filename)) file_dict[filename.encode('utf8')] = file_attachment #merge the file name and the storage dir in the submission_dict full_file_name = os.path.join(storage_dir, filename.encode('utf8')) file_list.append(full_file_name) submission_dict['files'] = file_list def create_files((path, dictionary_or_data)): """ Take any dictionary, eg.: { 'title' : 'The loveliest title.', 'name' : 'Pete the dog.', 'info' : 'pdf file content' } and create a set of files in the given directory: directory/title directory/name directory/info so that each filename is a dictionary key, and the contents of each file is the value that the key pointed to. """ fullpath = os.path.join(storage_dir, path) try: dictionary_or_data.has_key except AttributeError: open(fullpath, 'wb').write(dictionary_or_data) try: map(create_files, file_dict.items()) except EnvironmentError: response_email = elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS[ 'temp_problem'] admin_response_email = "There was a problem writing data to directory %s." % ( storage_dir) error = elmsubmitError( "There was a problem writing data to directory %s." % (storage_dir)) return (response_email, admin_response_email, error) return None