def f(bytstr): rand_name = _random_alphanum_string(num_random_bits, chars=rename_from_set) tries = 1 while rand_name in seen_rand_names: rand_name = _random_alphanum_string(num_random_bits, chars=rename_from_set) tries += 1 if tries > 20: raise EZArchiveError('20 random filename selections collided: perhaps you need to increase num_rand_bits?') seen_rand_names.append(rand_name) return [rand_name, bytstr]
def _verify_filename(name, seen_filenames, filename_collision, num_random_bits, rename_from_set): # name could be a filename or directory. if seen_filenames.has_key(name): seen_filenames[name] += 1 times = seen_filenames[name] (dirname, basename) = os.path.split(name) if filename_collision == "throw_error": raise EZArchiveError("filename collision: %s" % (name)) elif filename_collision == "rename_similar": # Just in case archive contains a list of # filenames that follow the same pattern as this # incrementing, we need to check the increment # doesn't collide as well: incremented_basename = str(times) + "." + basename while seen_filenames.has_key(os.path.join(dirname, incremented_basename)): times += 1 incremented_basename = str(times) + "." + basename # Make a note of how many increments we've had to # do: seen_filenames[name] = times name = os.path.join(dirname, incremented_basename) elif filename_collision == "rename_random": # Just in case of random collision, we introduce the while loop. randbasename = _random_alphanum_string(num_random_bits, chars=rename_from_set) tries = 1 while seen_filenames.has_key(os.path.join(dirname, randbasename)): randbasename = _random_alphanum_string(num_random_bits, chars=rename_from_set) # If user gives small set rename_from_set and low number of bits, # then it is possible we will exhaust all posibile combinations: tries += 1 if tries > 20: raise EZArchiveError( "20 random filename selections collided: perhaps you need to increase num_rand_bits?" ) seen_filenames[os.path.join(dirname, randbasename)] = 0 name = os.path.join(dirname, randbasename) elif filename_collision == "overwrite": pass elif filename_collision == "skip": return ["skip"] else: seen_filenames[name] = 0 return name
def _verify_filename(name, seen_filenames, filename_collision, num_random_bits, rename_from_set): # name could be a filename or directory. if seen_filenames.has_key(name): seen_filenames[name] += 1 times = seen_filenames[name] (dirname, basename) = os.path.split(name) if filename_collision == 'throw_error': raise EZArchiveError('filename collision: %s' % (name)) elif filename_collision == 'rename_similar': # Just in case archive contains a list of # filenames that follow the same pattern as this # incrementing, we need to check the increment # doesn't collide as well: incremented_basename = str(times) + '.' + basename while seen_filenames.has_key(os.path.join(dirname, incremented_basename)): times += 1 incremented_basename = str(times) + '.' + basename # Make a note of how many increments we've had to # do: seen_filenames[name] = times name = os.path.join(dirname, incremented_basename) elif filename_collision == 'rename_random': # Just in case of random collision, we introduce the while loop. randbasename = _random_alphanum_string(num_random_bits, chars=rename_from_set) tries = 1 while seen_filenames.has_key(os.path.join(dirname, randbasename)): randbasename = _random_alphanum_string(num_random_bits, chars=rename_from_set) # If user gives small set rename_from_set and low number of bits, # then it is possible we will exhaust all posibile combinations: tries += 1 if tries > 20: raise EZArchiveError('20 random filename selections collided: perhaps you need to increase num_rand_bits?') seen_filenames[os.path.join(dirname, randbasename)] = 0 name = os.path.join(dirname, randbasename) elif filename_collision == 'overwrite': pass elif filename_collision == 'skip': return ['skip'] else: seen_filenames[name] = 0 return name
def generate_filename(filename=None, file=None, content_type=None, no_rand_chars=8, prefix='', postfix=''): name_stub = _random_alphanum_string(no_rand_chars) name_ext = calculate_filename_extension(filename, file, content_type) return prefix + name_stub + postfix + '.' + name_ext
def process_files(msg, submission_dict, storage_dir): """ extract the files out of the email and include them in the submission dict """ files = map(lambda filename: filename.decode('utf8'), submission_dict['files']) # Check for the special filename 'all': if we find it, add all of # the files attached to the email to the list of files to submit: if 'all' in files: f = lambda attachment: attachment['filename'] is not None g = lambda attachment: attachment['filename'].lower() attached_filenames = map(g, filter(f, msg.attachments)) files.extend(attached_filenames) files = filter(lambda name: name != 'all', files) # Filter out duplicate filenames: _temp = {} map(lambda filename: _temp.update({ filename : 1}), files) files = _temp.keys() # Get the files out of the mail message: # file dictionary with file content needed for saving the file to proper directory file_dict = {} # file list needed to be included in submission_dict file_list = [] for filename in files: # See if we have special keyword self (which uses the mail message itself as the file): if filename == 'self': file_attachment = msg.original_message filename = _random_alphanum_string(8) + '_' + msg.date_sent_utc.replace(' ', '_').replace(':', '-') + '.msg' else: nominal_attachments = filter(lambda attachment: attachment['filename'].lower() == filename, msg.attachments) try: file_attachment = nominal_attachments[0]['file'] except IndexError: _notify(msg=msg, response=elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['missing_attachment'] + ' ' + filename) raise elmsubmitSubmissionError("Submission is missing attached file: %s" % (filename)) file_dict[filename.encode('utf8')] = file_attachment #merge the file name and the storage dir in the submission_dict full_file_name = os.path.join(storage_dir, filename.encode('utf8')) file_list.append(full_file_name) submission_dict['files'] = file_list def create_files((path, dictionary_or_data)): """ Take any dictionary, eg.: { 'title' : 'The loveliest title.', 'name' : 'Pete the dog.', 'info' : 'pdf file content' } and create a set of files in the given directory: directory/title directory/name directory/info so that each filename is a dictionary key, and the contents of each file is the value that the key pointed to. """ fullpath = os.path.join(storage_dir, path) try: dictionary_or_data.has_key except AttributeError: open(fullpath, 'wb').write(dictionary_or_data) try: map(create_files, file_dict.items()) except EnvironmentError: response_email = elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['temp_problem'] admin_response_email = "There was a problem writing data to directory %s." % (storage_dir) error = elmsubmitError("There was a problem writing data to directory %s." % (storage_dir)) return (response_email, admin_response_email, error) return None
# Check we have been given the required fields: available_fields = submission_dict.keys() if not len(filter(lambda x: x in available_fields, elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS)) == len(elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS): response = elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['missing_fields_1'] + elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['missing_fields_2'] + "\n\n" + repr(elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS) _notify(msg=e, response=response) raise elmsubmitSubmissionError("Submission does not contain the required fields %s." % (elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS)) # Check that the fields we have been given validate OK: map(lambda field: validate_submission_field(e, submission_dict, field, submission_dict[field]), elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS) # Get a submission directory: folder_name = 'elmsubmit_' + _random_alphanum_string(15) storage_dir = os.path.join(CFG_TMPDIR, folder_name) try: os.makedirs(storage_dir) except EnvironmentError: _notify(msg=e, response=elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['temp_problem']) _notify_admin(response="Could not create directory: %s" % (storage_dir)) raise elmsubmitError("Could not create directory: %s" % (storage_dir)) # Process the files list: process_files(e, submission_dict, storage_dir) #generate the appropriate Marc_XML for the submission
def process_files(msg, submission_dict, storage_dir): """ extract the files out of the email and include them in the submission dict """ files = map(lambda filename: filename.decode('utf8'), submission_dict['files']) # Check for the special filename 'all': if we find it, add all of # the files attached to the email to the list of files to submit: if 'all' in files: f = lambda attachment: attachment['filename'] is not None g = lambda attachment: attachment['filename'].lower() attached_filenames = map(g, filter(f, msg.attachments)) files.extend(attached_filenames) files = filter(lambda name: name != 'all', files) # Filter out duplicate filenames: _temp = {} map(lambda filename: _temp.update({filename: 1}), files) files = _temp.keys() # Get the files out of the mail message: # file dictionary with file content needed for saving the file to proper directory file_dict = {} # file list needed to be included in submission_dict file_list = [] for filename in files: # See if we have special keyword self (which uses the mail message itself as the file): if filename == 'self': file_attachment = msg.original_message filename = _random_alphanum_string( 8) + '_' + msg.date_sent_utc.replace(' ', '_').replace( ':', '-') + '.msg' else: nominal_attachments = filter( lambda attachment: attachment['filename'].lower() == filename, msg.attachments) try: file_attachment = nominal_attachments[0]['file'] except IndexError: _notify(msg=msg, response=elmsubmit_config. CFG_ELMSUBMIT_NOLANGMSGS['missing_attachment'] + ' ' + filename) raise elmsubmitSubmissionError( "Submission is missing attached file: %s" % (filename)) file_dict[filename.encode('utf8')] = file_attachment #merge the file name and the storage dir in the submission_dict full_file_name = os.path.join(storage_dir, filename.encode('utf8')) file_list.append(full_file_name) submission_dict['files'] = file_list def create_files((path, dictionary_or_data)): """ Take any dictionary, eg.: { 'title' : 'The loveliest title.', 'name' : 'Pete the dog.', 'info' : 'pdf file content' } and create a set of files in the given directory: directory/title directory/name directory/info so that each filename is a dictionary key, and the contents of each file is the value that the key pointed to. """ fullpath = os.path.join(storage_dir, path) try: dictionary_or_data.has_key except AttributeError: open(fullpath, 'wb').write(dictionary_or_data) try: map(create_files, file_dict.items()) except EnvironmentError: response_email = elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS[ 'temp_problem'] admin_response_email = "There was a problem writing data to directory %s." % ( storage_dir) error = elmsubmitError( "There was a problem writing data to directory %s." % (storage_dir)) return (response_email, admin_response_email, error) return None
elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS) _notify(msg=e, response=response) raise elmsubmitSubmissionError( "Submission does not contain the required fields %s." % (elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS)) # Check that the fields we have been given validate OK: map( lambda field: validate_submission_field(e, submission_dict, field, submission_dict[field]), elmsubmit_config.CFG_ELMSUBMIT_REQUIRED_FIELDS) # Get a submission directory: folder_name = 'elmsubmit_' + _random_alphanum_string(15) storage_dir = os.path.join(CFG_TMPDIR, folder_name) try: os.makedirs(storage_dir) except EnvironmentError: _notify( msg=e, response=elmsubmit_config.CFG_ELMSUBMIT_NOLANGMSGS['temp_problem']) _notify_admin(response="Could not create directory: %s" % (storage_dir)) raise elmsubmitError("Could not create directory: %s" % (storage_dir)) # Process the files list: