def setUp(self): self.sample_file = 'test_file.ogg' self.sample_algo = 'sha512' self.sample_size = os.path.getsize(self.sample_file) with open(self.sample_file) as f: self.sample_hash = tooltool.digest_file(f, self.sample_algo) self.test_record = tooltool.FileRecord(filename=self.sample_file, size=self.sample_size, digest=self.sample_hash, algorithm=self.sample_algo) # using mkstemp to ensure that the filename generated # isn't actually on the system. (tmpfd, filename) = tempfile.mkstemp() os.close(tmpfd) os.remove(filename) if os.path.exists(filename): self.fail('did not remove %s' % filename) self.absent_file = filename
def setUp(self): self.sample_file = 'test_file.ogg' self.sample_algo = 'sha512' self.sample_size = os.path.getsize(self.sample_file) with open(self.sample_file) as f: self.sample_hash = tooltool.digest_file(f, self.sample_algo) self.test_record = tooltool.FileRecord( filename=self.sample_file, size=self.sample_size, digest=self.sample_hash, algorithm=self.sample_algo ) # using mkstemp to ensure that the filename generated # isn't actually on the system. (tmpfd, filename) = tempfile.mkstemp() os.close(tmpfd) os.remove(filename) if os.path.exists(filename): self.fail('did not remove %s' % filename) self.absent_file = filename
def test_digest_file(self): test_digest = tooltool.digest_file(self.sample_data, self.sample_algo) # If this assertion fails, verify that test_file.ogg is an ogg file # of Linus Torvalds explaining how he pronounces 'Linux' self.assertEqual(test_digest, self.sample_digest)
def main(): root, matching, smtp_server, smtp_port, smtp_from, email_addresses, default_domain = load_config() notifier = Notifier(smtp_server, smtp_port, smtp_from, email_addresses, default_domain) users = [] for (_dirpath, dirnames, _files) in os.walk(root): users.extend(dirnames) break # not to navigate subfolders for user in users: for distribution_type in matching: files = [] upload_folder = get_upload_folder(root, user, distribution_type) for (dirpath, _dirnames, _files) in os.walk(upload_folder): files.extend(_files) break # not to navigate subfolders # "new" means that it has to be processed new_manifests = [filename for filename in files if (filename.endswith(".tt") and not begins_with_timestamp(filename))] for new_manifest in new_manifests: new_manifest_path = os.path.join(upload_folder, new_manifest) destination = matching[distribution_type] timestamp = datetime.datetime.now().strftime(STRFRTIME) timestamped_manifest_name = "%s.%s" % (timestamp, new_manifest) comment_filename = new_manifest.replace(".tt", ".txt") comment_filepath = os.path.join(upload_folder, new_manifest.replace(".tt", ".txt")) manifestOK = True allFilesAreOK = True content_folder_path = new_manifest_path.replace(".tt", tooltool.TOOLTOOL_PACKAGE_SUFFIX) digests = () try: digests = getDigests(new_manifest_path) except tooltool.InvalidManifest: manifestOK = False if manifestOK: # checking that ALL files mentioned in the manifest are in the upload folder, otherwise I cannot proceed copying if not os.path.exists(content_folder_path) or not os.path.isdir(content_folder_path): allFilesAreOK = False log.error("Impossible to process manifest %s because content has not been uploaded" % content_folder_path) else: for digest, algorithm in digests: digest_path = os.path.join(content_folder_path, digest) if not os.path.exists(digest_path): allFilesAreOK = False log.error("Impossible to process manifest %s because one of the mentioned file does not exist" % new_manifest) else: log.debug("Found file %s, let's check the content" % digest) with open(digest_path, 'rb') as f: d = tooltool.digest_file(f, algorithm) if d == digest: log.debug("Checksum is OK") else: allFilesAreOK = False log.error("Impossible to process manifest %s because the mentioned file %s has an incorrect content" % (new_manifest, digest)) if allFilesAreOK: # copying the digest files to destination copyOK = True for digest, _algorithm in digests: digest_path = os.path.join(content_folder_path, digest) try: shutil.copy(digest_path, os.path.join(destination, "temp%s" % digest)) except IOError as e: log.error("Impossible to copy file %s to %s; I/O error(%s): %s" % (digest_path, destination, e.errno, e.strerror)) copyOK = False break if copyOK: renamingOK = True for digest, _algorithm in digests: try: os.rename(os.path.join(destination, "temp%s" % digest), os.path.join(destination, digest)) except: log.error("Impossible to rename file %s to %s;" % (os.path.join(destination, "temp%s" % digest), os.path.join(destination, digest))) renamingOK = False if renamingOK: for digest, _algorithm in digests: # update digest with new manifest dealing with it with open("%s.MANIFESTS" % digest, 'a') as file: stored_manifest_name = "%s.%s.%s" % (user, distribution_type, timestamped_manifest_name) file.write("%s\n" % stored_manifest_name) # if renaming is not successful, there's probably some problem in the upload server # keep a local copy of the processed manifest shutil.copy(new_manifest_path, os.path.join(os.getcwd(), stored_manifest_name)) if os.path.exists(comment_filepath): shutil.copy(comment_filepath, os.path.join(os.getcwd(), "%s.%s.%s.%s" % (user, distribution_type, timestamp, comment_filename))) #rename original comment file os.rename(comment_filepath, os.path.join(upload_folder, "%s.%s" % (timestamp, comment_filename))) # rename original manifest name os.rename(new_manifest_path, os.path.join(upload_folder, timestamped_manifest_name)) else: #TODO: cleanup removing copied files beginning with "temp" pass if manifestOK and allFilesAreOK: if copyOK: if renamingOK: # cleaning up source directory of copied files shutil.rmtree(content_folder_path) notifier.sendmail(user, "TOOLTOOL UPLOAD COMPLETED! Tooltool package %s has been correctly processed by the tooltool sync script!" % new_manifest, "") else: notifier.sendmail("", "INTERNAL ERROR - sync script could not rename files in package %s" % new_manifest, "") else: notifier.sendmail("", "INTERNAL ERROR - sync script could not copy files in package %s" % new_manifest, "") # TODO: notify internal error both to uploader and to sync maintainer else: # general cleanup: the uploader will need to re-upload the package if os.path.exists(content_folder_path): shutil.rmtree(content_folder_path) os.remove(comment_filepath) os.remove(new_manifest_path) #TODO: notify error to user: a new upload needs to be made! log.error("Manifest %s has NOT been processed and will need to be re-uploaded by the user" % new_manifest) msg = "Dear tooltool user,\n\nThe upload of the tooltool package %s was unsuccessful because of the following reason:\n\n" % new_manifest if not manifestOK: msg = msg + "- The uploaded manifest was invalid and could not be correctly parsed.\n" if not allFilesAreOK: msg = msg + "- Some of the files mentioned in the manifest were either missing or their content was corrupted.\n" msg = msg + "\nPlease try again with a new upload.\n\n" msg = msg + "Kind regards,\n\nThe Tooltool sync script" notifier.sendmail(user, "TOOLTOOL UPLOAD FAILURE! - the tooltool sync script could not process manifest %s" % new_manifest, msg)
def main(): root, matching, smtp_server, smtp_port, smtp_from, email_addresses, default_domain = load_config( ) notifier = Notifier(smtp_server, smtp_port, smtp_from, email_addresses, default_domain) users = [] for (_dirpath, dirnames, _files) in os.walk(root): users.extend(dirnames) break # not to navigate subfolders for user in users: for distribution_type in matching: files = [] upload_folder = get_upload_folder(root, user, distribution_type) for (dirpath, _dirnames, _files) in os.walk(upload_folder): files.extend(_files) break # not to navigate subfolders # "new" means that it has to be processed new_manifests = [ filename for filename in files if (filename.endswith(".tt") and not begins_with_timestamp(filename)) ] for new_manifest in new_manifests: new_manifest_path = os.path.join(upload_folder, new_manifest) destination = matching[distribution_type] timestamp = datetime.datetime.now().strftime(STRFRTIME) timestamped_manifest_name = "%s.%s" % (timestamp, new_manifest) new_manifest_path_after_processing = os.path.join( upload_folder, timestamped_manifest_name) comment_filename = new_manifest.replace(".tt", ".txt") comment_filepath = os.path.join( upload_folder, new_manifest.replace(".tt", ".txt")) manifestOK = True allFilesAreOK = True content_folder_path = new_manifest_path.replace( ".tt", tooltool.TOOLTOOL_PACKAGE_SUFFIX) digests = () try: digests = getDigests(new_manifest_path) except tooltool.InvalidManifest: manifestOK = False if manifestOK: # checking that ALL files mentioned in the manifest are in the upload folder, otherwise I cannot proceed copying if not os.path.exists( content_folder_path) or not os.path.isdir( content_folder_path): allFilesAreOK = False log.error( "Impossible to process manifest %s because content has not been uploaded" % content_folder_path) else: for digest, algorithm in digests: digest_path = os.path.join(content_folder_path, digest) if not os.path.exists(digest_path): allFilesAreOK = False log.error( "Impossible to process manifest %s because one of the mentioned file does not exist" % new_manifest) else: log.debug( "Found file %s, let's check the content" % digest) with open(digest_path, 'rb') as f: d = tooltool.digest_file(f, algorithm) if d == digest: log.debug("Checksum is OK") else: allFilesAreOK = False log.error( "Impossible to process manifest %s because the mentioned file %s has an incorrect content" % (new_manifest, digest)) if allFilesAreOK: # copying the digest files to destination copyOK = True for digest, _algorithm in digests: digest_path = os.path.join(content_folder_path, digest) try: target_filename = os.path.join( destination, "temp%s" % digest) shutil.copy(digest_path, target_filename) # sets permission to -rw-r--r-- to make sure the file is readable os.chmod(target_filename, 0644) except IOError as e: log.error( "Impossible to copy file %s to %s; I/O error(%s): %s" % (digest_path, destination, e.errno, e.strerror)) copyOK = False break if copyOK: renamingOK = True for digest, _algorithm in digests: try: os.rename( os.path.join(destination, "temp%s" % digest), os.path.join(destination, digest)) except: log.error( "Impossible to rename file %s to %s;" % (os.path.join(destination, "temp%s" % digest), os.path.join(destination, digest))) renamingOK = False if renamingOK: for digest, _algorithm in digests: # update digest with new manifest dealing with it with open("%s.MANIFESTS" % digest, 'a') as file: stored_manifest_name = "%s.%s.%s" % ( user, distribution_type, timestamped_manifest_name) file.write("%s\n" % stored_manifest_name) # if renaming is not successful, there's probably some problem in the upload server # keep a local copy of the processed manifest shutil.copy( new_manifest_path, os.path.join(os.getcwd(), stored_manifest_name)) if os.path.exists(comment_filepath): shutil.copy( comment_filepath, os.path.join( os.getcwd(), "%s.%s.%s.%s" % (user, distribution_type, timestamp, comment_filename))) #rename original comment file os.rename( comment_filepath, os.path.join( upload_folder, "%s.%s" % (timestamp, comment_filename))) # rename original manifest name os.rename(new_manifest_path, new_manifest_path_after_processing) else: #TODO: cleanup removing copied files beginning with "temp" pass if manifestOK and allFilesAreOK: if copyOK: if renamingOK: # cleaning up source directory of copied files shutil.rmtree(content_folder_path) inputFile = open( new_manifest_path_after_processing, 'r') body = inputFile.read() inputFile.close() notifier.sendmail( user, "TOOLTOOL UPLOAD COMPLETED! Tooltool package %s has been correctly processed by the tooltool sync script!" % new_manifest, body) else: notifier.sendmail( "", "INTERNAL ERROR - sync script could not rename files in package %s" % new_manifest, "") else: notifier.sendmail( "", "INTERNAL ERROR - sync script could not copy files in package %s" % new_manifest, "") # TODO: notify internal error both to uploader and to sync maintainer else: # general cleanup: the uploader will need to re-upload the package if os.path.exists(content_folder_path): shutil.rmtree(content_folder_path) os.remove(comment_filepath) os.remove(new_manifest_path) #TODO: notify error to user: a new upload needs to be made! log.error( "Manifest %s has NOT been processed and will need to be re-uploaded by the user" % new_manifest) msg = "Dear tooltool user,\n\nThe upload of the tooltool package %s was unsuccessful because of the following reason:\n\n" % new_manifest if not manifestOK: msg = msg + "- The uploaded manifest was invalid and could not be correctly parsed.\n" if not allFilesAreOK: msg = msg + "- Some of the files mentioned in the manifest were either missing or their content was corrupted.\n" msg = msg + "\nPlease try again with a new upload.\n\n" msg = msg + "Kind regards,\n\nThe Tooltool sync script" notifier.sendmail( user, "TOOLTOOL UPLOAD FAILURE! - the tooltool sync script could not process manifest %s" % new_manifest, msg)