def setUp(self):
self.sample_file = 'test_file.ogg'
self.sample_algo = 'sha512'
self.sample_size = os.path.getsize(self.sample_file)
with open(self.sample_file) as f:
self.sample_hash = tooltool.digest_file(f, self.sample_algo)
self.test_record = tooltool.FileRecord(filename=self.sample_file,
size=self.sample_size,
digest=self.sample_hash,
algorithm=self.sample_algo)
# using mkstemp to ensure that the filename generated
# isn't actually on the system.
(tmpfd, filename) = tempfile.mkstemp()
os.close(tmpfd)
os.remove(filename)
if os.path.exists(filename):
self.fail('did not remove %s' % filename)
self.absent_file = filename
def setUp(self):
self.sample_file = 'test_file.ogg'
self.sample_algo = 'sha512'
self.sample_size = os.path.getsize(self.sample_file)
with open(self.sample_file) as f:
self.sample_hash = tooltool.digest_file(f, self.sample_algo)
self.test_record = tooltool.FileRecord(
filename=self.sample_file,
size=self.sample_size,
digest=self.sample_hash,
algorithm=self.sample_algo
)
# using mkstemp to ensure that the filename generated
# isn't actually on the system.
(tmpfd, filename) = tempfile.mkstemp()
os.close(tmpfd)
os.remove(filename)
if os.path.exists(filename):
self.fail('did not remove %s' % filename)
self.absent_file = filename
def test_digest_file(self):
test_digest = tooltool.digest_file(self.sample_data, self.sample_algo)
# If this assertion fails, verify that test_file.ogg is an ogg file
# of Linus Torvalds explaining how he pronounces 'Linux'
self.assertEqual(test_digest, self.sample_digest)
def test_digest_file(self):
test_digest = tooltool.digest_file(self.sample_data, self.sample_algo)
# If this assertion fails, verify that test_file.ogg is an ogg file
# of Linus Torvalds explaining how he pronounces 'Linux'
self.assertEqual(test_digest, self.sample_digest)
def main():
root, matching, smtp_server, smtp_port, smtp_from, email_addresses, default_domain = load_config()
notifier = Notifier(smtp_server, smtp_port, smtp_from, email_addresses, default_domain)
users = []
for (_dirpath, dirnames, _files) in os.walk(root):
users.extend(dirnames)
break # not to navigate subfolders
for user in users:
for distribution_type in matching:
files = []
upload_folder = get_upload_folder(root, user, distribution_type)
for (dirpath, _dirnames, _files) in os.walk(upload_folder):
files.extend(_files)
break # not to navigate subfolders
# "new" means that it has to be processed
new_manifests = [filename for filename in files if (filename.endswith(".tt") and not begins_with_timestamp(filename))]
for new_manifest in new_manifests:
new_manifest_path = os.path.join(upload_folder, new_manifest)
destination = matching[distribution_type]
timestamp = datetime.datetime.now().strftime(STRFRTIME)
timestamped_manifest_name = "%s.%s" % (timestamp, new_manifest)
comment_filename = new_manifest.replace(".tt", ".txt")
comment_filepath = os.path.join(upload_folder, new_manifest.replace(".tt", ".txt"))
manifestOK = True
allFilesAreOK = True
content_folder_path = new_manifest_path.replace(".tt", tooltool.TOOLTOOL_PACKAGE_SUFFIX)
digests = ()
try:
digests = getDigests(new_manifest_path)
except tooltool.InvalidManifest:
manifestOK = False
if manifestOK:
# checking that ALL files mentioned in the manifest are in the upload folder, otherwise I cannot proceed copying
if not os.path.exists(content_folder_path) or not os.path.isdir(content_folder_path):
allFilesAreOK = False
log.error("Impossible to process manifest %s because content has not been uploaded" % content_folder_path)
else:
for digest, algorithm in digests:
digest_path = os.path.join(content_folder_path, digest)
if not os.path.exists(digest_path):
allFilesAreOK = False
log.error("Impossible to process manifest %s because one of the mentioned file does not exist" % new_manifest)
else:
log.debug("Found file %s, let's check the content" % digest)
with open(digest_path, 'rb') as f:
d = tooltool.digest_file(f, algorithm)
if d == digest:
log.debug("Checksum is OK")
else:
allFilesAreOK = False
log.error("Impossible to process manifest %s because the mentioned file %s has an incorrect content" % (new_manifest, digest))
if allFilesAreOK:
# copying the digest files to destination
copyOK = True
for digest, _algorithm in digests:
digest_path = os.path.join(content_folder_path, digest)
try:
shutil.copy(digest_path, os.path.join(destination, "temp%s" % digest))
except IOError as e:
log.error("Impossible to copy file %s to %s; I/O error(%s): %s" % (digest_path, destination, e.errno, e.strerror))
copyOK = False
break
if copyOK:
renamingOK = True
for digest, _algorithm in digests:
try:
os.rename(os.path.join(destination, "temp%s" % digest), os.path.join(destination, digest))
except:
log.error("Impossible to rename file %s to %s;" % (os.path.join(destination, "temp%s" % digest), os.path.join(destination, digest)))
renamingOK = False
if renamingOK:
for digest, _algorithm in digests:
# update digest with new manifest dealing with it
with open("%s.MANIFESTS" % digest, 'a') as file:
stored_manifest_name = "%s.%s.%s" % (user, distribution_type, timestamped_manifest_name)
file.write("%s\n" % stored_manifest_name)
# if renaming is not successful, there's probably some problem in the upload server
# keep a local copy of the processed manifest
shutil.copy(new_manifest_path, os.path.join(os.getcwd(), stored_manifest_name))
if os.path.exists(comment_filepath):
shutil.copy(comment_filepath, os.path.join(os.getcwd(), "%s.%s.%s.%s" % (user, distribution_type, timestamp, comment_filename)))
#rename original comment file
os.rename(comment_filepath, os.path.join(upload_folder, "%s.%s" % (timestamp, comment_filename)))
# rename original manifest name
os.rename(new_manifest_path, os.path.join(upload_folder, timestamped_manifest_name))
else:
#TODO: cleanup removing copied files beginning with "temp"
pass
if manifestOK and allFilesAreOK:
if copyOK:
if renamingOK:
# cleaning up source directory of copied files
shutil.rmtree(content_folder_path)
notifier.sendmail(user, "TOOLTOOL UPLOAD COMPLETED! Tooltool package %s has been correctly processed by the tooltool sync script!" % new_manifest, "")
else:
notifier.sendmail("", "INTERNAL ERROR - sync script could not rename files in package %s" % new_manifest, "")
else:
notifier.sendmail("", "INTERNAL ERROR - sync script could not copy files in package %s" % new_manifest, "")
# TODO: notify internal error both to uploader and to sync maintainer
else:
# general cleanup: the uploader will need to re-upload the package
if os.path.exists(content_folder_path):
shutil.rmtree(content_folder_path)
os.remove(comment_filepath)
os.remove(new_manifest_path)
#TODO: notify error to user: a new upload needs to be made!
log.error("Manifest %s has NOT been processed and will need to be re-uploaded by the user" % new_manifest)
msg = "Dear tooltool user,\n\nThe upload of the tooltool package %s was unsuccessful because of the following reason:\n\n" % new_manifest
if not manifestOK:
msg = msg + "- The uploaded manifest was invalid and could not be correctly parsed.\n"
if not allFilesAreOK:
msg = msg + "- Some of the files mentioned in the manifest were either missing or their content was corrupted.\n"
msg = msg + "\nPlease try again with a new upload.\n\n"
msg = msg + "Kind regards,\n\nThe Tooltool sync script"
notifier.sendmail(user, "TOOLTOOL UPLOAD FAILURE! - the tooltool sync script could not process manifest %s" % new_manifest, msg)
def main():
root, matching, smtp_server, smtp_port, smtp_from, email_addresses, default_domain = load_config(
)
notifier = Notifier(smtp_server, smtp_port, smtp_from, email_addresses,
default_domain)
users = []
for (_dirpath, dirnames, _files) in os.walk(root):
users.extend(dirnames)
break # not to navigate subfolders
for user in users:
for distribution_type in matching:
files = []
upload_folder = get_upload_folder(root, user, distribution_type)
for (dirpath, _dirnames, _files) in os.walk(upload_folder):
files.extend(_files)
break # not to navigate subfolders
# "new" means that it has to be processed
new_manifests = [
filename for filename in files
if (filename.endswith(".tt")
and not begins_with_timestamp(filename))
]
for new_manifest in new_manifests:
new_manifest_path = os.path.join(upload_folder, new_manifest)
destination = matching[distribution_type]
timestamp = datetime.datetime.now().strftime(STRFRTIME)
timestamped_manifest_name = "%s.%s" % (timestamp, new_manifest)
new_manifest_path_after_processing = os.path.join(
upload_folder, timestamped_manifest_name)
comment_filename = new_manifest.replace(".tt", ".txt")
comment_filepath = os.path.join(
upload_folder, new_manifest.replace(".tt", ".txt"))
manifestOK = True
allFilesAreOK = True
content_folder_path = new_manifest_path.replace(
".tt", tooltool.TOOLTOOL_PACKAGE_SUFFIX)
digests = ()
try:
digests = getDigests(new_manifest_path)
except tooltool.InvalidManifest:
manifestOK = False
if manifestOK:
# checking that ALL files mentioned in the manifest are in the upload folder, otherwise I cannot proceed copying
if not os.path.exists(
content_folder_path) or not os.path.isdir(
content_folder_path):
allFilesAreOK = False
log.error(
"Impossible to process manifest %s because content has not been uploaded"
% content_folder_path)
else:
for digest, algorithm in digests:
digest_path = os.path.join(content_folder_path,
digest)
if not os.path.exists(digest_path):
allFilesAreOK = False
log.error(
"Impossible to process manifest %s because one of the mentioned file does not exist"
% new_manifest)
else:
log.debug(
"Found file %s, let's check the content" %
digest)
with open(digest_path, 'rb') as f:
d = tooltool.digest_file(f, algorithm)
if d == digest:
log.debug("Checksum is OK")
else:
allFilesAreOK = False
log.error(
"Impossible to process manifest %s because the mentioned file %s has an incorrect content"
% (new_manifest, digest))
if allFilesAreOK:
# copying the digest files to destination
copyOK = True
for digest, _algorithm in digests:
digest_path = os.path.join(content_folder_path,
digest)
try:
target_filename = os.path.join(
destination, "temp%s" % digest)
shutil.copy(digest_path, target_filename)
# sets permission to -rw-r--r-- to make sure the file is readable
os.chmod(target_filename, 0644)
except IOError as e:
log.error(
"Impossible to copy file %s to %s; I/O error(%s): %s"
% (digest_path, destination, e.errno,
e.strerror))
copyOK = False
break
if copyOK:
renamingOK = True
for digest, _algorithm in digests:
try:
os.rename(
os.path.join(destination,
"temp%s" % digest),
os.path.join(destination, digest))
except:
log.error(
"Impossible to rename file %s to %s;" %
(os.path.join(destination,
"temp%s" % digest),
os.path.join(destination, digest)))
renamingOK = False
if renamingOK:
for digest, _algorithm in digests:
# update digest with new manifest dealing with it
with open("%s.MANIFESTS" % digest,
'a') as file:
stored_manifest_name = "%s.%s.%s" % (
user, distribution_type,
timestamped_manifest_name)
file.write("%s\n" %
stored_manifest_name)
# if renaming is not successful, there's probably some problem in the upload server
# keep a local copy of the processed manifest
shutil.copy(
new_manifest_path,
os.path.join(os.getcwd(),
stored_manifest_name))
if os.path.exists(comment_filepath):
shutil.copy(
comment_filepath,
os.path.join(
os.getcwd(), "%s.%s.%s.%s" %
(user, distribution_type,
timestamp, comment_filename)))
#rename original comment file
os.rename(
comment_filepath,
os.path.join(
upload_folder, "%s.%s" %
(timestamp, comment_filename)))
# rename original manifest name
os.rename(new_manifest_path,
new_manifest_path_after_processing)
else:
#TODO: cleanup removing copied files beginning with "temp"
pass
if manifestOK and allFilesAreOK:
if copyOK:
if renamingOK:
# cleaning up source directory of copied files
shutil.rmtree(content_folder_path)
inputFile = open(
new_manifest_path_after_processing, 'r')
body = inputFile.read()
inputFile.close()
notifier.sendmail(
user,
"TOOLTOOL UPLOAD COMPLETED! Tooltool package %s has been correctly processed by the tooltool sync script!"
% new_manifest, body)
else:
notifier.sendmail(
"",
"INTERNAL ERROR - sync script could not rename files in package %s"
% new_manifest, "")
else:
notifier.sendmail(
"",
"INTERNAL ERROR - sync script could not copy files in package %s"
% new_manifest, "")
# TODO: notify internal error both to uploader and to sync maintainer
else:
# general cleanup: the uploader will need to re-upload the package
if os.path.exists(content_folder_path):
shutil.rmtree(content_folder_path)
os.remove(comment_filepath)
os.remove(new_manifest_path)
#TODO: notify error to user: a new upload needs to be made!
log.error(
"Manifest %s has NOT been processed and will need to be re-uploaded by the user"
% new_manifest)
msg = "Dear tooltool user,\n\nThe upload of the tooltool package %s was unsuccessful because of the following reason:\n\n" % new_manifest
if not manifestOK:
msg = msg + "- The uploaded manifest was invalid and could not be correctly parsed.\n"
if not allFilesAreOK:
msg = msg + "- Some of the files mentioned in the manifest were either missing or their content was corrupted.\n"
msg = msg + "\nPlease try again with a new upload.\n\n"
msg = msg + "Kind regards,\n\nThe Tooltool sync script"
notifier.sendmail(
user,
"TOOLTOOL UPLOAD FAILURE! - the tooltool sync script could not process manifest %s"
% new_manifest, msg)
