def test_003_temporary_file_avoid_delete(self):
a = GLSecureTemporaryFile(GLSetting.tmp_upload_path)
a.avoid_delete()
antani = "0123456789" * 10000
a.write(antani)
a.close()
self.assertTrue(os.path.exists(a.filepath))
b = GLSecureFile(a.filepath)
self.assertTrue(antani == b.read())
def fsops_pgp_encrypt(fpath, recipient_pgp):
"""
return
path of encrypted file,
length of the encrypted file
this function is used to encrypt a file for a specific recipient.
commonly 'receiver_desc' is expected as second argument;
anyhow a simpler dict can be used.
required keys are checked on top
"""
gpoj = GLBPGP()
try:
gpoj.load_key(recipient_pgp['pgp_key_public'])
filepath = os.path.join(GLSettings.submission_path, fpath)
with GLSecureFile(filepath) as f:
encrypted_file_path = os.path.join(os.path.abspath(GLSettings.submission_path), "pgp_encrypted-%s" % generateRandomKey(16))
_, encrypted_file_size = gpoj.encrypt_file(recipient_pgp['pgp_key_fingerprint'], f, encrypted_file_path)
except:
raise
finally:
# the finally statement is always called also if
# except contains a return or a raise
gpoj.destroy_environment()
return encrypted_file_path, encrypted_file_size
def fsops_gpg_encrypt(fpath, recipient_gpg):
"""
return
path of encrypted file,
length of the encrypted file
this function is used to encrypt a file for a specific recipient.
commonly 'receiver_desc' is expected as second argument;
anyhow a simpler dict can be used.
required keys are checked on top
"""
gpoj = GLBGPG()
try:
gpoj.load_key(recipient_gpg['gpg_key_armor'])
filepath = os.path.join(GLSetting.submission_path, fpath)
with GLSecureFile(filepath) as f:
encrypted_file_path, encrypted_file_size = \
gpoj.encrypt_file(recipient_gpg['gpg_key_fingerprint'], filepath, f, GLSetting.submission_path)
except:
raise
finally:
# the finally statement is always called also if
# except contains a return or a raise
gpoj.destroy_environment()
return encrypted_file_path, encrypted_file_size
def fsops_gpg_encrypt(fpath, recipient_gpg):
"""
return
path of encrypted file,
length of the encrypted file
this function is used to encrypt a file for a specific recipient.
commonly 'receiver_desc' is expected as second argument;
anyhow a simpler dict can be used.
required keys are checked on top
"""
assert isinstance(recipient_gpg, dict), "invalid recipient"
assert recipient_gpg.has_key('gpg_key_armor'), "missing key"
assert recipient_gpg.has_key('gpg_key_status'), "missing status"
assert recipient_gpg['gpg_key_status'] == u'Enabled', "GPG not enabled"
assert recipient_gpg.has_key('name'), "missing recipient Name"
gpoj = GLBGPG(recipient_gpg)
if not gpoj.validate_key(recipient_gpg['gpg_key_armor']):
raise Exception("Unable to validate key")
filepath = os.path.join(GLSetting.submission_path, fpath)
with GLSecureFile(filepath) as f:
encrypted_file_path, encrypted_file_size = \
gpoj.encrypt_file(filepath, f, GLSetting.submission_path)
gpoj.destroy_environment()
assert (encrypted_file_size > 1), "File generated is empty or size is 0"
assert os.path.isfile(
encrypted_file_path), "Output generated is not a file!"
return encrypted_file_path, encrypted_file_size
def process_files(receiverfiles_maps):
"""
@param receiverfiles_maps: the mapping of ifile/rfiles to be created on filesystem
@return: return None
"""
for ifile_id, receiverfiles_map in receiverfiles_maps.iteritems():
ifile_path = receiverfiles_map['ifile_path']
ifile_name = os.path.basename(ifile_path).split('.')[0]
plain_path = os.path.join(GLSettings.submission_path, "%s.plain" % ifile_name)
receiverfiles_map['plaintext_file_needed'] = False
for rcounter, rfileinfo in enumerate(receiverfiles_map['rfiles']):
if len(rfileinfo['receiver']['pgp_key_public']):
try:
new_path, new_size = fsops_pgp_encrypt(rfileinfo['path'], rfileinfo['receiver'])
log.debug("%d# Switch on Receiver File for %s path %s => %s size %d => %d" %
(rcounter, rfileinfo['receiver']['name'], rfileinfo['path'],
new_path, rfileinfo['size'], new_size))
rfileinfo['path'] = new_path
rfileinfo['size'] = new_size
rfileinfo['status'] = u'encrypted'
except Exception as excep:
log.err("%d# Unable to complete PGP encrypt for %s on %s: %s. marking the file as unavailable." % (
rcounter, rfileinfo['receiver']['name'], rfileinfo['path'], excep)
)
rfileinfo['status'] = u'unavailable'
elif GLSettings.memory_copy.allow_unencrypted:
receiverfiles_map['plaintext_file_needed'] = True
rfileinfo['status'] = u'reference'
rfileinfo['path'] = plain_path
else:
rfileinfo['status'] = u'nokey'
if receiverfiles_map['plaintext_file_needed']:
log.debug(":( NOT all receivers support PGP and the system allows plaintext version of files: %s saved as plaintext file %s" %
(ifile_path, plain_path))
try:
with open(plain_path, "wb") as plaintext_f, GLSecureFile(ifile_path) as encrypted_file:
chunk_size = 4096
written_size = 0
while True:
chunk = encrypted_file.read(chunk_size)
if len(chunk) == 0:
if written_size != receiverfiles_map['ifile_size']:
log.err("Integrity error on rfile write for ifile %s; ifile_size(%d), rfile_size(%d)" %
(ifile_id, receiverfiles_map['ifile_size'], written_size))
break
written_size += len(chunk)
plaintext_f.write(chunk)
receiverfiles_map['ifile_path'] = plain_path
except Exception as excep:
log.err("Unable to create plaintext file %s: %s" % (plain_path, excep))
else:
log.debug("All Receivers support PGP or the system denies plaintext version of files: marking internalfile as removed")
# the original AES file should always be deleted
log.debug("Deleting the submission AES encrypted file: %s" % ifile_path)
# Remove the AES file
try:
os.remove(ifile_path)
except OSError as ose:
log.err("Unable to remove %s: %s" % (ifile_path, ose.message))
# Remove the AES file key
try:
os.remove(os.path.join(GLSettings.ramdisk_path, ("%s%s" % (GLSettings.AES_keyfile_prefix, ifile_name))))
except OSError as ose:
log.err("Unable to remove keyfile associated with %s: %s" % (ifile_path, ose.message))
def operation(self):
"""
Goal of this function is to process/validate files, compute their checksums and
apply the configured delivery method.
"""
try:
# ==> Submission && Escalation
info_created_tips = yield tip_creation()
if info_created_tips:
log.debug("Delivery job: created %d tips" %
len(info_created_tips))
except Exception as excep:
log.err("Exception in asyncronous delivery job: %s" % excep)
sys.excepthook(*sys.exc_info())
# ==> Files && Files update,
# InternalFile is set as 'locked' status
# and would be unlocked at the end.
# TODO xxx limit of file number per operation
filemap = yield receiverfile_planning()
# the function returns a dict of lists with dicts:
# {
# 'ifile_path' : [
# { 'receiver' : receiver_desc, 'path': file_path,
# 'size' : file_size, 'status': XXX },
# { 'receiver' : receiver_desc, 'path': file_path,
# 'size' : file_size, 'status': YYY }, ... ]
# }, { }, ...
#
if not filemap:
return
# Here the files received are encrypted (if the receiver has PGP key)
log.debug("Delivery task: Iterate over %d ReceiverFile(s)" %
len(filemap.keys()))
for ifile_path, receivermap in filemap.iteritems():
plain_path = os.path.join(GLSetting.submission_path,
"%s.plain" % xeger(r'[A-Za-z0-9]{16}'))
create_plaintextfile = encrypt_where_available(receivermap)
for rfileinfo in receivermap:
if not create_plaintextfile and rfileinfo[
'status'] == u'reference':
rfileinfo['path'] = plain_path
try:
yield receiverfile_create(ifile_path, rfileinfo['path'],
rfileinfo['status'],
rfileinfo['size'],
rfileinfo['receiver'])
except Exception as excep:
log.err(
"Unable to create ReceiverFile from %s for %s: %s" %
(ifile_path, rfileinfo['receiver']['name'], excep))
continue
if not create_plaintextfile:
log.debug(
":( NOT all receivers support PGP and the system allows plaintext version of files: %s saved in plaintext file %s"
% (ifile_path, plain_path))
try:
with open(plain_path, "wb") as plain_f_is_sad_f, \
GLSecureFile(ifile_path) as encrypted_file:
chunk_size = 4096
while True:
chunk = encrypted_file.read(chunk_size)
if len(chunk) == 0:
break
plain_f_is_sad_f.write(chunk)
yield do_final_internalfile_update(ifile_path, u'ready',
plain_path)
except Exception as excep:
log.err("Unable to create plaintext file %s: %s" %
(plain_path, excep))
else: # create_plaintextfile
log.debug(
"All Receivers support PGP or the system denys plaintext version of files: marking internalfile as removed"
)
yield do_final_internalfile_update(ifile_path,
u'delivered') # Removed
# the original AES file need always to be deleted
log.debug("Deleting the submission AES encrypted file: %s" %
ifile_path)
try:
os.remove(ifile_path)
except OSError as ose:
log.err("Unable to remove %s: %s" % (ifile_path, ose.message))
try:
key_id = os.path.basename(ifile_path).split('.')[0]
keypath = os.path.join(
GLSetting.ramdisk_path,
("%s%s" % (GLSetting.AES_keyfile_prefix, key_id)))
os.remove(keypath)
except Exception as excep:
log.err("Unable to remove keyfile associated with %s: %s" %
(ifile_path, excep))