def write_empty_config(self, config, conffile):
rootdir = BaseDirectory.save_config_path('tuyau')
with file(conffile, 'wb') as fp:
config.write(fp)
g = GPG(gnupghome=os.path.join(rootdir, 'gnupg'))
g.list_keys()
def decrypt(file):
gpg_home = find_gpg_keys()
if ( gpg_home == ''):
print 'GPG keys not found'
sys.exit(1)
gpg = GPG(gnupghome=gpg_home, use_agent=True)
public_keys = gpg.list_keys()
key_id = public_keys[0]['keyid']
if ( os.path.isfile(file)):
if ( file.endswith('.gpg')):
stream = open(file, 'rb')
status = gpg.decrypt_file(stream, output=file[:-4])
if ( status.ok):
os.remove(file)
print file[:-4] + ' succesfully decrypted'
else:
print file + ' not encrypted'
elif ( os.path.isdir(file) ):
for root, dirs, files in os.walk(file, topdown=True):
for name in files:
current_file = (os.path.join(root, name))
if ( current_file.endswith('.gpg')):
stream = open(current_file, "rb")
status = gpg.decrypt_file(stream, output=current_file[:-4])
if ( status.ok ):
os.remove(current_file)
print current_file[:-4] + ' successfully decrypted'
else:
print current_file + ' not encrypted'
else:
print 'ERROR: file or directory not found'
def clean_key(self):
"""
Raises ValidationError if the entered key is invalid or includes a GPG private key.
"""
data = self.cleaned_data['key']
with TemporaryDirectory() as temp_dir:
gpg_keychain = GPG(gnupghome=temp_dir)
res = gpg_keychain.import_keys(data)
if not res:
errors = [
forms.ValidationError(_("Invalid key."), code='invalid')
]
for attr in ['status',
'stderr']: # not all fields are always present
if hasattr(res, attr):
errors.append(
forms.ValidationError("%(name)s: %(value)s",
params={
'name': attr,
'value':
getattr(res, attr)
},
code=attr))
raise forms.ValidationError(errors)
if len(gpg_keychain.list_keys(
True)) > 0: # check existance of private keys
raise forms.ValidationError(_(
"Import public keys only, no private keys! "
"You should consider the private key(s) compromised."),
code='private_key')
return data
def verify_gpg_signature(f):
gpg = GPG(homedir='~/.gnupg')
# search public key 0x90C8019E36C2E964
bitcoin_core_pgp_key_found = False
keys = gpg.list_keys()
for key in keys:
if key['keyid'] == '90C8019E36C2E964':
bitcoin_core_pgp_key_found = True
if not bitcoin_core_pgp_key_found == True:
print(
'* Warning: bitcoin-core GPG key not found, trying to find and import...'
)
import_key = gpg.recv_keys('90C8019E36C2E964',
keyserver='hkp://pool.sks-keyservers.net')
print('* Status: ' + import_key.summary())
print('* Fingerprint: ' + import_key.fingerprints[0])
with open(f) as fi:
verif = gpg.verify_file(fi)
print('* Verifying ' + f + ': ' + verif.status)
if not verif.valid:
print('* Impossible to compare checksums, quitting!')
return verif.valid
def get_expirations(keylist):
"""
This function is not implemented in GPG object class because need to operate
on the whole keys
"""
try:
temp_gpgroot = os.path.join(
GLSetting.gpgroot,
"-expiration_check-%s" % xeger(r'[A-Za-z0-9]{8}'))
os.makedirs(temp_gpgroot, mode=0700)
gpexpire = GPG(gnupghome=temp_gpgroot,
options=['--trust-model', 'always'])
except Exception as excep:
log.err("Unable to setup expiration check environment: %s" % excep)
raise excep
try:
for key in keylist:
gpexpire.import_keys(key)
except Exception as excep:
log.err("Error in GPG import_keys: %s" % excep)
raise excep
try:
all_keys = gpexpire.list_keys()
except Exception as excep:
log.err("Error in GPG list_keys: %s" % excep)
raise excep
expirations = {}
for ak in all_keys:
expirations.update({ak['fingerprint']: ak['date']})
return expirations
def get_expirations(keylist):
"""
This function is not implemented in GPG object class because need to operate
on the whole keys
"""
atfork()
try:
temp_gpgroot = os.path.join(GLSetting.gpgroot, "-expiration_check-%s" % random.randint(0, 0xFFFF) )
os.makedirs(temp_gpgroot, mode=0700)
gpexpire= GPG(gnupghome=temp_gpgroot, options="--trust-model always")
except Exception as excep:
log.err("Unable to setup expiration check environment: %s" % excep)
raise excep
try:
for key in keylist:
gpexpire.import_keys(key)
except Exception as excep:
log.err("Error in GPG import_keys: %s" % excep)
raise excep
try:
all_keys = gpexpire.list_keys()
except Exception as excep:
log.err("Error in GPG list_keys: %s" % excep)
raise excep
expirations = {}
for ak in all_keys:
expirations.update({ ak['fingerprint'] : ak['date']})
return expirations
def encrypt(file):
gpg_home = find_gpg_keys()
if ( gpg_home == ''):
print 'GPG keys not found'
sys.exit(1)
gpg = GPG(gnupghome=gpg_home, use_agent=True)
public_keys = gpg.list_keys()
key_id = public_keys[0]['keyid']
if ( os.path.isfile(file)):
if ( file.endswith('.gpg')):
print file + ' is already encrypted'
else:
stream = open(file, "rb")
status = gpg.encrypt_file(stream, key_id, passphrase='default_key', armor=False, always_trust=True,
output=file+'.gpg', symmetric=False)
stream.close()
if ( status.ok):
os.remove(file)
print file , ' successfully encrypted'
elif (os.path.isdir(file)):
for root, dirs, files in os.walk(file, topdown=True):
for name in files:
current_file = (os.path.join(root, name))
if ( current_file.endswith('.gpg') ):
print current_file + ' : is already encrypted'
else:
stream = open(current_file, "rb")
status = gpg.encrypt_file(stream, key_id, armor=True, always_trust=True, symmetric=False, output=current_file+'.gpg')
stream.close()
if ( status.ok ):
os.remove(current_file)
print current_file + ' successfully encrypted'
else:
print 'ERROR, file or directory not found'
def gnupg_key_fingerprint(gnupg_instance: GPG, private_gpg_key: str):
keys_imported = gnupg_instance.import_keys(private_gpg_key)
key_fingerprint = keys_imported.fingerprints[0]
yield key_fingerprint
if key_fingerprint in gnupg_instance.list_keys(secret=True).fingerprints:
remove_gpg_key_pair(gpg_binary=gnupg_instance.gpgbinary,
fingerprint=key_fingerprint)
def delete(self):
"""
Remove any keys for this address.
"""
gpg = GPG(gnupghome=GNUPG_HOME)
for key in gpg.list_keys():
if self.address in addresses_for_key(gpg, key):
gpg.delete_keys(key["fingerprint"], True)
gpg.delete_keys(key["fingerprint"])
super(Address, self).delete()
def delete(self):
"""
Remove any keys for this address.
"""
gpg = GPG(gnupghome=GNUPG_HOME)
for key in gpg.list_keys():
if self.address in addresses_for_key(gpg, key):
gpg.delete_keys(key["fingerprint"], True)
gpg.delete_keys(key["fingerprint"])
super(Address, self).delete()
def delete(self): """ remove any keys for this address """ from email_extras.utils import addresses_for_key gpg = GPG(gnupghome=GNUPG_HOME) for key in gpg.list_keys(): if self.address in addresses_for_key(gpg, key): gpg.delete_keys(key["fingerprint"], True) gpg.delete_keys(key["fingerprint"]) super(Address, self).delete()
def _generate_key(username):
gpg = GPG()
key_input = gpg.gen_key_input(key_type="RSA", key_length=1024, name_email=username+"@node.org", name_real=username)
entropy_thread = Process(target=generate_entropy)
entropy_thread.start()
key = gpg.gen_key(key_input)
entropy_thread.terminate()
keys = gpg.list_keys(True)
for k in keys:
if k.get("fingerprint") == key.fingerprint:
return k['keyid']
def encrypt_file(gpg: gnupg.GPG,
targer_folder: str):
configuration_kwargs = utils.get_configuratation(
path=os.path.join(
targer_folder,
'configuration'
)
)
value = _get_decryption(configuration_kwargs=configuration_kwargs)
gpg.import_keys(value)
keys = gpg.list_keys()
gpg.trust_keys(
keys.fingerprints,
'TRUST_ULTIMATE'
)
decrypted_filepaths = utils.get_files(
path=os.path.join(
targer_folder,
'decrypted_files'
)
)
encrypted_path = os.path.join(
targer_folder,
'encrypted_files'
)
for decryped_file in decrypted_filepaths:
base_name = __set_file_name(path=decryped_file)
logger.info(f'encrypting file {base_name}')
output_file = os.path.join(
encrypted_path,
base_name
)
with open(decryped_file, 'rb') as connection:
status = gpg.encrypt_file(
file=connection,
recipients=keys.fingerprints,
output=output_file,
)
logger.info(f'encrypting done')
if not status.ok:
logger.info(f'{status.stderr}')
else:
logger.info(f'Status [{status.status}]')
def _sign(self, data, digest_algo):
gpg = GPG(homedir=settings.GPG_SIGN_DIR)
if not gpg.list_keys():
# Import key if no private key key in keyring
with open(settings.GPG_SIGN_KEY, 'r') as f:
key = f.read()
gpg.import_keys(key)
signature = gpg.sign(data,
passphrase=settings.GPG_SIGN_KEY_PASSPHRASE,
clearsign=False,
detach=True,
digest_algo=digest_algo)
return str(signature)
def save(self, *args, **kwargs):
with TemporaryDirectory() as temp_dir:
gpg_keychain = GPG(gnupghome=temp_dir)
res = gpg_keychain.import_keys(self.key)
if not res:
handle_gpg_error(res, 'import')
if len(gpg_keychain.list_keys(True)) > 0:
raise GPGException("Will not import GPG private key!")
addresses = []
for key in res.results:
addresses.extend(addresses_for_key(gpg_keychain, key))
self.addresses = ', '.join(addresses)
operation = "Updating" if self.pk else "Creating"
logger.info("%s GPG key for: %s", operation, self.addresses)
super(Key, self).save(*args, **kwargs)
def gpg_keys(self):
"""
The GPG keyring path and list of key IDs.
:return: A tuple of: (path, key_ids)
The *path* is the absolute path to a keyring.
The *key_ids* is a list of key IDs added to the keyring.
:rtype: tuple
"""
home = self.working_dir
path = os.path.join(home, 'pubring.gpg')
key_list = self.config.get(constants.IMPORTER_CONFIG_KEY_GPG_KEYS, [])
gpg = GPG(gnupghome=home)
map(gpg.import_keys, key_list)
key_ids = [key['keyid'] for key in gpg.list_keys()]
return path, key_ids
def gpg_keys(self):
"""
The GPG keyring path and list of key IDs.
:return: A tuple of: (path, key_ids)
The *path* is the absolute path to a keyring.
The *key_ids* is a list of key IDs added to the keyring.
:rtype: tuple
"""
home = self.working_dir
path = os.path.join(home, 'pubring.gpg')
key_list = self.config.get(constants.IMPORTER_CONFIG_KEY_GPG_KEYS, [])
gpg = GPG(gnupghome=home)
map(gpg.import_keys, key_list)
key_ids = [key['keyid'] for key in gpg.list_keys()]
return path, key_ids
def load_local_pub_keys(local_path):
gpg = GPG(gnupghome = local_path)
# Get GPG public keys
public_keys = gpg.list_keys()
key_info = {'GPG':[], 'Ed25519':[]}
# Collect key info for GPG keys
for key in public_keys:
uids = []
for uid in key['uids']:
name, email = re.search(
'(.*) <(.*)>', uid).groups()
uids.append({'name': name,
'email': email})
key_info['GPG'].append({
'key_id': key['keyid'],
'uids': uids})
# Search for any Ed25519 verify key files
# and store the created VerifyKey objects
ed25519_key_path = f"{local_path}/{ED25519_KEY}"
try:
with open(ed25519_key_path, 'r') as f:
content = f.read()
# Extract the Base64 string
# representing the serialized verify key
# and encode it back to bytes
key_b64 = re.search(
"-----BEGIN PUBLIC KEY-----\n"
"(.*)\n-----END PUBLIC KEY-----",
content
).group(1).encode()
# Create a verify key object from
# the Base64 serialized bytes
key_info['Ed25519'].append(
VerifyKey(key_b64, encoder=Base64Encoder)
)
except:
pass
return key_info
def clean_key(self):
"""
Raises ValidationError if the entered key is invalid or includes a GPG private key.
"""
data = self.cleaned_data['key']
with TemporaryDirectory() as temp_dir:
gpg_keychain = GPG(gnupghome=temp_dir)
res = gpg_keychain.import_keys(data)
if not res:
errors = [forms.ValidationError(_("Invalid key."), code='invalid')]
for attr in ['status', 'stderr']: # not all fields are always present
if hasattr(res, attr):
errors.append(forms.ValidationError("%(name)s: %(value)s",
params={'name': attr, 'value': getattr(res, attr)},
code=attr))
raise forms.ValidationError(errors)
if len(gpg_keychain.list_keys(True)) > 0: # check existance of private keys
raise forms.ValidationError(_("Import public keys only, no private keys! "
"You should consider the private key(s) compromised."),
code='private_key')
return data
def _get_decryption(gpg: gnupg.GPG, configuration_kwargs: Dict) -> gnupg.GPG:
ssm_name = '{environment}.{name}-{company_name}'.format(
environment=settings.ENVIRONMENT,
name=configuration_kwargs.get('secret_name'),
company_name=configuration_kwargs.get('company_name'))
secret_key = []
for key in ['public', 'private']:
logger.info(f'Handling {key} for decryption')
ssm_name_extended = f'{ssm_name}-{key}'
value = parameter_store.get_keys(profile_name='ssm',
name=ssm_name_extended)
secret_key.append(value)
secret_keys = ''.join(secret_key)
gpg.import_keys(secret_keys)
keys = gpg.list_keys(True)
gpg.trust_keys(keys.fingerprints, 'TRUST_ULTIMATE')
return gpg
class GLBPGP(object):
"""
PGP has not a dedicated class, because one of the function is called inside a transact, and
I'm not quite confident on creating an object that operates on the filesystem knowing
that would be run also on the Storm cycle.
"""
def __init__(self):
"""
every time is needed, a new keyring is created here.
"""
try:
temp_pgproot = os.path.join(GLSettings.pgproot,
"%s" % generateRandomKey(8))
os.makedirs(temp_pgproot, mode=0700)
self.gnupg = GPG(gnupghome=temp_pgproot,
options=['--trust-model', 'always'])
self.gnupg.encoding = "UTF-8"
except OSError as ose:
log.err("Critical, OS error in operating with GnuPG home: %s" %
ose)
raise
except Exception as excep:
log.err("Unable to instance PGP object: %s" % excep)
raise
def load_key(self, key):
"""
@param key:
@return: True or False, True only if a key is effectively importable and listed.
"""
try:
import_result = self.gnupg.import_keys(key)
except Exception as excep:
log.err("Error in PGP import_keys: %s" % excep)
raise errors.PGPKeyInvalid
if len(import_result.fingerprints) == 0:
raise errors.PGPKeyInvalid
fingerprint = import_result.fingerprints[0]
# looking if the key is effectively reachable
try:
all_keys = self.gnupg.list_keys()
except Exception as excep:
log.err("Error in PGP list_keys: %s" % excep)
raise errors.PGPKeyInvalid
expiration = datetime.utcfromtimestamp(0)
for key in all_keys:
if key['fingerprint'] == fingerprint:
if key['expires']:
expiration = datetime.utcfromtimestamp(int(key['expires']))
break
return {
'fingerprint': fingerprint,
'expiration': expiration,
}
def encrypt_file(self, key_fingerprint, input_file, output_path):
"""
Encrypt a file with the specified PGP key
"""
encrypted_obj = self.gnupg.encrypt_file(input_file,
str(key_fingerprint),
output=output_path)
if not encrypted_obj.ok:
raise errors.PGPKeyInvalid
return encrypted_obj, os.stat(output_path).st_size
def encrypt_message(self, key_fingerprint, plaintext):
"""
Encrypt a text message with the specified key
"""
encrypted_obj = self.gnupg.encrypt(plaintext, str(key_fingerprint))
if not encrypted_obj.ok:
raise errors.PGPKeyInvalid
return str(encrypted_obj)
def destroy_environment(self):
try:
shutil.rmtree(self.gnupg.gnupghome)
except Exception as excep:
log.err("Unable to clean temporary PGP environment: %s: %s" %
(self.gnupg.gnupghome, excep))
class PGPContext(object):
"""
PGP does not have a dedicated class, because one of the function is called inside a transact.
I'm not confident creating an object that operates on the filesystem knowing that
would be run also on the Storm cycle.
"""
def __init__(self, tempdirprefix=None):
"""
every time is needed, a new keyring is created here.
"""
if tempdirprefix is None:
tempdir = tempfile.mkdtemp()
else:
tempdir = tempfile.mkdtemp(prefix=tempdirprefix)
try:
gpgbinary = 'gpg'
if os.path.exists('/usr/bin/gpg1'):
gpgbinary = 'gpg1'
self.gnupg = GPG(gpgbinary=gpgbinary,
gnupghome=tempdir,
options=['--trust-model', 'always'])
self.gnupg.encoding = "UTF-8"
except OSError as excep:
log.err("Critical, OS error in operating with GnuPG home: %s",
excep)
raise
except Exception as excep:
log.err("Unable to instance PGP object: %s" % excep)
raise
def load_key(self, key):
"""
@param key
@return: a dict with the expiration date and the key fingerprint
"""
try:
import_result = self.gnupg.import_keys(key)
except Exception as excep:
log.err("Error in PGP import_keys: %s", excep)
raise errors.InputValidationError
if not import_result.fingerprints:
raise errors.InputValidationError
fingerprint = import_result.fingerprints[0]
# looking if the key is effectively reachable
try:
all_keys = self.gnupg.list_keys()
except Exception as excep:
log.err("Error in PGP list_keys: %s", excep)
raise errors.InputValidationError
expiration = datetime.utcfromtimestamp(0)
for k in all_keys:
if k['fingerprint'] == fingerprint:
if k['expires']:
expiration = datetime.utcfromtimestamp(int(k['expires']))
break
return {'fingerprint': fingerprint, 'expiration': expiration}
def encrypt_file(self, key_fingerprint, input_file, output_path):
"""
Encrypt a file with the specified PGP key
"""
encrypted_obj = self.gnupg.encrypt_file(input_file,
str(key_fingerprint),
output=output_path)
if not encrypted_obj.ok:
raise errors.InputValidationError
return encrypted_obj, os.stat(output_path).st_size
def encrypt_message(self, key_fingerprint, plaintext):
"""
Encrypt a text message with the specified key
"""
encrypted_obj = self.gnupg.encrypt(plaintext, str(key_fingerprint))
if not encrypted_obj.ok:
raise errors.InputValidationError
return str(encrypted_obj)
def __del__(self):
try:
shutil.rmtree(self.gnupg.gnupghome)
except Exception as excep:
log.err("Unable to clean temporary PGP environment: %s: %s",
self.gnupg.gnupghome, excep)
class GLBPGP(object):
"""
PGP has not a dedicated class, because one of the function is called inside a transact, and
I'm not quite confident on creating an object that operates on the filesystem knowing
that would be run also on the Storm cycle.
"""
def __init__(self):
"""
every time is needed, a new keyring is created here.
"""
try:
temp_pgproot = os.path.join(GLSettings.pgproot, "%s" % rstr.xeger(r'[A-Za-z0-9]{8}'))
os.makedirs(temp_pgproot, mode=0700)
self.pgph = GPG(gnupghome=temp_pgproot, options=['--trust-model', 'always'])
self.pgph.encoding = "UTF-8"
except OSError as ose:
log.err("Critical, OS error in operating with GnuPG home: %s" % ose)
raise
except Exception as excep:
log.err("Unable to instance PGP object: %s" % excep)
raise
def load_key(self, key):
"""
@param key:
@return: True or False, True only if a key is effectively importable and listed.
"""
try:
import_result = self.pgph.import_keys(key)
except Exception as excep:
log.err("Error in PGP import_keys: %s" % excep)
raise errors.PGPKeyInvalid
if len(import_result.fingerprints) != 1:
raise errors.PGPKeyInvalid
fingerprint = import_result.fingerprints[0]
# looking if the key is effectively reachable
try:
all_keys = self.pgph.list_keys()
except Exception as excep:
log.err("Error in PGP list_keys: %s" % excep)
raise errors.PGPKeyInvalid
info = u""
expiration = datetime.utcfromtimestamp(0)
for key in all_keys:
if key['fingerprint'] == fingerprint:
if key['expires']:
expiration = datetime.utcfromtimestamp(int(key['expires']))
exp_date = datetime_to_day_str(expiration)
else:
exp_date = u'Never'
info += "Key length: %s\n" % key['length']
info += "Key expiration: %s\n" % exp_date
try:
for uid in key['uids']:
info += "\t%s\n" % uid
except Exception as excep:
log.err("Error in PGP key format/properties: %s" % excep)
raise errors.PGPKeyInvalid
break
if not len(info):
log.err("Key apparently imported but unable to reload it")
raise errors.PGPKeyInvalid
return {
'fingerprint': fingerprint,
'expiration': expiration,
'info': info
}
def encrypt_file(self, key_fingerprint, plainpath, filestream, output_path):
"""
@param pgp_key_public:
@param plainpath:
@return:
"""
encrypt_obj = self.pgph.encrypt_file(filestream, str(key_fingerprint))
if not encrypt_obj.ok:
raise errors.PGPKeyInvalid
log.debug("Encrypting for key %s file %s (%d bytes)" %
(key_fingerprint,
plainpath, len(str(encrypt_obj))))
encrypted_path = os.path.join(os.path.abspath(output_path),
"pgp_encrypted-%s" % rstr.xeger(r'[A-Za-z0-9]{16}'))
try:
with open(encrypted_path, "w+") as f:
f.write(str(encrypt_obj))
return encrypted_path, len(str(encrypt_obj))
except Exception as excep:
log.err("Error in writing PGP file output: %s (%s) bytes %d" %
(excep.message, encrypted_path, len(str(encrypt_obj)) ))
raise errors.InternalServerError("Error in writing [%s]" % excep.message)
def encrypt_message(self, key_fingerprint, plaintext):
"""
@param plaindata:
An arbitrary long text that would be encrypted
@param receiver_desc:
The output of
globaleaks.handlers.admin.admin_serialize_receiver()
dictionary. It contain the fingerprint of the Receiver PUBKEY
@return:
The unicode of the encrypted output (armored)
"""
# This second argument may be a list of fingerprint, not just one
encrypt_obj = self.pgph.encrypt(plaintext, str(key_fingerprint))
if not encrypt_obj.ok:
raise errors.PGPKeyInvalid
log.debug("Encrypting for key %s %d byte of plain data (%d cipher output)" %
(key_fingerprint,
len(plaintext), len(str(encrypt_obj))))
return str(encrypt_obj)
def destroy_environment(self):
try:
shutil.rmtree(self.pgph.gnupghome)
except Exception as excep:
log.err("Unable to clean temporary PGP environment: %s: %s" % (self.pgph.gnupghome, excep))
class Test(unittest.TestCase):
'''Unit test cases for signing envelopes from the Learning Registry'''
def __init__(self, methodName="runTest"):
self.gpgbin="/usr/local/bin/gpg"
self.gnupgHome = os.path.expanduser(os.path.join("~", ".gnupg"))
self.gpg = GPG(gpgbinary=self.gpgbin, gnupghome=self.gnupgHome)
unittest.TestCase.__init__(self, methodName)
self.testDataDir = None
configFile = os.path.join("config.cfg")
if os.path.exists(configFile):
config = json.load(file(configFile))
if config.has_key("global"):
if config["global"].has_key("testdata") and os.path.exists(config["global"]["testdata"]):
self.testDataDir = config["global"]["testdata"]
def setUp(self):
now = time.localtime()
now = calendar.timegm(now)
self.goodEmail = "signTest-{0}@learningregistry.org".format(now)
self.goodRealName = "Autogenerated Sign Test"
self.goodpassphrase = "supersecret"
input = self.gpg.gen_key_input(name_real=self.goodRealName, name_email=self.goodEmail, passphrase=self.goodpassphrase)
self.goodPrivateKey = self.gpg.gen_key(input)
privateKeyAvailable = False
privateKeys = self.gpg.list_keys(secret=True)
for skey in privateKeys:
if skey["keyid"] == self.goodPrivateKey.fingerprint:
privateKeyAvailable = True
self.privateKeyInfo = skey
break
if skey["fingerprint"] == self.goodPrivateKey.fingerprint:
privateKeyAvailable = True
self.privateKeyInfo = skey
break
assert privateKeyAvailable == True, "Could not locate generated Private Key"
self.goodkeyid = self.privateKeyInfo["keyid"]
self.goodowner = self.privateKeyInfo["uids"][0]
self.badkeyid = "XXXXXXXXXXXXXXXX"
self.badpassphrase = "bad passphrase"
self.sampleJSON = '''
{
"_id":"00e3f67232e743b6bc2a079bd98ff55a",
"_rev":"1-8163d32f6cc9996f2b7228d8b5db7962",
"doc_type":"resource_data",
"update_timestamp":"2011-03-14 13:36:04.617999",
"resource_data":"<oai_dc:dc xmlns:oai_dc=\\"http://www.openarchives.org/OAI/2.0/oai_dc/\\" xmlns:dc=\\"http://purl.org/dc/elements/1.1/\\" xmlns:xsi=\\"http://www.w3.org/2001/XMLSchema-instance\\" xmlns=\\"http://www.openarchives.org/OAI/2.0/\\" xsi:schemaLocation=\\"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd\\">\\n<dc:title>A chat about America. October and November, 1884.</dc:title>\\n<dc:creator>J. P.</dc:creator>\\n<dc:subject>United States--Description and travel.</dc:subject>\\n<dc:description>\\"Printed for private circulation only.\\"</dc:description>\\n<dc:description>Electronic reproduction. Washington, D.C. : Library of Congress, [2002-2003]</dc:description>\\n<dc:publisher>Manchester, Palmer & Howe</dc:publisher>\\n<dc:date>1885</dc:date>\\n<dc:type>text</dc:type>\\n<dc:identifier>http://hdl.loc.gov/loc.gdc/lhbtn.12281</dc:identifier>\\n<dc:language>eng</dc:language>\\n<dc:coverage>United States</dc:coverage>\\n</oai_dc:dc>\\n ",
"keys":["United States--Description and travel.","eng"],
"submitter_type":"agent",
"resource_data_type":"metadata",
"payload_schema_locator":"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd",
"payload_placement":"inline",
"submitter":"NSDL 2 LR Data Pump",
"payload_schema":["oai_dc"],
"node_timestamp":"2011-03-14 13:36:04.617999",
"doc_version":"0.10.0",
"create_timestamp":"2011-03-14 13:36:04.617999",
"active":true,
"publishing_node":"string",
"resource_locator":"http://hdl.loc.gov/loc.gdc/lhbtn.12281",
"doc_ID":"00e3f67232e743b6bc2a079bd98ff55a",
"TOS": {
"submission_TOS": "http://example.com/tos/unknown",
"submission_attribution": "unidentified"
}
}
'''
self.sampleJSON_strip = '''{"keys": ["United States--Description and travel.", "eng"], "TOS": {"submission_attribution": "unidentified", "submission_TOS": "http://example.com/tos/unknown"}, "payload_placement": "inline", "active": true, "resource_locator": "http://hdl.loc.gov/loc.gdc/lhbtn.12281", "doc_type": "resource_data", "resource_data": "<oai_dc:dc xmlns:oai_dc=\\"http://www.openarchives.org/OAI/2.0/oai_dc/\\" xmlns:dc=\\"http://purl.org/dc/elements/1.1/\\" xmlns:xsi=\\"http://www.w3.org/2001/XMLSchema-instance\\" xmlns=\\"http://www.openarchives.org/OAI/2.0/\\" xsi:schemaLocation=\\"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd\\">\\n<dc:title>A chat about America. October and November, 1884.</dc:title>\\n<dc:creator>J. P.</dc:creator>\\n<dc:subject>United States--Description and travel.</dc:subject>\\n<dc:description>\\"Printed for private circulation only.\\"</dc:description>\\n<dc:description>Electronic reproduction. Washington, D.C. : Library of Congress, [2002-2003]</dc:description>\\n<dc:publisher>Manchester, Palmer & Howe</dc:publisher>\\n<dc:date>1885</dc:date>\\n<dc:type>text</dc:type>\\n<dc:identifier>http://hdl.loc.gov/loc.gdc/lhbtn.12281</dc:identifier>\\n<dc:language>eng</dc:language>\\n<dc:coverage>United States</dc:coverage>\\n</oai_dc:dc>\\n ", "submitter_type": "agent", "resource_data_type": "metadata", "payload_schema_locator": "http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd", "payload_schema": ["oai_dc"], "doc_version": "0.10.0", "submitter": "NSDL 2 LR Data Pump"}'''
self.sampleJSON_strip_normal = '''{"keys": ["United States--Description and travel.", "eng"], "TOS": {"submission_attribution": "unidentified", "submission_TOS": "http://example.com/tos/unknown"}, "payload_placement": "inline", "active": "true", "resource_locator": "http://hdl.loc.gov/loc.gdc/lhbtn.12281", "doc_type": "resource_data", "resource_data": "<oai_dc:dc xmlns:oai_dc=\\"http://www.openarchives.org/OAI/2.0/oai_dc/\\" xmlns:dc=\\"http://purl.org/dc/elements/1.1/\\" xmlns:xsi=\\"http://www.w3.org/2001/XMLSchema-instance\\" xmlns=\\"http://www.openarchives.org/OAI/2.0/\\" xsi:schemaLocation=\\"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd\\">\\n<dc:title>A chat about America. October and November, 1884.</dc:title>\\n<dc:creator>J. P.</dc:creator>\\n<dc:subject>United States--Description and travel.</dc:subject>\\n<dc:description>\\"Printed for private circulation only.\\"</dc:description>\\n<dc:description>Electronic reproduction. Washington, D.C. : Library of Congress, [2002-2003]</dc:description>\\n<dc:publisher>Manchester, Palmer & Howe</dc:publisher>\\n<dc:date>1885</dc:date>\\n<dc:type>text</dc:type>\\n<dc:identifier>http://hdl.loc.gov/loc.gdc/lhbtn.12281</dc:identifier>\\n<dc:language>eng</dc:language>\\n<dc:coverage>United States</dc:coverage>\\n</oai_dc:dc>\\n ", "submitter_type": "agent", "resource_data_type": "metadata", "payload_schema_locator": "http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd", "payload_schema": ["oai_dc"], "doc_version": "0.10.0", "submitter": "NSDL 2 LR Data Pump"}'''
self.sampleJSON_strip_normal_bencode = '''d3:TOSd14:submission_TOS30:http://example.com/tos/unknown22:submission_attribution12:unidentifiede6:active4:true8:doc_type13:resource_data11:doc_version6:0.10.04:keysl38:United States--Description and travel.3:enge17:payload_placement6:inline14:payload_schemal6:oai_dce22:payload_schema_locator90:http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd13:resource_data968:<oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://www.openarchives.org/OAI/2.0/" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">\n<dc:title>A chat about America. October and November, 1884.</dc:title>\n<dc:creator>J. P.</dc:creator>\n<dc:subject>United States--Description and travel.</dc:subject>\n<dc:description>"Printed for private circulation only."</dc:description>\n<dc:description>Electronic reproduction. Washington, D.C. : Library of Congress, [2002-2003]</dc:description>\n<dc:publisher>Manchester, Palmer & Howe</dc:publisher>\n<dc:date>1885</dc:date>\n<dc:type>text</dc:type>\n<dc:identifier>http://hdl.loc.gov/loc.gdc/lhbtn.12281</dc:identifier>\n<dc:language>eng</dc:language>\n<dc:coverage>United States</dc:coverage>\n</oai_dc:dc>\n 18:resource_data_type8:metadata16:resource_locator38:http://hdl.loc.gov/loc.gdc/lhbtn.122819:submitter19:NSDL 2 LR Data Pump14:submitter_type5:agente'''
self.sampleJSON_sha256 = '''ef1b3b63adc663602c7a3c7595951b2761b34f5f6490ea1acee3df0fd97db03c'''
self.sampleKeyLocations = [
"http://example.com/mykey",
"http://example2.com/mykey"
]
self.signatureTemplate = '{{"key_location": [{0}], "key_owner": "'+self.goodowner+'", "signing_method": "LR-PGP.1.0", "signature": "{1}"}}'
def tearDown(self):
self.gpg.delete_keys([self.goodPrivateKey.fingerprint, ], secret=True)
self.gpg.delete_keys([self.goodPrivateKey.fingerprint, ], secret=False)
pass
def testMissingPrivateKey(self):
def missingKey():
try:
sign = Sign_0_21(self.badkeyid)
except UnknownKeyException as e:
assert e.keyid == self.badkeyid, "keyid in exception doesn't match key passed to sign."
raise e
self.assertRaises(UnknownKeyException, missingKey)
def testPresentPrivateKey(self):
sign = Sign_0_21(self.goodkeyid)
assert sign.privateKeyID == self.goodkeyid
def testStrip(self):
origJson = json.loads(self.sampleJSON)
benchmark = json.loads(self.sampleJSON_strip)
signer = Sign_0_21(self.goodkeyid)
stripJson = signer._stripEnvelope(origJson)
assert benchmark == stripJson
def testStripNormal(self):
origJson = json.loads(self.sampleJSON)
benchmark = json.loads(self.sampleJSON_strip_normal)
signer = Sign_0_21(self.goodkeyid)
stripJson = signer._stripEnvelope(origJson)
normalJson = signer._bnormal(stripJson)
assert benchmark == normalJson
def testStripNormalBencode(self):
origJson = json.loads(self.sampleJSON)
benchmark = self.sampleJSON_strip_normal_bencode
signer = Sign_0_21(self.goodkeyid)
stripJson = signer._stripEnvelope(origJson)
normalJson = signer._bnormal(stripJson)
bencodeJson = signer._buildCanonicalString(normalJson)
assert benchmark == bencodeJson
def testStripNormalBencodeHash(self):
origJson = json.loads(self.sampleJSON)
benchmark = self.sampleJSON_sha256
signer = Sign_0_21(self.goodkeyid)
stripJson = signer._stripEnvelope(origJson)
normalJson = signer._bnormal(stripJson)
bencodeJson = signer._buildCanonicalString(normalJson)
hash = signer._hash(bencodeJson)
assert benchmark == hash
def testGetMessage(self):
origJson = json.loads(self.sampleJSON)
benchmark = self.sampleJSON_sha256
signer = Sign_0_21(self.goodkeyid)
message = signer.get_message(origJson)
assert benchmark == message
def testPrivateKeyOwner(self):
benchmark = self.goodowner
signer = Sign_0_21(self.goodkeyid)
assert benchmark == signer._get_privatekey_owner()
def testSigBlock(self):
origJson = json.loads(self.sampleJSON)
arbitrarySigdata = "ABCDEF0123456789-abcdef"
arbitraryKeyLoc = self.sampleKeyLocations
keyloc = ",".join(map(lambda x: '"{0}"'.format(x), arbitraryKeyLoc))
benchmark = json.loads(self.signatureTemplate.format(keyloc, arbitrarySigdata))
signer = Sign_0_21(self.goodkeyid, passphrase=self.goodpassphrase, publicKeyLocations=arbitraryKeyLoc)
assert benchmark == signer._get_sig_block(arbitrarySigdata)
def testSign(self):
origJson = json.loads(self.sampleJSON)
arbitraryKeyLoc = self.sampleKeyLocations
signer = Sign_0_21(self.goodkeyid, passphrase=self.goodpassphrase, publicKeyLocations=arbitraryKeyLoc)
signed = signer.sign(origJson)
assert signed.has_key("digital_signature")
sig = signed["digital_signature"]
assert sig.has_key("signature")
assert sig["signature"] != None and len(sig["signature"]) > 0
def testSignUnicode(self):
if self.testDataDir == None:
log.info("Skipping test, test data directory not set.")
return
import codecs
fileName = "2011-02-28Metadata1004.json"
unsigned = json.load(codecs.open(os.path.join(self.testDataDir, fileName), "r", "utf-8-sig"))
arbitraryKeyLoc = self.sampleKeyLocations
signer = Sign_0_21(self.goodkeyid, passphrase=self.goodpassphrase, publicKeyLocations=arbitraryKeyLoc)
signed = signer.sign(unsigned)
assert signed.has_key("digital_signature")
sig = signed["digital_signature"]
assert sig.has_key("signature")
assert sig["signature"] != None and len(sig["signature"]) > 0
def testSignLRTestData(self):
if self.testDataDir == None:
log.info("Skipping test, test data directory not set.")
return
import codecs
allfiles = os.listdir(self.testDataDir)
for fileName in allfiles:
log.info("Trying to sign %s" % (fileName, ))
unsigned = json.load(codecs.open(os.path.join(self.testDataDir, fileName), "r", "utf-8-sig"))
arbitraryKeyLoc = self.sampleKeyLocations
signer = Sign_0_21(self.goodkeyid, passphrase=self.goodpassphrase, publicKeyLocations=arbitraryKeyLoc)
signed = signer.sign(unsigned)
assert signed.has_key("digital_signature")
sig = signed["digital_signature"]
assert sig.has_key("signature")
assert sig["signature"] != None and len(sig["signature"]) > 0
class PGPContext(object):
def __init__(self, tempdirprefix=None):
if tempdirprefix is None:
tempdir = tempfile.mkdtemp()
else:
tempdir = tempfile.mkdtemp(prefix=tempdirprefix)
try:
self.gnupg = GPG(gnupghome=tempdir,
options=['--trust-model', 'always'])
self.gnupg.encoding = "UTF-8"
except OSError as excep:
log.err("Critical, OS error in operating with GnuPG home: %s",
excep)
raise
except Exception as excep:
log.err("Unable to instance PGP object: %s" % excep)
raise
def load_key(self, key):
"""
@param key
@return: a dict with the expiration date and the key fingerprint
"""
try:
import_result = self.gnupg.import_keys(key)
except Exception as excep:
log.err("Error in PGP import_keys: %s", excep)
raise errors.InputValidationError
if not import_result.fingerprints:
raise errors.InputValidationError
fingerprint = import_result.fingerprints[0]
# looking if the key is effectively reachable
try:
all_keys = self.gnupg.list_keys()
except Exception as excep:
log.err("Error in PGP list_keys: %s", excep)
raise errors.InputValidationError
expiration = datetime.utcfromtimestamp(0)
for k in all_keys:
if k['fingerprint'] == fingerprint:
if k['expires']:
expiration = datetime.utcfromtimestamp(int(k['expires']))
break
return {'fingerprint': fingerprint, 'expiration': expiration}
def encrypt_file(self, key_fingerprint, input_file, output_path):
"""
Encrypt a file with the specified PGP key
"""
encrypted_obj = self.gnupg.encrypt_file(input_file,
key_fingerprint,
output=output_path)
if not encrypted_obj.ok:
raise errors.InputValidationError
return encrypted_obj, os.stat(output_path).st_size
def encrypt_message(self, key_fingerprint, plaintext):
"""
Encrypt a text message with the specified key
"""
encrypted_obj = self.gnupg.encrypt(plaintext, key_fingerprint)
if not encrypted_obj.ok:
raise errors.InputValidationError
return str(encrypted_obj)
def __del__(self):
try:
shutil.rmtree(self.gnupg.gnupghome)
except Exception as excep:
log.err("Unable to clean temporary PGP environment: %s: %s",
self.gnupg.gnupghome, excep)
class OpenPgpFactory(GenericFactory):
"""Provides OpenPGP functionality based on GnuPG."""
implements(ICipherModule)
gpg_binary = Option('crypto', 'gpg_binary', 'gpg',
"""GnuPG binary name, allows for full path too.
Value 'gpg' is same default as in python-gnupg itself.
For usual installations location of the gpg binary is auto-detected.
""")
gpg_home = Option('crypto', 'gpg_home', '.gpg',
"""Directory containing keyring files.
In case of wrong configuration missing keyring files without content
will be created in the configured location, provided necessary
write permssion is granted for the corresponding parent directory.
""")
priv_key = Option('crypto', 'gpg_private_key', '',
"""Key ID of private key (last 8 chars or more).
If unset, a private key will be selected from keyring automagicly.
The password must be available i.e. provided by running gpg-agent
(not yet available) or empty (security trade-off). No private key
operation can happen before the key has been unlocked successfully.
""")
name = 'openpgp'
_supported_keys = ['name_real', 'name_comment', 'name_email',
'key_type', 'key_length',
'subkey_type', 'subkey_length',
'expire_date', 'passphrase']
def __init__(self):
try:
from gnupg import GPG
except ImportError:
raise TracError(_("""Unable to load the python-gnupg module.
Please check and correct your installation."""))
try:
# Initialize the GnuPG instance.
path = os.path.join(os.path.abspath(self.env.path), self.gpg_home)
self.env.log.debug('Using GnuPG home dir: ' + str(path))
self.gpg = GPG(gpgbinary=self.gpg_binary, gnupghome=path)
except ValueError:
raise TracError(_("""Missing the crypto binary. Please check and
set full path with option 'gpg_binary'."""))
# IKeyVault methods
def keys(self, private=False, id_only=False):
"""Returns the list of all available keys."""
# DEVEL: Cache this for performance.
if not id_only:
return self.gpg.list_keys(private) # same as gpg.list_keys(False)
keys = []
for key in self.gpg.list_keys(private):
keys.append(key['fingerprint'])
return keys
def create_key(self, **kwargs):
"""Generate a new OpenPGP key pair."""
# Sanitize input.
for k in kwargs.keys():
if k not in self._supported_keys:
kwargs.pop(k)
input_data = self.gpg.gen_key_input(**kwargs)
try:
return self.gpg.gen_key(input_data)
except ValueError, e:
return False, e
def import_from_dump(file):
"""Import from an SKS dump."""
# First, load a new GPG instance.
gpg = GPG(gnupghome="/tmp/skier-import")
# Read data in
dumpf = open(file, 'rb')
data = dumpf.read()
dump = pgpdump.BinaryData(data)
dumpf.close()
# Import the keydump into gpg
try:
gpg.import_keys(data)
except:
# Yeah, whatever.
pass
"""
# Read in packets
packets_flat = []
try:
for packet in dump.packets():
packets_flat.append(packet)
except Exception as e:
# Some SKS data is malformed.
print("SKS data in {} is malformed - truncating at packet {}".format(file, len(packets_flat)+1))
# Unflatten list into sublists, starting at each new Publickey packet.
packets = []
tmpl = []
for packet in packets_flat:
if isinstance(packet, pgpdump.packet.PublicKeyPacket) and not isinstance(packet, pgpdump.packet.PublicSubkeyPacket) and tmpl:
tmpl.insert(0, packet)
gpg_output = gpg.export_keys(keyids=[packet.fingerprint.decode()])
if gpg_output:
tmpl.append(gpg_output)
packets.append(tmpl)
tmpl = []
continue
else:
tmpl.append(packet)
packets.append(tmpl)
"""
armored = []
for key in gpg.list_keys():
tmpkey = gpg.export_keys(keyids=key['fingerprint'])
if not tmpkey:
print("{} is malformed - not importing".format(key['fingerprint']))
armored.append(tmpkey)
print("Importing {} keys".format(len(armored)))
"""
for gpack in packets:
keyinfo = KeyInfo.pgp_dump(None, packets=gpack)
dbob = db.Key.from_keyinfo(keyinfo)
db.db.session.add(dbob)
db.db.session.commit()
"""
for key in armored:
kyinfo = KeyInfo.pgp_dump(key)
if not kyinfo.fingerprint:
print("Key {} is malformed - cannot be imported".format(kyinfo))
else:
dbob = db.Key.from_keyinfo(kyinfo)
db.db.session.add(dbob)
db.db.session.commit()
shutil.rmtree("/tmp/skier-import")
from gnupg import GPG
gpg_test_fingerprint = "0E315D7E40CE9FDC28F9E7C6CFE237CCFF068D3E"
passphrase = "mate1234"
gpg = GPG(binary='/usr/bin/gpg2',
homedir='~/.gnupg',
keyring='pubring.gpg',
secring='secring.gpg')
stored_keys = gpg.list_keys()
message = "This is a test message to be encripted."
options_enc = {
'passphrase': passphrase,
'armor': True,
'default_key': gpg_test_fingerprint
}
encrypted_msg = gpg.encrypt(message, gpg_test_fingerprint, **options_enc)
print(encrypted_msg)
options_dec = {
'passphrase': passphrase
}
decrypted_msg = gpg.decrypt(str(encrypted_msg), **options_dec)
assert message == str(decrypted_msg)
print("This is the original message:", decrypted_msg)
class CryptoTxt:
"""Crypto operation provider for plaintext.
We use GnuPG for now. Support for X.509 and other options might
appear in the future.
"""
def __init__(self, gpg_binary, gpg_home):
"""Initialize the GnuPG instance."""
self.gpg_binary = gpg_binary
self.gpg_home = gpg_home
if not GPG:
raise TracError(_("Unable to load the python-gnupg module. "
"Please check and correct your installation."))
try:
self.gpg = GPG(gpgbinary=self.gpg_binary, gnupghome=self.gpg_home)
except ValueError:
raise TracError(_("Missing the crypto binary. Please check and "
"set full path with option 'gpg_binary'."))
else:
# get list of available public keys once for later use
self.pub_keys = self.gpg.list_keys()
def sign(self, content, private_key=None):
private_key = self._get_private_key(private_key)
cipher = self.gpg.sign(content, keyid=private_key, passphrase='')
return str(cipher)
def encrypt(self, content, pubkeys):
# always_trust needed for making it work with just any pubkey
cipher = self.gpg.encrypt(content, pubkeys, always_trust=True)
return str(cipher)
def sign_encrypt(self, content, pubkeys, private_key=None):
private_key = self._get_private_key(private_key)
# always_trust needed for making it work with just any pubkey
cipher = self.gpg.encrypt(content, pubkeys, always_trust=True,
sign=private_key, passphrase='')
return str(cipher)
def get_pubkey_ids(self, addr):
"""Find public key with UID matching address to encrypt to."""
pubkey_ids = []
if self.pub_keys and 'uids' in self.pub_keys[-1] and \
'fingerprint' in self.pub_keys[-1]:
# compile pattern before use for better performance
rcpt_re = re.compile(addr)
for k in self.pub_keys:
for uid in k['uids']:
match = rcpt_re.search(uid)
if match is not None:
# check for key expiration
if k['expires'] == '':
pubkey_ids.append(k['fingerprint'][-16:])
elif (time.time() + 60) < float(k['expires']):
pubkey_ids.append(k['fingerprint'][-16:])
break
return pubkey_ids
def _get_private_key(self, privkey=None):
"""Find private (secret) key to sign with."""
# read private keys from keyring
privkeys = self.gpg.list_keys(True) # True => private keys
if privkeys > 0 and 'fingerprint' in privkeys[-1]:
fingerprints = []
for k in privkeys:
fingerprints.append(k['fingerprint'])
else:
# no private key in keyring
return None
if privkey:
# check for existence of private key received as argument
# DEVEL: check for expiration as well
if 7 < len(privkey) <= 40:
for fp in fingerprints:
if fp.endswith(privkey):
# work with last 16 significant chars internally,
# even if only 8 are required in trac.ini
privkey = fp[-16:]
break
# no fingerprint matching key ID
else:
privkey = None
else:
# reset invalid key ID
privkey = None
else:
# select (last) private key from keyring
privkey = fingerprints[-1][-16:]
return privkey
class PGPContext(object):
def __init__(self, tempdirprefix=None):
if tempdirprefix is None:
tempdir = tempfile.mkdtemp()
else:
tempdir = tempfile.mkdtemp(prefix=tempdirprefix)
try:
gpgbinary='gpg'
if os.path.exists('/usr/bin/gpg1'):
gpgbinary='gpg1'
self.gnupg = GPG(gpgbinary=gpgbinary, gnupghome=tempdir, options=['--trust-model', 'always'])
self.gnupg.encoding = "UTF-8"
except OSError as excep:
log.err("Critical, OS error in operating with GnuPG home: %s", excep)
raise
except Exception as excep:
log.err("Unable to instance PGP object: %s" % excep)
raise
def load_key(self, key):
"""
@param key
@return: a dict with the expiration date and the key fingerprint
"""
try:
import_result = self.gnupg.import_keys(key)
except Exception as excep:
log.err("Error in PGP import_keys: %s", excep)
raise errors.InputValidationError
if not import_result.fingerprints:
raise errors.InputValidationError
fingerprint = import_result.fingerprints[0]
# looking if the key is effectively reachable
try:
all_keys = self.gnupg.list_keys()
except Exception as excep:
log.err("Error in PGP list_keys: %s", excep)
raise errors.InputValidationError
expiration = datetime.utcfromtimestamp(0)
for k in all_keys:
if k['fingerprint'] == fingerprint:
if k['expires']:
expiration = datetime.utcfromtimestamp(int(k['expires']))
break
return {
'fingerprint': fingerprint,
'expiration': expiration
}
def encrypt_file(self, key_fingerprint, input_file, output_path):
"""
Encrypt a file with the specified PGP key
"""
encrypted_obj = self.gnupg.encrypt_file(input_file, str(key_fingerprint), output=output_path)
if not encrypted_obj.ok:
raise errors.InputValidationError
return encrypted_obj, os.stat(output_path).st_size
def encrypt_message(self, key_fingerprint, plaintext):
"""
Encrypt a text message with the specified key
"""
encrypted_obj = self.gnupg.encrypt(plaintext, str(key_fingerprint))
if not encrypted_obj.ok:
raise errors.InputValidationError
return str(encrypted_obj)
def __del__(self):
try:
shutil.rmtree(self.gnupg.gnupghome)
except Exception as excep:
log.err("Unable to clean temporary PGP environment: %s: %s", self.gnupg.gnupghome, excep)
def test_check_git_commits_ok():
with TemporaryDirectory() as tmp_git_repo:
gpg_tmp_home = os.path.join(tmp_git_repo, 'gpg_tmp_home')
# Included in the repository to avoid problems with low entropy in CI
# environments.
# input_data = gpg.gen_key_input(
# key_type='RSA',
# subkey_type='RSA',
# key_length=1024,
# subkey_length=1024,
# name_comment='This is only a test key who’s private key is publicly know. Don’t use this key for anything!1!',
# name_email='*****@*****.**',
# # Has already expired at the time of creation to ensure no one will ever use the key.
# expire_date='2012-12-24', # Needs to be set to the 24 for the key to expire on 23.
# # Hm, Ok, gpg does not do that by default. `--faked-system-time`
# # could be used to force it but that would require cmd access.
# # https://www.gnupg.org/documentation/manuals/gnupg/Unattended-GPG-key-generation.html
# # "gpg: Invalid option "--faked-system-time"" :(
# # Only: gpg2 --batch --gen-key --debug=0 --faked-system-time '2342-05-23'
# # has been confirmed to work from current Debian Stretch.
# # Faking it manually anyway …
# )
# print(input_data)
# print(gpg.gen_key(input_data))
shutil.copytree(debops_keyring_fake_gnupg_home, gpg_tmp_home)
gpg = GPG(gnupghome=gpg_tmp_home)
os.chmod(gpg_tmp_home, 0o700)
for r, d, f in os.walk(gpg_tmp_home):
os.chmod(r, 0o700)
gpg_key_fingerprint = gpg.list_keys()[0]['fingerprint']
gpg_edit_key_cmd = subprocess.Popen(
[
'gpg', '--homedir', gpg_tmp_home, '--command-fd', '0',
'--batch', '--edit-key', gpg_key_fingerprint
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
(gpg_edit_key_cmd_stdout,
gpg_edit_key_cmd_stderr) = gpg_edit_key_cmd.communicate(
input='expire\n0\nsave\n'.encode(),
timeout=5,
)
# print(gpg_edit_key_cmd_stderr.decode())
# print(subprocess.check_output(['gpg', '--homedir', gpg_tmp_home, '--list-public-keys']).decode())
if 'expires: never' not in str(gpg_edit_key_cmd_stderr):
raise Exception("Could not change expiration date.")
tmp_keyring_dir = os.path.join(tmp_git_repo, 'tmp-keyring-gpg')
tmp_pubkey_file = os.path.join(
tmp_keyring_dir, '0x' + gpg_key_fingerprint[-16:].upper())
os.mkdir(tmp_keyring_dir)
with open(tmp_pubkey_file, 'w') as tmp_pubkey_fh:
tmp_pubkey_fh.write(gpg.export_keys(gpg_key_fingerprint))
git_cmd = git.Git(tmp_git_repo)
git_cmd.update_environment(GNUPGHOME=debops_keyring_fake_gnupg_home, )
git_cmd.init()
git_cmd.config(['user.signingkey', gpg_key_fingerprint])
git_cmd.config(['user.email', '*****@*****.**'])
git_cmd.config(['user.name', 'debops-keyring-test'])
git_cmd.update_environment(GNUPGHOME=os.path.join(
tmp_git_repo, 'gpg_tmp_home'), )
tmp_git_file = os.path.join(tmp_git_repo, 'new-file')
with open(tmp_git_file, 'w') as tmp_git_fh:
tmp_git_fh.write(str(time.time()))
git_cmd.add([tmp_git_file])
git_cmd.commit(['--gpg-sign', '--message', 'Signed commit'])
debops_keyring = Keyring(keyring_name=tmp_keyring_dir, )
debops_keyring.check_git_commits(tmp_git_repo)
# Now make an unsigned commit to ensure that this raises an exception.
with open(tmp_git_file, 'w') as tmp_git_fh:
tmp_git_fh.write(str(time.time()))
git_cmd.add([tmp_git_file])
git_cmd.commit(['--no-gpg-sign', '--message', 'Unsigned commit'])
try:
debops_keyring.check_git_commits(tmp_git_repo)
assert False
except Exception as e:
if 'OpenPGP signature of commit could not be verified' in str(
e) and 'Unsigned commit' in str(e):
assert True
else:
assert False
class TempGPGWrapper(object):
"""
A context manager that wraps a temporary GPG keyring which only contains
the keys given at object creation.
"""
log = Logger()
def __init__(self, keys=None, gpgbinary=None):
"""
Create an empty temporary keyring and import any given C{keys} into
it.
:param keys: OpenPGP key, or list of.
:type keys: OpenPGPKey or list of OpenPGPKeys
:param gpgbinary: Name for GnuPG binary executable.
:type gpgbinary: C{str}
"""
self._gpg = None
self._gpgbinary = gpgbinary
if not keys:
keys = list()
if not isinstance(keys, list):
keys = [keys]
self._keys = keys
def __enter__(self):
"""
Build and return a GPG keyring containing the keys given on
object creation.
:return: A GPG instance containing the keys given on object creation.
:rtype: gnupg.GPG
"""
self._build_keyring()
return self._gpg
def __exit__(self, exc_type, exc_value, traceback):
"""
Ensure the gpg is properly destroyed.
"""
# TODO handle exceptions and log here
self._destroy_keyring()
def _build_keyring(self):
"""
Create a GPG keyring containing the keys given on object creation.
:return: A GPG instance containing the keys given on object creation.
:rtype: gnupg.GPG
"""
privkeys = [key for key in self._keys if key and key.private is True]
publkeys = [key for key in self._keys if key and key.private is False]
# here we filter out public keys that have a correspondent
# private key in the list because the private key_data by
# itself is enough to also have the public key in the keyring,
# and we want to count the keys afterwards.
privfps = map(lambda privkey: privkey.fingerprint, privkeys)
publkeys = filter(
lambda pubkey: pubkey.fingerprint not in privfps, publkeys)
listkeys = lambda: self._gpg.list_keys()
listsecretkeys = lambda: self._gpg.list_keys(secret=True)
try:
self._gpg = GPG(binary=self._gpgbinary,
homedir=tempfile.mkdtemp())
except TypeError:
# compat-mode with python-gnupg until windows
# support is fixed in gnupg-ng
self._gpg = GPG(gpgbinary=self._gpgbinary,
gnupghome=tempfile.mkdtemp(),
options=[])
leap_assert(len(listkeys()) is 0, 'Keyring not empty.')
# import keys into the keyring:
# concatenating ascii-armored keys, which is correctly
# understood by GPG.
self._gpg.import_keys("".join(
[x.key_data for x in publkeys + privkeys]))
# assert the number of keys in the keyring
leap_assert(
len(listkeys()) == len(publkeys) + len(privkeys),
'Wrong number of public keys in keyring: %d, should be %d)' %
(len(listkeys()), len(publkeys) + len(privkeys)))
leap_assert(
len(listsecretkeys()) == len(privkeys),
'Wrong number of private keys in keyring: %d, should be %d)' %
(len(listsecretkeys()), len(privkeys)))
def _destroy_keyring(self):
"""
Securely erase the keyring.
"""
# TODO: implement some kind of wiping of data or a more
# secure way that
# does not write to disk.
try:
for secret in [True, False]:
for key in self._gpg.list_keys(secret=secret):
self._gpg.delete_keys(
key['fingerprint'],
secret=secret)
leap_assert(len(self._gpg.list_keys()) is 0, 'Keyring not empty!')
except:
raise
finally:
try:
homedir = self._gpg.homedir
except AttributeError:
homedir = self._gpg.gnupghome
leap_assert(homedir != os.path.expanduser('~/.gnupg'),
"watch out! Tried to remove default gnupg home!")
# TODO some windows debug ....
homedir = os.path.normpath(homedir).replace("\\", "/")
homedir = str(homedir.replace("c:/", "c://"))
if platform.system() == "Windows":
self.log.error("BUG! Not erasing folder in Windows")
return
shutil.rmtree(homedir)
args = parser.parse_args()
gpg_home, pass_loc = get_env_vars()
gpg = GPG(gnupghome=gpg_home)
with open(pass_loc, mode='rb') as f:
decoded = gpg.decrypt_file(f, passphrase=getpass())
if decoded.data == b'':
print("Incorrect Passphrase")
exit()
passes = loads(clean_gpg_json(decoded.data))
if args.list:
for key in passes.keys(): print(key)
if args.key is not None:
if args.password:
print_pass(passes, args.key)
if args.user:
print_user(passes, args.key)
if args.set or args.remove:
keys = gpg.list_keys()
for index, key in enumerate(keys):
print(index, key['uids'])
fp = keys[int(input('Pick gpg key: '))]['fingerprint']
key = input('Specify key: ')
if args.set:
passes[key] = getpass()
else:
del passes[key]
with open(pass_loc, 'wb') as f:
f.write(gpg.encrypt(dumps(passes), fp).data)
class Test(unittest.TestCase):
'''Unit test cases for signing envelopes from the Learning Registry'''
def __init__(self, methodName="runTest"):
self.gpgbin = "gpg"
self.gnupgHome = os.path.expanduser(
os.path.abspath(os.path.join("..", "gnupg_home")))
try:
os.makedirs(self.gnupghome)
except:
pass
self.gpg = None
unittest.TestCase.__init__(self, methodName)
self.testDataDir = None
self.testDataUnicode = None
configFile = os.path.join("config.cfg")
if os.path.exists(configFile):
config = json.load(file(configFile))
if "global" in config:
if "testdata" in config["global"] and os.path.exists(
config["global"]["testdata"]):
self.testDataDir = config["global"]["testdata"]
if "testdata_unicode" in config["global"] and os.path.exists(
config["global"]["testdata_unicode"]):
self.testDataUnicode = config["global"]["testdata_unicode"]
def setUp(self):
now = time.localtime()
now = calendar.timegm(now)
self.goodEmail = "signTest-{0}@learningregistry.org".format(now)
self.goodRealName = "Autogenerated Sign Test"
self.goodpassphrase = "supersecret"
try:
for root, dirs, files in os.walk(self.gnupgHome):
for filename in files:
try:
os.unlink(os.path.join(root, filename))
except:
pass
os.removedirs(root)
except:
pass
os.makedirs(self.gnupgHome)
self.gpg = GPG(gpgbinary=self.gpgbin, gnupghome=self.gnupgHome)
input = self.gpg.gen_key_input(name_real=self.goodRealName,
name_email=self.goodEmail,
passphrase=self.goodpassphrase)
self.goodPrivateKey = self.gpg.gen_key(input)
privateKeyAvailable = False
privateKeys = self.gpg.list_keys(secret=True)
for skey in privateKeys:
if skey["keyid"] == self.goodPrivateKey.fingerprint:
privateKeyAvailable = True
self.privateKeyInfo = skey
break
if skey["fingerprint"] == self.goodPrivateKey.fingerprint:
privateKeyAvailable = True
self.privateKeyInfo = skey
break
assert privateKeyAvailable == True, "Could not locate generated Private Key"
self.goodkeyid = self.privateKeyInfo["keyid"]
self.goodowner = self.privateKeyInfo["uids"][0]
self.badkeyid = "XXXXXXXXXXXXXXXX"
self.badpassphrase = "bad passphrase"
self.sampleJSON = '''
{
"_id":"00e3f67232e743b6bc2a079bd98ff55a",
"_rev":"1-8163d32f6cc9996f2b7228d8b5db7962",
"doc_type":"resource_data",
"update_timestamp":"2011-03-14 13:36:04.617999",
"resource_data":"<oai_dc:dc xmlns:oai_dc=\\"http://www.openarchives.org/OAI/2.0/oai_dc/\\" xmlns:dc=\\"http://purl.org/dc/elements/1.1/\\" xmlns:xsi=\\"http://www.w3.org/2001/XMLSchema-instance\\" xmlns=\\"http://www.openarchives.org/OAI/2.0/\\" xsi:schemaLocation=\\"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd\\">\\n<dc:title>A chat about America. October and November, 1884.</dc:title>\\n<dc:creator>J. P.</dc:creator>\\n<dc:subject>United States--Description and travel.</dc:subject>\\n<dc:description>\\"Printed for private circulation only.\\"</dc:description>\\n<dc:description>Electronic reproduction. Washington, D.C. : Library of Congress, [2002-2003]</dc:description>\\n<dc:publisher>Manchester, Palmer & Howe</dc:publisher>\\n<dc:date>1885</dc:date>\\n<dc:type>text</dc:type>\\n<dc:identifier>http://hdl.loc.gov/loc.gdc/lhbtn.12281</dc:identifier>\\n<dc:language>eng</dc:language>\\n<dc:coverage>United States</dc:coverage>\\n</oai_dc:dc>\\n ",
"keys":["United States--Description and travel.","eng"],
"submitter_type":"agent",
"resource_data_type":"metadata",
"payload_schema_locator":"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd",
"payload_placement":"inline",
"submitter":"NSDL 2 LR Data Pump",
"payload_schema":["oai_dc"],
"node_timestamp":"2011-03-14 13:36:04.617999",
"doc_version":"0.10.0",
"create_timestamp":"2011-03-14 13:36:04.617999",
"active":true,
"publishing_node":"string",
"resource_locator":"http://hdl.loc.gov/loc.gdc/lhbtn.12281",
"doc_ID":"00e3f67232e743b6bc2a079bd98ff55a",
"TOS": {
"submission_TOS": "http://example.com/tos/unknown",
"submission_attribution": "unidentified"
}
}
'''
self.sampleJSON__0_23_0 = '''
{
"_id":"00e3f67232e743b6bc2a079bd98ff55a",
"_rev":"1-8163d32f6cc9996f2b7228d8b5db7962",
"doc_type":"resource_data",
"update_timestamp":"2011-03-14 13:36:04.617999",
"resource_data":"<oai_dc:dc xmlns:oai_dc=\\"http://www.openarchives.org/OAI/2.0/oai_dc/\\" xmlns:dc=\\"http://purl.org/dc/elements/1.1/\\" xmlns:xsi=\\"http://www.w3.org/2001/XMLSchema-instance\\" xmlns=\\"http://www.openarchives.org/OAI/2.0/\\" xsi:schemaLocation=\\"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd\\">\\n<dc:title>A chat about America. October and November, 1884.</dc:title>\\n<dc:creator>J. P.</dc:creator>\\n<dc:subject>United States--Description and travel.</dc:subject>\\n<dc:description>\\"Printed for private circulation only.\\"</dc:description>\\n<dc:description>Electronic reproduction. Washington, D.C. : Library of Congress, [2002-2003]</dc:description>\\n<dc:publisher>Manchester, Palmer & Howe</dc:publisher>\\n<dc:date>1885</dc:date>\\n<dc:type>text</dc:type>\\n<dc:identifier>http://hdl.loc.gov/loc.gdc/lhbtn.12281</dc:identifier>\\n<dc:language>eng</dc:language>\\n<dc:coverage>United States</dc:coverage>\\n</oai_dc:dc>\\n ",
"keys":["United States--Description and travel.","eng"],
"identity":{
"submitter_type":"agent",
"submitter":"NSDL 2 LR Data Pump"
},
"resource_data_type":"metadata",
"payload_schema_locator":"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd",
"payload_placement":"inline",
"payload_schema":["oai_dc"],
"node_timestamp":"2011-03-14 13:36:04.617999",
"doc_version":"0.23.0",
"create_timestamp":"2011-03-14 13:36:04.617999",
"active":true,
"publishing_node":"string",
"resource_locator":"http://hdl.loc.gov/loc.gdc/lhbtn.12281",
"doc_ID":"00e3f67232e743b6bc2a079bd98ff55a",
"TOS": {
"submission_TOS": "http://example.com/tos/unknown",
"submission_attribution": "unidentified"
}
}
'''
self.sampleJSON__0_51_0 = '''
{
"_id":"00e3f67232e743b6bc2a079bd98ff55a",
"_rev":"1-8163d32f6cc9996f2b7228d8b5db7962",
"doc_type":"resource_data",
"update_timestamp":"2014-04-01 13:36:04.617999",
"resource_data":"<oai_dc:dc xmlns:oai_dc=\\"http://www.openarchives.org/OAI/2.0/oai_dc/\\" xmlns:dc=\\"http://purl.org/dc/elements/1.1/\\" xmlns:xsi=\\"http://www.w3.org/2001/XMLSchema-instance\\" xmlns=\\"http://www.openarchives.org/OAI/2.0/\\" xsi:schemaLocation=\\"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd\\">\\n<dc:title>A chat about America. October and November, 1884.</dc:title>\\n<dc:creator>J. P.</dc:creator>\\n<dc:subject>United States--Description and travel.</dc:subject>\\n<dc:description>\\"Printed for private circulation only.\\"</dc:description>\\n<dc:description>Electronic reproduction. Washington, D.C. : Library of Congress, [2002-2003]</dc:description>\\n<dc:publisher>Manchester, Palmer & Howe</dc:publisher>\\n<dc:date>1885</dc:date>\\n<dc:type>text</dc:type>\\n<dc:identifier>http://hdl.loc.gov/loc.gdc/lhbtn.12281</dc:identifier>\\n<dc:language>eng</dc:language>\\n<dc:coverage>United States</dc:coverage>\\n</oai_dc:dc>\\n ",
"keys":["United States--Description and travel.","eng"],
"identity":{
"submitter_type":"agent",
"submitter":"NSDL 2 LR Data Pump"
},
"resource_data_type":"metadata",
"payload_schema_locator":"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd",
"payload_placement":"inline",
"payload_schema":["oai_dc"],
"node_timestamp":"2014-04-01 13:32:04.617999",
"doc_version":"0.51.0",
"create_timestamp":"2014-04-01 13:36:04.617999",
"active":true,
"publishing_node":"string",
"resource_locator":"http://hdl.loc.gov/loc.gdc/lhbtn.12281",
"doc_ID":"00e3f67232e743b6bc2a079bd98ff55a",
"TOS": {
"submission_TOS": "http://example.com/tos/unknown",
"submission_attribution": "unidentified"
}
}
'''
self.sampleJSON_no_coercion = '''
{
"_id":"00e3f67232e743b6bc2a079bd98ff55a",
"_rev":"1-8163d32f6cc9996f2b7228d8b5db7962",
"doc_type":"resource_data",
"update_timestamp":"2011-03-14 13:36:04.617999",
"resource_data": {
"name": "Test coersion",
"nullable": null,
"booleanT": true,
"booleanF": false
},
"keys":["United States--Description and travel.","eng"],
"identity":{
"submitter_type":"agent",
"submitter":"NSDL 2 LR Data Pump"
},
"resource_data_type":"metadata",
"payload_schema_locator":"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd",
"payload_placement":"inline",
"payload_schema":["oai_dc"],
"node_timestamp":"2011-03-14 13:36:04.617999",
"doc_version":"0.23.0",
"create_timestamp":"2011-03-14 13:36:04.617999",
"active":true,
"publishing_node":"string",
"resource_locator":"http://hdl.loc.gov/loc.gdc/lhbtn.12281",
"doc_ID":"00e3f67232e743b6bc2a079bd98ff55a",
"TOS": {
"submission_TOS": "http://example.com/tos/unknown",
"submission_attribution": "unidentified"
}
}
'''
self.sampleJSON_lossy_sha256 = [
'''
{
"_id":"00e3f67232e743b6bc2a079bd98ff55a",
"_rev":"1-8163d32f6cc9996f2b7228d8b5db7962",
"doc_type":"resource_data",
"update_timestamp":"2011-03-14 13:36:04.617999",
"resource_data": {
"name": "Test Lossy"
"integer": 1
},
"keys":["United States--Description and travel.","eng"],
"identity":{
"submitter_type":"agent",
"submitter":"NSDL 2 LR Data Pump"
},
"resource_data_type":"metadata",
"payload_schema_locator":"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd",
"payload_placement":"inline",
"payload_schema":["oai_dc"],
"node_timestamp":"2011-03-14 13:36:04.617999",
"doc_version":"0.23.0",
"create_timestamp":"2011-03-14 13:36:04.617999",
"active":true,
"publishing_node":"string",
"resource_locator":"http://hdl.loc.gov/loc.gdc/lhbtn.12281",
"doc_ID":"00e3f67232e743b6bc2a079bd98ff55a",
"TOS": {
"submission_TOS": "http://example.com/tos/unknown",
"submission_attribution": "unidentified"
}
}
''', '''
{
"_id":"00e3f67232e743b6bc2a079bd98ff55a",
"_rev":"1-8163d32f6cc9996f2b7228d8b5db7962",
"doc_type":"resource_data",
"update_timestamp":"2011-03-14 13:36:04.617999",
"resource_data": {
"name": "Test Lossy"
"float": 1.1
},
"keys":["United States--Description and travel.","eng"],
"identity":{
"submitter_type":"agent",
"submitter":"NSDL 2 LR Data Pump"
},
"resource_data_type":"metadata",
"payload_schema_locator":"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd",
"payload_placement":"inline",
"payload_schema":["oai_dc"],
"node_timestamp":"2011-03-14 13:36:04.617999",
"doc_version":"0.23.0",
"create_timestamp":"2011-03-14 13:36:04.617999",
"active":true,
"publishing_node":"string",
"resource_locator":"http://hdl.loc.gov/loc.gdc/lhbtn.12281",
"doc_ID":"00e3f67232e743b6bc2a079bd98ff55a",
"TOS": {
"submission_TOS": "http://example.com/tos/unknown",
"submission_attribution": "unidentified"
}
}
''', '''
{
"_id":"00e3f67232e743b6bc2a079bd98ff55a",
"_rev":"1-8163d32f6cc9996f2b7228d8b5db7962",
"doc_type":"resource_data",
"update_timestamp":"2011-03-14 13:36:04.617999",
"resource_data": {
"name": "Test Lossy"
"max_int": 9007199254740990
},
"keys":["United States--Description and travel.","eng"],
"identity":{
"submitter_type":"agent",
"submitter":"NSDL 2 LR Data Pump"
},
"resource_data_type":"metadata",
"payload_schema_locator":"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd",
"payload_placement":"inline",
"payload_schema":["oai_dc"],
"node_timestamp":"2011-03-14 13:36:04.617999",
"doc_version":"0.23.0",
"create_timestamp":"2011-03-14 13:36:04.617999",
"active":true,
"publishing_node":"string",
"resource_locator":"http://hdl.loc.gov/loc.gdc/lhbtn.12281",
"doc_ID":"00e3f67232e743b6bc2a079bd98ff55a",
"TOS": {
"submission_TOS": "http://example.com/tos/unknown",
"submission_attribution": "unidentified"
}
}
''', '''
{
"_id":"00e3f67232e743b6bc2a079bd98ff55a",
"_rev":"1-8163d32f6cc9996f2b7228d8b5db7962",
"doc_type":"resource_data",
"update_timestamp":"2011-03-14 13:36:04.617999",
"resource_data": {
"name": "Test Lossy"
"min_int": -9007199254740990
},
"keys":["United States--Description and travel.","eng"],
"identity":{
"submitter_type":"agent",
"submitter":"NSDL 2 LR Data Pump"
},
"resource_data_type":"metadata",
"payload_schema_locator":"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd",
"payload_placement":"inline",
"payload_schema":["oai_dc"],
"node_timestamp":"2011-03-14 13:36:04.617999",
"doc_version":"0.23.0",
"create_timestamp":"2011-03-14 13:36:04.617999",
"active":true,
"publishing_node":"string",
"resource_locator":"http://hdl.loc.gov/loc.gdc/lhbtn.12281",
"doc_ID":"00e3f67232e743b6bc2a079bd98ff55a",
"TOS": {
"submission_TOS": "http://example.com/tos/unknown",
"submission_attribution": "unidentified"
}
}
'''
]
self.sampleJSON_strip = '''{"keys": ["United States--Description and travel.", "eng"], "TOS": {"submission_attribution": "unidentified", "submission_TOS": "http://example.com/tos/unknown"}, "payload_placement": "inline", "active": true, "resource_locator": "http://hdl.loc.gov/loc.gdc/lhbtn.12281", "doc_type": "resource_data", "resource_data": "<oai_dc:dc xmlns:oai_dc=\\"http://www.openarchives.org/OAI/2.0/oai_dc/\\" xmlns:dc=\\"http://purl.org/dc/elements/1.1/\\" xmlns:xsi=\\"http://www.w3.org/2001/XMLSchema-instance\\" xmlns=\\"http://www.openarchives.org/OAI/2.0/\\" xsi:schemaLocation=\\"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd\\">\\n<dc:title>A chat about America. October and November, 1884.</dc:title>\\n<dc:creator>J. P.</dc:creator>\\n<dc:subject>United States--Description and travel.</dc:subject>\\n<dc:description>\\"Printed for private circulation only.\\"</dc:description>\\n<dc:description>Electronic reproduction. Washington, D.C. : Library of Congress, [2002-2003]</dc:description>\\n<dc:publisher>Manchester, Palmer & Howe</dc:publisher>\\n<dc:date>1885</dc:date>\\n<dc:type>text</dc:type>\\n<dc:identifier>http://hdl.loc.gov/loc.gdc/lhbtn.12281</dc:identifier>\\n<dc:language>eng</dc:language>\\n<dc:coverage>United States</dc:coverage>\\n</oai_dc:dc>\\n ", "submitter_type": "agent", "resource_data_type": "metadata", "payload_schema_locator": "http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd", "payload_schema": ["oai_dc"], "doc_version": "0.10.0", "submitter": "NSDL 2 LR Data Pump"}'''
self.sampleJSON_strip_normal = '''{"keys": ["United States--Description and travel.", "eng"], "TOS": {"submission_attribution": "unidentified", "submission_TOS": "http://example.com/tos/unknown"}, "payload_placement": "inline", "active": "true", "resource_locator": "http://hdl.loc.gov/loc.gdc/lhbtn.12281", "doc_type": "resource_data", "resource_data": "<oai_dc:dc xmlns:oai_dc=\\"http://www.openarchives.org/OAI/2.0/oai_dc/\\" xmlns:dc=\\"http://purl.org/dc/elements/1.1/\\" xmlns:xsi=\\"http://www.w3.org/2001/XMLSchema-instance\\" xmlns=\\"http://www.openarchives.org/OAI/2.0/\\" xsi:schemaLocation=\\"http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd\\">\\n<dc:title>A chat about America. October and November, 1884.</dc:title>\\n<dc:creator>J. P.</dc:creator>\\n<dc:subject>United States--Description and travel.</dc:subject>\\n<dc:description>\\"Printed for private circulation only.\\"</dc:description>\\n<dc:description>Electronic reproduction. Washington, D.C. : Library of Congress, [2002-2003]</dc:description>\\n<dc:publisher>Manchester, Palmer & Howe</dc:publisher>\\n<dc:date>1885</dc:date>\\n<dc:type>text</dc:type>\\n<dc:identifier>http://hdl.loc.gov/loc.gdc/lhbtn.12281</dc:identifier>\\n<dc:language>eng</dc:language>\\n<dc:coverage>United States</dc:coverage>\\n</oai_dc:dc>\\n ", "submitter_type": "agent", "resource_data_type": "metadata", "payload_schema_locator": "http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd", "payload_schema": ["oai_dc"], "doc_version": "0.10.0", "submitter": "NSDL 2 LR Data Pump"}'''
self.sampleJSON_strip_normal_bencode = '''d3:TOSd14:submission_TOS30:http://example.com/tos/unknown22:submission_attribution12:unidentifiede6:active4:true8:doc_type13:resource_data11:doc_version6:0.10.04:keysl38:United States--Description and travel.3:enge17:payload_placement6:inline14:payload_schemal6:oai_dce22:payload_schema_locator90:http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd13:resource_data968:<oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://www.openarchives.org/OAI/2.0/" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">\n<dc:title>A chat about America. October and November, 1884.</dc:title>\n<dc:creator>J. P.</dc:creator>\n<dc:subject>United States--Description and travel.</dc:subject>\n<dc:description>"Printed for private circulation only."</dc:description>\n<dc:description>Electronic reproduction. Washington, D.C. : Library of Congress, [2002-2003]</dc:description>\n<dc:publisher>Manchester, Palmer & Howe</dc:publisher>\n<dc:date>1885</dc:date>\n<dc:type>text</dc:type>\n<dc:identifier>http://hdl.loc.gov/loc.gdc/lhbtn.12281</dc:identifier>\n<dc:language>eng</dc:language>\n<dc:coverage>United States</dc:coverage>\n</oai_dc:dc>\n 18:resource_data_type8:metadata16:resource_locator38:http://hdl.loc.gov/loc.gdc/lhbtn.122819:submitter19:NSDL 2 LR Data Pump14:submitter_type5:agente'''
self.sampleJSON_sha256 = '''ef1b3b63adc663602c7a3c7595951b2761b34f5f6490ea1acee3df0fd97db03c'''
self.sampleKeyLocations = [
"http://example.com/mykey", "http://example2.com/mykey"
]
self.signatureTemplate = '{{"key_location": [{0}], "key_owner": "' + self.goodowner + '", "signing_method": "LR-PGP.1.0", "signature": "{1}"}}'
def tearDown(self):
self.gpg.delete_keys([
self.goodPrivateKey.fingerprint,
], secret=True)
self.gpg.delete_keys([
self.goodPrivateKey.fingerprint,
], secret=False)
pass
def testMissingPrivateKey(self):
def missingKey():
try:
sign = Sign_0_21(self.badkeyid,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
except UnknownKeyException as e:
assert e.keyid == self.badkeyid, "keyid in exception doesn't match key passed to sign."
raise e
self.assertRaises(UnknownKeyException, missingKey)
def testPresentPrivateKey(self):
sign = Sign_0_21(self.goodkeyid,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
assert sign.privateKeyID == self.goodkeyid
def testStrip(self):
origJson = json.loads(self.sampleJSON)
benchmark = json.loads(self.sampleJSON_strip)
signer = Sign_0_21(self.goodkeyid,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
stripJson = signer._stripEnvelope(origJson)
assert benchmark == stripJson
def testStripNormal(self):
origJson = json.loads(self.sampleJSON)
benchmark = json.loads(self.sampleJSON_strip_normal)
signer = Sign_0_21(self.goodkeyid,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
stripJson = signer._stripEnvelope(origJson)
normalJson = signer._bnormal(stripJson)
assert benchmark == normalJson
def testStripNormalBencode(self):
origJson = json.loads(self.sampleJSON)
benchmark = self.sampleJSON_strip_normal_bencode
signer = Sign_0_21(self.goodkeyid,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
stripJson = signer._stripEnvelope(origJson)
normalJson = signer._bnormal(stripJson)
bencodeJson = signer._buildCanonicalString(normalJson)
assert benchmark == bencodeJson
def testStripNormalBencodeHash(self):
origJson = json.loads(self.sampleJSON)
benchmark = self.sampleJSON_sha256
signer = Sign_0_21(self.goodkeyid,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
stripJson = signer._stripEnvelope(origJson)
normalJson = signer._bnormal(stripJson)
bencodeJson = signer._buildCanonicalString(normalJson)
hashed = signer._hash(bencodeJson)
assert benchmark == hashed
def testKnownLossyHash(self):
for idx, lossyJSON in enumerate(self.sampleJSON_lossy_sha256):
origJson = json.loads(self.sampleJSON)
signer = Sign_0_21(self.goodkeyid,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
stripJson = signer._stripEnvelope(origJson)
normalJson = signer._bnormal(stripJson)
bencodeJson = signer._buildCanonicalString(normalJson)
hashed = signer._hash(bencodeJson)
if idx > 0:
assert prev_hashed == hashed, "Hashes should match - algorithm is lossy"
assert prev_bencodeJson == bencodeJson, "Bencode strings should match - algorithm is lossy"
prev_bencodeJson = bencodeJson
prev_hashed = hashed
def testGetMessage(self):
origJson = json.loads(self.sampleJSON)
benchmark = self.sampleJSON_sha256
signer = Sign_0_21(self.goodkeyid,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
message = signer.get_message(origJson)
assert benchmark == message
def testPrivateKeyOwner(self):
benchmark = self.goodowner
signer = Sign_0_21(self.goodkeyid,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
assert benchmark == signer._get_privatekey_owner()
def testSigBlock(self):
origJson = json.loads(self.sampleJSON)
arbitrarySigdata = "ABCDEF0123456789-abcdef"
arbitraryKeyLoc = self.sampleKeyLocations
keyloc = ",".join(['"{0}"'.format(x) for x in arbitraryKeyLoc])
benchmark = json.loads(
self.signatureTemplate.format(keyloc, arbitrarySigdata))
signer = Sign_0_21(self.goodkeyid,
passphrase=self.goodpassphrase,
publicKeyLocations=arbitraryKeyLoc,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
assert benchmark == signer._get_sig_block(arbitrarySigdata)
def testSignNoEnvelopeCoercion(self):
origJson = json.loads(self.sampleJSON_no_coercion)
arbitraryKeyLoc = self.sampleKeyLocations
assert origJson["resource_data"]["nullable"] == None, "Expected null"
assert origJson["resource_data"]["booleanT"] == True, "Expected true"
assert origJson["resource_data"]["booleanF"] == False, "Expected false"
signer = Sign_0_21(self.goodkeyid,
passphrase=self.goodpassphrase,
publicKeyLocations=arbitraryKeyLoc,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
signed = signer.sign(origJson)
assert "digital_signature" in signed
assert origJson["resource_data"]["nullable"] == None, "Expected null"
assert origJson["resource_data"]["booleanT"] == True, "Expected true"
assert origJson["resource_data"]["booleanF"] == False, "Expected false"
def testSign_0_10__no_passthru(self):
origJson = json.loads(self.sampleJSON)
arbitraryKeyLoc = self.sampleKeyLocations
signer = Sign_0_21(self.goodkeyid,
passphrase=self.goodpassphrase,
publicKeyLocations=arbitraryKeyLoc,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
signed = signer.sign(origJson)
assert "digital_signature" in signed
sig = signed["digital_signature"]
assert "signature" in sig
assert sig["signature"] != None and len(sig["signature"]) > 0
def test_Sign__0_10__passthru(self):
origJson = json.loads(self.sampleJSON)
arbitraryKeyLoc = self.sampleKeyLocations
signer = Sign_0_21(self.goodkeyid,
passphrase=self.goodpassphrase,
publicKeyLocations=arbitraryKeyLoc,
sign_everything=False,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
signed = signer.sign(origJson)
assert "digital_signature" not in signed
# sig = signed["digital_signature"]
# assert sig.has_key("signature")
# assert sig["signature"] != None and len(sig["signature"]) > 0
def test_Sign__0_23__passthru(self):
origJson = json.loads(self.sampleJSON__0_23_0)
arbitraryKeyLoc = self.sampleKeyLocations
signer = Sign_0_23(self.goodkeyid,
passphrase=self.goodpassphrase,
publicKeyLocations=arbitraryKeyLoc,
sign_everything=False,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
signed = signer.sign(origJson)
assert "digital_signature" in signed
sig = signed["digital_signature"]
assert "signature" in sig
assert sig["signature"] != None and len(sig["signature"]) > 0
def test_Sign__0_23__no_passthru(self):
origJson = json.loads(self.sampleJSON__0_23_0)
arbitraryKeyLoc = self.sampleKeyLocations
signer = Sign_0_23(self.goodkeyid,
passphrase=self.goodpassphrase,
publicKeyLocations=arbitraryKeyLoc,
sign_everything=True,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
signed = signer.sign(origJson)
assert "digital_signature" in signed
sig = signed["digital_signature"]
assert "signature" in sig
assert sig["signature"] != None and len(sig["signature"]) > 0
def test_Sign__0_51__no_passthru(self):
origJson = json.loads(self.sampleJSON__0_51_0)
arbitraryKeyLoc = self.sampleKeyLocations
signer = Sign_0_51(self.goodkeyid,
passphrase=self.goodpassphrase,
publicKeyLocations=arbitraryKeyLoc,
sign_everything=True,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
signed = signer.sign(origJson)
assert "digital_signature" in signed
sig = signed["digital_signature"]
assert "signature" in sig
assert sig["signature"] != None and len(sig["signature"]) > 0
def testSignUnicode(self):
if self.testDataUnicode == None:
log.info("Skipping test, unicode test data file not set.")
return
import codecs
# fileName = "2011-02-28Metadata1004.json"
# unsigned = json.load(codecs.open(os.path.join(self.testDataDir, fileName), "r", "utf-8-sig"))
unsigned = json.load(
codecs.open(self.testDataUnicode, "r", "utf-8-sig"))
arbitraryKeyLoc = self.sampleKeyLocations
signer = Sign_0_21(self.goodkeyid,
passphrase=self.goodpassphrase,
publicKeyLocations=arbitraryKeyLoc,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
signed = signer.sign(unsigned)
assert "digital_signature" in signed
sig = signed["digital_signature"]
assert "signature" in sig
assert sig["signature"] != None and len(sig["signature"]) > 0
def testSignLRTestData(self):
if self.testDataDir == None:
log.info("Skipping test, test data directory not set.")
return
import codecs
# allfiles = os.listdir(self.testDataDir)
for root, dirs, files in os.walk(self.testDataDir):
for fileName in files:
log.info("Trying to sign %s" % (fileName, ))
unsigned = json.load(
codecs.open(os.path.join(root, fileName), "r",
"utf-8-sig"))
arbitraryKeyLoc = self.sampleKeyLocations
signer = Sign_0_21(self.goodkeyid,
passphrase=self.goodpassphrase,
publicKeyLocations=arbitraryKeyLoc,
gnupgHome=self.gnupgHome,
gpgbin=self.gpgbin)
signed = signer.sign(unsigned)
assert "digital_signature" in signed
sig = signed["digital_signature"]
assert "signature" in sig
assert sig["signature"] != None and len(sig["signature"]) > 0
class GLBGPG:
"""
GPG has not a dedicated class, because one of the function is callend inside a transact, and
I'm not quite confident on creating an object that operates on the filesystem knowing
that would be run also on the Storm cycle.
"""
def __init__(self, receiver_desc):
"""
every time is needed, a new keyring is created here.
"""
atfork()
if receiver_desc.has_key('gpg_key_status') and \
receiver_desc['gpg_key_status'] != Receiver._gpg_types[1]: # Enabled
log.err("Requested GPG initialization for a receiver without GPG configured! %s" %
receiver_desc['username'])
raise Exception("Requested GPG init for user without GPG [%s]" % receiver_desc['username'])
try:
temp_gpgroot = os.path.join(GLSetting.gpgroot, "%s" % random.randint(0, 0xFFFF) )
os.makedirs(temp_gpgroot, mode=0700)
self.gpgh = GPG(gnupghome=temp_gpgroot, options="--trust-model always")
except Exception as excep:
log.err("Unable to instance GPG object: %s" % excep)
raise excep
self.receiver_desc = receiver_desc
log.debug("GPG object initialized for receiver %s" % receiver_desc['username'])
def sanitize_gpg_string(self, received_gpgasc):
"""
@param received_gpgasc: A gpg armored key
@return: Sanitized string or raise InvalidInputFormat
This function validate the integrity of a GPG key
"""
lines = received_gpgasc.split("\n")
sanitized = ""
start = 0
if not len(lines[start]):
start += 1
if lines[start] != '-----BEGIN PGP PUBLIC KEY BLOCK-----':
raise errors.InvalidInputFormat("GPG invalid format")
else:
sanitized += lines[start] + "\n"
i = 0
while i < len(lines):
# the C language as left some archetypes in my code
# [ITA] https://www.youtube.com/watch?v=7jI4DnRJP3k
i += 1
try:
if len(lines[i]) < 2:
continue
except IndexError:
continue
main_content = re.compile( r"^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" , re.UNICODE)
base64only = main_content.findall(lines[i])
if len(base64only) == 1:
sanitized += str(base64only[0]) + "\n"
# this GPG/PGP format it's different from the common base64 ? dunno
if len(lines[i]) == 5 and lines[i][0] == '=':
sanitized += str(lines[i]) + "\n"
if lines[i] == '-----END PGP PUBLIC KEY BLOCK-----':
sanitized += lines[i] + "\n"
return sanitized
raise errors.InvalidInputFormat("Malformed PGP key block")
def validate_key(self, armored_key):
"""
@param armored_key:
@return: True or False, True only if a key is effectively importable and listed.
"""
# or raise InvalidInputFormat
sanitized_gpgasc = self.sanitize_gpg_string(armored_key)
try:
self.ke = self.gpgh.import_keys(sanitized_gpgasc)
except Exception as excep:
log.err("Error in GPG import_keys: %s" % excep)
return False
# Error reported in stderr may just be warning, this is because is not raise an exception here
# if self.ke.stderr:
# log.err("Receiver %s in uploaded GPG key has raise and alarm:\n< %s >" %
# (self.receiver_desc['username'], (self.ke.stderr.replace("\n", "\n "))[:-3]))
if not (hasattr(self.ke, 'results') and len(self.ke.results) == 1 and self.ke.results[0].has_key('fingerprint')):
log.err("User error: unable to import GPG key in the keyring")
return False
# else, the key has been loaded and we extract info about that:
self.fingerprint = self.ke.results[0]['fingerprint']
# looking if the key is effectively reachable
try:
all_keys = self.gpgh.list_keys()
except Exception as excep:
log.err("Error in GPG list_keys: %s" % excep)
return False
self.keyinfo = u""
for key in all_keys:
if key['fingerprint'] == self.fingerprint:
self.keyinfo += "Key length %s" % key['length']
try:
for uid in key['uids']:
self.keyinfo += "\n\t%s" % uid
except Exception as excep:
log.err("Error in GPG key format/properties: %s" % excep)
return False
if not len(self.keyinfo):
log.err("Key apparently imported but unable to be extracted info")
return False
return True
def encrypt_file(self, plainpath, filestream, output_path):
"""
@param gpg_key_armor:
@param plainpath:
@return:
"""
if not self.validate_key(self.receiver_desc['gpg_key_armor']):
raise errors.GPGKeyInvalid
encrypt_obj = self.gpgh.encrypt_file(filestream, str(self.receiver_desc['gpg_key_fingerprint']))
if not encrypt_obj.ok:
# continue here if is not ok
log.err("Falure in encrypting file %s %s (%s)" % ( plainpath,
self.receiver_desc['username'], self.receiver_desc['gpg_key_fingerprint']) )
log.err(encrypt_obj.stderr)
raise errors.GPGKeyInvalid
log.debug("Encrypting for %s (%s) file %s (%d bytes)" %
(self.receiver_desc['username'], self.receiver_desc['gpg_key_fingerprint'],
plainpath, len(str(encrypt_obj))) )
encrypted_path = os.path.join( os.path.abspath(output_path),
"gpg_encrypted-%d-%d" %
(random.randint(0, 0xFFFF), random.randint(0, 0xFFFF)))
if os.path.isfile(encrypted_path):
log.err("Unexpected unpredictable unbelievable error! %s" % encrypted_path)
raise errors.InternalServerError("File conflict in GPG encrypted output")
try:
with open(encrypted_path, "w+") as f:
f.write(str(encrypt_obj))
return encrypted_path, len(str(encrypt_obj))
except Exception as excep:
log.err("Error in writing GPG file output: %s (%s) bytes %d" %
(excep.message, encrypted_path, len(str(encrypt_obj)) ))
raise errors.InternalServerError("Error in writing [%s]" % excep.message )
def encrypt_message(self, plaintext):
"""
@param plaindata:
An arbitrary long text that would be encrypted
@param receiver_desc:
The output of
globaleaks.handlers.admin.admin_serialize_receiver()
dictionary. It contain the fingerprint of the Receiver PUBKEY
@return:
The unicode of the encrypted output (armored)
"""
if not self.validate_key(self.receiver_desc['gpg_key_armor']):
raise errors.GPGKeyInvalid
# This second argument may be a list of fingerprint, not just one
encrypt_obj = self.gpgh.encrypt(plaintext, str(self.receiver_desc['gpg_key_fingerprint']) )
if not encrypt_obj.ok:
# else, is not .ok
log.err("Falure in encrypting %d bytes for %s (%s)" % (len(plaintext),
self.receiver_desc['username'], self.receiver_desc['gpg_key_fingerprint']) )
log.err(encrypt_obj.stderr)
raise errors.GPGKeyInvalid
log.debug("Encrypting for %s (%s) %d byte of plain data (%d cipher output)" %
(self.receiver_desc['username'], self.receiver_desc['gpg_key_fingerprint'],
len(plaintext), len(str(encrypt_obj))) )
return str(encrypt_obj)
def destroy_environment(self):
try:
shutil.rmtree(self.gpgh.gnupghome)
except Exception as excep:
log.err("Unable to clean temporary GPG environment: %s: %s" % (self.gpgh.gnupghome, excep))
class OpenPgpFactory(GenericFactory):
"""Provides OpenPGP functionality based on GnuPG."""
implements(ICipherModule)
gpg_binary = Option(
'crypto', 'gpg_binary', 'gpg',
"""GnuPG binary name, allows for full path too.
Value 'gpg' is same default as in python-gnupg itself.
For usual installations location of the gpg binary is auto-detected.
""")
gpg_home = Option(
'crypto', 'gpg_home', '.gpg', """Directory containing keyring files.
In case of wrong configuration missing keyring files without content
will be created in the configured location, provided necessary
write permssion is granted for the corresponding parent directory.
""")
priv_key = Option(
'crypto', 'gpg_private_key', '',
"""Key ID of private key (last 8 chars or more).
If unset, a private key will be selected from keyring automagicly.
The password must be available i.e. provided by running gpg-agent
(not yet available) or empty (security trade-off). No private key
operation can happen before the key has been unlocked successfully.
""")
name = 'openpgp'
_supported_keys = [
'name_real', 'name_comment', 'name_email', 'key_type', 'key_length',
'subkey_type', 'subkey_length', 'expire_date', 'passphrase'
]
def __init__(self):
try:
from gnupg import GPG
except ImportError:
raise TracError(
_("""Unable to load the python-gnupg module.
Please check and correct your installation."""))
try:
# Initialize the GnuPG instance.
path = os.path.join(os.path.abspath(self.env.path), self.gpg_home)
self.env.log.debug('Using GnuPG home dir: ' + str(path))
self.gpg = GPG(gpgbinary=self.gpg_binary, gnupghome=path)
except ValueError:
raise TracError(
_("""Missing the crypto binary. Please check and
set full path with option 'gpg_binary'."""))
# IKeyVault methods
def keys(self, private=False, id_only=False):
"""Returns the list of all available keys."""
# DEVEL: Cache this for performance.
if not id_only:
return self.gpg.list_keys(private) # same as gpg.list_keys(False)
keys = []
for key in self.gpg.list_keys(private):
keys.append(key['fingerprint'])
return keys
def create_key(self, **kwargs):
"""Generate a new OpenPGP key pair."""
# Sanitize input.
for k in kwargs.keys():
if k not in self._supported_keys:
kwargs.pop(k)
input_data = self.gpg.gen_key_input(**kwargs)
try:
return self.gpg.gen_key(input_data)
except ValueError, e:
return False, e
class GLBPGP(object):
"""
PGP has not a dedicated class, because one of the function is called inside a transact, and
I'm not quite confident on creating an object that operates on the filesystem knowing
that would be run also on the Storm cycle.
"""
def __init__(self):
"""
every time is needed, a new keyring is created here.
"""
try:
temp_pgproot = os.path.join(GLSettings.pgproot, "%s" % generateRandomKey(8))
os.makedirs(temp_pgproot, mode=0700)
self.gnupg = GPG(gnupghome=temp_pgproot, options=['--trust-model', 'always'])
self.gnupg.encoding = "UTF-8"
except OSError as ose:
log.err("Critical, OS error in operating with GnuPG home: %s" % ose)
raise
except Exception as excep:
log.err("Unable to instance PGP object: %s" % excep)
raise
def load_key(self, key):
"""
@param key:
@return: True or False, True only if a key is effectively importable and listed.
"""
try:
import_result = self.gnupg.import_keys(key)
except Exception as excep:
log.err("Error in PGP import_keys: %s" % excep)
raise errors.PGPKeyInvalid
if len(import_result.fingerprints) == 0:
raise errors.PGPKeyInvalid
fingerprint = import_result.fingerprints[0]
# looking if the key is effectively reachable
try:
all_keys = self.gnupg.list_keys()
except Exception as excep:
log.err("Error in PGP list_keys: %s" % excep)
raise errors.PGPKeyInvalid
expiration = datetime.utcfromtimestamp(0)
for key in all_keys:
if key['fingerprint'] == fingerprint:
if key['expires']:
expiration = datetime.utcfromtimestamp(int(key['expires']))
break
return {
'fingerprint': fingerprint,
'expiration': expiration,
}
def encrypt_file(self, key_fingerprint, input_file, output_path):
"""
Encrypt a file with the specified PGP key
"""
encrypted_obj = self.gnupg.encrypt_file(input_file, str(key_fingerprint), output=output_path)
if not encrypted_obj.ok:
raise errors.PGPKeyInvalid
return encrypted_obj, os.stat(output_path).st_size
def encrypt_message(self, key_fingerprint, plaintext):
"""
Encrypt a text message with the specified key
"""
encrypted_obj = self.gnupg.encrypt(plaintext, str(key_fingerprint))
if not encrypted_obj.ok:
raise errors.PGPKeyInvalid
return str(encrypted_obj)
def destroy_environment(self):
try:
shutil.rmtree(self.gnupg.gnupghome)
except Exception as excep:
log.err("Unable to clean temporary PGP environment: %s: %s" % (self.gnupg.gnupghome, excep))
class GLBGPG:
"""
GPG has not a dedicated class, because one of the function is called inside a transact, and
I'm not quite confident on creating an object that operates on the filesystem knowing
that would be run also on the Storm cycle.
"""
def __init__(self, receiver_desc):
"""
every time is needed, a new keyring is created here.
"""
if receiver_desc.has_key('gpg_key_status') and \
receiver_desc['gpg_key_status'] != Receiver._gpg_types[1]: # Enabled
log.err(
"Requested GPG initialization for a receiver without GPG configured! %s"
% receiver_desc['username'])
raise Exception("Requested GPG init for user without GPG [%s]" %
receiver_desc['username'])
try:
temp_gpgroot = os.path.join(GLSetting.gpgroot,
"%s" % xeger(r'[A-Za-z0-9]{8}'))
os.makedirs(temp_gpgroot, mode=0700)
self.gpgh = GPG(gnupghome=temp_gpgroot,
options=['--trust-model', 'always'])
except Exception as excep:
log.err("Unable to instance GPG object: %s" % excep)
raise excep
self.receiver_desc = receiver_desc
log.debug("GPG object initialized for receiver %s" %
receiver_desc['username'])
def sanitize_gpg_string(self, received_gpgasc):
"""
@param received_gpgasc: A gpg armored key
@return: Sanitized string or raise InvalidInputFormat
This function validate the integrity of a GPG key
"""
lines = received_gpgasc.split("\n")
sanitized = ""
start = 0
if not len(lines[start]):
start += 1
if lines[start] != '-----BEGIN PGP PUBLIC KEY BLOCK-----':
raise errors.InvalidInputFormat("GPG invalid format")
else:
sanitized += lines[start] + "\n"
i = 0
while i < len(lines):
# the C language as left some archetypes in my code
# [ITA] https://www.youtube.com/watch?v=7jI4DnRJP3k
i += 1
try:
if len(lines[i]) < 2:
continue
except IndexError:
continue
main_content = re.compile(
r"^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$",
re.UNICODE)
base64only = main_content.findall(lines[i])
if len(base64only) == 1:
sanitized += str(base64only[0]) + "\n"
# this GPG/PGP format it's different from the common base64 ? dunno
if len(lines[i]) == 5 and lines[i][0] == '=':
sanitized += str(lines[i]) + "\n"
if lines[i] == '-----END PGP PUBLIC KEY BLOCK-----':
sanitized += lines[i] + "\n"
return sanitized
raise errors.InvalidInputFormat("Malformed PGP key block")
def validate_key(self, armored_key):
"""
@param armored_key:
@return: True or False, True only if a key is effectively importable and listed.
"""
# or raise InvalidInputFormat
sanitized_gpgasc = self.sanitize_gpg_string(armored_key)
try:
key = self.gpgh.import_keys(sanitized_gpgasc)
except Exception as excep:
log.err("Error in GPG import_keys: %s" % excep)
return False
# Error reported in stderr may just be warning, this is because is not raise an exception here
# if self.ke.stderr:
# log.err("Receiver %s in uploaded GPG key has raise and alarm:\n< %s >" %
# (self.receiver_desc['username'], (self.ke.stderr.replace("\n", "\n "))[:-3]))
if not (hasattr(key, 'results') and len(key.results) >= 1
and key.results[0].has_key('fingerprint')):
log.err("User error: unable to import GPG key in the keyring")
return False
# else, the key has been loaded and we extract info about that:
self.fingerprint = key.results[0]['fingerprint']
# looking if the key is effectively reachable
try:
all_keys = self.gpgh.list_keys()
except Exception as excep:
log.err("Error in GPG list_keys: %s" % excep)
return False
self.keyinfo = u""
for key in all_keys:
if key['fingerprint'] == self.fingerprint:
self.keyinfo += "Key length %s" % key['length']
try:
for uid in key['uids']:
self.keyinfo += "\n\t%s" % uid
except Exception as excep:
log.err("Error in GPG key format/properties: %s" % excep)
return False
if not len(self.keyinfo):
log.err("Key apparently imported but unable to be extracted info")
return False
return True
def encrypt_file(self, plainpath, filestream, output_path):
"""
@param gpg_key_armor:
@param plainpath:
@return:
"""
if not self.validate_key(self.receiver_desc['gpg_key_armor']):
raise errors.GPGKeyInvalid
encrypt_obj = self.gpgh.encrypt_file(
filestream, str(self.receiver_desc['gpg_key_fingerprint']))
if not encrypt_obj.ok:
# continue here if is not ok
log.err("Falure in encrypting file %s %s (%s)" %
(plainpath, self.receiver_desc['username'],
self.receiver_desc['gpg_key_fingerprint']))
log.err(encrypt_obj.stderr)
raise errors.GPGKeyInvalid
log.debug("Encrypting for %s (%s) file %s (%d bytes)" %
(self.receiver_desc['username'],
self.receiver_desc['gpg_key_fingerprint'], plainpath,
len(str(encrypt_obj))))
encrypted_path = os.path.join(
os.path.abspath(output_path),
"gpg_encrypted-%s" % xeger(r'[A-Za-z0-9]{8}'))
if os.path.isfile(encrypted_path):
log.err("Unexpected unpredictable unbelievable error! %s" %
encrypted_path)
raise errors.InternalServerError(
"File conflict in GPG encrypted output")
try:
with open(encrypted_path, "w+") as f:
f.write(str(encrypt_obj))
return encrypted_path, len(str(encrypt_obj))
except Exception as excep:
log.err("Error in writing GPG file output: %s (%s) bytes %d" %
(excep.message, encrypted_path, len(str(encrypt_obj))))
raise errors.InternalServerError("Error in writing [%s]" %
excep.message)
def encrypt_message(self, plaintext):
"""
@param plaindata:
An arbitrary long text that would be encrypted
@param receiver_desc:
The output of
globaleaks.handlers.admin.admin_serialize_receiver()
dictionary. It contain the fingerprint of the Receiver PUBKEY
@return:
The unicode of the encrypted output (armored)
"""
if not self.validate_key(self.receiver_desc['gpg_key_armor']):
raise errors.GPGKeyInvalid
# This second argument may be a list of fingerprint, not just one
encrypt_obj = self.gpgh.encrypt(
plaintext, str(self.receiver_desc['gpg_key_fingerprint']))
if not encrypt_obj.ok:
# else, is not .ok
log.err("Falure in encrypting %d bytes for %s (%s)" %
(len(plaintext), self.receiver_desc['username'],
self.receiver_desc['gpg_key_fingerprint']))
log.err(encrypt_obj.stderr)
raise errors.GPGKeyInvalid
log.debug(
"Encrypting for %s (%s) %d byte of plain data (%d cipher output)" %
(self.receiver_desc['username'],
self.receiver_desc['gpg_key_fingerprint'], len(plaintext),
len(str(encrypt_obj))))
return str(encrypt_obj)
def destroy_environment(self):
try:
shutil.rmtree(self.gpgh.gnupghome)
except Exception as excep:
log.err("Unable to clean temporary GPG environment: %s: %s" %
(self.gpgh.gnupghome, excep))
class CryptoTxt:
"""Crypto operation provider for plaintext.
We use GnuPG for now. Support for X.509 and other options might
appear in the future.
"""
def __init__(self, gpg_binary, gpg_home):
"""Initialize the GnuPG instance."""
self.gpg_binary = gpg_binary
self.gpg_home = gpg_home
if not GPG:
raise TracError(
_("Unable to load the python-gnupg module. "
"Please check and correct your installation."))
try:
self.gpg = GPG(gpgbinary=self.gpg_binary, gnupghome=self.gpg_home)
except ValueError:
raise TracError(
_("Missing the crypto binary. Please check and "
"set full path with option 'gpg_binary'."))
else:
# get list of available public keys once for later use
self.pub_keys = self.gpg.list_keys()
def sign(self, content, private_key=None):
private_key = self._get_private_key(private_key)
cipher = self.gpg.sign(content, keyid=private_key, passphrase='')
return str(cipher)
def encrypt(self, content, pubkeys):
# always_trust needed for making it work with just any pubkey
cipher = self.gpg.encrypt(content, pubkeys, always_trust=True)
return str(cipher)
def sign_encrypt(self, content, pubkeys, private_key=None):
private_key = self._get_private_key(private_key)
# always_trust needed for making it work with just any pubkey
cipher = self.gpg.encrypt(content,
pubkeys,
always_trust=True,
sign=private_key,
passphrase='')
return str(cipher)
def get_pubkey_ids(self, addr):
"""Find public key with UID matching address to encrypt to."""
pubkey_ids = []
if self.pub_keys and 'uids' in self.pub_keys[-1] and \
'fingerprint' in self.pub_keys[-1]:
# compile pattern before use for better performance
rcpt_re = re.compile(addr)
for k in self.pub_keys:
for uid in k['uids']:
match = rcpt_re.search(uid)
if match is not None:
# check for key expiration
if k['expires'] == '':
pubkey_ids.append(k['fingerprint'][-16:])
elif (time.time() + 60) < float(k['expires']):
pubkey_ids.append(k['fingerprint'][-16:])
break
return pubkey_ids
def _get_private_key(self, privkey=None):
"""Find private (secret) key to sign with."""
# read private keys from keyring
privkeys = self.gpg.list_keys(True) # True => private keys
if privkeys > 0 and 'fingerprint' in privkeys[-1]:
fingerprints = []
for k in privkeys:
fingerprints.append(k['fingerprint'])
else:
# no private key in keyring
return None
if privkey:
# check for existence of private key received as argument
# DEVEL: check for expiration as well
if 7 < len(privkey) <= 40:
for fp in fingerprints:
if fp.endswith(privkey):
# work with last 16 significant chars internally,
# even if only 8 are required in trac.ini
privkey = fp[-16:]
break
# no fingerprint matching key ID
else:
privkey = None
else:
# reset invalid key ID
privkey = None
else:
# select (last) private key from keyring
privkey = fingerprints[-1][-16:]
return privkey
def _check_openpgp_pubkey_from_file(self, pubkey_file, long_key_id):
with TemporaryDirectory() as temp_gpg_home:
gpg = GPG(gnupghome=temp_gpg_home)
with open(pubkey_file, 'rb') as pubkey_fh:
import_result = gpg.import_keys(pubkey_fh.read())
if len(import_result.results) == 0:
raise Exception(
"The OpenPGP file {} contains no OpenPGP keys."
" Keys: {}".format(
pubkey_file,
import_result.results,
)
)
logging.info("OK - OpenPGP file {pubkey_file} contains one or more OpenPGP key.".format(
pubkey_file=pubkey_file,
))
fingerprint = [x['fingerprint'] for x in import_result.results if x.get('fingerprint')][0]
actual_long_key_id = fingerprint[-16:]
given_long_key_id = re.sub(r'^0x', '', long_key_id)
if actual_long_key_id.lower() != given_long_key_id.lower():
raise Exception(
textwrap.dedent(
"""
The OpenPGP file {given_long_key_id} contains a different key than what the file name suggests.
Key ID from file name: {given_long_key_id},
Key ID from pubkey in file: {actual_long_key_id}
"""
).lstrip().format(
given_long_key_id=given_long_key_id,
actual_long_key_id=actual_long_key_id,
)
)
logging.info(
"OK - OpenPGP file {pubkey_file} contains a OpenPGP public key"
" whose long key ID matching the file name.".format(
pubkey_file=pubkey_file,
)
)
list_key = gpg.list_keys()[0]
epoch_time = int(time.time())
expires_time = int(list_key['expires'])
if self._strict:
if expires_time < epoch_time:
raise Exception(
textwrap.dedent(
"""
The OpenPGP file {} contains a expired OpenPGP key.
Current date: {}
Expiration date: {}
"""
).lstrip().format(
pubkey_file,
datetime.fromtimestamp(epoch_time),
datetime.fromtimestamp(expires_time),
)
)
else:
logging.info(
"OK - OpenPGP public key from {pubkey_file} is not expired."
" Expiration date: {expiration_date}".format(
pubkey_file=pubkey_file,
expiration_date=datetime.fromtimestamp(expires_time),
)
)
# https://keyring.debian.org/creating-key.html
if self._strict:
if int(list_key['length']) < self._OPENPGP_MIN_KEY_SIZE:
raise Exception(
textwrap.dedent(
"""
The OpenPGP file {} contains a weak OpenPGP key.
Current key length in bits: {}
Expected at least (inclusive): {}
"""
).lstrip().format(
pubkey_file,
list_key['length'],
self._OPENPGP_MIN_KEY_SIZE,
)
)
else:
logging.info(
"OK - The key length of the OpenPGP public key from {pubkey_file} is not considered to be weak."
" Key length in bits: {key_size}".format(
pubkey_file=pubkey_file,
key_size=list_key['length'],
)
)
return True
class GLBGPG(object):
"""
GPG has not a dedicated class, because one of the function is called inside a transact, and
I'm not quite confident on creating an object that operates on the filesystem knowing
that would be run also on the Storm cycle.
"""
def __init__(self):
"""
every time is needed, a new keyring is created here.
"""
try:
temp_gpgroot = os.path.join(GLSetting.gpgroot,
"%s" % xeger(r'[A-Za-z0-9]{8}'))
os.makedirs(temp_gpgroot, mode=0700)
self.gpgh = GPG(gnupghome=temp_gpgroot,
options=['--trust-model', 'always'])
self.gpgh.encoding = "UTF-8"
except Exception as excep:
log.err("Unable to instance GPG object: %s" % excep)
raise excep
def sanitize_gpg_string(self, key):
"""
@param key: A pgp armored key
@return: Sanitized string or raise InvalidInputFormat
This function validate the integrity of a GPG key
"""
lines = key.split("\n")
sanitized = ""
start = 0
if not len(lines[start]):
start += 1
if lines[start] != '-----BEGIN PGP PUBLIC KEY BLOCK-----':
raise errors.InvalidInputFormat("GPG invalid format")
else:
sanitized += lines[start] + "\n"
i = 0
while i < len(lines):
# the C language has left some archetypes in my code
# [ITA] https://www.youtube.com/watch?v=7jI4DnRJP3k
i += 1
try:
if len(lines[i]) < 2:
continue
except IndexError:
continue
main_content = re.compile(
r"^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$",
re.UNICODE)
base64only = main_content.findall(lines[i])
if len(base64only) == 1:
sanitized += str(base64only[0]) + "\n"
# this GPG/PGP format it's different from the common base64 ? dunno
if len(lines[i]) == 5 and lines[i][0] == '=':
sanitized += str(lines[i]) + "\n"
if lines[i] == '-----END PGP PUBLIC KEY BLOCK-----':
sanitized += lines[i] + "\n"
return sanitized
raise errors.InvalidInputFormat("Malformed PGP key block")
def load_key(self, key):
"""
@param key:
@return: True or False, True only if a key is effectively importable and listed.
"""
try:
sanitized_key = self.sanitize_gpg_string(key)
import_result = self.gpgh.import_keys(sanitized_key)
except Exception as excep:
log.err("Error in GPG import_keys: %s" % excep)
raise errors.GPGKeyInvalid
if len(import_result.fingerprints) != 1:
raise errors.GPGKeyInvalid
fingerprint = import_result.fingerprints[0]
# looking if the key is effectively reachable
try:
all_keys = self.gpgh.list_keys()
except Exception as excep:
log.err("Error in GPG list_keys: %s" % excep)
raise errors.GPGKeyInvalid
info = u""
expiration = datetime.utcfromtimestamp(0)
for key in all_keys:
if key['fingerprint'] == fingerprint:
if key['expires']:
expiration = datetime.utcfromtimestamp(int(key['expires']))
exp_date = datetime_to_day_str(expiration)
else:
exp_date = u'Never'
info += "Key length: %s\n" % key['length']
info += "Key expiration: %s\n" % exp_date
try:
for uid in key['uids']:
info += "\t%s\n" % uid
except Exception as excep:
log.err("Error in GPG key format/properties: %s" % excep)
raise errors.GPGKeyInvalid
break
if not len(info):
log.err("Key apparently imported but unable to reload it")
raise errors.GPGKeyInvalid
ret = {
'fingerprint': fingerprint,
'expiration': expiration,
'info': info
}
return ret
def encrypt_file(self, key_fingerprint, plainpath, filestream,
output_path):
"""
@param gpg_key_armor:
@param plainpath:
@return:
"""
encrypt_obj = self.gpgh.encrypt_file(filestream, str(key_fingerprint))
if not encrypt_obj.ok:
raise errors.GPGKeyInvalid
log.debug("Encrypting for key %s file %s (%d bytes)" %
(key_fingerprint, plainpath, len(str(encrypt_obj))))
encrypted_path = os.path.join(
os.path.abspath(output_path),
"gpg_encrypted-%s" % xeger(r'[A-Za-z0-9]{8}'))
try:
with open(encrypted_path, "w+") as f:
f.write(str(encrypt_obj))
return encrypted_path, len(str(encrypt_obj))
except Exception as excep:
log.err("Error in writing GPG file output: %s (%s) bytes %d" %
(excep.message, encrypted_path, len(str(encrypt_obj))))
raise errors.InternalServerError("Error in writing [%s]" %
excep.message)
def encrypt_message(self, key_fingerprint, plaintext):
"""
@param plaindata:
An arbitrary long text that would be encrypted
@param receiver_desc:
The output of
globaleaks.handlers.admin.admin_serialize_receiver()
dictionary. It contain the fingerprint of the Receiver PUBKEY
@return:
The unicode of the encrypted output (armored)
"""
# This second argument may be a list of fingerprint, not just one
encrypt_obj = self.gpgh.encrypt(plaintext, str(key_fingerprint))
if not encrypt_obj.ok:
raise errors.GPGKeyInvalid
log.debug(
"Encrypting for key %s %d byte of plain data (%d cipher output)" %
(key_fingerprint, len(plaintext), len(str(encrypt_obj))))
return str(encrypt_obj)
def destroy_environment(self):
try:
shutil.rmtree(self.gpgh.gnupghome)
except Exception as excep:
log.err("Unable to clean temporary GPG environment: %s: %s" %
(self.gpgh.gnupghome, excep))
class TempGPGWrapper(object):
"""
A context manager that wraps a temporary GPG keyring which only contains
the keys given at object creation.
"""
def __init__(self, keys=None, gpgbinary=None):
"""
Create an empty temporary keyring and import any given C{keys} into
it.
:param keys: OpenPGP key, or list of.
:type keys: OpenPGPKey or list of OpenPGPKeys
:param gpgbinary: Name for GnuPG binary executable.
:type gpgbinary: C{str}
"""
self._gpg = None
self._gpgbinary = gpgbinary
if not keys:
keys = list()
if not isinstance(keys, list):
keys = [keys]
self._keys = keys
for key in keys:
leap_assert_type(key, OpenPGPKey)
def __enter__(self):
"""
Build and return a GPG keyring containing the keys given on
object creation.
:return: A GPG instance containing the keys given on object creation.
:rtype: gnupg.GPG
"""
self._build_keyring()
return self._gpg
def __exit__(self, exc_type, exc_value, traceback):
"""
Ensure the gpg is properly destroyed.
"""
# TODO handle exceptions and log here
self._destroy_keyring()
def _build_keyring(self):
"""
Create a GPG keyring containing the keys given on object creation.
:return: A GPG instance containing the keys given on object creation.
:rtype: gnupg.GPG
"""
privkeys = [key for key in self._keys if key and key.private is True]
publkeys = [key for key in self._keys if key and key.private is False]
# here we filter out public keys that have a correspondent
# private key in the list because the private key_data by
# itself is enough to also have the public key in the keyring,
# and we want to count the keys afterwards.
privfps = map(lambda privkey: privkey.fingerprint, privkeys)
publkeys = filter(
lambda pubkey: pubkey.fingerprint not in privfps, publkeys)
listkeys = lambda: self._gpg.list_keys()
listsecretkeys = lambda: self._gpg.list_keys(secret=True)
self._gpg = GPG(binary=self._gpgbinary,
homedir=tempfile.mkdtemp())
leap_assert(len(listkeys()) is 0, 'Keyring not empty.')
# import keys into the keyring:
# concatenating ascii-armored keys, which is correctly
# understood by GPG.
self._gpg.import_keys("".join(
[x.key_data for x in publkeys + privkeys]))
# assert the number of keys in the keyring
leap_assert(
len(listkeys()) == len(publkeys) + len(privkeys),
'Wrong number of public keys in keyring: %d, should be %d)' %
(len(listkeys()), len(publkeys) + len(privkeys)))
leap_assert(
len(listsecretkeys()) == len(privkeys),
'Wrong number of private keys in keyring: %d, should be %d)' %
(len(listsecretkeys()), len(privkeys)))
def _destroy_keyring(self):
"""
Securely erase the keyring.
"""
# TODO: implement some kind of wiping of data or a more
# secure way that
# does not write to disk.
try:
for secret in [True, False]:
for key in self._gpg.list_keys(secret=secret):
self._gpg.delete_keys(
key['fingerprint'],
secret=secret)
leap_assert(len(self._gpg.list_keys()) is 0, 'Keyring not empty!')
except:
raise
finally:
leap_assert(self._gpg.homedir != os.path.expanduser('~/.gnupg'),
"watch out! Tried to remove default gnupg home!")
shutil.rmtree(self._gpg.homedir)
def _check_openpgp_pubkey_from_file(self, pubkey_file, long_key_id):
with TemporaryDirectory() as temp_gpg_home:
gpg = GPG(gnupghome=temp_gpg_home)
with open(pubkey_file, 'rb') as pubkey_fh:
import_result = gpg.import_keys(pubkey_fh.read())
if len(import_result.results) == 0:
raise Exception("The OpenPGP file {} contains no OpenPGP keys."
" Keys: {}".format(
pubkey_file,
import_result.results,
))
logging.info(
"OK - OpenPGP file {pubkey_file} contains one or more OpenPGP key."
.format(pubkey_file=pubkey_file, ))
fingerprint = [
x['fingerprint'] for x in import_result.results
if x.get('fingerprint')
][0]
actual_long_key_id = fingerprint[-16:]
given_long_key_id = re.sub(r'^0x', '', long_key_id)
if actual_long_key_id.lower() != given_long_key_id.lower():
raise Exception(
textwrap.dedent("""
The OpenPGP file {given_long_key_id} contains a different key than what the file name suggests.
Key ID from file name: {given_long_key_id},
Key ID from pubkey in file: {actual_long_key_id}
""").lstrip().format(
given_long_key_id=given_long_key_id,
actual_long_key_id=actual_long_key_id,
))
logging.info(
"OK - OpenPGP file {pubkey_file} contains a OpenPGP public key"
" whose long key ID matching the file name.".format(
pubkey_file=pubkey_file, ))
list_key = gpg.list_keys()[0]
epoch_time = int(time.time())
expires_time = int(list_key['expires'])
if self._strict:
if expires_time < epoch_time:
raise Exception(
textwrap.dedent("""
The OpenPGP file {} contains a expired OpenPGP key.
Current date: {}
Expiration date: {}
""").lstrip().format(
pubkey_file,
datetime.fromtimestamp(epoch_time),
datetime.fromtimestamp(expires_time),
))
else:
logging.info(
"OK - OpenPGP public key from {pubkey_file} is not expired."
" Expiration date: {expiration_date}".format(
pubkey_file=pubkey_file,
expiration_date=datetime.fromtimestamp(
expires_time),
))
# https://keyring.debian.org/creating-key.html
if self._strict:
if int(list_key['length']) < self._OPENPGP_MIN_KEY_SIZE:
raise Exception(
textwrap.dedent("""
The OpenPGP file {} contains a weak OpenPGP key.
Current key length in bits: {}
Expected at least (inclusive): {}
""").lstrip().format(
pubkey_file,
list_key['length'],
self._OPENPGP_MIN_KEY_SIZE,
))
else:
logging.info(
"OK - The key length of the OpenPGP public key from {pubkey_file} is not considered to be weak."
" Key length in bits: {key_size}".format(
pubkey_file=pubkey_file,
key_size=list_key['length'],
))
return True
def test_check_git_commits_ok():
with TemporaryDirectory() as tmp_git_repo:
gpg_tmp_home = os.path.join(tmp_git_repo, 'gpg_tmp_home')
# Included in the repository to avoid problems with low entropy in CI
# environments.
# input_data = gpg.gen_key_input(
# key_type='RSA',
# subkey_type='RSA',
# key_length=1024,
# subkey_length=1024,
# name_comment='This is only a test key who’s private key is publicly know. Don’t use this key for anything!1!',
# name_email='*****@*****.**',
# # Has already expired at the time of creation to ensure no one will ever use the key.
# expire_date='2012-12-24', # Needs to be set to the 24 for the key to expire on 23.
# # Hm, Ok, gpg does not do that by default. `--faked-system-time`
# # could be used to force it but that would require cmd access.
# # https://www.gnupg.org/documentation/manuals/gnupg/Unattended-GPG-key-generation.html
# # "gpg: Invalid option "--faked-system-time"" :(
# # Only: gpg2 --batch --gen-key --debug=0 --faked-system-time '2342-05-23'
# # has been confirmed to work from current Debian Stretch.
# # Faking it manually anyway …
# )
# print(input_data)
# print(gpg.gen_key(input_data))
shutil.copytree(debops_keyring_fake_gnupg_home, gpg_tmp_home)
gpg = GPG(gnupghome=gpg_tmp_home)
os.chmod(gpg_tmp_home, 0o700)
for r, d, f in os.walk(gpg_tmp_home):
os.chmod(r, 0o700)
gpg_key_fingerprint = gpg.list_keys()[0]['fingerprint']
gpg_edit_key_cmd = subprocess.Popen(
['gpg', '--homedir', gpg_tmp_home, '--command-fd', '0', '--batch', '--edit-key', gpg_key_fingerprint],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
)
(gpg_edit_key_cmd_stdout, gpg_edit_key_cmd_stderr) = gpg_edit_key_cmd.communicate(
input='expire\n0\nsave\n'.encode(),
timeout=5,
)
# print(gpg_edit_key_cmd_stderr.decode())
# print(subprocess.check_output(['gpg', '--homedir', gpg_tmp_home, '--list-public-keys']).decode())
if 'expires: never' not in str(gpg_edit_key_cmd_stderr):
raise Exception("Could not change expiration date.")
tmp_keyring_dir = os.path.join(tmp_git_repo, 'tmp-keyring-gpg')
tmp_pubkey_file = os.path.join(
tmp_keyring_dir,
'0x' + gpg_key_fingerprint[-16:].upper()
)
os.mkdir(tmp_keyring_dir)
with open(tmp_pubkey_file, 'w') as tmp_pubkey_fh:
tmp_pubkey_fh.write(gpg.export_keys(gpg_key_fingerprint))
git_cmd = git.Git(tmp_git_repo)
git_cmd.update_environment(
GNUPGHOME=debops_keyring_fake_gnupg_home,
)
git_cmd.init()
git_cmd.config(['user.signingkey', gpg_key_fingerprint])
git_cmd.config(['user.email', '*****@*****.**'])
git_cmd.config(['user.name', 'debops-keyring-test'])
git_cmd.update_environment(
GNUPGHOME=os.path.join(tmp_git_repo, 'gpg_tmp_home'),
)
tmp_git_file = os.path.join(tmp_git_repo, 'new-file')
with open(tmp_git_file, 'w') as tmp_git_fh:
tmp_git_fh.write(str(time.time()))
git_cmd.add([tmp_git_file])
git_cmd.commit(['--gpg-sign', '--message', 'Signed commit'])
debops_keyring = Keyring(
keyring_name=tmp_keyring_dir,
)
debops_keyring.check_git_commits(tmp_git_repo)
# Now make an unsigned commit to ensure that this raises an exception.
with open(tmp_git_file, 'w') as tmp_git_fh:
tmp_git_fh.write(str(time.time()))
git_cmd.add([tmp_git_file])
git_cmd.commit(['--no-gpg-sign', '--message', 'Unsigned commit'])
try:
debops_keyring.check_git_commits(tmp_git_repo)
assert False
except Exception as e:
if 'OpenPGP signature of commit could not be verified' in str(e) and 'Unsigned commit' in str(e):
assert True
else:
assert False
class GLBPGP(object):
"""
PGP has not a dedicated class, because one of the function is called inside a transact, and
I'm not quite confident on creating an object that operates on the filesystem knowing
that would be run also on the Storm cycle.
"""
def __init__(self):
"""
every time is needed, a new keyring is created here.
"""
try:
temp_pgproot = os.path.join(GLSettings.pgproot, "%s" % xeger(r'[A-Za-z0-9]{8}'))
os.makedirs(temp_pgproot, mode=0700)
self.pgph = GPG(gnupghome=temp_pgproot, options=['--trust-model', 'always'])
self.pgph.encoding = "UTF-8"
except OSError as ose:
log.err("Critical, OS error in operating with GnuPG home: %s" % ose)
raise
except Exception as excep:
log.err("Unable to instance PGP object: %s" % excep)
raise
def load_key(self, key):
"""
@param key:
@return: True or False, True only if a key is effectively importable and listed.
"""
try:
import_result = self.pgph.import_keys(key)
except Exception as excep:
log.err("Error in PGP import_keys: %s" % excep)
raise errors.PGPKeyInvalid
if len(import_result.fingerprints) != 1:
raise errors.PGPKeyInvalid
fingerprint = import_result.fingerprints[0]
# looking if the key is effectively reachable
try:
all_keys = self.pgph.list_keys()
except Exception as excep:
log.err("Error in PGP list_keys: %s" % excep)
raise errors.PGPKeyInvalid
info = u""
expiration = datetime.utcfromtimestamp(0)
for key in all_keys:
if key['fingerprint'] == fingerprint:
if key['expires']:
expiration = datetime.utcfromtimestamp(int(key['expires']))
exp_date = datetime_to_day_str(expiration)
else:
exp_date = u'Never'
info += "Key length: %s\n" % key['length']
info += "Key expiration: %s\n" % exp_date
try:
for uid in key['uids']:
info += "\t%s\n" % uid
except Exception as excep:
log.err("Error in PGP key format/properties: %s" % excep)
raise errors.PGPKeyInvalid
break
if not len(info):
log.err("Key apparently imported but unable to reload it")
raise errors.PGPKeyInvalid
ret = {
'fingerprint': fingerprint,
'expiration': expiration,
'info': info
}
return ret
def encrypt_file(self, key_fingerprint, plainpath, filestream, output_path):
"""
@param pgp_key_public:
@param plainpath:
@return:
"""
encrypt_obj = self.pgph.encrypt_file(filestream, str(key_fingerprint))
if not encrypt_obj.ok:
raise errors.PGPKeyInvalid
log.debug("Encrypting for key %s file %s (%d bytes)" %
(key_fingerprint,
plainpath, len(str(encrypt_obj))))
encrypted_path = os.path.join(os.path.abspath(output_path),
"pgp_encrypted-%s" % xeger(r'[A-Za-z0-9]{16}'))
try:
with open(encrypted_path, "w+") as f:
f.write(str(encrypt_obj))
return encrypted_path, len(str(encrypt_obj))
except Exception as excep:
log.err("Error in writing PGP file output: %s (%s) bytes %d" %
(excep.message, encrypted_path, len(str(encrypt_obj)) ))
raise errors.InternalServerError("Error in writing [%s]" % excep.message)
def encrypt_message(self, key_fingerprint, plaintext):
"""
@param plaindata:
An arbitrary long text that would be encrypted
@param receiver_desc:
The output of
globaleaks.handlers.admin.admin_serialize_receiver()
dictionary. It contain the fingerprint of the Receiver PUBKEY
@return:
The unicode of the encrypted output (armored)
"""
# This second argument may be a list of fingerprint, not just one
encrypt_obj = self.pgph.encrypt(plaintext, str(key_fingerprint))
if not encrypt_obj.ok:
raise errors.PGPKeyInvalid
log.debug("Encrypting for key %s %d byte of plain data (%d cipher output)" %
(key_fingerprint,
len(plaintext), len(str(encrypt_obj))))
return str(encrypt_obj)
def destroy_environment(self):
try:
shutil.rmtree(self.pgph.gnupghome)
except Exception as excep:
log.err("Unable to clean temporary PGP environment: %s: %s" % (self.pgph.gnupghome, excep))
