def update_ldap_users_from_ad():
# Setup the ldap connection
dn = "cn=admin,dc=canvas,dc=virtue,dc=com"
ldap = LDAP('', '')
ldap.get_ldap_connection()
# FIXME - Figure out a soluton for the Hardcoded password
# TODO - Fix Hardcoded password for ldap and AD connection
ldap.conn.simple_bind_s(dn, "Test123!")
# Bind the AD connection
ad = LDAP('*****@*****.**', 'Test123!')
ad.bind_ad()
# Query AD for all users
ad_users = ad.query_ad('objectClass', 'user')
ldap_usernames = get_ldap_usernames(ldap)
# Go through all users in AD
for user in ad_users:
cn = user[1]['cn'][0]
ad_username = user[1]['sAMAccountName'][0]
# If a username in AD is not found in ldap then add it
if ad_username not in ldap_usernames:
new_user = {
'username': ad_username,
'authorizedRoleIds': [],
'name': cn
}
new_ldap_user = ldap_tools.to_ldap(new_user, 'OpenLDAPuser')
ldap.add_obj(new_ldap_user, 'users', 'cusername', throw_error=True)
# Check to make sure the key directory exists
if (not os.path.exists('{0}/user-keys'.format(
os.environ['HOME']))):
os.mkdir('{0}/user-keys'.format(os.environ['HOME']))
# Create/Add keys for each user
subprocess.check_call([
'ssh-keygen', '-t', 'rsa', '-f',
'{}/user-keys/{}.pem'.format(os.environ['HOME'], ad_username),
'-C', '"For Virtue user {0}"'.format(ad_username), '-N', ''
])
# Create a dummy kerberos file for the user.
# It'll be regenerated as soon as the user logs in.
with open('/tmp/krb5cc_{}'.format(ad_username), 'w') as f:
f.write("Error 404: Easter Egg text not found")
return ldap
class EndPoint_Security:
rethinkdb_host = 'rethinkdb.galahad.com'
ca_cert = '/var/private/ssl/rethinkdb_cert.pem'
excalibur_key_file = '/var/private/ssl/excalibur_private_key.pem'
virtue_key_dir = os.environ['HOME'] + '/user-keys'
wait_for_ack = 30 # seconds
def __init__(self, user, password):
self.inst = LDAP(user, password)
dn = 'cn=admin,dc=canvas,dc=virtue,dc=com'
self.inst.get_ldap_connection()
self.inst.conn.simple_bind_s(dn, 'Test123!')
self.rdb_manager = RethinkDbManager()
self.conn = None
def transducer_list(self):
'''
Lists all transducers currently available in the system
Args:
Returns:
list: Transducer objects for known transducers
'''
try:
transducers_raw = self.inst.get_objs_of_type('OpenLDAPtransducer')
if transducers_raw is None:
return self.__error(
'unspecifiedError',
details='Unable to get transducer objects from LDAP')
transducers_ret = []
for transducer in transducers_raw:
ldap_tools.parse_ldap(transducer[1])
transducers_ret.append(transducer[1])
return json.dumps(transducers_ret)
except Exception as e:
return self.__error('unspecifiedError',
details='Unable to get transducer objects: ' +
str(e))
def transducer_get(self, transducerId):
'''
Gets information about the indicated Transducer. Does not include information about any
instantiation in Virtues.
Args:
transducerId (str): The ID of the Transducer to get.
Returns:
Transducer: information about the indicated transducer
'''
try:
transducer = self.inst.get_obj('cid', transducerId,
'openLDAPtransducer', True)
if transducer is None or transducer == ():
return self.__error('invalidTransducerId')
ldap_tools.parse_ldap(transducer)
if DEBUG_PERMISSIONS:
return json.dumps(transducer)
return json.dumps(transducer)
except Exception as e:
return self.__error('unspecifiedError',
details='Unable to get transducer object: ' +
str(e))
def transducer_enable(self, transducerId, virtueId, configuration):
'''
Enables the indicated Transducer in the indicated Virtue.
Args:
transducerId (str) : The ID of the Transducer to enable.
virtueId (str) : The ID of the Virtue in which to enable the Transducer.
configuration (object): The configuration to apply to the Transducer when it is enabled. Format
is Transducer-specific. This overrides any existing configuration with the same keys.
Returns:
bool: True if the transducer was enabled, false otherwise
'''
ret = self.__enable_disable(transducerId, virtueId, configuration,
True)
if type(ret) is bool and ret == True:
return self.__error('success')
return ret
def transducer_disable(self, transducerId, virtueId):
'''
Disables the indicated Transducer in the indicated Virtue
Args:
transducerId (str) : The ID of the Transducer to disable
virtueId (str) : The ID of the Virtue in which to enable the Transducer.
Returns:
bool: True if the transducer was enabled, false otherwise
'''
ret = self.__enable_disable(transducerId, virtueId, None, False)
if type(ret) is bool and ret == True:
return self.__error('success')
return ret
def transducer_get_enabled(self, transducerId, virtueId):
'''
Gets the current enabled status for the indicated Transducer in the indicated Virtue.
Args:
transducerId (str) : The ID of the Transducer
virtueId (str) : The ID of the Virtue in which to enable the Transducer.
Returns:
bool: True if the Transducer is enabled in the Virtue, false if it is not
'''
if self.conn is None:
self.__connect_rethinkdb()
try:
row = r.db('transducers').table('acks')\
.get([virtueId, transducerId]).run(self.conn)
except r.ReqlError as e:
return self.__error(
'unspecifiedError',
details='Failed to get info about transducer: ' + str(e))
# Transducers are enabled by default
if row is None:
return json.dumps({'enabled': True})
verified = self.__verify_message(row)
if (verified != True):
return verified
return json.dumps({'enabled': row['enabled']})
def transducer_get_configuration(self, transducerId, virtueId):
'''
Gets the current configuration for the indicated Transducer in the indicated Virtue.
Args:
transducerId (str) : The ID of the Transducer
virtueId (str) : The ID of the Virtue in which to enable the Transducer.
Returns:
TransducerConfig: Configuration information for the indicated Transducer in the indicated Virtue
'''
if self.conn is None:
self.__connect_rethinkdb()
try:
row = r.db('transducers').table('acks')\
.get([virtueId, transducerId]).run(self.conn)
except r.ReqlError as e:
return self.__error(
'unspecifiedError',
details='Failed to get info about transducer: ' + str(e))
# If there's no row, there's no set config
if row is None:
return json.dumps(None)
verified = self.__verify_message(row)
if (verified != True):
return verified
# By definition, the configuration is a JSON object
return row['configuration']
def transducer_list_enabled(self, virtueId):
'''
Lists all Transducers currently that are currently enabled in the indicated Virtue.
Args:
virtueId (str) : The ID of the Virtue in which to enable the Transducer.
Returns:
list(Transducer): List of enabled transducers within the specified Virtue
'''
enabled_transducers = []
if self.conn is None:
self.__connect_rethinkdb()
try:
for row in r.db('transducers').table('acks')\
.filter({'virtue_id': virtueId}).run(self.conn):
verified = self.__verify_message(row)
if (verified != True):
return verified
if ('enabled' in row) and row['enabled']:
enabled_transducers.append(row['transducer_id'])
except r.ReqlError as e:
return self.__error('unspecifiedError',
details='Failed to get enabled transducers: ' +
str(e))
return json.dumps(enabled_transducers)
def transducer_all_virtues(self, transducerId, configuration, isEnabled):
'''
Sets the given transducer to true for all running and stopped virtues.
Updates roles and starting configuration to match, so additionally created
or started virtues will share the same config.
:param transducerId:
:param isEnabled::o
:return:
'''
try:
ret = self.__update_roles_transducer(transducerId, isEnabled)
if ret != True:
return ret
ret = self.__update_transducer_starting_config(
transducerId, configuration)
if ret != True:
return ret
virtues_raw = self.inst.get_objs_of_type('OpenLDAPvirtue')
if (virtues_raw == None):
return json.dumps(ErrorCodes.user['unspecifiedError'])
for virtue in virtues_raw:
ldap_tools.parse_ldap(virtue[1])
ret = self.__enable_disable(transducerId, virtue[1]['id'],
configuration, isEnabled)
if (ret != True):
return json.dumps(ret)
return self.__error('success')
except:
print('Error:\n{0}'.format(traceback.format_exc()))
return json.dumps(ErrorCodes.user['unspecifiedError'])
def __connect_rethinkdb(self):
# RethinkDB connection
# This connection will fail if setup_rethinkdb.py hasn't been run, because
# there won't be an excalibur user and it won't have the specified password.
with open(self.excalibur_key_file, 'r') as f:
key = f.read()
self.excalibur_key = RSA.importKey(key)
try:
self.conn = r.connect(host=self.rethinkdb_host,
user='******',
password=key,
ssl={'ca_certs': self.ca_cert})
except r.ReqlDriverError as e:
return self.__error('unspecifiedError', details=\
'Failed to connect to RethinkDB at host: ' + \
self.rethinkdb_host + ' because: ' + str(e))
return True
def __enable_disable(self, transducerId, virtueId, configuration,
isEnable):
# Make sure transducer exists
transducer = self.inst.get_obj('cid', transducerId,
'openLDAPtransducer', True)
if transducer is None or transducer == ():
return self.__error('invalidTransducerId')
ldap_tools.parse_ldap(transducer)
# Make sure virtue exists
virtue = self.inst.get_obj('cid', virtueId, 'openLDAPvirtue', True)
if virtue is None or virtue == ():
return self.__error('invalidVirtueId')
ldap_tools.parse_ldap(virtue)
virtue_running = (virtue['state'] == 'RUNNING')
# Change the ruleset
ret = self.__change_ruleset(virtueId,
transducerId,
transducer['type'],
isEnable,
virtue_running,
config=configuration)
if ret != True:
return ret
if virtue_running:
# Update the virtue's list of transducers
# Call loads because transducer_list_enabled returns a string
new_t_list = json.loads(self.transducer_list_enabled(virtueId))
if type(new_t_list) is dict and new_t_list['status'] == 'failed':
# Couldn't retrieve new list of transducers
return self.__error(
'unspecifiedError',
details='Unable to update virtue\'s list of transducers')
virtue['transducerIds'] = new_t_list
ret = self.inst.modify_obj(
'cid', virtueId, ldap_tools.to_ldap(virtue, 'OpenLDAPvirtue'),
'OpenLDAPvirtue', True)
else:
# Update list of transducers in LDAP without syncing it
# with rethink (Non-running virutes are not in rethink)
new_t_list = virtue['transducerIds']
if isEnable:
if transducerId not in new_t_list:
new_t_list.append(transducerId)
else:
if transducerId in new_t_list:
new_t_list.remove(transducerId)
virtue['transducerIds'] = new_t_list
ret = self.inst.modify_obj(
'cid', virtueId, ldap_tools.to_ldap(virtue, 'OpenLDAPvirtue'),
'OpenLDAPvirtue', True)
if ret != 0:
return self.__error(
'unspecifiedError',
details='Unable to update virtue\'s list of transducers')
return True
def __sign_message(self, row):
required_keys = [
'virtue_id', 'transducer_id', 'type', 'configuration', 'enabled',
'timestamp'
]
if not all([(key in row) for key in required_keys]):
return (False,
self.__error('unspecifiedError', details='Missing required keys in row: ' +\
str(filter((lambda key: key not in row), required_keys))))
message = '|'.join([
row['virtue_id'], row['transducer_id'], row['type'],
str(row['configuration']),
str(row['enabled']),
str(row['timestamp'])
])
h = SHA.new(str(message))
signer = PKCS1_v1_5.new(self.excalibur_key)
signature = signer.sign(h)
return (True, signature)
def __verify_message(self, row):
if row is None:
return self.__error('unspecifiedError',
details='No match found in database')
required_keys = [
'virtue_id', 'transducer_id', 'type', 'configuration', 'enabled',
'timestamp', 'signature'
]
if not all([(key in row) for key in required_keys]):
return self.__error('unspecifiedError', details='Missing required keys in row: ' +\
str(filter((lambda key: key not in row), required_keys)))
message = '|'.join([
row['virtue_id'], row['transducer_id'], row['type'],
str(row['configuration']),
str(row['enabled']),
str(row['timestamp'])
])
virtue_public_key = os.path.join(self.virtue_key_dir,
row['virtue_id'] + '.pem.pub')
if not os.path.isfile(virtue_public_key):
return self.__error('invalidOrMissingParameters', details=\
'No file found for Virtue public key at: ' + \
virtue_public_key)
with open(virtue_public_key) as f:
virtue_key = RSA.importKey(f.read())
h = SHA.new(str(message))
verifier = PKCS1_v1_5.new(virtue_key)
verified = verifier.verify(h, row['signature'])
if not verified:
printable_msg = deepcopy(row)
del printable_msg['signature']
return self.__error('unspecifiedError', details=\
'Unable to validate signature of ACK message: ' + \
json.dumps(printable_msg, indent=2))
return True
def __change_ruleset(self,
virtue_id,
trans_id,
transducer_type,
enable,
virtue_running,
config=None):
if self.conn is None:
ret = self.__connect_rethinkdb()
# Return if error
if ret != True:
return ret
if type(transducer_type) is list:
transducer_type = transducer_type[0]
timestamp = int(time.time())
row = {
'id': [virtue_id, trans_id],
'virtue_id': virtue_id,
'transducer_id': trans_id,
'type': transducer_type,
'configuration': config,
'enabled': enable,
'timestamp': timestamp
}
(success, signature) = self.__sign_message(row)
if not success:
# Return error code
return signature
row['signature'] = r.binary(signature)
# Send command to change ruleset
try:
res = r.db('transducers').table('commands')\
.insert(row, conflict='replace').run(self.conn)
if res['errors'] > 0:
return self.__error(
'unspecifiedError',
details=
'Failed to insert into commands table; first error: ' +
res['first_error'])
except r.ReqlError as e:
return self.__error(
'unspecifiedError',
details='Failed to insert into commands table: ' + str(e))
# If the virtue isn't running yet, don't bother waiting for an ACK
if not virtue_running:
return True
# Wait for ACK from the virtue that the ruleset has been changed
# try:
cursor = r.db('transducers').table('acks')\
.get([virtue_id, trans_id])\
.changes(squash=False).run(self.conn)
# except r.ReqlError as e:
# print 'ERROR: Failed to read from the ACKs table because:', e
# return False
retry = True
while retry:
try:
retry = False
# Wait max 30 seconds - if we miss the real ACK, hopefully
# at least the next heartbeat will suffice
print 'INFO: Waiting for ACK'
change = cursor.next(wait=self.wait_for_ack)
row = change['new_val']
verified = self.__verify_message(row)
if (verified != True):
return verified
if row['timestamp'] >= timestamp:
if row['enabled'] == enable:
print 'INFO: ACK received!'
return True
else:
return self.__error(
'unspecifiedError',
details=
'Received ACK with incorrect value for enabled: ' +
str(enable) + ' vs ' + str(row['enabled']))
else:
print 'WARN: Timestamp incorrect:', timestamp, row[
'timestamp']
# Retry once in case that was just a wayward ACK
retry = True
except (r.ReqlCursorEmpty, r.ReqlDriverError) as e:
return self.__error(
'unspecifiedError',
details='Failed to receive ACK before timeout')
finally:
cursor.close()
return self.__error('unspecifiedError',
details='Failed to receive ACK before timeout')
def __error(self, key, details=None):
if key not in ErrorCodes.security:
print "Error type '" + key + "' not found! Using UnspecifiedError."
key = 'unspecifiedError'
e = deepcopy(ErrorCodes.security[key])
if details is not None:
# e['details'] = details
e['result'].append(details)
return json.dumps(e)
def __update_roles_transducer(self, transducerId, isEnabled):
# Make sure transducer exists
transducer = self.inst.get_obj('cid', transducerId,
'openLDAPtransducer', True)
if transducer is None or transducer == ():
return self.__error('invalidTransducerId')
try:
ldap_roles = self.inst.get_objs_of_type('OpenLDAProle')
assert ldap_roles != None
roles = ldap_tools.parse_ldap_list(ldap_roles)
for role in roles:
new_t_list = role['startingTransducerIds']
if isEnabled:
if transducerId not in new_t_list:
new_t_list.append(transducerId)
else:
if transducerId in new_t_list:
new_t_list.remove(transducerId)
role['startingTransducerIds'] = new_t_list
ret = self.inst.modify_obj(
'cid', role['id'],
ldap_tools.to_ldap(role,
'OpenLDAProle'), 'OpenLDAProle', True)
if ret != 0:
return self.__error(
'unspecifiedError',
details='Unable to update virtue\'s list of transducers'
)
return True
except Exception as e:
print('Error:\n{0}'.format(traceback.format_exc()))
return json.dumps(ErrorCodes.admin['unspecifiedError'])
def __update_transducer_starting_config(self, transducerId, configuration):
transducer = self.inst.get_obj('cid', transducerId,
'openLDAPtransducer', True)
if transducer is None or transducer == ():
return self.__error('invalidTransducerId')
ldap_tools.parse_ldap(transducer)
transducer['startingConfiguration'] = configuration
ret = self.inst.modify_obj(
'cid', transducer['id'],
ldap_tools.to_ldap(transducer, 'openLDAPtransducer'),
'openLDAPtransducer', True)
if ret != 0:
return self.__error(
'unspecifiedError',
details='Unable to update virtue\'s list of transducers')
return True
class AssembleRoleThread(threading.Thread):
def __init__(self,
ldap_user,
ldap_password,
role,
base_img_name,
use_ssh=True):
super(AssembleRoleThread, self).__init__()
self.inst = LDAP(ldap_user, ldap_password)
# Going to need to write to LDAP
# self.inst.bind_ldap()
dn = 'cn=admin,dc=canvas,dc=virtue,dc=com'
self.inst.get_ldap_connection()
self.inst.conn.simple_bind_s(dn, 'Test123!')
self.role = role
self.base_img_name = base_img_name
self.use_ssh = use_ssh
def run(self):
self.role['state'] = 'CREATING'
ldap_role = ldap_tools.to_ldap(self.role, 'OpenLDAProle')
ret = self.inst.add_obj(ldap_role, 'roles', 'cid', throw_error=True)
assert ret == 0
virtue_path = 'images/non_provisioned_virtues/' + self.role[
'id'] + '.img'
key_path = USER_KEY_DIR + '/default-virtue-key.pem'
valor_manager = ValorManager()
try:
# Create the role standby image files
standby_roles = StandbyRoles(self.base_img_name, self.role)
standby_roles.create_role_image_file()
if (self.use_ssh):
# Launch by adding a 'virtue' to RethinkDB
valor = valor_manager.get_empty_valor()
virtue_ip = valor_manager.add_virtue(
valor['address'], valor['valor_id'], self.role['id'],
virtue_path, True) # send role_create=True)
time.sleep(5)
# Get Docker login command
# Use the AWS Account ID of the Account which has the docker registry,
# currently this is in the StarLab account.
docker_cmd = subprocess.check_output(
shlex.split(
'aws ecr get-login --registry-ids {} --no-include-email '
'--region us-east-2'.format(AWS_ECR_ACCOUNT_NUMBER)))
print('docker_cmd: ' + docker_cmd)
# Run assembler
assembler = Assembler(work_dir='{0}/.{1}_assembly'.format(
os.environ['HOME'], self.role['id']))
assembler.assemble_running_vm(self.role['applicationIds'],
docker_cmd, key_path, virtue_ip)
# Create the virtue standby image files
standby_virtues = StandbyVirtues(self.role['id'])
standby_virtues.create_standby_virtues()
self.role['state'] = 'CREATED'
ldap_role = ldap_tools.to_ldap(self.role, 'OpenLDAProle')
ret = self.inst.modify_obj('cid',
self.role['id'],
ldap_role,
objectClass='OpenLDAProle',
throw_error=True)
assert ret == 0
except:
print('Error while assembling role {0}:\n{1}'.format(
self.role['id'], traceback.format_exc()))
self.role['state'] = 'FAILED'
ldap_role = ldap_tools.to_ldap(self.role, 'OpenLDAProle')
ret = self.inst.modify_obj('cid',
self.role['id'],
ldap_role,
objectClass='OpenLDAProle',
throw_error=True)
finally:
if valor_manager.rethinkdb_manager.get_virtue(self.role['id']):
valor_manager.rethinkdb_manager.remove_virtue(self.role['id'])
class CreateVirtueThread(threading.Thread):
def __init__(self,
ldap_user,
ldap_password,
role_id,
user,
virtue_id,
role=None):
super(CreateVirtueThread, self).__init__()
self.inst = LDAP(ldap_user, ldap_password)
self.role_id = role_id
self.role = role
self.username = user
self.virtue_id = virtue_id
def run(self):
thread_list.append(self)
# Going to need to write to LDAP
# self.inst.bind_ldap()
dn = 'cn=admin,dc=canvas,dc=virtue,dc=com'
self.inst.get_ldap_connection()
self.inst.conn.simple_bind_s(dn, 'Test123!')
if (self.role == None):
role = self.inst.get_obj('cid',
self.role_id,
objectClass='OpenLDAProle')
if (role == () or role == None):
return
ldap_tools.parse_ldap(role)
else:
role = self.role
virtue = {
'id': self.virtue_id,
'username': self.username,
'roleId': self.role_id,
'applicationIds': [],
'resourceIds': role['startingResourceIds'],
'transducerIds': role['startingTransducerIds'],
'networkRules': role['networkRules'],
'state': 'CREATING',
'ipAddress': 'NULL'
}
ldap_virtue = ldap_tools.to_ldap(virtue, 'OpenLDAPvirtue')
assert self.inst.add_obj(ldap_virtue, 'virtues', 'cid') == 0
virtue_path = 'images/provisioned_virtues/' + virtue['id'] + '.img'
try:
# For now generate keys and store in local dir
subprocess.check_output(
shlex.split(('ssh-keygen -t rsa -f {0}/{1}.pem -C'
' "Virtue Key for {1}" -N ""').format(
USER_KEY_DIR, virtue['id'])))
with open(USER_KEY_DIR + '/' + virtue['id'] + '.pem',
'r') as virtue_key_file:
virtue_key = virtue_key_file.read().strip()
with open(GALAHAD_KEY_DIR + '/excalibur_pub.pem',
'r') as excalibur_key_file:
excalibur_key = excalibur_key_file.read().strip()
user_key = subprocess.check_output([
'ssh-keygen', '-y', '-f',
'{0}/{1}.pem'.format(USER_KEY_DIR, self.username)
])
with open(GALAHAD_KEY_DIR + '/rethinkdb_cert.pem',
'r') as rdb_cert_file:
rdb_cert = rdb_cert_file.read().strip()
with open('/tmp/networkRules-' + virtue['id'],
'w+') as iprules_file:
for rule in role['networkRules']:
iprules_file.write(rule + '\n')
# Create the virtue standby image files
standby_virtues = StandbyVirtues(self.role_id)
standby_virtues.create_virtue_image_file(self.virtue_id)
subprocess.check_call(['sudo', 'python',
os.environ['HOME'] + '/galahad/excalibur/'
+ \
'call_provisioner.py',
'-u', self.username,
'-i', virtue['id'],
'-n', '/tmp/networkRules-' + virtue['id'],
'-b',
'/mnt/efs/images/non_provisioned_virtues/' +
role['id'] + '.img',
'-o', '/mnt/efs/' + virtue_path,
'-v', virtue_key,
'-e', excalibur_key,
'--user_key', user_key,
'-r', rdb_cert])
# Enable/disable sensors as specified by the role
transducers = {}
for tid in role['startingTransducerIds']:
transducer = self.inst.get_obj('cid', tid,
'OpenLDAPtransducer', True)
if (transducer == ()):
continue
ldap_tools.parse_ldap(transducer)
transducers[tid] = transducer
eps = EndPoint_Security(self.inst.email, self.inst.password)
all_transducers = json.loads(eps.transducer_list())
for transducer in all_transducers:
if transducer['id'] in transducers:
config = transducer['startingConfiguration']
ret = eps.transducer_enable(transducer['id'],
self.virtue_id, config)
else:
ret = eps.transducer_disable(transducer['id'],
self.virtue_id)
virtue['state'] = 'STOPPED'
ldap_virtue = ldap_tools.to_ldap(virtue, 'OpenLDAPvirtue')
assert self.inst.modify_obj('cid', virtue['id'], ldap_virtue) == 0
except:
print('Error while creating Virtue for role {0}:\n{1}'.format(
role['id'], traceback.format_exc()))
assert self.inst.del_obj('cid',
virtue['id'],
objectClass='OpenLDAPvirtue',
throw_error=True) == 0
os.remove('{0}/{1}.pem'.format(USER_KEY_DIR, virtue['id']))
os.remove('{0}/{1}.pem.pub'.format(USER_KEY_DIR, virtue['id']))
os.remove('/mnt/efs/' + virtue_path)