def test_stream_logger_return(self):
from cis.libs import utils
u = utils.StructuredLogger('foo', 2)
logger = u.get_logger()
assert logger is not None
def handle(event, context):
utils.StructuredLogger(name='cis-idvtoidv', level=logging.INFO)
# Initialize Stream Logger
# Log level can be environment driven later in development.
logger = logging.getLogger('cis-idvtoidv')
logger.info("Stream Processor initialized.")
for record in event['Records']:
pass
return None
def __init__(self, config):
self.default_headers = {'content-type': "application/json"}
self.uri = config.uri
self.client_id = config.client_id
self.client_secret = config.client_secret
self.access_token = None
self.access_token_scope = None
self.access_token_valid_until = 0
self.conn = http.client.HTTPSConnection(config.uri)
utils.StructuredLogger(name='cis-idvtoauth0', level=logging.INFO)
self.logger = logging.getLogger('CISAuthZero')
def handle(event, context):
"""This is the main handler called during function invocation."""
utils.StructuredLogger(name='cis-validator', level=logging.INFO)
logger = logging.getLogger('cis-validator')
encrypted_profile_data = json.loads(base64.b64decode(event.get('profile')))
p = processor.ValidatorOperation(
boto_session=boto3.Session(region_name='us-west-2'),
publisher=event.get('publisher'),
signature=event.get('signature'),
encrypted_profile_data=encrypted_profile_data)
result = p.run()
logger.info('The result of the change operation was {r}'.format(r=result))
return result
def handle(event, context):
utils.StructuredLogger(name='cis-streamtoidv', level=logging.INFO)
logger = logging.getLogger('cis-streamtoidv')
logger.info("Stream Processor initialized.")
for record in event['Records']:
# Kinesis data is base64 encoded so decode here
logger.info("Record is loaded.")
payload = json.loads(
base64.b64decode(record['kinesis']['data']).decode('utf-8'))
logger.info("Initial payload decoded.")
try:
p = processor.StreamtoVaultOperation(
boto_session=boto3.Session(region_name='us-west-2'),
publisher=record['kinesis']['partitionKey'],
signature=None,
encrypted_profile_data=payload)
res = p.run()
except Exception as e:
logger.error('Error writing to dynamo: {}'.format(e))
res = False
logger.info("Payload decrypted. Attempting storage in the vault.")
# Store the result of the event in the identity vault
logger.info("Vault storage status is {status}".format(status=res))
logger.info('Successfully processed {} records.'.format(
len(event['Records'])))
return 'Successfully processed {} records.'.format(len(event['Records']))
import boto3
import logging
import os
import hris
import task
from cis.libs import utils
from cis.libs.api import Person
from cis.settings import get_config
from utils import get_secret
sl = utils.StructuredLogger(name='cis_hris', level=logging.DEBUG)
# Logger setUp
logger = logging.getLogger('cis_hris')
def dead_letter():
"""Send the profile to the dead letter queue for manual inspection."""
# XXX TBD
pass
def assume_role_session():
# Load config
config = get_config()
sts = boto3.client('sts')
sts_response = sts.assume_role(RoleArn=config('iam_role_arn',
namespace='cis'),
RoleSessionName=config(
'iam_role_session_name',
def handle(event, context):
utils.StructuredLogger(name='cis-idvtoauth0', level=logging.INFO)
logger = logging.getLogger('cis-idvtoauth0')
logger.info("Stream Processor initialized.")
environment = os.getenv('ENVIRONMENT', 'dev')
if environment == 'production':
environment = 'prod'
else:
logger.info('Development stage recognized. Applying to credstash.')
environment = 'dev'
# New up the config object for CISAuthZero
config = authzero.DotDict(dict())
config.client_id = credstash.getSecret(name="cis.client_id",
context={
'app': 'cis',
'environment': environment
},
region="us-west-2")
config.client_secret = credstash.getSecret(name="cis.client_secret",
context={
'app': 'cis',
'environment': environment
},
region="us-west-2")
config.uri = credstash.getSecret(name="cis.uri",
context={
'app': 'cis',
'environment': environment
},
region="us-west-2")
client = authzero.CISAuthZero(config)
client.get_access_token()
for record in event['Records']:
# Kinesis data is base64 encoded so decode here
logger.info("Record is loaded.")
logger.info("Processing {record}".format(record=record))
user_id = record['dynamodb']['Keys']['user_id']['S']
logger.info("Initial payload decoded.")
logger.info("Searching for dynamo record for {u}".format(u=user_id))
profile = find_user(user_id)
logger.info("Status of profile search is {s}".format(s=profile))
if profile is not None:
# Profile whitelisting. This allows to select which user profiles are
# to be integrated using CIS, mainly for transitioning purposes.
# See also: https://mozillians.org/en-US/group/cis_whitelist
if 'mozilliansorg_cis_whitelist' not in profile['groups']:
continue
# XXX Force-integrate LDAP groups as these are synchronized
# from LDAP to Auth0 directly.
# This is to be removed when LDAP feeds CIS.
try:
upstream_user = client.get_user(user_id)
if 'groups' in upstream_user.keys():
for g in upstream_user['groups']:
if g not in profile['groups']:
profile['groups'].append(g)
logger.info(
"Forced re-integration of LDAP group {}".
format(g))
res = client.update_user(user_id, profile)
except Exception as e:
"""Temporarily patch around raising inside loop until authzero.py can become part of CIS core."""
res = e
logger.info("Status of message processing is {s}".format(s=res))
else:
logger.critical(
"User could not be matched in vault for userid : {user_id}".
format(user_id=user_id))
logger.info('Successfully processed {} records.'.format(
len(event['Records'])))
return 'Successfully processed {} records.'.format(len(event['Records']))
import boto3
import json
import logging
import os
import unicodedata
from cis.libs import utils
from botocore.exceptions import ClientError
from jsonschema import validate as jsonschema_validate
from jsonschema.exceptions import ValidationError
sl = utils.StructuredLogger(name=__name__, level=logging.DEBUG)
logger = logging.getLogger('cis_hris')
class HrisJSON(object):
"""Retrieval and serialization object for the current HRIS data file."""
def __init__(self, boto_session):
self.boto_session = boto_session
self.from_file = False
self.file_name = 'workday.json'
self.s3_bucket = None
self.bucket_name = None
def load(self):
"""Fetch the file as stream from the s3 bucket."""
if self.from_file is True:
# From file set only for development purposes.
hris_json = os.path.join(
os.path.abspath(os.path.dirname(__file__)),
'../sample_data/{file_name}'.format(file_name=self.file_name))
def test_stream_logger_init(self):
from cis.libs import utils
u = utils.StructuredLogger('foo', 2)
assert u is not None