def configure_flywheel_engine(graph):
"""
Create the flywheel engine.
"""
namespace = ()
if graph.metadata.testing:
namespace = 'test'
engine = Engine(namespace=namespace)
engine.connect_to_region(graph.config.dynamodb.region)
return engine
def connect(*args, **kwargs):
engine = getattr(current_module, 'engine')
session = getattr(current_module, 'session') or kwargs.pop('session', None)
if engine:
return engine
else:
engine = Engine()
# Connect the engine to either a local DynamoDB or a particular region.
if ('USE_LOCAL_DB' in os.environ
and os.environ['USE_LOCAL_DB'] == 'True'):
engine.connect(os.environ['AWS_REGION'],
host='localhost',
port=8000,
access_key='anything',
secret_key='anything',
is_secure=False,
session=session)
elif ('CI' in os.environ and os.environ['CI'] == 'True'):
engine.connect_to_region(os.environ['AWS_REGION'], session=session)
else:
engine.connect_to_region(os.environ['AWS_REGION'])
setattr(current_module, 'engine', engine)
return engine
# Import flask and template operators
from flask import Flask
from flask_cors import CORS
from flywheel import Engine
SUPER_ADMIN = '*****@*****.**'
ALL_PARENTS = '*****@*****.**'
#dynamodb move
engine = Engine()
#engine.connect(region='dummy', host='localhost', port=8000, access_key='dummy', secret_key='dummy', is_secure=False, session = None)
engine.connect_to_region('eu-west-1')
def create_app():
# Define the WSGI application object
app = Flask(__name__, instance_relative_config=True)
CORS(app)
# Configurations
app.config.from_object('config')
# Import a module / component using its blueprint handler variable (mod_group)
from base.mod_draft.controllers import mod_draft as draft_module
from base.mod_group.controllers import mod_group as group_module
from base.mod_member.controllers import mod_member as member_module
from base.mod_workday.controllers import mod_workday as workday_module
from base.mod_rule.controllers import mod_rule as rule_module
from base.mod_term.controllers import mod_term as term_module
from base.mod_switchday.controllers import mod_switchday as switchday_module
from base.mod_communicate.controllers import mod_communicate as communication_module
class CLI():
def __init__(self, args):
self.engine = Engine()
self.engine.connect_to_region('eu-west-1')
# Register our model with the engine so it can create the Dynamo table
self.engine.register(Approval)
# Create the dynamo table for our registered model
self.engine.create_schema()
self.args = args
# Setup logging
if args.verbose:
loglevel = logging.DEBUG
else:
loglevel = logging.INFO
logging.basicConfig(format="%(levelname)s: %(message)s",
level=loglevel)
def main(self):
if self.args.action == 'list':
self.list()
elif self.args.action == 'approve':
self.approve()
elif self.args.action == 'reject':
self.reject()
else:
logging.error('Please use a correct argument')
exit(1)
def approve(self):
if 'id' not in self.args:
logging.error('Please give an id')
exit(1)
approval_lock = self.engine.scan(Approval).filter(
id=self.args.id).all()
if not approval_lock:
logging.info('No lock with the id %s has been found' %
self.args.id)
exit(1)
approval_lock = approval_lock[0]
approval_lock.approved = True
approval_lock.timestamp = datetime.utcnow()
self.engine.save(approval_lock, overwrite=True)
logging.info('The lock %s has been approved' % self.args.id)
def reject(self):
if 'id' not in self.args:
logging.error('Please give an id')
exit(1)
approval_lock = self.engine.scan(Approval).filter(
id=self.args.id).all()
if not approval_lock:
logging.info('No lock with the id %s has been found' %
self.args.id)
exit(1)
approval_lock = approval_lock[0]
approval_lock.approved = False
approval_lock.timestamp = datetime.utcnow()
self.engine.save(approval_lock, overwrite=True)
logging.info('The lock %s has been rejected' % self.args.id)
def list(self):
approval_locks = self.engine.scan(Approval).filter(claimed=True).all()
table = []
if approval_locks:
headers = sorted(approval_locks[0].keys_())
else:
headers = None
if approval_locks:
for item in approval_locks:
row = []
for key in sorted(item.keys_()):
row.append(getattr(item, key))
table.append(row)
print(tabulate(table, headers))
else:
print('There is no waiting approval')
title = Field()
date = Field()
scrobble = Field() #should have made this NUMBER, int
def __init__(self, artist, ts, title, album, date, scrobble):
self.artist = artist
self.ts = ts
self.album = album
self.title = title
self.date = date
self.scrobble = scrobble
engine = Engine()
#engine.connect_to_host(host='localhost', port=8000)
engine.connect_to_region('us-east-1')
engine.register(scrobble)
# uncomment the following if you actually want to create the database for the first time
#engine.create_schema()
# below is an example of how you would write to the the DynamoDB if you wanted to create a record
# in tft_sonos_instagram.py I am using boto but if it was using flywheel, it would look like the following
#z = scrobble("Patty Griffin", datetime.now(), "Making Pies", "Children Running through it", "Date: 1234", "14")
#engine.save(z)
days = input("How many days do you want to go back? ")
# scan may be slow but not looking for this to be particularly fast and can't query with no hash key
z = engine.scan(scrobble).filter(
album = Field()
title = Field()
date = Field()
scrobble = Field() #should have made this NUMBER, int
def __init__(self, artist, ts, title, album, date, scrobble):
self.artist = artist
self.ts = ts
self.album = album
self.title = title
self.date = date
self.scrobble = scrobble
engine = Engine()
#engine.connect_to_host(host='localhost', port=8000)
engine.connect_to_region('us-east-1')
engine.register(scrobble)
# uncomment the following if you actually want to create the database for the first time
#engine.create_schema()
# below is an example of how you would write to the the DynamoDB if you wanted to create a record
# in tft_sonos_instagram.py I am using boto but if it was using flywheel, it would look like the following
#z = scrobble("Patty Griffin", datetime.now(), "Making Pies", "Children Running through it", "Date: 1234", "14")
#engine.save(z)
days = input("How many days do you want to go back? ")
# scan may be slow but not looking for this to be particularly fast and can't query with no hash key
z = engine.scan(scrobble).filter(scrobble.ts > datetime.now()-timedelta(days=int(days))).all()
# parsing arguments
PARSER = argparse.ArgumentParser(description='Client message processor')
PARSER.add_argument('API_token', help="the individual API token given to your team")
PARSER.add_argument('API_base', help="the base URL for the game API")
ARGS = PARSER.parse_args()
# defining global vars
API_BASE = ARGS.API_base
# 'https://csm45mnow5.execute-api.us-west-2.amazonaws.com/dev'
APP = Flask(__name__)
DDB_ENGINE = Engine()
DDB_ENGINE.connect_to_region('eu-central-1')
Message.meta_.name = 'gameday-production'
DDB_ENGINE.register(Message)
#ELASTICACHE_CLIENT = boto3.client('elasticache')
# creating flask route for type argument
@APP.route('/ping', methods=['GET', 'POST'])
def healthcheck():
return "ok"
@APP.route('/', methods=['GET', 'POST'])
def main_handler():
"""
class ApprovalResource:
"""
Approval resource implementation.
This python script is the target of symbolic links and is used for check, in and out.
These three commands are defined as methods on this class and common parameters live in the constructor.
To enable the debug output, the resource source configuration must have the debug parameter.
"""
def __init__(self, command_name, json_data, command_argument):
self.command_name = command_name
self.command_argument = command_argument
# Namespace the approval lock
self.data = json.loads(json_data)
self.wait_lock = 10
self.pool = ''
self.engine = Engine()
# allow debug logging to console for tests
if os.getenv('RESOURCE_DEBUG', False) or self.data.get('source', {}).get('debug', False):
log.basicConfig(level=log.DEBUG)
else:
logfile = tempfile.NamedTemporaryFile(delete=False, prefix='log')
log.basicConfig(level=log.DEBUG, filename=logfile.name)
stderr = log.StreamHandler()
stderr.setLevel(log.INFO)
log.getLogger().addHandler(stderr)
log.debug('command: %s', command_name)
log.debug('input: %s', self.data)
log.debug('args: %s', command_argument)
log.debug('environment: %s', os.environ)
def check_cmd(self, source, version):
"""
Check for new version(s)
This function will look on Dynamodb if there's a lock within the pool
and will return the last timestamps associated.
:param source: is an arbitrary JSON object which specifies the location of the resource,
including any credentials. This is passed verbatim from the pipeline configuration.
:param version: is a JSON object with string fields, used to uniquely identify an instance of the resource.
:return: a dict with the version fetched
"""
log.debug('version: %s', version)
if not version:
version = {"timestamp": '0'}
log.debug('source: %s', source)
log.debug('version: %s', version)
approval_locks = self.engine.query(Approval)\
.filter(
Approval.timestamp >= datetime.fromtimestamp(Decimal(version.get('timestamp'))),
pool=self.pool) \
.all()
versions_list = []
for lock in approval_locks:
versions_list.append({"timestamp": "{timestamp}".format(timestamp=Decimal(lock.timestamp.timestamp()))})
if not approval_locks:
versions_list.append(version)
log.debug(versions_list)
return versions_list
def in_cmd(self, target_dir, source, version, params):
"""
This function will fetch a lock in dynamodb an write it in the target directory.
If parameters lock_name and need_approval are passed, then the function will wait for a change
on the dynamodb lock item.
:param target_dir: a temporary directory which will be exposed as an output
:param source: is the same value as passed to check
:param version: is the same type of value passed to check, and specifies the version to fetch.
:param params: is an arbitrary JSON object passed along verbatim from params on a get.
:return: a dict with the version fetched and the metadata of the lock
"""
log.debug('source: %s', source)
log.debug('version: %s', version)
if not version:
version = {"timestamp": 0}
# Does the get should wait for an approval or not ?
if 'lock_name' in params and 'need_approval' in params:
log.debug('Looking for the lock %s in the pool %s' % (params.get('lock_name'), self.pool))
approval_lock = self.query_lock(params.get('lock_name'))
if approval_lock:
# We want to wait until the approve is done
while approval_lock.approved is None and approval_lock.need_approval:
# Query the lock item in the loop
refresh_approval = self.query_lock(lock_name=params['lock_name'])
# If the lock has timed out, then we override the refresh_approval to simulate a reject
if 'timeout' in params:
if approval_lock.timestamp + timedelta(minutes=params['timeout']) <= datetime.now():
refresh_approval.approved = False
countdown = (approval_lock.timestamp.replace(microsecond=0) +
timedelta(minutes=params['timeout'])) - datetime.now().replace(microsecond=0)
timedelta(minutes=params['timeout']).total_seconds() / self.wait_lock
if countdown.days >= 0:
log.info("The lock %s is waiting for an approval. There is %s left" %
(params['lock_name'], str(countdown)))
else:
log.info("The lock %s is waiting for an approval" % params['lock_name'])
# If hasn't been approved or rejected, waiting a bit more
if refresh_approval.approved is None:
time.sleep(self.wait_lock)
continue
# Is it approved ?
if refresh_approval.approved:
approval_lock.approved = True
else:
approval_lock.approved = False
# If the lock has been rejected we should fail the job and release the lock
if not approval_lock.approved and approval_lock.need_approval:
log.info("The lock hasn't been approved, exiting")
approval_lock.claimed = False
approval_lock.approved = None
self.engine.save(approval_lock, overwrite=True)
exit(1)
elif 'lock_name' in params:
approval_lock = self.query_lock(params.get('lock_name'))
else:
# There is no approval, we have just a normal lock. Let's fetch the lock
approval_lock = self.engine.query(Approval)\
.filter(
Approval.timestamp >= datetime.fromtimestamp(Decimal(version.get('timestamp'))),
pool=self.pool)\
.all()
if approval_lock:
approval_lock = approval_lock[0]
metadata = []
if not approval_lock:
log.info("No lock have been found")
exit(0)
for key in approval_lock.keys_():
value = getattr(approval_lock, key)
if type(value) is datetime:
value = str(Decimal(value.timestamp()))
elif type(value) is bool:
value = str(value)
metadata.append(
{
'name': key,
'value': value
}
)
name_path = os.path.join(target_dir, 'name')
with open(name_path, 'w') as name:
name.write(getattr(approval_lock, 'lockname'))
metadata_path = os.path.join(target_dir, 'metadata')
with open(metadata_path, 'w') as metadata_file:
json.dump(metadata, metadata_file)
return {
'version': {"timestamp": "{timestamp}".format(
timestamp=Decimal(getattr(approval_lock, 'timestamp').timestamp()))},
'metadata': metadata,
}
def _do_claim(self, params):
"""
This method handle the claiming of a lock. If the lock is already claimed, it wait until the lock is
available. Else, it create the lock.
:param params: the params passed as parameters of the resource
:return: the approval_lock item in dynamodb
"""
approval_lock = self.query_lock(lock_name=params['lock_name'])
need_approval = params.get('need_approval', False)
override_approval = params.get('override_approval', False)
if approval_lock:
# To override the previous approval, we need to reject the previous one
# Then the get will see it was rejected, will release the lock and fail the job
if override_approval:
approval_lock.approved = False
approval_lock.timestamp = datetime.now()
self.engine.save(approval_lock, overwrite=True)
log.info("Rejecting the previous approval")
# Let the get fail before acquiring the new lock
time.sleep(self.wait_lock + 5)
# We want to wait until the lock is not claimed
log_only_once = False
while approval_lock.claimed:
if not log_only_once:
log_only_once = True
log.info("The lock %s is already claimed" % params['lock_name'])
refresh_approval = self.query_lock(lock_name=params['lock_name'])
if not refresh_approval:
log.info("The lock does not exist")
exit(1)
if not refresh_approval.claimed:
approval_lock.claimed = False
else:
time.sleep(self.wait_lock)
else:
approval_lock = Approval(
id=uuid.uuid4().urn[9:],
lockname=params['lock_name'],
pool=self.pool,
claimed=True,
team=os.getenv('BUILD_TEAM_NAME', "team"),
pipeline=os.getenv('BUILD_PIPELINE_NAME', "pipeline"),
description=params.get('description', None)
)
if need_approval:
approval_lock.need_approval = True
approval_lock.claimed = True
approval_lock.approved = None
approval_lock.timestamp = datetime.now()
self.engine.save(approval_lock, overwrite=True)
log.info("Claiming the lock %s" % params['lock_name'])
return approval_lock
def _do_release(self, params):
"""
This method handle the release of a claimed lock
:param params: the params passed as parameters of the resource
:return: the approval_lock item in dynamodb
"""
approval_lock = self.query_lock(lock_name=params['lock_name'])
if not approval_lock:
log.info("The lock does not exist")
exit(1)
approval_lock.claimed = False
approval_lock.approved = None
approval_lock.timestamp = datetime.now()
self.engine.save(approval_lock, overwrite=True)
log.info("Releasing the lock %s" % params['lock_name'])
return approval_lock
def query_lock(self, lock_name):
"""
This method is used to query the lock in the approval loop to check if there is a change on it
:param lock_name: The name of the lock to fetch
:return: the dynamodb item
"""
approval_lock = self.engine.query(Approval) \
.filter(
lockname=lock_name,
pool=self.pool) \
.all()
if approval_lock:
approval_lock = approval_lock[0]
return approval_lock
def out_cmd(self, target_dir, source, params):
"""
This method is responsible to acquire or release a lock. If the lock doesn't exist yet, then the method
create it automatically.
If a lock is already acquired, the method will wait indefinitely until being able to acquire it.
:param target_dir:
:param source: is the same value as passed to check.
:param params: is an arbitrary JSON object passed along verbatim from params on a put.
:return: a dict with the version fetched and the metadata of the lock
"""
metadata = []
if 'lock_name' not in params:
log.error('You must set a lock_name on params')
if 'action' not in params:
log.error('You must set an action on params')
if 'claim' in params['action']:
approval_lock = self._do_claim(params=params)
elif 'release' in params['action']:
approval_lock = self._do_release(params=params)
else:
log.error('Please use an available action')
exit(1)
metadata = []
for key in approval_lock.keys_():
value = getattr(approval_lock, key)
if type(value) is datetime:
value = str(Decimal(value.timestamp()))
elif type(value) is bool:
value = str(value)
metadata.append(
{
'name': key,
'value': value
}
)
name_path = os.path.join(target_dir, 'name')
with open(name_path, 'w') as name:
name.write(approval_lock.lockname)
metadata_path = os.path.join(target_dir, 'metadata')
with open(metadata_path, 'w') as metadata_file:
json.dump(metadata, metadata_file)
return {
'version': {"timestamp": "{timestamp}".format(
timestamp=Decimal(approval_lock.timestamp.timestamp()))},
'metadata': metadata,
}
def run(self):
"""Parse input/arguments, perform requested command return output."""
# Extract informations from the json
source = self.data.get('source', {})
params = self.data.get('params', {})
version = self.data.get('version', {})
# To use AWS with efficiency, we are pushing the AWS credentials into environment
os.environ['AWS_ACCESS_KEY_ID'] = source.get('AWS_ACCESS_KEY_ID', '')
os.environ['AWS_SECRET_ACCESS_KEY'] = source.get('AWS_SECRET_ACCESS_KEY', '')
os.environ['AWS_DEFAULT_REGION'] = source.get('AWS_DEFAULT_REGION', "eu-west-1")
self.wait_lock = source.get('wait_lock', 10)
# Ensure we are receiving the required parameters on the configuration
if 'pool' not in source:
log.error("pool must exist in the source configuration")
exit(1)
else:
self.pool = source.get('pool')
# Configure the connection to Dynamodb
self.engine.connect_to_region(os.environ.get('AWS_DEFAULT_REGION', 'eu-west-1'))
# Register our model with the engine so it can create the Dynamo table
self.engine.register(Approval)
# Create the dynamo table for our registered model
self.engine.create_schema()
# Define which operation to perform
if self.command_name == 'check':
response = self.check_cmd(source, version)
elif self.command_name == 'in':
response = self.in_cmd(self.command_argument[0], source, version, params)
else:
response = self.out_cmd(self.command_argument[0], source, params)
return json.dumps(response)
os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = "1"
api_url = os.environ.get('api_url')
client_id = os.environ.get('client_id')
client_secret = os.environ.get('client_secret') # TODO: put in SSM?
scope = 'openid offline'
secret_base64_encoded = base64.b64encode(f"{client_id}:{client_secret}".encode())
authorization_header = {"Authorization": f"Basic {secret_base64_encoded}"}
region = os.environ.get('AWS_REGION', 'eu-west-1')
sessions_tablename = os.environ.get('SESSIONS_TABLENAME', 'apigw-bucko-dev')
engine = Engine()
engine.connect_to_region(region)
# Set up our data model
class SessionObject(Model):
__metadata__ = {
'_name': sessions_tablename,
}
key = Field(hash_key=True)
state = Field()
token = Field()
def __init__(self, key, state):
self.key = key
