def _handle(self, event, context):
"""
:param dict event:
:param context:
"""
deployer = ProjectDeployer()
prefix = ''
try:
if 'prefix' in event:
prefix = event['prefix']
if 'Records' in event:
# If we got 'Records' that means a template change was uploaded to S3 and we got the trigger
for record in event['Records']:
# See if it is a notification from an S3 bucket
if 's3' in record:
bucket_name = record['s3']['bucket']['name']
if '-' in bucket_name:
prefix = bucket_name.split('-')[0] + '-'
App(prefix=prefix)
key = record['s3']['object']['key']
deployer.deploy_revision_to_door43(key)
elif 'build_log_key' in event:
App(prefix=prefix)
deployer.deploy_revision_to_door43(event['build_log_key'])
else:
App(prefix=prefix)
# this is triggered manually through AWS Lambda console to update all projects
deploy_function = '{0}tx_door43_deploy'.format(App.prefix)
deployer.redeploy_all_projects(deploy_function)
except Exception as e:
App.logger.debug("Project Deployer Error: " + str(e))
deployer.close()
def test_reset_app(self):
default_name = App.name
App(name='test-name')
App()
self.assertEqual(App.name, default_name)
App.name = 'test-name-2'
App(name='test-name-2', reset=False)
self.assertNotEqual(App.name, default_name)
def _handle(self, event, context):
"""
:param dict event:
:param context:
"""
deployer = ProjectDeployer()
prefix = ''
deploy_bucket = os.environ.get('DEPLOYBUCKET')
cdn_bucket = os.environ.get('CDNBUCKET')
deploy_function = os.environ.get('LAMBDA_FUNCTION_NAME')
try:
if 'prefix' in event:
prefix = event['prefix']
if 'Records' in event:
# If we got 'Records' that means a template change was uploaded to S3 and we got the trigger
for record in event['Records']:
# See if it is a notification from an S3 bucket
if 's3' in record:
bucket_name = record['s3']['bucket']['name']
if '-' in bucket_name:
prefix = bucket_name.split('-')[0] + '-'
App(prefix=prefix)
App.aws_region_name = record['awsRegion']
if cdn_bucket is not None:
App.cdn_bucket = cdn_bucket
if deploy_bucket is not None:
App.door43_bucket = deploy_bucket
if bucket_name == deploy_bucket:
deployer.redeploy_all_projects(
deploy_function, True)
else:
key = record['s3']['object']['key']
deployer.deploy_revision_to_door43(key)
elif 'build_log_key' in event:
App(prefix=prefix)
if deploy_bucket is not None:
App.door43_bucket = deploy_bucket
if cdn_bucket is not None:
App.cdn_bucket = cdn_bucket
deployer.deploy_revision_to_door43(event['build_log_key'])
else:
App(prefix=prefix)
if deploy_bucket is not None:
App.door43_bucket = deploy_bucket
if cdn_bucket is not None:
App.cdn_bucket = cdn_bucket
# this is triggered manually through AWS Lambda console to update all projects
deployer.redeploy_all_projects(deploy_function)
except Exception:
App.logger.exception("Project Deployer Error")
deployer.close()
def test_prefix_vars(self):
App(prefix='')
self.assertEqual(App.cdn_bucket, 'cdn.door43.org')
self.assertEqual(App.api_url, 'https://api.door43.org')
App(prefix='test-')
self.assertEqual(App.cdn_bucket, 'test-cdn.door43.org')
self.assertEqual(App.api_url, 'https://test-api.door43.org')
App(prefix='test2-')
self.assertEqual(App.cdn_bucket, 'test2-cdn.door43.org')
self.assertEqual(App.api_url, 'https://test2-api.door43.org')
App(prefix='')
self.assertEqual(App.cdn_bucket, 'cdn.door43.org')
self.assertEqual(App.api_url, 'https://api.door43.org')
def setUp(self):
"""Runs before each test."""
App(prefix='{0}-'.format(self._testMethodName),
db_connection_string='sqlite:///:memory:')
self.items = {}
self.init_items()
self.populate_table()
def setUp(self):
"""Runs before each test."""
App(prefix='{0}-'.format(self._testMethodName),
db_connection_string='sqlite:///:memory:')
App.cdn_s3_handler().create_bucket()
App.pre_convert_s3_handler().create_bucket()
App.cdn_s3_handler().upload_file = self.mock_cdn_upload_file
App.cdn_s3_handler().get_json = self.mock_cdn_get_json
App.pre_convert_s3_handler().upload_file = self.mock_s3_upload_file
try:
os.makedirs(ClientWebhookTest.base_temp_dir)
except:
pass
self.temp_dir = tempfile.mkdtemp(dir=self.base_temp_dir,
prefix='webhookTest_')
self.job_converter_count = 0
self.job_linter_count = 0
self.uploaded_files = []
self.job_data = {
'job_id': '123456890',
'status': 'started',
'success': False,
'resource_type': 'obs',
'input_format': 'md',
'output_format': 'html',
'convert_module': 'module1',
'created_at': datetime.utcnow(),
'errors': []
}
self.register_modules()
def handle(self, event, context):
"""
:param dict event:
:param context:
:return dict:
"""
if 'vars' in event:
App(**event['vars'])
App.logger.debug("EVENT:")
App.logger.debug(json.dumps(self.mask_event(event)))
self.data = {}
if 'data' in event and isinstance(event['data'], dict):
self.data = event['data']
if 'body-json' in event and isinstance(event['body-json'], dict):
self.data.update(event['body-json'])
try:
return self._handle(event, context)
except Exception as e:
App.logger.error(e.message)
App.logger.error('{0}: {1}'.format(str(e), traceback.format_exc()))
raise EnvironmentError('Bad Request: {}'.format(e.message))
finally:
App.db_close()
def setUp(self):
"""Runs before each test."""
App(prefix='{0}-'.format(self._testMethodName),
db_connection_string='sqlite:///:memory:')
App.cdn_s3_handler().create_bucket()
self.temp_dir = tempfile.mkdtemp(prefix="test_project_printer")
self.printer = ProjectPrinter()
self.mock_s3_obs_project()
def test_db(self):
App(db_connection_string='sqlite:///:memory:')
App.db_create_tables([User.__table__])
user = User(name='ed', fullname='Edward Scissorhands', password='******')
user.insert()
user_from_db = User.get(name='ed')
self.assertIsNotNone(user_from_db)
self.assertEqual(user_from_db.password, '12345')
def test_construction_connection_string(self):
"""
Test the construction of the connection string with multiple attributes
"""
App(db_protocol='protocol', db_user='******', db_pass='******', db_end_point='my.endpoint.url', db_port='9999',
db_name='db', db_connection_string_params='charset=utf8', auto_setup_db=False)
expected = "protocol://*****:*****@my.endpoint.url:9999/db?charset=utf8"
connection_str = App.construct_connection_string()
self.assertEqual(connection_str, expected)
def test_setup_db_with_connection_string_parts(self):
App(db_protocol='sqlite', db_user=None, db_pass=None, db_end_point=None, db_port=None, db_name=':memory:',
db_connection_string_params=None)
App.db_create_tables([User.__table__])
user = User(name='ed', fullname='Edward Scissorhands', password='******')
user.insert()
user_from_db = User.get(name='ed')
self.assertIsNotNone(user_from_db)
self.assertEqual(user_from_db.password, '12345')
App.db_close()
def init_table(self, view_count):
App(db_connection_string='sqlite:///:memory:')
tx_manifest = TxManifest(repo_name=ViewCountTest.REPO_NAME,
user_name=ViewCountTest.USER_NAME,
lang_code='lang',
resource_id='redID',
resource_type='resType',
title='title',
last_updated=datetime.utcnow(),
manifest='{}',
views=view_count)
tx_manifest.insert()
def setUp(self):
"""Runs before each test."""
App(prefix='{0}-'.format(self._testMethodName))
App.cdn_s3_handler().create_bucket()
App.door43_s3_handler().create_bucket()
self.temp_dir = tempfile.mkdtemp(prefix="test_project_deployer")
self.deployer = ProjectDeployer()
TdLanguage.language_list = {
'aa':
TdLanguage({
'gw': False,
'ld': 'ltr',
'ang': 'Afar',
'lc': 'aa',
'ln': 'Afaraf',
'lr': 'Africa',
'pk': 6
}),
'en':
TdLanguage({
'gw': True,
'ld': 'ltr',
'ang': 'English',
'lc': 'en',
'ln': 'English',
'lr': 'Europe',
'pk': 1747
}),
'es':
TdLanguage({
'gw': True,
'ld': 'ltr',
'ang': 'Spanish',
'lc': 'es',
'ln': 'espa\xf1ol',
'lr': 'Europe',
'pk': 1776
}),
'fr':
TdLanguage({
'gw': True,
'ld': 'ltr',
'ang': 'French',
'lc': 'fr',
'ln': 'fran\xe7ais, langue fran\xe7aise',
'lr': 'Europe',
'pk': 1868
})
}
def setUp(self):
"""Runs before each test."""
App(prefix='{0}-'.format(self._testMethodName),
db_connection_string='sqlite:///:memory:')
App.cdn_s3_handler().create_bucket()
App.cdn_s3_handler().upload_file = self.mock_cdn_upload_file
App.cdn_s3_handler().get_json = self.mock_cdn_get_json
App.cdn_s3_handler().key_exists = self.mock_cdn_key_exists
self.init_items()
self.populate_table()
try:
os.makedirs(self.base_temp_dir)
except:
pass
self.temp_dir = tempfile.mkdtemp(dir=self.base_temp_dir,
prefix='callbackTest_')
self.transferred_files = []
self.raiseDownloadException = False
self.s3_results_key = 'u/results'
self.source_folder = tempfile.mkdtemp(dir=self.temp_dir,
prefix='sources_')
def setUpClass(cls):
"""Runs before all tests."""
branch = os.environ.get(
'TRAVIS_BRANCH',
'develop') # default is testing develop branch (dev)
print("Testing on branch: {0}".format(branch))
gogs_user_token = os.environ.get('GOGS_USER_TOKEN', '')
db_pass = os.environ.get('DB_PASS', '')
if not db_pass:
db_pass = os.environ.get('{0}_DB_PASS'.format(branch.upper()), '')
if branch == 'master':
prefix = '' # no prefix for production
elif branch == 'test':
prefix = 'test-' # For running on test
else:
prefix = 'dev-' # default
App(prefix=prefix, gogs_user_token=gogs_user_token, db_pass=db_pass)
App.logger.debug('Testing on \'' + branch + '\' branch, e.g.: ' +
App.api_url)
def setUp(self):
"""Runs before each test."""
App(prefix='{0}-'.format(self._testMethodName),
db_connection_string='sqlite:///:memory:')
App.cdn_s3_handler().create_bucket()
App.cdn_s3_handler().upload_file = self.mock_cdn_upload_file
App.cdn_s3_handler().get_json = self.mock_cdn_get_json
App.cdn_s3_handler().key_exists = self.mock_cdn_key_exists
try:
os.makedirs(self.base_temp_dir)
except:
pass
self.temp_dir = tempfile.mkdtemp(dir=self.base_temp_dir,
prefix='callbackTest_')
self.transferred_files = []
self.raiseDownloadException = False
self.source_folder = None
self.results_key = 'u/results'
self.lint_callback_data = {
'identifier': 'dummy_id',
's3_results_key': self.results_key,
'success': True,
'info': [],
'warnings': [],
'errors': []
}
self.expected_error_count = 0
self.expected_warning_count = 0
self.expected_log_count = 0
self.expected_status = "success"
self.expected_success = True
self.expected_all_parts_completed = True
self.expected_multipart = False
def setUp(self):
"""Runs before each test."""
App(prefix='{0}-'.format(self._testMethodName))
self.out_dir = ''
self.out_zip_file = ''
def test_init(self):
gogs_url = 'https://my.gogs.org'
App(gogs_url=gogs_url)
self.assertEqual(App.gogs_url, gogs_url)
def setUp(self):
"""Runs before each test."""
App(prefix='{0}-'.format(self._testMethodName), db_connection_string='sqlite:///:memory:')
self.init_table(ViewCountTest.INITIAL_VIEW_COUNT)
def setUp(self):
"""Runs before each test."""
App(prefix='{0}-'.format(self._testMethodName))
self.temp_dir = tempfile.mkdtemp(prefix='TestUsfmHtmlConverter')
def setUp(self):
"""Runs before each test."""
App(prefix='{0}-'.format(self._testMethodName),
db_connection_string='sqlite:///:memory:')
App.pre_convert_s3_handler().create_bucket()