def create_app(environment): app = Flask(__name__) app.config.from_object(get_config(environment)) app.register_blueprint(api) return app
def mock_config(): """get config, used for testing default config""" _config = config.get_config() if _config is None: raise EnvironmentError("_config environment is not set.") return _config
def test_get_config(self): variables = (("CONFIG_SETTING", REQUIRED, ""), ("CONFIG_OPTIONAL", "default", "")) source = dict(CONFIG_SETTING="hello") config = get_config(variables, source) self.assertEqual(config["CONFIG_SETTING"], "hello") self.assertEqual(config["CONFIG_OPTIONAL"], "default") source = dict(CONFIG_SETTING="hello", CONFIG_OPTIONAL="override") config = get_config(variables, source) self.assertEqual(config["CONFIG_SETTING"], "hello") self.assertEqual(config["CONFIG_OPTIONAL"], "override") with self.assertRaises(ConfigException): variables = (("CONFIG_SETTING", REQUIRED, ""),) source = dict() config = get_config(variables, source)
async def init_app(): """ Initializes web application. Gets config file, init routes... """ _app = web.Application(logger=_logger) _app['config'] = config.get_config() executor = await worker.init_workers(_app) init_routes(_app, executor) return _app
def test_get_config(self): variables = ( ('CONFIG_SETTING', REQUIRED, ''), ('CONFIG_OPTIONAL', 'default', ''), ) source = dict(CONFIG_SETTING='hello') config = get_config(variables, source) self.assertEqual(config['CONFIG_SETTING'], 'hello') self.assertEqual(config['CONFIG_OPTIONAL'], 'default') source = dict(CONFIG_SETTING='hello', CONFIG_OPTIONAL='override') config = get_config(variables, source) self.assertEqual(config['CONFIG_SETTING'], 'hello') self.assertEqual(config['CONFIG_OPTIONAL'], 'override') with self.assertRaises(ConfigException): variables = (('CONFIG_SETTING', REQUIRED, ''), ) source = dict() config = get_config(variables, source)
def test_virus_scan_handler(mocker, data_builder, as_public, file_form): # prepare mock_set_virus_scan_state = mocker.patch( 'api.callbacks.virus_scan_handler.liststorage.FileStorage.set_virus_scan_state' ) mock_exec_op = mocker.patch( 'api.callbacks.virus_scan_handler.liststorage.FileStorage.exec_op', return_value={'virus_scan': { 'state': 'quarantined' }}) acquisition_id = '000000000000000000000000' endpoint = 'callbacks/virus-scan/acquisitions/{}/files/test.csv'.format( acquisition_id) url = urljoin(config.get_config()['site']['api_url'].rstrip('/') + '/', endpoint) # permission denied without signature r = as_public.post('/' + endpoint, json={'state': 'clean'}) assert not r.ok assert r.status_code == 403 signed_url = signed_urls.generate_signed_url(url, method='POST') parsed_url = urlparse.urlparse(signed_url) # valid request, clean r = as_public.post('/{}?{}'.format(endpoint, parsed_url.query), json={'state': 'clean'}) assert r.ok mock_set_virus_scan_state.assert_called_with( _id='000000000000000000000000', query_params={'name': 'test.csv'}, state='clean') mock_set_virus_scan_state.reset_mock() # valid request, virus r = as_public.post('/{}?{}'.format(endpoint, parsed_url.query), json={'state': 'virus'}) assert r.ok mock_set_virus_scan_state.assert_called_with( _id='000000000000000000000000', query_params={'name': 'test.csv'}, state='virus') # invalid payload r = as_public.post('/{}?{}'.format(endpoint, parsed_url.query), json={'state': 'wrong state'}) assert not r.ok # state already set mock_exec_op.return_value = {'virus_scan': {'state': 'clean'}} r = as_public.post('/{}?{}'.format(endpoint, parsed_url.query), json={'state': 'virus'}) assert not r.ok
def create_app(): app = Flask(__name__) config_object = get_config() app.config.from_object(config_object) db.init_app(app) mail.init_app(app) cors.init_app(app, resources = {"*": {"origins": "*"}}) from api.routes import university_api app.register_blueprint(university_api, url_prefix = os.environ.get("API_PREFIX", "/api")) return app
""" @author:xda @file:Msg.py @time:2020/12/15 """ import re from api import receive from api import reply from api.model_process import accessIncrement from api.config import get_config from abc import abstractmethod, ABCMeta from datahub import cb_info from datahub.cb_info import CBInfo CONFIG = get_config() token = CONFIG['token'] MAX_COUNT = CONFIG['access_max'] IGNORE_RESPONSE = CONFIG['ignore'] LIMIT_RESPONSE = CONFIG['limit_response'] class MsgBase: def __init__(self, msg): self.msg = msg self.recMsg = self.parse(self.msg) def parse(self, msg): return receive.parse_xml(msg)
def collect_db_metrics(): """Collect metrics from mongodb, including version and job states""" try: # Get version info epoch = datetime(1970, 1, 1) version_info = config.get_version() if version_info: values.DB_VERSION.set(version_info.get('database', 0)) release = version_info.get('release', 'UNKNOWN') values.RELEASE_VERSION.labels(release).set(1) flywheel_version = version_info.get('flywheel_release', 'UNKNOWN') values.FLYWHEEL_VERSION.labels(flywheel_version).set(1) # Get raw DB stats db_stats = config.db.command('dbstats') values.DB_DATA_SIZE.set(db_stats['dataSize']) values.DB_STORAGE_SIZE.set(db_stats['storageSize']) values.DB_OBJECTS.set(db_stats['objects']) # Get jobs info for state in JOB_STATES: count = config.db.jobs.count({'state': state}) values.JOBS_BY_STATE.labels(state).set(count) # Find the oldest pending job oldest_jobs = list(config.db.jobs.find({'state': 'pending', 'created': {'$exists': 1}}, {'created': 1}, sort=[('created', 1)], limit=1)) if oldest_jobs: oldest_job_timestamp = (oldest_jobs[0]['created'] - epoch).total_seconds() else: oldest_job_timestamp = float('NaN') values.OLDEST_PENDING_JOB.set(oldest_job_timestamp) # Get raw collection stats for collection_name in config.db.list_collection_names(): stats = config.db.command('collstats', collection_name) values.COLLECTION_COUNT.labels(collection_name).set(stats['count']) values.COLLECTION_SIZE.labels(collection_name).set(stats['size']) values.COLLECTION_STORAGE_SIZE.labels(collection_name).set(stats['storageSize']) for index_name, index_size in stats.get('indexSizes', {}).items(): values.COLLECTION_INDEX_SIZE.labels(collection_name, index_name).set(index_size) values.COLLECTION_TOTAL_INDEX_SIZE.labels(collection_name).set(stats['totalIndexSize']) # Get access logs of type user login login_count = config.log_db.access_log.find({'access_type': 'user_login', 'origin.id': {'$regex': '@(?!flywheel\\.io)'}}).count() values.USER_LOGIN_COUNT.set(login_count) ### Last Event Times Collection # Get the last user_login last_event = config.log_db.access_log.find_one({'access_type': 'user_login', 'origin.id': {'$regex': '@(?!flywheel\\.io)'}}, sort=[('timestamp', -1)]) if last_event: time_since = last_event['timestamp'] - epoch values.LAST_EVENT_TIME.labels('user_login').set(time_since.total_seconds()) # Get the last session_creation last_event = config.db.sessions.find_one({}, sort=[('created', -1)]) if last_event: time_since = last_event['created'] - epoch values.LAST_EVENT_TIME.labels('session_created').set(time_since.total_seconds()) # Get the last job_queued by system and user last_event = config.db.jobs.find_one({'origin.type': 'system'}, sort=[('created', -1)]) if last_event: time_since = last_event['created'] - epoch values.LAST_EVENT_TIME.labels('job_queued_by_system').set(time_since.total_seconds()) last_event = config.db.jobs.find_one({'origin.type': 'user'}, sort=[('created', -1)]) if last_event: time_since = last_event['created'] - epoch values.LAST_EVENT_TIME.labels('job_queued_by_user').set(time_since.total_seconds()) # Get gear versions gear_count = 0 job_count_by_gear = { d['_id']: d['count'] for d in config.db.jobs.aggregate([ { '$group': { '_id': '$gear_id', 'count': {'$sum':1} } } ])} for gear_doc in config.db.gears.find(): gear = gear_doc.get('gear', {}) name = gear.get('name', 'UNKNOWN') version = gear.get('version', 'UNKNOWN') created = str(gear_doc.get('created', 'UNKNOWN')) count = job_count_by_gear.get(str(gear_doc['_id']), 0) values.GEAR_VERSIONS.labels(name, version, created).set(count) gear_count = gear_count + 1 values.COLLECTION_COUNT.labels('gears').set(gear_count) # Auth provider types auth_config = config.get_config()['auth'] for key in AuthProviders.keys(): values.AUTH_PROVIDER_TYPES.labels(key).set(int(key in auth_config)) # Get devices device_count = 0 device_storage = containerstorage.ContainerStorage('devices', use_object_id=True) devices = device_storage.get_all_el(None, None, None) device_statuses = get_device_statuses(devices) status_counts = {} for device in devices: device_id = str(device['_id']) device_type = device.get('type') or device.get('method') or 'UNKNOWN' device_name = device.get('name', 'UNKNOWN') last_seen = device.get('last_seen') if last_seen: since_last_seen = (datetime.now() - last_seen).total_seconds() else: since_last_seen = -1 interval = device.get('interval', -1) # Set device_label = [device_type, device_name, device_id] values.DEVICE_TIME_SINCE_LAST_SEEN.labels(*device_label).set(since_last_seen) values.DEVICE_INTERVAL.labels(*device_label).set(interval) # Increment count by type device_status = device_statuses[device_id]['status'] status_key = (device_type, device_status) current_count = status_counts.setdefault(status_key, 0) status_counts[status_key] = current_count + 1 device_count = device_count + 1 # Device count values.COLLECTION_COUNT.labels('devices').set(device_count) # Status count for label, count in status_counts.items(): values.DEVICE_STATUS_COUNT.labels(*label).set(count) except: # pylint: disable=bare-except log.critical('Error collecting db metrics', exc_info=True)
def test_upload_with_virus_scan_enabled(mocker, as_public, as_user, as_drone, as_admin, data_builder, file_form, api_db): # setup mock_get_feature = mocker.patch('api.placer.config.get_feature', return_value={'virus_scan': True}) mock_config = config.get_config() mock_config['webhooks']['virus_scan'] = None mock_get_feature = mocker.patch('api.placer.config.get_config', return_value=mock_config) mock_webhook_post = mocker.patch('api.webhooks.base.Session.post') orig_find = config.db['acquisitions'].find def wrap_find(*args, **kwargs): return orig_find(args[0]) mocker.patch.object(config.db['acquisitions'], 'find', wraps=wrap_find) project = data_builder.create_project() session = data_builder.create_session() acquisition = data_builder.create_acquisition(session=session) gear_config = {'param': {'type': 'string', 'pattern': '^default|custom$', 'default': 'default'}} gear = data_builder.create_gear(gear={'version': '0.0.1', 'config': gear_config}) # create rule rule_json = { 'gear_id': gear, 'name': 'csv-job-trigger-rule', 'any': [], 'not': [], 'all': [ {'type': 'file.type', 'value': 'tabular data'}, ] } r = as_admin.post('/projects/' + project + '/rules', json=rule_json) assert r.ok rule = r.json['_id'] # upload file as drone file_name = 'test.csv' r = as_drone.post('/acquisitions/' + acquisition + '/files', POST=file_form(file_name)) assert r.ok # job was created via rule since the file was uploaded by a trusted origin gear_jobs = [job for job in api_db.jobs.find({'gear_id': gear})] assert len(gear_jobs) == 1 # file uploaded by drone won't be quarantined r = as_drone.get('/acquisitions/' + acquisition + '/files/test.csv') assert r.ok uid = as_user.get('/users/self').json['_id'] r = as_drone.post('/projects/' + project + '/permissions', json={'_id': uid, 'access': 'admin'}) assert r.ok # user uploads fails if not webhook configure r = as_user.post('/acquisitions/' + acquisition + '/files', POST=file_form( file_name, meta={'name': file_name, 'type': 'csv'})) assert not r.ok assert r.status_code == 500 # set webhook mock_config['webhooks']['virus_scan'] = 'http://localhost' mock_get_feature.return_value = mock_config # upload file as user r = as_user.post('/acquisitions/' + acquisition + '/files', POST=file_form((file_name, 'some;content'))) assert r.ok # job was not created via rule since the file is quarantined gear_jobs = [job for job in api_db.jobs.find({'gear_id': gear})] assert len(gear_jobs) == 1 # user uploaded file is quarantined r = as_user.get('/acquisitions/' + acquisition + '/files/test.csv') assert not r.ok assert r.status_code == 400 _, kwargs = mock_webhook_post.call_args_list[0] webhook_payload = json.loads(kwargs['data']) # can download the file using the signed url parsed_url = urlparse.urlparse(webhook_payload['file_download_url']) download_endpoint = parsed_url.path.replace('/api', '') r = as_public.get('{}?{}'.format(download_endpoint, parsed_url.query)) assert r.ok assert r.body == 'some;content' # can use the signed response url to send back the virus scan result parsed_url = urlparse.urlparse(webhook_payload['response_url']) response_endpoint = parsed_url.path.replace('/api', '') # mark the file as clean using r = as_public.post('{}?{}'.format(response_endpoint, parsed_url.query), json={'state': 'clean'}) assert r.ok # now the file is accessible r = as_user.get('/acquisitions/' + acquisition + '/files/test.csv') assert r.ok # job was created via rule since the file is marked as clean gear_jobs = [job for job in api_db.jobs.find({'gear_id': gear})] assert len(gear_jobs) == 2
from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from starlette_exporter import PrometheusMiddleware, handle_metrics from api.config import get_config from api.db import db from api.public import api as public_api from api.utils.logger import logger_config logger = logger_config(__name__) settings = get_config() app = FastAPI( title=settings.PROJECT_NAME, version=settings.VERSION, description=settings.DESCRIPTION, docs_url="/", # openapi_tags=tags_metadata, # to provide custom information on the swagger ) app.include_router(public_api) app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) # prometheus metrics: https://prometheus.io/