def main(ctx, verbosity): configure_logger(verbosity) # Load the configurations from file config = Config(root_path=".") config.from_object(default_settings) config.from_envvar(ZUBBI_SETTINGS_ENV) # Validate the configuration tenant_sources_repo = config.get("TENANT_SOURCES_REPO") tenant_sources_file = config.get("TENANT_SOURCES_FILE") # Fail if both are set or none of both is set if ( not tenant_sources_file and not tenant_sources_repo or (tenant_sources_file and tenant_sources_repo) ): raise ScraperConfigurationError( "Either one of 'TENANT_SOURCES_REPO' " "and 'TENANT_SOURCES_FILE' must be set, " "but not both." ) # Store the config in click's context object to be available for subcommands ctx.obj = {"config": config} if ctx.invoked_subcommand is None: ctx.invoke(scrape)
from flask.config import Config import boto.sqs from boto.sqs.message import RawMessage from boto import exception LOG = logging.getLogger('alerta.sqs') config = Config('/') config.from_pyfile('/etc/alertad.conf', silent=True) config.from_envvar('ALERTA_SVR_CONF_FILE', silent=True) DEFAULT_AWS_REGION = 'eu-west-1' DEFAULT_AWS_SQS_QUEUE = 'alerts' AWS_REGION = os.environ.get('AWS_REGION') or config.get( 'AWS_REGION', DEFAULT_AWS_REGION) AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') or config.get( 'AWS_ACCESS_KEY_ID') AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') or config.get( 'AWS_SECRET_ACCESS_KEY') AWS_SQS_QUEUE = os.environ.get('AWS_SQS_QUEUE') or config.get( 'AWS_SQS_QUEUE', DEFAULT_AWS_SQS_QUEUE) class Worker(object): def __init__(self): try: connection = boto.sqs.connect_to_region( AWS_REGION, aws_access_key_id=AWS_ACCESS_KEY_ID,
def get(self, key, default=None): """Gets config from dynaconf variables if variables does not exists in dynaconf try getting from app.config to support runtime settings.""" return self._settings.get(key, Config.get(self, key, default))
from flask.config import Config import boto.sqs from boto.sqs.message import RawMessage from boto import exception LOG = logging.getLogger('alerta.sqs') config = Config('/') config.from_pyfile('/etc/alertad.conf', silent=True) config.from_envvar('ALERTA_SVR_CONF_FILE', silent=True) DEFAULT_AWS_REGION = 'eu-west-1' DEFAULT_AWS_SQS_QUEUE = 'alerts' AWS_REGION = os.environ.get('AWS_REGION') or config.get('AWS_REGION', DEFAULT_AWS_REGION) AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') or config.get('AWS_ACCESS_KEY_ID') AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') or config.get('AWS_SECRET_ACCESS_KEY') AWS_SQS_QUEUE = os.environ.get('AWS_SQS_QUEUE') or config.get('AWS_SQS_QUEUE', DEFAULT_AWS_SQS_QUEUE) class Worker(object): def __init__(self): try: connection = boto.sqs.connect_to_region( AWS_REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY )
class Spider(metaclass=abc.ABCMeta): def __init__(self, import_name, middlewares=None, loop=None, session=None): self.import_name = import_name self.root_path = get_root_path(import_name) self.config = Config(self.root_path, default_config) self._context = {} self._loop = loop or asyncio.get_event_loop() self._middlewares = SpiderMiddlewareManager(self, middlewares) self._session = session or aiohttp.ClientSession(loop=self._loop) def enqueue_request(self, **kwargs): context = self._context[self._task] max_depth = self.config.get('MAX_DEPTH') if max_depth and context['request'].depth > max_depth: return request = Request(referer=context['response'], **kwargs) if request.url in self._seen: return if not self._url_allowed(request): return request.depth = context['response'].request.depth + 1 self._queue.put_nowait(request) def run(self): self._loop.run_until_complete(self.start()) @asyncio.coroutine def start(self): self._seen = set() self._queue = Queue(loop=self._loop) for url in self.config['URLS']: self._queue.put_nowait(Request(url)) workers = [asyncio.Task(self._work()) for _ in range(self.config['CONCURRENCY'])] yield from self._queue.join() for worker in workers: worker.cancel() @asyncio.coroutine def _work(self): while True: request = yield from self._queue.get() yield from self._fetch(request) self._queue.task_done() @asyncio.coroutine def _fetch(self, request): for callback in self._middlewares['before_request']: request = callback(self, request) resp = yield from self._session.request('get', **request.params) body = yield from resp.read_and_close() response = Response(request, resp, body) for callback in self._middlewares['after_response']: response = callback(self, response) with self._request_context(request, response): self.parse(response) @property def _task(self): return asyncio.Task.current_task(loop=self._loop) @contextlib.contextmanager def _request_context(self, request, response): self._context[self._task] = {'request': request, 'response': response} try: yield finally: del self._context[self._task] def _url_allowed(self, request): return next( ( True for domain in self.config['DOMAINS'] if request.furl.host.endswith(domain) ), False, ) @abc.abstractmethod def parse(self, response): pass
def PISS(instance_path): # Create a configuration object and read the configuration file app_config = Config(instance_path) app_config.from_pyfile('piss.cfg') # If `EVE_SETTINGS` exists and points to valid file, use that instead of # the default `settings.py` alt_settings = app_config.get('EVE_SETTINGS', None) if alt_settings and os.path.isfile(alt_settings): settings_file = alt_settings else: settings_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'settings.py') # Create the app instance app = Eve(settings=settings_file, json_encoder=NewBase60Encoder, validator=NewBase60Validator, auth=HawkAuth, instance_path=instance_path, static_folder=None) # Update the app's config object with our settings app.config.update(**dict(app_config)) # Add event hooks app.on_insert_posts += before_insert_posts app.on_update_posts += before_update_posts app.on_fetched_item_posts += after_fetched_item_posts app.on_pre_GET_posts += before_GET_posts app.on_pre_POST_posts += before_POST_posts app.on_post_POST_posts += after_POST_posts # Make sure necessary settings exist missing_settings = [] for setting in ('META_POST', 'ROOT_CREDENTIALS', 'SECRET_KEY', 'MENU_ITEMS', 'SERVER_NAME'): if not app.config.get(setting, None): missing_settings.append(setting) if missing_settings: raise SystemExit('Missing configuration settings! (%s)' % (','.join(missing_settings),)) # Check that a `meta.json` file exists in `types` and that you can read from it meta_schema = '' meta_schema_file = os.path.join(os.path.dirname(instance_path), 'types/meta.json') try: with open(meta_schema_file, 'r') as f: meta_schema = f.read() except IOError as e: raise SystemExit('Could not find `meta` post schema file at %s!' % (meta_schema_file,)) if not meta_schema: raise SystemExit('No data in `meta` post schema at %s!' % (meta_schema_file,)) # Validate the data in `META_POST` against the `meta` post schema v = Validator(json.loads(meta_schema)) if not v.validate(app.config['META_POST']): raise SystemExit('Invalid `META_POST` configuration! \ Validator returned the following errors: \n%s' % (str(v.errors),)) # Make sure necessary directories exist if not os.path.isdir(os.path.join(instance_path, 'attachments')): os.makedirs(os.path.join(instance_path, 'attachments')) if not os.path.isdir(os.path.join(instance_path, 'tmp')): os.makedirs(os.path.join(instance_path, 'tmp')) # Add some additional services besides the REST API app.register_blueprint(attachments) app.register_blueprint(server_info) app.register_blueprint(syndication) # Override some of Eve's default methods in order to handle HTML requests eve_override(app) # Override Jinja defaults, filters, and template loaders jinja_override(app) return app