def main(ctx, verbosity):
configure_logger(verbosity)
# Load the configurations from file
config = Config(root_path=".")
config.from_object(default_settings)
config.from_envvar(ZUBBI_SETTINGS_ENV)
# Validate the configuration
tenant_sources_repo = config.get("TENANT_SOURCES_REPO")
tenant_sources_file = config.get("TENANT_SOURCES_FILE")
# Fail if both are set or none of both is set
if (
not tenant_sources_file
and not tenant_sources_repo
or (tenant_sources_file and tenant_sources_repo)
):
raise ScraperConfigurationError(
"Either one of 'TENANT_SOURCES_REPO' "
"and 'TENANT_SOURCES_FILE' must be set, "
"but not both."
)
# Store the config in click's context object to be available for subcommands
ctx.obj = {"config": config}
if ctx.invoked_subcommand is None:
ctx.invoke(scrape)
from flask.config import Config
import boto.sqs
from boto.sqs.message import RawMessage
from boto import exception
LOG = logging.getLogger('alerta.sqs')
config = Config('/')
config.from_pyfile('/etc/alertad.conf', silent=True)
config.from_envvar('ALERTA_SVR_CONF_FILE', silent=True)
DEFAULT_AWS_REGION = 'eu-west-1'
DEFAULT_AWS_SQS_QUEUE = 'alerts'
AWS_REGION = os.environ.get('AWS_REGION') or config.get(
'AWS_REGION', DEFAULT_AWS_REGION)
AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') or config.get(
'AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') or config.get(
'AWS_SECRET_ACCESS_KEY')
AWS_SQS_QUEUE = os.environ.get('AWS_SQS_QUEUE') or config.get(
'AWS_SQS_QUEUE', DEFAULT_AWS_SQS_QUEUE)
class Worker(object):
def __init__(self):
try:
connection = boto.sqs.connect_to_region(
AWS_REGION,
aws_access_key_id=AWS_ACCESS_KEY_ID,
def get(self, key, default=None):
"""Gets config from dynaconf variables
if variables does not exists in dynaconf try getting from
app.config to support runtime settings."""
return self._settings.get(key, Config.get(self, key, default))
from flask.config import Config
import boto.sqs
from boto.sqs.message import RawMessage
from boto import exception
LOG = logging.getLogger('alerta.sqs')
config = Config('/')
config.from_pyfile('/etc/alertad.conf', silent=True)
config.from_envvar('ALERTA_SVR_CONF_FILE', silent=True)
DEFAULT_AWS_REGION = 'eu-west-1'
DEFAULT_AWS_SQS_QUEUE = 'alerts'
AWS_REGION = os.environ.get('AWS_REGION') or config.get('AWS_REGION', DEFAULT_AWS_REGION)
AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') or config.get('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') or config.get('AWS_SECRET_ACCESS_KEY')
AWS_SQS_QUEUE = os.environ.get('AWS_SQS_QUEUE') or config.get('AWS_SQS_QUEUE', DEFAULT_AWS_SQS_QUEUE)
class Worker(object):
def __init__(self):
try:
connection = boto.sqs.connect_to_region(
AWS_REGION,
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY
)
class Spider(metaclass=abc.ABCMeta):
def __init__(self, import_name, middlewares=None, loop=None, session=None):
self.import_name = import_name
self.root_path = get_root_path(import_name)
self.config = Config(self.root_path, default_config)
self._context = {}
self._loop = loop or asyncio.get_event_loop()
self._middlewares = SpiderMiddlewareManager(self, middlewares)
self._session = session or aiohttp.ClientSession(loop=self._loop)
def enqueue_request(self, **kwargs):
context = self._context[self._task]
max_depth = self.config.get('MAX_DEPTH')
if max_depth and context['request'].depth > max_depth:
return
request = Request(referer=context['response'], **kwargs)
if request.url in self._seen:
return
if not self._url_allowed(request):
return
request.depth = context['response'].request.depth + 1
self._queue.put_nowait(request)
def run(self):
self._loop.run_until_complete(self.start())
@asyncio.coroutine
def start(self):
self._seen = set()
self._queue = Queue(loop=self._loop)
for url in self.config['URLS']:
self._queue.put_nowait(Request(url))
workers = [asyncio.Task(self._work()) for _ in range(self.config['CONCURRENCY'])]
yield from self._queue.join()
for worker in workers:
worker.cancel()
@asyncio.coroutine
def _work(self):
while True:
request = yield from self._queue.get()
yield from self._fetch(request)
self._queue.task_done()
@asyncio.coroutine
def _fetch(self, request):
for callback in self._middlewares['before_request']:
request = callback(self, request)
resp = yield from self._session.request('get', **request.params)
body = yield from resp.read_and_close()
response = Response(request, resp, body)
for callback in self._middlewares['after_response']:
response = callback(self, response)
with self._request_context(request, response):
self.parse(response)
@property
def _task(self):
return asyncio.Task.current_task(loop=self._loop)
@contextlib.contextmanager
def _request_context(self, request, response):
self._context[self._task] = {'request': request, 'response': response}
try:
yield
finally:
del self._context[self._task]
def _url_allowed(self, request):
return next(
(
True for domain in self.config['DOMAINS']
if request.furl.host.endswith(domain)
),
False,
)
@abc.abstractmethod
def parse(self, response):
pass
def PISS(instance_path):
# Create a configuration object and read the configuration file
app_config = Config(instance_path)
app_config.from_pyfile('piss.cfg')
# If `EVE_SETTINGS` exists and points to valid file, use that instead of
# the default `settings.py`
alt_settings = app_config.get('EVE_SETTINGS', None)
if alt_settings and os.path.isfile(alt_settings):
settings_file = alt_settings
else:
settings_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'settings.py')
# Create the app instance
app = Eve(settings=settings_file,
json_encoder=NewBase60Encoder,
validator=NewBase60Validator,
auth=HawkAuth,
instance_path=instance_path,
static_folder=None)
# Update the app's config object with our settings
app.config.update(**dict(app_config))
# Add event hooks
app.on_insert_posts += before_insert_posts
app.on_update_posts += before_update_posts
app.on_fetched_item_posts += after_fetched_item_posts
app.on_pre_GET_posts += before_GET_posts
app.on_pre_POST_posts += before_POST_posts
app.on_post_POST_posts += after_POST_posts
# Make sure necessary settings exist
missing_settings = []
for setting in ('META_POST', 'ROOT_CREDENTIALS', 'SECRET_KEY', 'MENU_ITEMS', 'SERVER_NAME'):
if not app.config.get(setting, None):
missing_settings.append(setting)
if missing_settings:
raise SystemExit('Missing configuration settings! (%s)' % (','.join(missing_settings),))
# Check that a `meta.json` file exists in `types` and that you can read from it
meta_schema = ''
meta_schema_file = os.path.join(os.path.dirname(instance_path), 'types/meta.json')
try:
with open(meta_schema_file, 'r') as f:
meta_schema = f.read()
except IOError as e:
raise SystemExit('Could not find `meta` post schema file at %s!' % (meta_schema_file,))
if not meta_schema:
raise SystemExit('No data in `meta` post schema at %s!' % (meta_schema_file,))
# Validate the data in `META_POST` against the `meta` post schema
v = Validator(json.loads(meta_schema))
if not v.validate(app.config['META_POST']):
raise SystemExit('Invalid `META_POST` configuration! \
Validator returned the following errors: \n%s' % (str(v.errors),))
# Make sure necessary directories exist
if not os.path.isdir(os.path.join(instance_path, 'attachments')):
os.makedirs(os.path.join(instance_path, 'attachments'))
if not os.path.isdir(os.path.join(instance_path, 'tmp')):
os.makedirs(os.path.join(instance_path, 'tmp'))
# Add some additional services besides the REST API
app.register_blueprint(attachments)
app.register_blueprint(server_info)
app.register_blueprint(syndication)
# Override some of Eve's default methods in order to handle HTML requests
eve_override(app)
# Override Jinja defaults, filters, and template loaders
jinja_override(app)
return app
