def get_source_seed(**kwargs):
"""
Get the default configuration seed
Variables:
None
Arguments:
None
Data Block:
None
Result example:
{
"KEY": "value", # Dictionary of key/value pair
...
}
"""
seed_module = STORAGE.get_blob("seed_module")
if not seed_module:
return make_api_response({})
seed = module_attribute_by_name(seed_module)
services_to_register = seed['services']['master_list']
for service, svc_detail in services_to_register.iteritems():
seed['services']['master_list'][service] = get_merged_svc_config(service, svc_detail, LOGGER)
return make_api_response(seed)
def parse(self, request, hits, content):
parsed_configs = []
for func_name in set(
x.meta.get('al_configdumper', None) for x in hits):
if not func_name:
continue
name = func_name.split('.')[-2]
parsed = None
try:
config = module_attribute_by_name(func_name)(content)
if not config:
parsed = configparser.NullParsedConfig(self, name=name)
else:
parsed = configparser.ParsedConfig(self, "1.0", name=name)
for k, v in config.iteritems():
parsed.add_value(k, v)
except: # pylint: disable=W0702
request._svc.log.exception("Problem with %s:", func_name)
parsed = configparser.ParsedConfig(self, "1.0", name=name)
parsed_configs.append(parsed)
return parsed_configs
def get_config(force_refresh=False,
static_seed=os.getenv("AL_SEED_STATIC", None)):
global seed # pylint: disable=W0603
if force_refresh or not seed:
if static_seed:
seed = easydict.EasyDict(module_attribute_by_name(static_seed))
else:
seed = easydict.EasyDict(load_seed())
return seed
def reload_config(self):
self.seed_module = None
if isinstance(self.initial_seed, dict):
self.config = self.initial_seed
elif self.initial_seed:
self.config = module_attribute_by_name(self.initial_seed)
self.seed_module = self.initial_seed
services_to_register = self.config['services']['master_list']
for service, svc_detail in services_to_register.iteritems():
self.config['services']['master_list'][service] = get_merged_svc_config(service, svc_detail, self.log)
else:
from assemblyline.al.common import config_riak
self.config = config_riak.load_seed()
def install(alsi, include_worker_extensions=False):
if not alsi.config['system']['use_proxy']:
alsi.info("Proxy disabled in the seed. Skipping...")
return
import jinja2
alsi.sudo_apt_install(['haproxy',])
# build list of riak nodes to include in the haproxy configuration
nodes = []
count = 1
for riak_ip in alsi.config['datastore']['riak']['nodes']:
nodes.append({'hostname': 'riak-' + str(count), 'ip': riak_ip})
count += 1
tmpl_path = os.path.join(alsi.alroot, 'pkg', 'assemblyline/al/install/etc/haproxy/haproxy.cfg.core.tmpl')
alsi.info("Updating HAProxy configuration for nodes: " + pprint.pformat(nodes))
haproxy_cfg_tmpl = jinja2.Template(open(tmpl_path, 'r').read())
# we might need to pass alsi.config to the template in future if we have
# more advanced haproxy settings
redis_ip = alsi.config['core']['nodes'][0]
worker_ext = ''
if include_worker_extensions:
haproxy_ext = alsi.config['workers'].get('haproxy_extensions', None)
if haproxy_ext:
worker_ext = module_attribute_by_name(haproxy_ext)
include_redis = False
if alsi.get_ipaddress() != redis_ip:
include_redis = alsi.config['workers'].get('proxy_redis', True)
concrete_cfg = haproxy_cfg_tmpl.render(nodes=nodes,
include_redis=include_redis,
redis_ip=redis_ip,
worker_extensions=worker_ext,
datastore_port=alsi.config['datastore']['port'],
datastore_stream_port=alsi.config['datastore']['stream_port'],
datastore_solr_port=alsi.config['datastore']['solr_port'])
with open('/tmp/haproxy.cfg', 'w') as f:
f.write(concrete_cfg)
alsi.runcmd("sudo cp /tmp/haproxy.cfg /etc/haproxy/haproxy.cfg")
alsi.sudo_sed_inline('/etc/default/haproxy', ['s/ENABLED=0/ENABLED=1/'])
alsi.runcmd('sudo service haproxy restart')
alsi.info("HAProxy installation and configuration complete!")
# noinspection PyUnresolvedReferences
import pip
if pip.__version__ != "8.1.2":
raise ImportError("Upgrade PIP")
# noinspection PyUnresolvedReferences
import requests
if requests.__version__ != "2.10.0":
raise ImportError("Upgrade request")
except ImportError:
from assemblyline.common.importing import module_attribute_by_name
from assemblyline.al.install import PipInstaller
seed = os.environ.get('AL_SEED', None)
if seed:
temp_config = module_attribute_by_name(seed)
pip_installer = PipInstaller(
pypi_index_url=temp_config['installation']['pip_index_url'])
pip_installer.upgrade_all(['requests==2.10.0'])
# noinspection PyBroadException
try:
pip_installer.upgrade_all(['pip==8.1.2'])
except:
pass
else:
print "Cannot pip installer without the AL_SEED variable set"
exit(1)
# Start SiteInstaller
def _dynamic_import(path):
if not path:
ImportError("Error importing '%s'" % path or '')
return module_attribute_by_name(path)
for name, settings in config.datasources.iteritems():
name = name.lower()
classpath = 'unknown'
# noinspection PyBroadException
try:
classpath = settings['classpath']
cfg = settings['config']
if isinstance(cfg, basestring):
path = cfg
cfg = config
for point in path.split('.'):
if 'enabled' in cfg:
if not cfg['enabled']:
raise SkipDatasource()
cfg = cfg.get(point)
cls = module_attribute_by_name(classpath)
obj = cls(LOGGER, **cfg)
sources[name] = create_query_datasource(obj)
except SkipDatasource:
continue
except: # pylint: disable=W0702
LOGGER.exception("Problem creating %s datasource (%s)", name,
classpath)
except: # pylint: disable=W0702
LOGGER.exception("No datasources")
# noinspection PyUnusedLocal
@hash_search_api.route("/<file_hash>/", methods=["GET"])
@api_login(required_priv=['R'])
def search_hash(file_hash, *args, **kwargs):
except:
print "fix_seed <seed_path>"
exit(1)
if __name__ == "__main__":
import logging
from assemblyline.al.core.datastore import RiakStore
from assemblyline.common.importing import module_attribute_by_name
from assemblyline.al.service import register_service
log = logging.getLogger('assemblyline.datastore')
log.setLevel(logging.WARNING)
# noinspection PyBroadException
seed = module_attribute_by_name(seed_path)
services_to_register = seed['services']['master_list']
for service, svc_detail in services_to_register.iteritems():
classpath = svc_detail.get(
'classpath', "al_services.%s.%s" %
(svc_detail['repo'], svc_detail['class_name']))
config_overrides = svc_detail.get('config', {})
seed['services']['master_list'][service].update(
register_service.register(classpath,
config_overrides=config_overrides,
store_config=False,
enabled=svc_detail.get('enabled', True)))
ds = RiakStore(hosts=seed['datastore']['hosts'])