def get_datasource_settings(self, vendor=None):
# Container for datasource settings.
datasource_settings = SmartBunch({
'datasources': [],
'datasource': SmartBunch(),
'total': SmartBunch.bunchify({'fulltext_countries': [], 'details_countries': []}),
})
# Read datasource settings from configuration.
datasource_settings.datasources = read_list(self.application_settings.get('ip_navigator', {}).get('datasources'))
datasource_settings.protected_fields = read_list(self.application_settings.get('ip_navigator', {}).get('datasources_protected_fields'))
for datasource in datasource_settings.datasources:
datasource_info = SmartBunch()
if vendor is None:
settings_key = 'datasource:{name}'.format(name=datasource)
else:
settings_key = 'datasource:{name}:{vendor}'.format(name=datasource, vendor=vendor)
ds_settings = self.application_settings.get(settings_key, {})
datasource_info.setdefault('fulltext_enabled', asbool(ds_settings.get('fulltext_enabled', False)))
datasource_info.setdefault('fulltext_countries', read_list(ds_settings.get('fulltext_countries', '')))
datasource_info.setdefault('details_enabled', asbool(ds_settings.get('details_enabled', False)))
datasource_info.setdefault('details_countries', read_list(ds_settings.get('details_countries', '')))
for key, value in ds_settings.iteritems():
datasource_info.setdefault(key, value)
datasource_settings.datasource[datasource] = SmartBunch.bunchify(datasource_info)
# Aggregate data for all countries.
datasource_settings.total.fulltext_countries += datasource_info['fulltext_countries']
return datasource_settings
def get_datasource_settings(self):
# Container for datasource settings
datasource_settings = SmartBunch({
'datasources': [],
'datasource': SmartBunch(),
'total': SmartBunch.bunchify({'fulltext_countries': [], 'details_countries': []}),
})
# Read datasource settings from configuration
datasource_settings.datasources = read_list(self.application_settings.get('ip_navigator', {}).get('datasources'))
datasource_settings.protected_fields = read_list(self.application_settings.get('ip_navigator', {}).get('datasources_protected_fields'))
for datasource in datasource_settings.datasources:
settings_key = 'datasource:{name}'.format(name=datasource)
datasource_info = self.application_settings.get(settings_key, {})
datasource_info['fulltext_enabled'] = asbool(datasource_info.get('fulltext_enabled', False))
datasource_info['fulltext_countries'] = read_list(datasource_info.get('fulltext_countries', ''))
datasource_info['details_enabled'] = asbool(datasource_info.get('details_enabled', False))
datasource_info['details_countries'] = read_list(datasource_info.get('details_countries', ''))
datasource_settings.datasource[datasource] = SmartBunch.bunchify(datasource_info)
# Aggregate data for all countries
datasource_settings.total.fulltext_countries += datasource_info['fulltext_countries']
return datasource_settings
def read(self):
# Read metadata
"""
out:
"meta": {
"status": "success",
"params": {
"sort": "pd desc, ucid asc",
"rows": "250",
"indent": "true",
"qt": "premium",
"timeAllowed": "300000",
"q": "text:vibrat* AND (ic:G01F000184 OR cpc:G01F000184)",
"start": "0",
"wt": "json",
"fl": "ucid,fam"
},
"pager": {
"totalEntries": 6872,
"entriesOnThisPage": 250,
"firstPage": 1,
"lastPage": 28,
"previousPage": null,
"currentPage": 1,
"entriesPerPage": "250",
"nextPage": 2
},
"name": "ifi",
"time": "4.836163"
}
"""
self.meta.upstream.update({
'name':
'ifi',
'time':
self.input['time'],
'status':
self.input['status'],
'params':
SmartBunch.bunchify(
self.input['content']['responseHeader']['params']),
'pager':
SmartBunch.bunchify(self.input['content']['responseHeader'].get(
'pager', {})),
})
self.meta.navigator.count_total = int(
self.meta.upstream.pager.totalEntries)
self.meta.navigator.count_page = int(
self.meta.upstream.pager.entriesOnThisPage)
self.meta.navigator.offset = int(self.meta.upstream.params.start)
self.meta.navigator.limit = int(self.meta.upstream.params.rows)
self.meta.navigator.postprocess = SmartBunch()
# Read content
self.documents = self.input['content']['response']['docs']
self.read_documents()
def issue_reporter_handler(request):
targets = request.params.get('targets')
report_data = request.json
report_data.setdefault('application', {})
report = SmartBunch.bunchify(report_data)
# Add user information to issue report
user = request.user
if user:
# Anonymize sensitive user data
user.password = None
user.upstream_credentials = None
# Serialize user object and attach to report
report.application.user = SmartBunch(json.loads(user.to_json()))
# Send the whole beast to the standard application log
log.error('Issue report [{targets}]:\n{report}'.format(
report=report.pretty(),
targets=targets
))
# TODO: Store the issue report into database
# TODO: What about other targets like "log:error", "log:warning", "human:support", "human:user"?
# Send email report
for target in read_list(targets):
if target.startswith('email:'):
recipient = target.replace('email:', '')
email_issue_report(report, recipient)
def get_vendor_settings(self):
# Container for vendor settings
vendor_settings = SmartBunch({
'vendors': [],
'vendor': SmartBunch(),
})
# Read vendor settings from configuration
try:
vendor_settings.vendors = read_list(self.application_settings.ip_navigator.vendors)
assert vendor_settings.vendors
except:
raise ConfigurationError('No vendor configured in "{configfile}"'.format(configfile=self.configfile))
for vendor in vendor_settings.vendors:
settings_key = 'vendor:{name}'.format(name=vendor)
if settings_key not in self.application_settings:
raise ConfigurationError('Vendor "{vendor}" not configured in "{configfile}"'.format(
vendor=vendor, configfile=self.configfile))
vendor_info = self.application_settings.get(settings_key, {})
for key, value in vendor_info.iteritems():
vendor_info[key] = value.decode('utf-8')
if 'hostname_matches' in vendor_info:
vendor_info.hostname_matches = read_list(vendor_info.hostname_matches)
vendor_info.email = self.get_email_settings(vendor)
vendor_settings.vendor[vendor] = SmartBunch.bunchify(vendor_info)
return vendor_settings
def setup(self):
# Documents from upstream data source
self.documents = []
# Metadata information, upstream (raw) and downstream (unified)
self.meta = SmartBunch.bunchify({
'navigator': {},
'upstream': {},
})
# Output information, upstream (raw) and downstream (unified)
self.output = SmartBunch.bunchify({
'meta': {},
'numbers': [],
'details': [],
'navigator': {},
})
def datasource_settings(self):
"""
Return datasource settings while accounting for sensible settings like API URI and credentials.
"""
request = get_current_request()
datasource_settings = SmartBunch.bunchify(request.registry.datasource_settings)
if 'protected_fields' in datasource_settings:
for fieldname in datasource_settings.protected_fields:
for name, settings in datasource_settings.datasource.iteritems():
if fieldname in settings:
del settings[fieldname]
del datasource_settings['protected_fields']
return datasource_settings
def __init__(self, input, options=None):
# Input data and options
self.input = input
self.options = options and SmartBunch.bunchify(options) or SmartBunch()
# Setup data structures
self.setup()
# Read input information
self.read()
# Run data munging actions
if 'feature_family_remove' in self.options and self.options.feature_family_remove:
self.remove_family_members()