self.config.services.services_controller.service_list
):
services_list.append(
# a tuple associating a URI with a service class
(
uri,
# a binding of the service class with the configuration
# namespace for that service class
class_with_partial_init(
self.config.services[namespace]
.service_implementation_class,
self.config.services[namespace],
self.config
)
)
)
# initialize the wsgi server with the list of URI/service impl tuples
self.web_server = self.config.web_server.wsgi_server_class(
self.config, # needs the whole config not the local namespace
services_list
)
# for modwsgi the 'run' method returns the wsgi function that the web
# server will use. For other webservers, the 'run' method actually
# starts the standalone web server.
application = self.web_server.run()
if __name__ == '__main__':
main(CollectorApp)
import os
from socorrolib.app.generic_app import main
from socorro.middleware.middleware_app import MiddlewareApp
from socorro.webapi.servers import WSGIServer
import socorro.middleware.middleware_app
from configman import (
ConfigFileFutureProxy,
environment
)
if os.path.isfile('/etc/socorro/middleware.ini'):
config_path = '/etc/socorro'
else:
config_path = WSGIServer.get_socorro_config_path(__file__)
# invoke the generic main function to create the configman app class and which
# will then create the wsgi app object.
main(
MiddlewareApp, # the socorro app class
config_path=config_path,
values_source_list=[
ConfigFileFutureProxy,
environment
]
)
application = socorro.middleware.middleware_app.application
import os
from socorrolib.app.generic_app import main
from socorro.middleware.middleware_app import MiddlewareApp
from socorro.webapi.servers import WSGIServer
import socorro.middleware.middleware_app
from configman import (ConfigFileFutureProxy, environment)
if os.path.isfile('/etc/socorro/middleware.ini'):
config_path = '/etc/socorro'
else:
config_path = WSGIServer.get_socorro_config_path(__file__)
# invoke the generic main function to create the configman app class and which
# will then create the wsgi app object.
main(
MiddlewareApp, # the socorro app class
config_path=config_path,
values_source_list=[ConfigFileFutureProxy, environment])
application = socorro.middleware.middleware_app.application
doc='the class responsible for proving an hbase connection',
from_string_converter=class_converter
)
required_config.add_option(
'command',
default=help,
doc='command to use',
from_string_converter=lambda s: class_converter(__name__ + '.' + s)
)
required_config.add_option(
'json',
default=False,
short_form='j',
doc='json output instead of a pretty printed mapping',
)
def main(self):
self.storage = self.config.hbase_crash_storage_class(self.config)
self.config.command(self).run()
if __name__ == '__main__':
try:
generic_app.main(HBaseClientApp,
config_manager_cls=HBaseClientConfigurationManager)
except NotEnoughArguments as e:
print >> sys.stderr, "ERROR: was expecting another argument: " + e.arg
print >> sys.stderr, "Use the 'help' command to get help on commands."
services_list = []
# populate the 'services_list' with the tuples that will define the
# urls and services offered by dataservice.
for impl_class_namespace in (
self.config.services.service_list.subordinate_namespace_names
):
impl_class = self.config.services[impl_class_namespace].cls
services_list.append(impl_class)
if not services_list:
raise RuntimeError(
"No services configured."
"See the [services].service_list setting in the config file"
)
self.web_server = self.config.web_server.wsgi_server_class(
self.config,
services_list
)
# for modwsgi the 'run' method returns the wsgi function that
# will use. For other webservers, the 'run' method actually starts
# the standalone web server.
application = self.web_server.run()
#==============================================================================
if __name__ == '__main__':
main(DataserviceApp)
app_name = 'example'
app_version = '0.1'
app_description = __doc__
#--------------------------------------------------------------------------
# in this section, define any configuration requirements
required_config = Namespace()
required_config.add_option('name',
default='Wilma',
doc='a name to echo')
required_config.add_option('time',
default=datetime.datetime.now(),
doc='the time of day')
#--------------------------------------------------------------------------
# implementing this constructor is only necessary when there is more
# initialization to be done before main can be called
#def __init__(self, config):
#super(ExampleApp,self).__init__(config)
#--------------------------------------------------------------------------
def main(self):
# this is where we'd implement the app
# the configuraton is already setup as self.config
print 'hello, %s. The time is: %s' % (self.config.name,
self.config.time)
if __name__ == '__main__':
main(ExampleApp)
class MeasuringImplementationWrapper(ImplementationWrapper):
"""This class does nothing differently than its parent class except it
measures how long it takes to generated the data for every GET request.
This timing is just to get and collect the data. It excludes the time
to parse the request and to serialize to JSON.
Use it only for development purposes to give yourself a feel for how
much time is spent on gathering different pieces of data.
"""
def GET(self, *args, **kwargs):
t0 = time.time()
result = (
super(MeasuringImplementationWrapper, self)
.GET(*args, **kwargs)
)
t1 = time.time()
self.config.logger.info(
'measuringmiddleware:%.2f\t%s\t%s' % (
1000 * (t1 - t0),
web.ctx.path,
web.ctx.query
)
)
return result
if __name__ == '__main__':
main(MiddlewareApp)
for key, value in params.iteritems():
if value.count(terms_sep) and not DONT_TERM_SPLIT.match(value):
params[key] = value.split(terms_sep)
return params
class MeasuringImplementationWrapper(ImplementationWrapper):
"""This class does nothing differently than its parent class except it
measures how long it takes to generated the data for every GET request.
This timing is just to get and collect the data. It excludes the time
to parse the request and to serialize to JSON.
Use it only for development purposes to give yourself a feel for how
much time is spent on gathering different pieces of data.
"""
def GET(self, *args, **kwargs):
t0 = time.time()
result = (super(MeasuringImplementationWrapper,
self).GET(*args, **kwargs))
t1 = time.time()
self.config.logger.info('measuringmiddleware:%.2f\t%s\t%s' %
(1000 *
(t1 - t0), web.ctx.path, web.ctx.query))
return result
if __name__ == '__main__':
main(MiddlewareApp)
host=config.host,
port=config.port,
virtual_host=config.virtual_host,
credentials=pika.credentials.PlainCredentials(
config.rabbitmq_user, config.rabbitmq_password)))
except:
logging.error("Failed to connect")
raise
self.connection = connection
def main(self):
self.connect()
channel = self.connection.channel()
channel.queue_declare(queue='socorro.reprocessing', durable=True)
with open(self.config.reprocesscrashlist.crashes, 'r') as file:
for uuid in file.read().splitlines():
channel.basic_publish(
exchange='',
routing_key="socorro.reprocessing",
body=uuid,
properties=pika.BasicProperties(delivery_mode=2))
logging.debug('submitted %s' % uuid)
self.connection.close()
if __name__ == '__main__':
main(ReprocessCrashlistApp)
self.config.services.services_controller.service_list
):
services_list.append(
# a tuple associating a URI with a service class
(
uri,
# a binding of the service class with the configuration
# namespace for that service class
class_with_partial_init(
self.config.services[namespace]
.service_implementation_class,
self.config.services[namespace],
self.config
)
)
)
# initialize the wsgi server with the list of URI/service impl tuples
self.web_server = self.config.web_server.wsgi_server_class(
self.config, # needs the whole config not the local namespace
services_list
)
# for modwsgi the 'run' method returns the wsgi function that the web
# server will use. For other webservers, the 'run' method actually
# starts the standalone web server.
application = self.web_server.run()
if __name__ == '__main__':
main(CollectorApp)
bulk(
client=es_connection,
actions=actions,
)
# Verify data was correctly inserted.
index_client.refresh(index=[es_index])
total_indexed = es_connection.count(
index=es_index,
doc_type='supersearch_fields',
)['count']
total_expected = len(all_fields)
if total_expected > total_indexed:
indexed_fields = es_connection.search(
index=es_index,
doc_type='supersearch_fields',
size=total_indexed,
)
indexed_fields = [x['_id'] for x in indexed_fields['hits']['hits']]
self.config.logger.error(
'The SuperSearch fields data was not correctly indexed, '
'%s fields are missing from the database. Missing fields: %s',
total_expected - total_indexed,
list(set(all_fields.keys()) - set(indexed_fields)))
if __name__ == '__main__':
generic_app.main(SetupSuperSearchApp)
client=es_connection,
actions=actions,
)
# Verify data was correctly inserted.
index_client.refresh(index=[es_index])
total_indexed = es_connection.count(
index=es_index,
doc_type='supersearch_fields',
)['count']
total_expected = len(all_fields)
if total_expected > total_indexed:
indexed_fields = es_connection.search(
index=es_index,
doc_type='supersearch_fields',
size=total_indexed,
)
indexed_fields = [x['_id'] for x in indexed_fields['hits']['hits']]
self.config.logger.error(
'The SuperSearch fields data was not correctly indexed, '
'%s fields are missing from the database. Missing fields: %s',
total_expected - total_indexed,
list(set(all_fields.keys()) - set(indexed_fields))
)
if __name__ == '__main__':
generic_app.main(SetupSuperSearchApp)
