def app_main(self, config, options, args): db_url = db.create_url_from_config(config['db']) message_url = message_queue.create_url_from_config(config['amqp']['broker']) message_exchange = config['amqp']['exchange']['name'] maximum_priority = int(config['scanner']['maximum_priority']) iteration_minimum_duration = float(config['scanner']['iteration_minimum_duration']) iteration_minimum_duration = datetime.timedelta(seconds=iteration_minimum_duration) db.configure_session(db_url) message_client = message_queue.create_message_client(message_url) message_queue.create_queues_from_config(message_client, config['amqp']) message_queue.close_message_client(message_client) scanners = [] for priority in range(0, maximum_priority + 1): scanner = threading.Thread(target=scan_events, args=(message_url, message_exchange, priority, iteration_minimum_duration, maximum_priority)) scanner.start() scanners.append(scanner) for scanner in scanners: scanner.join()
def app_main(self, config, options, queues): TIM_DATA = 'TIM_DATA' data_directory = os.environ.get(TIM_DATA, None) if data_directory is None: logging.error('Environment variable %s not defined', TIM_DATA) sys.exit() file = options.message_file.format(tim_data=data_directory) if not os.path.exists(file): logging.warning('File "%s" does not exist', file) sys.exit() # read the message file try: messages = json_serializer.load(open(file, 'r')) except Exception: logging.error('Failed to read json file: %s', file) raise # create amqp connection client = message_queue.create_message_client(options.url) # create all of the required queues message_queue.create_queues_from_config(client, config['amqp']) # itereate and send all the interesting messages for message in messages: queue = message['header']['type'] if queue in queues: message_queue.send_messages(client, config['amqp']['exchange']['name'], [message]) sys.stdout.write('.') sys.stdout.write('\n')
def app_main(self, config, options, args): logging.info("Beginning...") # read the db url from the config db_url = db.create_url_from_config(config['db']) # get the broker and queue config broker_url = message_queue.create_url_from_config(config['amqp']['broker']) # get exchange information self.amqp_exchange = config['amqp']['exchange']['name'] # initialize the db engine & session db.configure_session(db_url) data_access.service.initialize() data_access.post_type.initialize() # get message broker client and store in instance -- used for both receiving and sending self.client = message_queue.create_message_client(broker_url) self.handlers = {} for service in _services_configuration(config): self.handlers[service['name']] = event_updater.from_service_name( service['name'], service['oauth']) logging.info('Queue broker URL: %s', broker_url) logging.debug('Active handlers: %s', self.handlers) message_queue.create_queues_from_config(self.client, config['amqp']) message_queue.join( self.client, config['amqp']['queues']['updater']['queue'], self.handler) logging.info("Finished...")
def app_main(self, config, options, args): logging.info("Beginning...") # read the db url from the config db_url = db.create_url_from_config(config['db']) # get the broker and queue config broker_url = message_queue.create_url_from_config(config['amqp']['broker']) # get the maximum priority max_priority = int(config['scanner']['maximum_priority']) min_duration = float(config['scanner']['iteration_minimum_duration']) min_duration = datetime.timedelta(seconds=min_duration) # initialize the db engine & session db.configure_session(db_url) service.initialize() post_type.initialize() # Create event processor self.processor = event_processor.EventProcessor( max_priority, min_duration, config['oauth']) logging.info('Queue broker URL: %s', broker_url) # get message broker client and store in instance -- used for both receiving and sending self.client = message_queue.create_message_client(broker_url) message_queue.create_queues_from_config(self.client, config['amqp']) message_queue.join( self.client, config['amqp']['queues']['processor']['queue'], self.handler) logging.info("Finished...")
def app_main(self, config, options, args): logging.info("Beginning...") db.configure_session(url=db.create_url_from_config(config['db'])) # if services option is empty default to all available configured services if not options.services: service_names = [] # generate the list of service name's from queue list for key in config['queues'].iterkeys(): service_names.append(key) else: service_names = options.services # if authors option is empty default to all authors if not options.authors: author_names = [author_name for author_name in db.Session().query(Author.author_name).all()] else: author_names = options.authors # get the broker and queue config broker_url = config['broker']['url'] # get message broker client and store in instance client = message_queue.create_message_client(broker_url) # for each specified service for service_name in service_names: # create the queue for this service message_queue.create_queues_from_config(client, config['amqp']) # for each specified author for author_name in author_names: # post a update message for service & author for asm, event in db.Session().query(AuthorServiceMap, ServiceEvent). \ join(ServiceEvent, and_(AuthorServiceMap.author_id == ServiceEvent.author_id, AuthorServiceMap.service_id == ServiceEvent.service_id)). \ join(Service, AuthorServiceMap.service_id == Service.id). \ join(Author, AuthorServiceMap.author_id == Author.id). \ filter(and_(Service.service_name == service_name, Author.author_name == author_name)).all(): if event.type_id == 2: continue message_queue.send_messages(client, [create_event_update_message(service_name, asm.service_author_id, event.event_id)]) logging.info("Finished...")
def app_main(self, config, options, args): logging.info("Beginning...") # read the db url from the config db_url = db.create_url_from_config(config['db']) # initialize the db engine & session db.configure_session(db_url) # if services option is empty default to all available configured services if not options.services: service_names = [] # generate the list of service name's from queue list for key in config['queues'].iterkeys(): service_names.append(key) else: service_names = options.services # get the broker and queue config broker_url = message_queue.create_url_from_config(config['amqp']['broker']) # get message broker client and store in instance client = message_queue.create_message_client(broker_url) # for each specified service for service_name in service_names: # create the queue for this service message_queue.create_queues_from_config(client, config['amqp']) # post a notification for each author subscribed to this service for asm in db.Session().query(AuthorServiceMap). \ join(Service, AuthorServiceMap.service_id == Service.id). \ filter(Service.service_name == service_name).all(): message_queue.send_messages(client, [create_notification_message(service_name, asm.service_author_id)]) logging.info("Finished...")
def create_feed_queue(event): environment = event.app.registry.settings[config.ENVIRONMENT_KEY] url = message_queue.create_url_from_config(environment['amqp']['broker']) client = message_queue.get_current_message_client(url) message_queue.create_queues_from_config(client, environment['amqp'])