def scan_events(
message_url,
message_exchange,
priority,
highest_priority_duration,
maximum_priority):
priority_duration = (2 ** priority) * highest_priority_duration
message_client = message_queue.create_message_client(message_url)
current_id = ""
while True:
with db.Context():
event_count = _query_event_count(priority)
event_interval = highest_priority_duration
if event_count != 0:
event_interval = priority_duration / event_count
logging.info("Sleepging for %s seconds for priorrity %s",
total_seconds(event_interval),
priority)
time.sleep(total_seconds(event_interval))
view_result = _query_event(current_id, priority)
current_id = _send_update_message_from_event(message_client, message_exchange, view_result)
_decrease_priority(view_result, maximum_priority)
def app_main(self, config, options, args):
db_url = db.create_url_from_config(config['db'])
message_url = message_queue.create_url_from_config(config['amqp']['broker'])
message_exchange = config['amqp']['exchange']['name']
maximum_priority = int(config['scanner']['maximum_priority'])
iteration_minimum_duration = float(config['scanner']['iteration_minimum_duration'])
iteration_minimum_duration = datetime.timedelta(seconds=iteration_minimum_duration)
db.configure_session(db_url)
message_client = message_queue.create_message_client(message_url)
message_queue.create_queues_from_config(message_client, config['amqp'])
message_queue.close_message_client(message_client)
scanners = []
for priority in range(0, maximum_priority + 1):
scanner = threading.Thread(target=scan_events,
args=(message_url,
message_exchange,
priority,
iteration_minimum_duration,
maximum_priority))
scanner.start()
scanners.append(scanner)
for scanner in scanners:
scanner.join()
def app_main(self, config, options, queues):
TIM_DATA = 'TIM_DATA'
data_directory = os.environ.get(TIM_DATA, None)
if data_directory is None:
logging.error('Environment variable %s not defined', TIM_DATA)
sys.exit()
file = options.message_file.format(tim_data=data_directory)
if not os.path.exists(file):
logging.warning('File "%s" does not exist', file)
sys.exit()
# read the message file
try:
messages = json_serializer.load(open(file, 'r'))
except Exception:
logging.error('Failed to read json file: %s', file)
raise
# create amqp connection
client = message_queue.create_message_client(options.url)
# create all of the required queues
message_queue.create_queues_from_config(client, config['amqp'])
# itereate and send all the interesting messages
for message in messages:
queue = message['header']['type']
if queue in queues:
message_queue.send_messages(client, config['amqp']['exchange']['name'], [message])
sys.stdout.write('.')
sys.stdout.write('\n')
def app_main(self, config, options, args):
logging.info("Beginning...")
# read the db url from the config
db_url = db.create_url_from_config(config['db'])
# get the broker and queue config
broker_url = message_queue.create_url_from_config(config['amqp']['broker'])
# get exchange information
self.amqp_exchange = config['amqp']['exchange']['name']
# initialize the db engine & session
db.configure_session(db_url)
data_access.service.initialize()
data_access.post_type.initialize()
# get message broker client and store in instance -- used for both receiving and sending
self.client = message_queue.create_message_client(broker_url)
self.handlers = {}
for service in _services_configuration(config):
self.handlers[service['name']] = event_updater.from_service_name(
service['name'],
service['oauth'])
logging.info('Queue broker URL: %s', broker_url)
logging.debug('Active handlers: %s', self.handlers)
message_queue.create_queues_from_config(self.client, config['amqp'])
message_queue.join(
self.client,
config['amqp']['queues']['updater']['queue'],
self.handler)
logging.info("Finished...")
def app_main(self, config, options, args):
logging.info("Beginning...")
# read the db url from the config
db_url = db.create_url_from_config(config['db'])
# get the broker and queue config
broker_url = message_queue.create_url_from_config(config['amqp']['broker'])
# get the maximum priority
max_priority = int(config['scanner']['maximum_priority'])
min_duration = float(config['scanner']['iteration_minimum_duration'])
min_duration = datetime.timedelta(seconds=min_duration)
# initialize the db engine & session
db.configure_session(db_url)
service.initialize()
post_type.initialize()
# Create event processor
self.processor = event_processor.EventProcessor(
max_priority,
min_duration,
config['oauth'])
logging.info('Queue broker URL: %s', broker_url)
# get message broker client and store in instance -- used for both receiving and sending
self.client = message_queue.create_message_client(broker_url)
message_queue.create_queues_from_config(self.client, config['amqp'])
message_queue.join(
self.client,
config['amqp']['queues']['processor']['queue'],
self.handler)
logging.info("Finished...")
def app_main(self, config, options, args):
logging.info("Beginning...")
db.configure_session(url=db.create_url_from_config(config['db']))
# if services option is empty default to all available configured services
if not options.services:
service_names = []
# generate the list of service name's from queue list
for key in config['queues'].iterkeys():
service_names.append(key)
else:
service_names = options.services
# if authors option is empty default to all authors
if not options.authors:
author_names = [author_name for author_name in db.Session().query(Author.author_name).all()]
else:
author_names = options.authors
# get the broker and queue config
broker_url = config['broker']['url']
# get message broker client and store in instance
client = message_queue.create_message_client(broker_url)
# for each specified service
for service_name in service_names:
# create the queue for this service
message_queue.create_queues_from_config(client, config['amqp'])
# for each specified author
for author_name in author_names:
# post a update message for service & author
for asm, event in db.Session().query(AuthorServiceMap, ServiceEvent). \
join(ServiceEvent, and_(AuthorServiceMap.author_id == ServiceEvent.author_id,
AuthorServiceMap.service_id == ServiceEvent.service_id)). \
join(Service, AuthorServiceMap.service_id == Service.id). \
join(Author, AuthorServiceMap.author_id == Author.id). \
filter(and_(Service.service_name == service_name,
Author.author_name == author_name)).all():
if event.type_id == 2:
continue
message_queue.send_messages(client,
[create_event_update_message(service_name,
asm.service_author_id,
event.event_id)])
logging.info("Finished...")
def app_main(self, config, options, args):
logging.info("Beginning...")
# read the db url from the config
db_url = db.create_url_from_config(config['db'])
# initialize the db engine & session
db.configure_session(db_url)
# if services option is empty default to all available configured services
if not options.services:
service_names = []
# generate the list of service name's from queue list
for key in config['queues'].iterkeys():
service_names.append(key)
else:
service_names = options.services
# get the broker and queue config
broker_url = message_queue.create_url_from_config(config['amqp']['broker'])
# get message broker client and store in instance
client = message_queue.create_message_client(broker_url)
# for each specified service
for service_name in service_names:
# create the queue for this service
message_queue.create_queues_from_config(client, config['amqp'])
# post a notification for each author subscribed to this service
for asm in db.Session().query(AuthorServiceMap). \
join(Service, AuthorServiceMap.service_id == Service.id). \
filter(Service.service_name == service_name).all():
message_queue.send_messages(client,
[create_notification_message(service_name,
asm.service_author_id)])
logging.info("Finished...")
