Exemplo n.º 1
0
  def app_main(self, config, options, args):
    db_url = db.create_url_from_config(config['db'])
    message_url = message_queue.create_url_from_config(config['amqp']['broker'])
    message_exchange = config['amqp']['exchange']['name']

    maximum_priority = int(config['scanner']['maximum_priority'])
    iteration_minimum_duration = float(config['scanner']['iteration_minimum_duration'])
    iteration_minimum_duration = datetime.timedelta(seconds=iteration_minimum_duration)

    db.configure_session(db_url)

    message_client = message_queue.create_message_client(message_url)
    message_queue.create_queues_from_config(message_client, config['amqp'])
    message_queue.close_message_client(message_client)

    scanners = []
    for priority in range(0, maximum_priority + 1):
      scanner = threading.Thread(target=scan_events,
                                 args=(message_url,
                                       message_exchange,
                                       priority,
                                       iteration_minimum_duration,
                                       maximum_priority))
      scanner.start()
      scanners.append(scanner)

    for scanner in scanners:
      scanner.join()
Exemplo n.º 2
0
  def app_main(self, config, options, args):
    logging.info("Beginning...")

    # read the db url from the config
    db_url = db.create_url_from_config(config['db'])
    # get the broker and queue config
    broker_url = message_queue.create_url_from_config(config['amqp']['broker'])
    # get exchange information
    self.amqp_exchange = config['amqp']['exchange']['name']

    # initialize the db engine & session
    db.configure_session(db_url)
    data_access.service.initialize()
    data_access.post_type.initialize()

    # get message broker client and store in instance -- used for both receiving and sending
    self.client = message_queue.create_message_client(broker_url)

    self.handlers = {}
    for service in _services_configuration(config):
      self.handlers[service['name']] = event_updater.from_service_name(
          service['name'],
          service['oauth'])

    logging.info('Queue broker URL: %s', broker_url)
    logging.debug('Active handlers: %s', self.handlers)

    message_queue.create_queues_from_config(self.client, config['amqp'])
    message_queue.join(
        self.client,
        config['amqp']['queues']['updater']['queue'],
        self.handler)

    logging.info("Finished...")
Exemplo n.º 3
0
  def app_main(self, config, options, args):
    logging.info("Beginning...")

    # read the db url from the config
    db_url = db.create_url_from_config(config['db'])
    # get the broker and queue config
    broker_url = message_queue.create_url_from_config(config['amqp']['broker'])
    # get the maximum priority
    max_priority = int(config['scanner']['maximum_priority'])
    min_duration = float(config['scanner']['iteration_minimum_duration'])
    min_duration = datetime.timedelta(seconds=min_duration)

    # initialize the db engine & session
    db.configure_session(db_url)
    service.initialize()
    post_type.initialize()

    # Create event processor
    self.processor = event_processor.EventProcessor(
        max_priority,
        min_duration,
        config['oauth'])

    logging.info('Queue broker URL: %s', broker_url)

    # get message broker client and store in instance -- used for both receiving and sending
    self.client = message_queue.create_message_client(broker_url)
    message_queue.create_queues_from_config(self.client, config['amqp'])

    message_queue.join(
        self.client,
        config['amqp']['queues']['processor']['queue'],
        self.handler)

    logging.info("Finished...")
Exemplo n.º 4
0
  def app_main(self, config, options, args):
    logging.info("Beginning...")

    db.configure_session(url=db.create_url_from_config(config['db']))

    # if services option is empty default to all available configured services
    if not options.services:
      service_names = []
      # generate the list of service name's from queue list
      for key in config['queues'].iterkeys():
        service_names.append(key)
    else:
      service_names = options.services

    # if authors option is empty default to all authors
    if not options.authors:
      author_names = [author_name for author_name in db.Session().query(Author.author_name).all()]
    else:
      author_names = options.authors

    # get the broker and queue config
    broker_url = config['broker']['url']

    # get message broker client and store in instance
    client = message_queue.create_message_client(broker_url)

    # for each specified service
    for service_name in service_names:

      # create the queue for this service
      message_queue.create_queues_from_config(client, config['amqp'])

      # for each specified author
      for author_name in author_names:

        # post a update message for service & author
        for asm, event in db.Session().query(AuthorServiceMap, ServiceEvent). \
                                       join(ServiceEvent, and_(AuthorServiceMap.author_id == ServiceEvent.author_id,
                                                               AuthorServiceMap.service_id == ServiceEvent.service_id)). \
                                       join(Service, AuthorServiceMap.service_id == Service.id). \
                                       join(Author, AuthorServiceMap.author_id == Author.id). \
                                       filter(and_(Service.service_name == service_name,
                                                   Author.author_name == author_name)).all():

          if event.type_id == 2:
            continue

          message_queue.send_messages(client,
                                      [create_event_update_message(service_name,
                                                                   asm.service_author_id,
                                                                   event.event_id)])

    logging.info("Finished...")
Exemplo n.º 5
0
    def app_main(self, config, options, args):

        logging.info("Beginning...")

        # initialize the db engine & session
        db.configure_session(url=db.create_url_from_config(config["db"]))

        # get the engine, bind it to the metadata base, and create
        # the tables
        engine = db.Session().get_bind()

        mi_schema.Base.metadata.bind = engine
        mi_schema.Base.metadata.create_all(engine)

        logging.info("Finished...")
Exemplo n.º 6
0
  def app_main(self, config, options, args):
    # TODO: deal with new user registrations by listening to amqp and schedule the rest

    observer = PythonLoggingObserver()
    observer.start()

    # initialize some important maps
    db.configure_session(db.create_url_from_config(config['db']))
    service.initialize()
    post_type.initialize()

    # Grab twitter consumer keys
    self.consumer_key = config['oauth']['twitter']['key']
    self.consumer_secret = config['oauth']['twitter']['secret']
    self.default_token_key = config['oauth']['twitter']['default_access_token']
    self.default_token_secret = config['oauth']['twitter']['default_access_token_secret']

    # Grab feed configuration
    self.wait_on_collector_query_delay = float(config['feed']['wait_on_collector_query_delay'])

    # Configure amqp
    amqp_host = config['amqp']['broker']['host']
    amqp_port = int(config['amqp']['broker']['port'])
    amqp_spec = message_queue.create_spec_path(config['amqp']['broker']['spec'])
    self.amqp_exchange = config['amqp']['exchange']['name']

    self.amqp = AmqpFactory(host=amqp_host, port=amqp_port, spec_file=amqp_spec)

    db_host = config['db']['host']
    db_user = config['db']['user']
    db_passwd = config['db']['password']
    db_database = config['db']['database']
    db_unicode = to_bool(config['db']['unicode'])

    self.db_pool = adbapi.ConnectionPool(
        'MySQLdb',
        host=db_host,
        user=db_user,
        passwd=db_passwd,
        db=db_database,
        use_unicode=db_unicode,
        cp_noisy=True)
    self._process_twitter_users()

    reactor.run()
Exemplo n.º 7
0
  def app_main(self, config, options, args):
    db.configure_session(url=db.create_url_from_config(config['db']))
    services = _lookup_services_with_event_queue(config)
    maximum_priority = int(config['scanner']['maximum_priority'])

    logging.info("Adding %s events to the scanner", options.events)
    with db.Context():
      for ignore in xrange(options.events):
        service_name = random.choice(services)
        priority = random.randint(0, maximum_priority)
        user_id = _random_string(32)
        event_id = _random_string(64)

        scanner_event = EventScannerPriority(event_id, user_id, service_name, priority)
        db.Session().add(scanner_event)
        sys.stdout.write('.')

    sys.stdout.write('\n')
Exemplo n.º 8
0
  def app_main(self, config, options, args):
    logging.info("Beginning...")

    # read the db url from the config
    db_url = db.create_url_from_config(config['db'])

    # initialize the db engine & session
    db.configure_session(db_url)

    # if services option is empty default to all available configured services
    if not options.services:
      service_names = []
      # generate the list of service name's from queue list
      for key in config['queues'].iterkeys():
        service_names.append(key)
    else:
      service_names = options.services

    # get the broker and queue config
    broker_url = message_queue.create_url_from_config(config['amqp']['broker'])

    # get message broker client and store in instance
    client = message_queue.create_message_client(broker_url)

    # for each specified service
    for service_name in service_names:

      # create the queue for this service
      message_queue.create_queues_from_config(client, config['amqp'])

      # post a notification for each author subscribed to this service
      for asm in db.Session().query(AuthorServiceMap). \
                              join(Service, AuthorServiceMap.service_id == Service.id). \
                              filter(Service.service_name == service_name).all():
        message_queue.send_messages(client,
                                    [create_notification_message(service_name,
                                                                 asm.service_author_id)])

    logging.info("Finished...")
  def app_main(self, config, options, args):

    logging.info("Beginning...")

    # read the db url from the config
    db_url = db.create_url_from_config(config['db'])

    # initialize the db engine & session
    db.configure_session(db_url)

    # if services option is empty default to all available configured services
    if not options.services:
      service_names = []
      # generate the list of service name's from queue list
      for key in config['queues'].iterkeys():
        service_names.append(key)
    else:
      service_names = options.services

    # for each specified service
    for service_name in service_names:

      fetcher = profile_fetcher.from_service_name(service_name, config['oauth'][service_name])

      # get each author subscribed to this service
      for asm in db.Session().query(AuthorServiceMap). \
                              join(Service, AuthorServiceMap.service_id == Service.id). \
                              filter(Service.service_name == service_name).all():

        profile_json = fetcher.get_author_profile(asm.service_author_id, asm)

        if 'picture_url' in profile_json:
          asm.profile_image_url = profile_json['picture_url']
          db.Session().flush()

    logging.info("Finished...")
Exemplo n.º 10
0
def main(global_config, **settings):

  """ This function returns a Pyramid WSGI application.
  """

  """ Setup the config
  """
  global tim_config
  tim_config = load_configuration('{TIM_CONFIG}/config.ini')

  """ Setup the database
  """
  db_url = db.create_url_from_config(tim_config['db'])
  db.configure_session(db_url)

  session_factory = pyramid_beaker.session_factory_from_settings(settings)

  authn_policy = AuthTktAuthenticationPolicy(tim_config['api']['authentication_secret'])
  authz_policy = ACLAuthorizationPolicy()

  config = Configurator(settings=settings,
                        authentication_policy=authn_policy,
                        authorization_policy=authz_policy,
                        session_factory=session_factory)

  config.add_static_view('img', 'timmobilev2:img', cache_max_age=0)
  config.add_static_view('css', 'timmobilev2:css', cache_max_age=0)
  config.add_static_view('js', 'timmobilev2:js', cache_max_age=0)

  config.add_route('index', '/')
  config.add_route('settings', '/settings')
  config.add_route('login', '/login')
  # config.add_route('app', '/{authorname}/')
  config.add_route('app', '/{authorname}')

  # twitter oauth
  config.add_route('twitter', '/oauth/twitter')
  config.add_route('twitter_callback', '/oauth/twitter/callback')

  # facebook auth
  config.add_route('facebook', '/oauth/facebook')
  config.add_route('facebook_callback', '/oauth/facebook/callback')

  # linkedin auth
  config.add_route('linkedin', '/oauth/linkedin')
  config.add_route('linkedin_callback', '/oauth/linkedin/callback')

  # google+ auth
  config.add_route('googleplus', '/oauth/googleplus')
  config.add_route('googleplus_callback', '/oauth/googleplus/callback')

  # instagram auth
  config.add_route('instagram', '/oauth/instagram')
  config.add_route('instagram_callback', '/oauth/instagram/callback')

  # flickr auth
  config.add_route('flickr', '/oauth/flickr')
  config.add_route('flickr_callback', '/oauth/flickr/callback')

  # foursquare auth
  config.add_route('foursquare', '/oauth/foursquare')
  config.add_route('foursquare_callback', '/oauth/foursquare/callback')

  # generic oauth
  config.add_route('oauth', '/oauth/{servicename}')
  config.add_route('oauth_callback', '/oauth/{servicename}/callback')

  config.add_route('resource.any', '/{authorname}/asset/{resource}.{ext}')

  config.scan()

  return config.make_wsgi_app()