Example #1
0
  def fromJSONFields(self, fromJSON, auxData=None):

    super(LinkedInConnectEvent, self).fromJSONFields(fromJSON, auxData)

    # query for the connection's profile image and bio
    connection = self.raw_json['updateContent']['person']['connections']['values'][0]

    if connection['id'] != 'private':

      url = 'http://api.linkedin.com/v1/people/id=%s:(headline,summary,picture-url)' % connection['id']

      try:
        # request the user's updates
        contentJSON = make_request(auxData, url, {'x-li-format': 'json'})
        contentObj = json_serializer.load_string(contentJSON)

        self.headline = contentObj.get('headline')
        self.summary = contentObj.get('summary')
        self.photo = contentObj.get('pictureUrl')

      except urllib2.URLError:
        self.log.error('***ERROR*** parse error')
        self.log.error(contentJSON)
        pass

    return self
  def get_author_profile(self,afm,db_session,oauth_config):
    
    # Create our OAuth consumer instance
    consumer = oauth.Consumer(oauth_config['key'], oauth_config['secret'])
    token = oauth.Token(key=afm.access_token,secret=afm.access_token_secret)
    client = oauth.Client(consumer, token)

    # request the user's profile
    content = make_request(client,'https://api.twitter.com/1/account/verify_credentials.json')
    str_content = content.decode('utf-8')

    respJSON = json.loads(str_content)

#    print json.dumps(respJSON, sort_keys=True, indent=2)
    
    profileJSON = {}

    if respJSON.has_key('name'):
      profileJSON['name'] = respJSON['name']

    if respJSON.has_key('location'):
      profileJSON['location'] = respJSON['location']

    if respJSON.has_key('profile_image_url'):
      profileJSON['picture_url'] = respJSON['profile_image_url']

    if respJSON.has_key('description'): 
      profileJSON['headline'] = respJSON['description']
      
    profileJSON['public_profile_url'] = 'https://twitter.com/#!/%s' % respJSON['screen_name']

    return profileJSON
  def fetch(self, service_id, service_author_id, service_event_id, callback):

    asm = self.get_author_service_map(service_author_id)

    # TODO - temporary until we figure out a better solution for
    # not over-driving Twitter with un-authenticated events
    if not asm.access_token:
      return

    if asm.access_token:
      consumer = oauth.Consumer(self.oauth_config['key'], self.oauth_config['secret'])
      token = oauth.Token(asm.access_token, asm.access_token_secret)
      client = oauth.Client(consumer, token)

    args = {'id': service_event_id,
            'include_entities': '1',
            'trim_user': '******'}

    # if not authenticated provide the user_id query arg
    if not asm.access_token:
      args['user_id'] = asm.service_author_id

    url = TWEET_STATUS % (self.oauth_config['endpoint'],
                          urllib.urlencode(args))

    # TODO - remove the try/except once figure out a better solution for not
    # exceeding Twitter's rate limits
    try:
      json_obj = json_serializer.load_string(make_request(client, url)) if asm.access_token \
                 else json_serializer.load(urllib2.urlopen(url))
    except urllib2.URLError, e:
      logging.error('ERROR REQUEST URL: {0}'.format(url))
      logging.error('ERROR REASON: {0}, {1}'.format(e.code, e.read()))
      raise
Example #4
0
def linkedin_callback(request):

  error_msg = None

  author_id = unauthenticated_userid(request)

  # the oauth_token is request as a query arg; the auth_token_secret is in session store
  oauth_token = request.params['oauth_token']
  oauth_token_secret = request.session['oauth_token_secret']

  oauth_verifier = request.params['oauth_verifier']

  # Step 3: Once the consumer has redirected the user back to the oauth_callback
  # URL you can request the access token the user has approved. You use the
  # request token to sign this request. After this is done you throw away the
  # request token and use the access token returned. You should store this
  # access token somewhere safe, like a database, for future use.
  consumer = oauth.Consumer(tim_config['oauth'][SERVICE]['key'], tim_config['oauth'][SERVICE]['secret'])

  token = oauth.Token(oauth_token, oauth_token_secret)
  token.set_verifier(oauth_verifier)

  client = oauth.Client(consumer, token)

  resp, content = client.request(tim_config['oauth'][SERVICE]['access_token_url'], "POST")
  if resp['status'] != '200':
      raise Exception("Invalid response %s (%s)." % (resp['status'], content))

  access_dict = dict(urlparse.parse_qsl(content))

  # these are the real deal and need to be stored securely in the DB
  access_token = access_dict['oauth_token']
  access_token_secret = access_dict['oauth_token_secret']

  # Create our OAuth consumer instance
  token = oauth.Token(key=access_token, secret=access_token_secret)
  client = oauth.Client(consumer, token)

  url = '{endpoint}{resource}'.format(endpoint=tim_config['oauth'][SERVICE]['endpoint'],
                                      resource='people/~:(id)')

  response = make_request(client, url, {'x-li-format': 'json'})
  json_dict = json.loads(response)

  linkedin_id = json_dict['id']

  url = '{endpoint}/v1/authors/{author}/services'.format(endpoint=tim_config['api']['endpoint'],
                                                         author=author_id)
  payload = {'name': SERVICE, 'access_token': access_token, 'access_token_secret': access_token_secret, 'service_author_id': linkedin_id}
  headers = {'content-type': 'application/json; charset=utf-8'}
  cookies = request.cookies

  try:
    r = requests.post(url, data=json.dumps(payload), headers=headers, cookies=cookies)
    r.raise_for_status()
  except requests.exceptions.RequestException, e:
    log.error(e.message)
    if e.response.status_code == 409:
      error_msg = 'Service already exists for this author ({message})'.format(message=e.message)
  def fetch(self, service_id, service_author_id, service_event_id, callback):

    asm = self.get_author_service_map(service_author_id)

    consumer = oauth.Consumer(self.oauth_config['key'], self.oauth_config['secret'])
    token = oauth.Token(asm.access_token, asm.access_token_secret)
    client = oauth.Client(consumer, token)

    # check if this event isCommentable or isLikable
    event_json, = db.Session().query(ServiceEvent.json). \
                               filter(and_(ServiceEvent.author_service_map_id == asm.id,
                                           ServiceEvent.event_id == service_event_id)).one()

    event_obj = json_serializer.load_string(event_json)

    update_obj = None
    if event_obj.get("isCommentable", False):
      url = UPDATE_COMMENTS % (self.oauth_config['endpoint'],
                               service_event_id)

      update_obj = json_serializer.load_string(make_request(client, url, {'x-li-format': 'json'}))

    likes_obj = None
    if event_obj.get("isLikable", False):

      url = UPDATE_LIKES % (self.oauth_config['endpoint'],
                            service_event_id)

      likes_obj = json_serializer.load_string(make_request(client, url, {'x-li-format': 'json'}))

    # merge update and likes together into one object
    if update_obj or likes_obj:

      if update_obj:
        event_obj['updateComments'] = update_obj

      if likes_obj:
        event_obj['isLiked'] = likes_obj['_total'] > 0
        event_obj['numLikes'] = likes_obj['_total']
        event_obj['likes'] = likes_obj

      interpreter = LinkedinEventInterpreter(event_obj, asm, self.oauth_config)

      callback(create_linkedin_event(asm.author_id, CURRENT_STATE, service_author_id, interpreter.get_id(), event_obj))
Example #6
0
def linkedin_callback(request):
  
  # the oauth_token is request as a query arg; the auth_token_secret is in session store
  oauth_token = request.params['oauth_token']
  oauth_token_secret = request.session['oauth_token_secret']

  oauth_verifier = request.params['oauth_verifier']
  
  # Step 3: Once the consumer has redirected the user back to the oauth_callback
  # URL you can request the access token the user has approved. You use the
  # request token to sign this request. After this is done you throw away the
  # request token and use the access token returned. You should store this
  # access token somewhere safe, like a database, for future use.
  consumer = oauth.Consumer(oauth_config[FEATURE]['key'], oauth_config[FEATURE]['secret'])
  
  token = oauth.Token(oauth_token,oauth_token_secret)
  token.set_verifier(oauth_verifier)

  client = oauth.Client(consumer, token)
  
  resp, content = client.request(oauth_config[FEATURE]['access_token_url'], "POST")
  access_token = dict(urlparse.parse_qsl(content))

  # these are the real deal and need to be stored securely in the DB
  accessToken = access_token['oauth_token']
  accessTokenSecret = access_token['oauth_token_secret']

  # Create our OAuth consumer instance
  token = oauth.Token(key=accessToken,secret=accessTokenSecret)
  client = oauth.Client(consumer, token)

  response = make_request(client,'http://api.linkedin.com/v1/people/~:(id)',{'x-li-format':'json'})
  respJSON = json.loads(response)
  linkedinId = respJSON['id']

  json_payload = json.dumps({'access_token': accessToken, 'access_token_secret': accessTokenSecret, 'service_author_id': linkedinId})
  headers = {'Content-Type':'application/json; charset=utf-8'}
  url = '{0}/v1/authors/{1}/services/{2}'.format(request.registry.settings['mi.api.endpoint'], authenticated_userid(request), FEATURE)
  log.info(url)
  req = RequestWithMethod(url,
                          'PUT',
                          json_payload,
                          headers)
  res = urllib2.urlopen(req)
  resJSON = json.loads(res.read())

  try:
    request.session['linkedin_access_token'] = accessToken
    request.session['linkedin_access_token_secret'] = accessTokenSecret
  except Exception, e:
    print e
Example #7
0
def twitter_callback(request):

  # the oauth_token is request as a query arg; the auth_token_secret is in session store
  oauth_token = request.params['oauth_token']
  oauth_token_secret = request.session['oauth_token_secret']

  oauth_verifier = request.params['oauth_verifier']

  # Step 3: Once the consumer has redirected the user back to the oauth_callback
  # URL you can request the access token the user has approved. You use the
  # request token to sign this request. After this is done you throw away the
  # request token and use the access token returned. You should store this
  # access token somewhere safe, like a database, for future use.
  consumer_key = oauth_config[FEATURE]['key']
  consumer_secret = oauth_config[FEATURE]['secret']
  consumer = oauth.Consumer(consumer_key, consumer_secret)
  token = oauth.Token(oauth_token, oauth_token_secret)
  client = oauth.Client(consumer, token)

  token.set_verifier(oauth_verifier)

  resp, content = client.request(oauth_config[FEATURE]['access_token_url'], "POST")
  access_token = dict(urlparse.parse_qsl(content))

  # these are the real deal and need to be stored securely in the DB
  oauth_token = access_token['oauth_token']
  oauth_token_secret = access_token['oauth_token_secret']

  token = oauth.Token(oauth_token, oauth_token_secret)
  client = oauth.Client(consumer, token)

  userInfoJSON = json.loads(make_request(client, 'https://api.twitter.com/1/account/verify_credentials.json').decode('utf-8'))

  json_payload = json.dumps({'access_token': oauth_token, 'access_token_secret': oauth_token_secret, 'service_author_id': userInfoJSON['id']})
  headers = {'Content-Type': 'application/json; charset=utf-8'}
  req = RequestWithMethod('%s/v1/authors/%s/services/%s' %
                            (request.registry.settings['mi.api.endpoint'], authenticated_userid(request), FEATURE),
                          'PUT',
                          json_payload,
                          headers)
  try:
    res = urllib2.urlopen(req)
    resJSON = json.loads(res.read())
  except HTTPError, e:
    # TODO: handle errors more gracefully here (caused by, e.g., API S3 bucket not existing)
    print e.read()
    def init_connection(self):

        connection = self.json["updateContent"]["person"]["connections"]["values"][0]

        if connection["id"] != "private" and self._client is None:
            url = "%speople/id=%s:(headline,summary,picture-url)" % (self.oauth_config["endpoint"], connection["id"])

            # request the user's updates
            json_str = make_request(self.get_oauth_client(), url, {"x-li-format": "json"})
            json_obj = json_serializer.load_string(json_str)

            self.headline = json_obj.get("headline")
            self.summary = json_obj.get("summary")
            self.photo = json_obj.get("pictureUrl")

        else:
            self.headline = None
            self.summary = None
            self.photo = None
  def get_author_profile(self, service_author_id, asm):

    asm = self.fetch_begin(service_author_id, asm)

    args = {'user_id': asm.service_author_id,
            'include_entities': True}

    # Create our OAuth consumer instance
    if asm.access_token:
      consumer = oauth.Consumer(self.oauth_config['key'], self.oauth_config['secret'])
      token = oauth.Token(key=asm.access_token, secret=asm.access_token_secret)
      client = oauth.Client(consumer, token)

    url = '%s%s?%s' % (self.oauth_config['endpoint'],
                       USER_INFO,
                       urllib.urlencode(args))

    # request the user's profile
    json_obj = json_serializer.load_string(make_request(client, url)) if asm.access_token \
               else json_serializer.load(urllib2.urlopen(url))

    profile_json = {}

    if 'name' in json_obj:
      profile_json['name'] = json_obj['name']

    if 'location' in json_obj:
      profile_json['location'] = json_obj['location']

    if 'profile_image_url' in json_obj:
      profile_json['picture_url'] = json_obj['profile_image_url']

    if 'description' in json_obj:
      profile_json['headline'] = json_obj['description']

    profile_json['public_profile_url'] = 'https://twitter.com/#!/%s' % json_obj['screen_name']

    return profile_json
Example #10
0
def twitter_callback(request):

    error_msg = None

    author_id = unauthenticated_userid(request)

    # the oauth_token is request as a query arg; the auth_token_secret is in session store
    oauth_token = request.params["oauth_token"]
    oauth_token_secret = request.session["oauth_token_secret"]

    oauth_verifier = request.params["oauth_verifier"]

    # Step 3: Once the consumer has redirected the user back to the oauth_callback
    # URL you can request the access token the user has approved. You use the
    # request token to sign this request. After this is done you throw away the
    # request token and use the access token returned. You should store this
    # access token somewhere safe, like a database, for future use.
    consumer_key = tim_config["oauth"][SERVICE]["key"]
    consumer_secret = tim_config["oauth"][SERVICE]["secret"]

    consumer = oauth.Consumer(consumer_key, consumer_secret)
    token = oauth.Token(oauth_token, oauth_token_secret)

    client = oauth.Client(consumer, token)

    token.set_verifier(oauth_verifier)

    resp, content = client.request(tim_config["oauth"][SERVICE]["access_token_url"], "POST")
    if resp["status"] != "200":
        raise Exception("Invalid response {status} ({message}).".format(status=resp["status"], message=content))

    access_token = dict(urlparse.parse_qsl(content))

    # these tokens are the real deal and need to be passed to the API
    oauth_token = access_token["oauth_token"]
    oauth_token_secret = access_token["oauth_token_secret"]

    token = oauth.Token(oauth_token, oauth_token_secret)
    client = oauth.Client(consumer, token)

    # get some information about the user from twitter
    url = "{endpoint}account/verify_credentials.json".format(endpoint=tim_config["oauth"][SERVICE]["endpoint"])
    json_dict = json.loads(make_request(client, url).decode("utf-8"))

    url = "{endpoint}/v1/authors/{author}/services".format(endpoint=tim_config["api"]["endpoint"], author=author_id)
    payload = {
        "name": SERVICE,
        "access_token": oauth_token,
        "access_token_secret": oauth_token_secret,
        "service_author_id": json_dict["id"],
    }
    headers = {"content-type": "application/json; charset=utf-8"}
    cookies = request.cookies

    try:
        r = requests.post(url, data=json.dumps(payload), headers=headers, cookies=cookies)
        r.raise_for_status()
    except requests.exceptions.RequestException, e:
        log.error(e.message)
        if e.response.status_code == 409:
            error_msg = "Service already exists for this author ({message})".format(message=e.message)
  def get_author_profile(self, service_author_id, asm):

    asm = self.fetch_begin(service_author_id, asm)

    # setup what we need for oauth
    consumer = oauth.Consumer(self.oauth_config['key'], self.oauth_config['secret'])
    if asm.access_token:
      token = oauth.Token(key=asm.access_token, secret=asm.access_token_secret)
    else:
      token = oauth.Token(self.oauth_config['user1_access_token'], self.oauth_config['user1_access_token_secret'])
    client = oauth.Client(consumer, token)

    url_path = self.PROFILE_INFO if asm.access_token \
                       else self.PUBLIC_PROFILE_INFO.format(urllib.quote(asm.service_author_id, ''))

    url = '%s%s' % (self.oauth_config['endpoint'], url_path)

    # request the user's profile
    json_obj = json_serializer.load_string(make_request(client, url, {'x-li-format': 'json'}))

    profile_json = {}

    firstName = lastName = ''

    if 'firstName' in json_obj:
      firstName = profile_json['first_name'] = json_obj['firstName']

    if 'lastName' in json_obj:
      lastName = profile_json['last_name'] = json_obj['lastName']

    # if we have a non-empty string add it to the json
    name = ('%s %s' % (firstName, lastName)).strip()
    if len(name) > 0:
      profile_json['name'] = name

    if 'industry' in json_obj:
      profile_json['industry'] = json_obj['industry']

    if 'headline' in json_obj:
      profile_json['headline'] = json_obj['headline']

    if 'pictureUrl' in json_obj:
      profile_json['picture_url'] = json_obj['pictureUrl']

    if 'location' in json_obj and 'name' in json_obj['location']:
      profile_json['location'] = json_obj['location']['name']

    if 'summary' in json_obj:
      profile_json['summary'] = json_obj['summary']

    if 'specialties' in json_obj:
      profile_json['specialties'] = json_obj['specialties']

    if 'publicProfileUrl' in json_obj:
      profile_json['public_profile_url'] = json_obj['publicProfileUrl']

    if 'positions' in json_obj and 'values' in json_obj['positions']:
      positions = []
      for position in json_obj['positions']['values']:

        position_json = {}

        if 'company' in position:

          if 'name' in position['company']:
            position_json['company'] = position['company']['name']

          if 'industry' in position['company']:
            position_json['industry'] = position['company']['industry']

        if 'summary' in position:
          position_json['summary'] = position['summary']

        if 'title' in position:
          position_json['title'] = position['title']

        positions.append(position_json)

      profile_json['positions'] = positions

    return profile_json
Example #12
0
  def build_one(self,afm,dbSession, oauthConfig, incremental):

    super(TwitterFullCollector, self).build_one(afm,dbSession,oauthConfig,incremental)

    # get the name of the author  
    authorName = dbSession.query(Author.author_name).filter_by(id=afm.author_id).one()

    consumer = oauth.Consumer(oauthConfig['key'], oauthConfig['secret'])
    token = oauth.Token(afm.access_token, afm.access_token_secret)
    
    client = oauth.Client(consumer, token)

    auxData = json.loads(afm.auxillary_data)
    userId = int(auxData['id'])

    try:
  
      # API endpoing for querying user info
      url = '%s%s' % (oauthConfig['endpoint'],USER_INFO)
      userInfoJSON = json.loads(make_request(client,url))
  
      twitterUserId = userInfoJSON['id']
      
      if twitterUserId != userId:
        raise Exception("Bad state - mis-matched twitter user ids")

      profileImageUrl = userInfoJSON['profile_image_url'] if userInfoJSON.has_key('profile_image_url') else None 

      traversal = self.beginTraversal(dbSession,afm,profileImageUrl)
      page = 1

      # API endpoint for getting user timeline
      url = '%s%s?%s' % (oauthConfig['endpoint'],USER_TIMELINE,urllib.urlencode({'include_rts':'1','include_entities':'1','count':'200','page':page}))

      while url and traversal.totalAccepted < 200:

        content = make_request(client,url)
        
        try:
          rawJSON = json.loads(content)
        except:
          self.log.error('***ERROR*** parse error')
          self.log.error(content)
          continue

        if len(rawJSON) == 0:
          url = None
          continue

        for post in rawJSON:
  
          # process the item
          #print json.dumps(post, sort_keys=True, indent=2)
  
          event = Event.TwitterEvent(afm.author_id).fromJSON(post)
  
          self.writeEvent(event,traversal)
      
        # setup for the next page (if any)
        page = page + 1
        url = '%s%s?%s' % (oauthConfig['endpoint'],USER_TIMELINE,urllib.urlencode({'include_rts':'1','include_entities':'1','count':'200','page':page}))

      self.endTraversal(traversal,authorName)
      
    except Exception, e:
      self.log.error('****ERROR****')
      self.log.error(e)
      dbSession.rollback()
      raise #continue
  def fetch(self, service_author_id, callback):

    super(TwitterEventCollector, self).fetch(service_author_id, callback)

    state = self.fetch_begin(service_author_id)

    self.fetch_log_info(state)

    asm = state['asm']

    args = {'include_rts': 1,
            'include_entities': 1,
            'trim_user': 1,
            'count': 200}

    # use authenticated access if we can
    if asm.access_token:
      consumer = oauth.Consumer(self.oauth_config['key'], self.oauth_config['secret'])
      token = oauth.Token(asm.access_token, asm.access_token_secret)
      client = oauth.Client(consumer, token)
    else:
      args['user_id'] = asm.service_author_id

    if asm.most_recent_event_id:
      args['since_id'] = asm.most_recent_event_id

    # API endpoint for getting user timeline
    url = '%s%s?%s' % (self.oauth_config['endpoint'],
                       USER_TIMELINE,
                       urllib.urlencode(args))

    min_age = datetime.utcnow() - self.NEW_LOOKBACK_WINDOW
    last_id = None
    while True:

      try:
        raw_json = json_serializer.load_string(make_request(client, url)) if asm.access_token \
                   else json_serializer.load(urllib2.urlopen(url))
      except urllib2.URLError, e:
        logging.error('ERROR REQUEST URL: {0}'.format(url))
        logging.error('ERROR REASON: {0}, {1}'.format(e.code, e.read()))
        raise

      # check if nothing returned and terminate loop if so
      if len(raw_json) == 0:
        break

      for post in raw_json:

        # process the item
        #print json.dumps(post, sort_keys=True, indent=2)

        interpreter = TwitterEventInterpreter(post, asm, self.oauth_config)
        last_id = interpreter.get_id()

        # terminate fetching any more events if we've gone beyond the lookback window
        if interpreter.get_create_time() < min_age:
          url = None
          break

        if self.screen_event(interpreter, state):
          callback(create_twitter_event(asm.author_id, CURRENT_STATE, service_author_id, interpreter.get_id(), post))

      if not url:
        break

      # setup for the next page (if any)
      args['max_id'] = long(last_id) - 1
      url = '%s%s?%s' % (self.oauth_config['endpoint'],
                         USER_TIMELINE,
                         urllib.urlencode(args))
Example #14
0
  def build_one(self, afm, dbSession, oauthConfig, incremental):

    super(LinkedInFullCollector, self).build_one(afm, dbSession, oauthConfig, incremental)

    # get the name of the author
    authorName = dbSession.query(Author.author_name).filter_by(id=afm.author_id).one()

    auxData = json.loads(afm.auxillary_data)
    userId = auxData['id']

    # setup what we need for oauth
    consumer = oauth.Consumer(oauthConfig['key'], oauthConfig['secret'])
    token = oauth.Token(key=afm.access_token, secret=afm.access_token_secret)
    client = oauth.Client(consumer, token)

    try:

      # request the user's profile
      response = make_request(client, 'http://api.linkedin.com/v1/people/~:(picture-url)', {'x-li-format': 'json'})
      respJSON = json.loads(response)

      profileImageURL = respJSON['pictureUrl'] if 'pictureUrl' in respJSON else None

      traversal = self.beginTraversal(dbSession, afm, profileImageURL)

      # optimization to request only those since we've last updated
      args = {'scope': 'self',
              'type': ['APPS', 'CMPY', 'CONN', 'JOBS', 'JGRP', 'PICT', 'PRFX', 'RECU', 'PRFU', 'QSTN', 'SHAR', 'VIRL'],
              'count': PAGE_SIZE}

      # incremental
      if traversal.baselineLastUpdateTime:
        # since a little before the last update time
        args['after'] = '%s000' % int(mktime(traversal.baselineLastUpdateTime.timetuple()))
      # full
      else:
        # limit to only one year of data
        args['after'] = '%s000' % int(mktime((traversal.now - FULL_LOOKBACK_WINDOW).timetuple()))

      offset = 0
#      args['start'] = offset

      url = '%s?%s' % ('http://api.linkedin.com/v1/people/~/network/updates', urllib.urlencode(args, True))

      while url and traversal.totalAccepted < 200:

        # request the user's updates
        content = make_request(client, url, {'x-li-format': 'json'})

        try:
          rawJSON = json.loads(content)
        except:
          self.log.error('***ERROR*** parse error')
          self.log.error(content)
          continue

#        print json.dumps(rawJSON, sort_keys=True, indent=2)

        if rawJSON.get('_total', 0) == 0:
          url = None
          continue

        LinkedInEvent.eventsFromJSON(self, rawJSON, traversal, afm.author_id, userId, client)

        # setup for the next page (if any)
        if rawJSON['_total'] < PAGE_SIZE:
          url = None
        else:
          offset = offset + PAGE_SIZE
#          args['start'] = offset
          url = '%s?%s' % ('http://api.linkedin.com/v1/people/~/network/updates', urllib.urlencode(args, True))

      self.endTraversal(traversal, authorName)

    except Exception, e:
      self.log.error('****ERROR****')
      self.log.error(e)
      dbSession.rollback()
      raise  # continue
  def get_author_profile(self,afm,db_session,oauth_config):
    
    # Create our OAuth consumer instance
    consumer = oauth.Consumer(oauth_config['key'], oauth_config['secret'])
    token = oauth.Token(key=afm.access_token,secret=afm.access_token_secret)
    client = oauth.Client(consumer, token)

    # request the user's profile
    response = make_request(client,'http://api.linkedin.com/v1/people/~:(first-name,last-name,headline,public-profile-url,picture-url,location:(name),industry,summary,specialties,associations,honors,interests,positions:(title,summary,company))',{'x-li-format':'json'})
    respJSON = json.loads(response)

#    print json.dumps(respJSON, sort_keys=True, indent=2)
    
    profileJSON = {}
    
    firstName = lastName = ''

    if respJSON.has_key('firstName'): 
      firstName = profileJSON['first_name'] = respJSON['firstName']
      
    if respJSON.has_key('lastName'): 
      lastName = profileJSON['last_name'] = respJSON['lastName']

    # if we have a non-empty string add it to the json
    name = ('%s %s' % (firstName,lastName)).strip()
    if len(name) > 0:
      profileJSON['name'] = name

    if respJSON.has_key('industry'): 
      profileJSON['industry'] = respJSON['industry']

    if respJSON.has_key('headline'): 
      profileJSON['headline'] = respJSON['headline']
      
    if respJSON.has_key('pictureUrl'):
      profileJSON['picture_url'] = respJSON['pictureUrl']

    if respJSON.has_key('location') and respJSON['location'].has_key('name'):
      profileJSON['location'] = respJSON['location']['name']

    if respJSON.has_key('summary'): 
      profileJSON['summary'] = respJSON['summary']

    if respJSON.has_key('specialties'): 
      profileJSON['specialties'] = respJSON['specialties']

    if respJSON.has_key('publicProfileUrl'): 
      profileJSON['public_profile_url'] = respJSON['publicProfileUrl']

    if respJSON.has_key('positions') and respJSON['positions'].has_key('values'):
      positions = []
      for position in respJSON['positions']['values']:
        
        positionJSON = {}
        
        if position.has_key('company'):
          
          if position['company'].has_key('name'):
            positionJSON['company'] = position['company']['name']

          if position['company'].has_key('industry'):
            positionJSON['industry'] = position['company']['industry']

        if position.has_key('summary'):
          positionJSON['summary'] = position['summary']

        if position.has_key('title'):
          positionJSON['title'] = position['title']
          
        positions.append(positionJSON)
  
      profileJSON['positions'] = positions
    
    return profileJSON
  def fetch(self, service_author_id, callback):

    super(LinkedinEventCollector, self).fetch(service_author_id, callback)

    state = self.fetch_begin(service_author_id)

    self.fetch_log_info(state)

    asm = state['asm']

    # if this author has no access_token they are unauthorized and we
    # don't collect LinkedIn events for them
    if not asm.access_token:
      return

    service_author_id = asm.service_author_id

    min_age = datetime.utcnow() - self.NEW_LOOKBACK_WINDOW

    # setup what we need for oauth
    consumer = oauth.Consumer(self.oauth_config['key'], self.oauth_config['secret'])
    token = oauth.Token(key=asm.access_token, secret=asm.access_token_secret)
    client = oauth.Client(consumer, token)

    args = {'scope': 'self',
            'count': self.PAGE_SIZE}

    # get only events since last update or past year depending on if this
    # is the first collection of not
    if asm.most_recent_event_timestamp:
      after = calendar.timegm((asm.most_recent_event_timestamp -
                               self.MOST_RECENT_OVERLAP).utctimetuple()) * 1000
    else:
      after = calendar.timegm((datetime.utcnow() -
                               self.NEW_LOOKBACK_WINDOW).utctimetuple()) * 1000
    args['after'] = after

    offset = 0
    args['start'] = offset

    url = '%s%s?%s' % (self.oauth_config['endpoint'],
                       UPDATE_RESOURCE,
                       urllib.urlencode(args, True))

    total_count = 0
    while url:

      # request the user's updates
      raw_json = json_serializer.load_string(make_request(client, url, {'x-li-format': 'json'}))

      if raw_json == None or raw_json.get('_total', 0) == 0:
        url = None
        break

      for post in raw_json.get('values', []):

        update_type = post['updateType']

        if update_type in self.SUPPORTED_TYPES:

          if update_type == 'CONN' and post['updateContent']['person']['id'] == service_author_id:

            # the response can contain multiple connections that the member has made.  We'll
            # separate them into individual responses
            postClone = copy.deepcopy(post)

            for connection in post['updateContent']['person']['connections']['values']:

              postClone['updateContent']['person']['connections'] = {"_total": 1, "values": [copy.deepcopy(connection)]}

              interpreter = LinkedinEventInterpreter(postClone, asm, self.oauth_config)

              if interpreter.get_create_time() < min_age:
                url = None
                break

              if self.screen_event(interpreter, state):
                callback(create_linkedin_event(asm.author_id, CURRENT_STATE, service_author_id, interpreter.get_id(), postClone))

          elif (update_type == 'PREC' or update_type == 'SVPR') and post['updateContent']['person']['id'] == service_author_id:

            interpreter = LinkedinEventInterpreter(post, asm, self.oauth_config)

            if interpreter.get_create_time() < min_age:
              url = None
              break

            if self.screen_event(interpreter, state):
              callback(create_linkedin_event(asm.author_id, CURRENT_STATE, service_author_id, interpreter.get_id(), post))

          elif update_type == 'SHAR':

            interpreter = LinkedinEventInterpreter(post, asm, self.oauth_config)

            if interpreter.get_create_time() < min_age:
              url = None
              break

            if self.screen_event(interpreter, state):
              callback(create_linkedin_event(asm.author_id, CURRENT_STATE, service_author_id, interpreter.get_id(), post))

          elif update_type == 'MSFC' and post['updateContent']['companyPersonUpdate']['person']['id'] == service_author_id:

            interpreter = LinkedinEventInterpreter(post, asm, self.oauth_config)

            if interpreter.get_create_time() < min_age:
              url = None
              break

            if self.screen_event(interpreter, state):
              callback(create_linkedin_event(asm.author_id, CURRENT_STATE, service_author_id, interpreter.get_id(), post))

          elif update_type == 'JOBP' and post['updateContent']['job']['jobPoster']['id'] == service_author_id:

            interpreter = LinkedinEventInterpreter(post, asm, self.oauth_config)

            if interpreter.get_create_time() < min_age:
              url = None
              break

            if self.screen_event(interpreter, state):
              callback(create_linkedin_event(asm.author_id, CURRENT_STATE, service_author_id, interpreter.get_id(), post))

          elif update_type == 'JGRP' and post['updateContent']['person']['id'] == service_author_id:

            # the response can contain multiple groups that the member has joined.  We'll
            # separate them into individual responses
            postClone = copy.deepcopy(post)

            for group in post['updateContent']['person']['memberGroups']['values']:

              postClone['updateContent']['person']['memberGroups'] = {"_total": 1, "values": [copy.deepcopy(group)]}

              interpreter = LinkedinEventInterpreter(postClone, asm, self.oauth_config)

              if interpreter.get_create_time() < min_age:
                url = None
                break

              if self.screen_event(interpreter, state):
                callback(create_linkedin_event(asm.author_id, CURRENT_STATE, service_author_id, interpreter.get_id(), postClone))

          elif update_type == 'STAT' and post['updateContent']['person']['id'] == service_author_id:

            interpreter = LinkedinEventInterpreter(post, asm, self.oauth_config)

            if interpreter.get_create_time() < min_age:
              url = None
              break

            if self.screen_event(interpreter, state):
              callback(create_linkedin_event(asm.author_id, CURRENT_STATE, service_author_id, interpreter.get_id(), post))

        else:
          if not update_type in self.IGNORED_TYPES:
            logging.warning('???? skipping linkedIn event: %s' % update_type)

        # if the url is None stop
        if not url:
          break

      # if the url is None stop
      if not url:
        break

      total_count = total_count + raw_json['_count'] if '_count' in raw_json else raw_json['_total']
      if raw_json['_total'] == total_count:
        url = None
        break

      offset = offset + self.PAGE_SIZE
      args['start'] = offset
      url = '%s%s?%s' % (self.oauth_config['endpoint'], UPDATE_RESOURCE, urllib.urlencode(args, True))

    print total_count

    # terminate the fetch
    self.fetch_end(state)