def _tags_for_track(artist_name, track_name): """ Get the top tags for a track. Also fetches tags for the artist. Only includes tracks that break a certain threshold of usage, defined by settings.LASTFM_TAG_USAGE_THRESHOLD (which defaults to 15). """ urls = [ ARTIST_TAGS_URL % (urllib.quote(artist_name)), TRACK_TAGS_URL % (urllib.quote(artist_name), urllib.quote(track_name)), ] tags = set() for url in urls: log.debug("Fetching tags from %r", url) try: xml = utils.getxml(url) except HttpLib2Error, e: if e.code == 408: return "" else: raise for t in xml.getiterator("tag"): count = utils.safeint(t.find("count").text) if count >= getattr(settings, 'LASTFM_TAG_USAGE_THRESHOLD', 15): tags.add(slugify(smart_unicode(t.find("name").text)))
def __call__(self, **params): # Enforce Yahoo's "no calls quicker than every 1 second" rule delta = time.time() - DeliciousClient.lastcall if delta < 2: time.sleep(2 - delta) DeliciousClient.lastcall = time.time() url = ("https://api.del.icio.us/%s?" % self.method) + urllib.urlencode(params) return utils.getxml(url, username=self.username, password=self.password)
def _tags_for_url(url): tags = set() try: xml = utils.getxml(url) except HttpLib2Error, e: if e.code == 408: return "" else: raise
def update(): last_update_date = Item.objects.get_last_update_of_model(Track) log.debug("Last update date: %s", last_update_date) xml = utils.getxml(RECENT_TRACKS_URL % settings.LASTFM_USERNAME) for track in xml.getiterator("track"): artist = track.find('artist') artist_name = smart_unicode(artist.text) artist_mbid = artist.get('mbid') track_name = smart_unicode(track.find('name').text) track_mbid = smart_unicode(track.find('mbid').text) url = smart_unicode(track.find('url').text) timestamp = datetime.datetime.fromtimestamp(int(track.find('date').get('uts'))) if not _track_exists(artist_name, track_name, timestamp): tags = _tags_for_track(artist_name, track_name) _handle_track(artist_name, artist_mbid, track_name, track_mbid, url, timestamp, tags)
def update(): last_update_date = Item.objects.get_last_update_of_model(Message) log.debug("Last update date: %s", last_update_date) xml = utils.getxml(RECENT_STATUSES_URL % settings.TWITTER_USERNAME) for status in xml.getiterator("item"): message = status.find('title') message_text = smart_unicode(message.text) url = smart_unicode(status.find('link').text) # pubDate delivered as UTC timestamp = dateutil.parser.parse(status.find('pubDate').text) if utils.JELLYROLL_ADJUST_DATETIME: timestamp = utils.utc_to_local_datetime(timestamp) if not _status_exists(message_text, url, timestamp): _handle_status(message_text, url, timestamp)
def __call__(self, **params): url = ("https://api.del.icio.us/%s?" % self.method) + urllib.urlencode(params) ctr = 0 # HACK: try three times before giving up :/ while True: try: # Enforce Yahoo's "no calls quicker than every 1 second" rule delta = time.time() - DeliciousClient.lastcall if delta < 2: time.sleep(2 - delta) return utils.getxml(url, username=self.username, password=self.password) except Exception, e: if ctr+1 < 3: log.debug("Fetching %s failed. Retrying" % url) ctr += 1 else: raise e finally:
def update(): last_update_date = Item.objects.get_last_update_of_model(Track) log.debug("Last update date: %s", last_update_date) xml = utils.getxml(RECENT_TRACKS_URL % settings.LASTFM_USERNAME) for track in xml.getiterator("track"): artist = track.find('artist') artist_name = smart_unicode(artist.text) artist_mbid = artist.get('mbid') track_name = smart_unicode(track.find('name').text) track_mbid = smart_unicode(track.find('mbid').text) url = smart_unicode(track.find('url').text) # date delivered as UTC timestamp = datetime.datetime.fromtimestamp( int(track.find('date').get('uts'))) if utils.JELLYROLL_ADJUST_DATETIME: timestamp = utils.utc_to_local_timestamp( int(track.find('date').get('uts'))) if not _track_exists(artist_name, track_name, timestamp): tags = _tags_for_track(artist_name, track_name) _handle_track(artist_name, artist_mbid, track_name, track_mbid, url, timestamp, tags)
def __call__(self, **params): params['api_key'] = self.api_key url = ("http://ma.gnolia.com/api/rest/1/%s/?" % (self.method)) + urllib.urlencode(params) return utils.getxml(url)