def save_feeds(request, username): new_feeds_to_save = [] params = request.POST.copy() log.info("Saving for stream with an id of") log.info(params['streams[]']) stream = get_object_or_404(lifestream.models.Stream, user=request.user, id=params['streams[]']) feed_url = request.POST['url'] possible_feed = is_possible_feed(feed_url) if possible_feed: new_feeds_to_save.append(possible_feed) else: pq = PyQuery(feed_url) pq('link[rel=alternate]').each( lambda link_el: append_if_dict(new_feeds_to_save, make_possible_feed(link_el))) new_feeds = [] forms = [] something_saved = False for new_feed_to_save in new_feeds_to_save: # This might have been 1 URL posted that turned into multiple embeded Feed links params['url'] = new_feed_to_save['feed_url'] feed_url_hash = hashcompat.md5_constructor( django.utils.encoding.smart_str(new_feed_to_save['feed_url'])).hexdigest() a_feed = lifestream.models.Feed(url_hash = feed_url_hash, title=new_feed_to_save['feed_title'], url = new_feed_to_save['feed_url'], etag='', last_modified=datetime.datetime(1975, 1, 10), enabled=True, disabled_reason='', fetch_error_count=0, user=request.user, created_date=datetime.datetime.today()) a_feed.streams.add(stream) form = lifestream.models.FeedForm(params, instance=a_feed) forms.append(form) if form.is_valid(): form.save() db_feed = lifestream.models.Feed.objects.get(pk=feed_url_hash) new_feeds.append(db_feed.to_primitives()) something_saved = True else: log.info("Error, couldn't save %s" % feed_url_hash) pass # Keep trying other feeds if something_saved: stream_config = StreamConfig(stream.config) stream_config.ensureFeedsConfig(stream.feed_set.all()) stream.config = stream_config.__unicode__() stream.save() return (True, new_feeds, forms) else: return (False, [], forms)
def manage_stream(request, username, streamname): if request.user.username == username: stream = get_object_or_404(lifestream.models.Stream, user=request.user, name=streamname) stream_config = StreamConfig(stream.config) feed_rows = stream.feed_set.all() stream_config.ensureFeedsConfig(feed_rows) feed_id_to_feed = {} for row in feed_rows: feed_id_to_feed[row.pk] = row feeds = [] for feed in stream_config.config['feeds']: feed_row = feed_id_to_feed[feed['url_hash']] feeds.append({'url': feed_row.url, 'title': feed_row.title, 'pk': feed_row.pk, 'entries_visible_default': feed['entries_visible_default']}) raw_entries = (lifestream.models.Entry.objects.order_by('-last_published_date') .filter(feed__user=request.user, feed__streams__name__exact = streamname))[:150] plugins = [StreamEditorPlugin(log)] entry_pair = zip(raw_entries, render_entries(request, raw_entries, plugins)) feed_model = lifestream.models.FeedForm() stream.url = "/u/%s/s/%s" % (username, stream.name) preferences = patchouli_auth.preferences.getPreferences(request.user) template_data = { 'feeds': feeds, 'entry_pair': entry_pair, 'unused_feeds': [], 'form': feed_model, 'request': request, 'stream': stream, 'stream_config': stream_config, 'username': request.user.username, 'preferences': preferences} [template_data.update(plugin.template_variables(template_data)) for plugin in plugins] return render_to_response('stream_editor.html', template_data, context_instance=django.template.RequestContext(request)) else: return django.http.HttpResponse(HACKING_MESSAGE, status=400)
def update_feed(feed_meta): """ Update a single feed. feed_meta - lifestream.models.Feed object Returns number of newly fetched entries. """ new_entry_count = 0 try: feed = fetch_feed(feed_meta) if feed: log.debug('Processing %s %s' % (feed_meta.url_hash, feed_meta.url)) if 'feed' in feed and 'title' in feed.feed: log.debug('Processing title: %s', feed.feed.title) dirty_feed = False if 'feed' in feed and 'title' in feed.feed and feed_meta.title != feed.feed.title: feed_meta.title = feed.feed.title dirty_feed = True else: if (not 'feed' in feed) or (not 'title' in feed.feed): log.warn("Feed doesn't have a title property") if dirty_feed: try: feed_meta.save() except KeyboardInterrupt: raise except Exception, x: log.error("Unable to update feed") log.exception(x) # A feed lives in several streams stream_feed_configs = [] for stream in feed_meta.streams.all(): stream_config = StreamConfig(stream.config) for feed_config in stream_config.config['feeds']: if feed_config['url_hash'] == feed_meta.url_hash: # We'll reuse this config but add which stream it's for... feed_config['stream'] = stream stream_feed_configs.append(feed_config) for entry in feed.entries: if save_entry(feed_meta, entry, stream_feed_configs): new_entry_count += 1 except KeyboardInterrupt: raise except Exception, e: log.error("General Error starting loop: %s", e) log.exception(e)
def edit_feed_show(request, username, stream_id, feed_id): feed = get_object_or_404(lifestream.models.Feed, url_hash=feed_id) stream = get_object_or_404(lifestream.models.Stream, id=stream_id) feed_config = feed_config_from_stream(StreamConfig(stream.config), feed_id) return render_to_response('feed_editor.html', { 'feed': feed, 'feed_id': feed_id, 'stream': stream, 'stream_id': stream_id, 'feed_config': feed_config, 'lang_dir': 'LTR', 'page_lang': 'en', #'unused_feeds': [], 'request': request, #'streams': streams, 'feed_id': feed_id, 'username': request.user.username,}, context_instance=django.template.RequestContext(request))
def edit_feed_update(request, username, stream_id, feed_id): stream = get_object_or_404(lifestream.models.Stream, id=stream_id) stream_config = StreamConfig(stream.config) stream_config.ensureFeedsConfig(stream.feed_set.all()) feed_config = feed_config_from_stream(stream_config, feed_id) # TODO hmmm this code sucks because I'm writing directly to the confic Dict # instead of an Abstract Data Type... fixme params = request.POST.copy() # TODO use per-stream feed title in editor and lifestream views if 'title' in params: feed_config['feed_title'] = params['title'] if 'entries_visible_default' in params: if 'True' == params['entries_visible_default']: feed_config['entries_visible_default'] = True else: feed_config['entries_visible_default'] = False if 'enable_num_entries_shown' in params and 'True' == params['enable_num_entries_shown']: feed_config['enable_num_entries_shown'] = True try: num_entries = int(params['num_entries_shown']) if num_entries > 50: num_entries = 50 if num_entries < 0: #WTF? num_entries = 5 feed_config['num_entries_shown'] = num_entries except ValueError: log.info("Unable to parse int out of %s" % params['num_entries_shown']) feed_config['num_entries_shown'] = 5 else: feed_config['enable_num_entries_shown'] = False stream_config.update_feed_config(feed_config) stream.config = stream_config.__unicode__() stream.save() return django.http.HttpResponseRedirect("/manage/account/%s/stream/%s/feed/%s" % (username, stream_id, feed_id))
def _recent_entries(user, stream, feed_id, count=150, only_visible=True): # Grab entries for each feed basedon the stream constraints per feed stream_id = stream.id stream_config = StreamConfig(stream.config) feed_ids = [] if 'ANY' == feed_id: for feed in stream_config.config['feeds']: feed_ids.append(feed['url_hash']) else: feed_ids.append(feed_id) entries = [] for a_feed_id in feed_ids: feed_config = stream_config.feed_config(a_feed_id) query = ( Entry.objects.order_by('-last_published_date').filter( feed__user=user, feed__url_hash=a_feed_id, # TODO .... Why constrain by Stream? We want to reuse feeds, right? # the cron somehow repairs the links betwee a feed and all the streams it lives in feed__streams__id=stream_id)) # Visible is special, we want to see hidden items in the editor if only_visible: query = query.filter(streamentry__visible=True) # Lastly... LIMIT if 'enable_num_entries_shown' in feed_config and True == feed_config[ 'enable_num_entries_shown'] and 'num_entries_shown' in feed_config: entries += query[0:feed_config['num_entries_shown']] else: entries += query[:count] entries = sorted(entries, compare_entries) entries = entries[:count] log.info("Got back %d entries", len(entries)) # Performance tweak... We always also want to load all StreamEntry objects in one go. This isn't # hooked up as a straight ont-to-one so select_related can't be used on Entry stream_entries = [] for a_feed_id in feed_ids: feed_config = stream_config.feed_config(a_feed_id) query = (StreamEntry.objects.select_related().order_by( '-entry__last_published_date').filter( entry__feed__url_hash=a_feed_id, stream__id=stream_id)) if only_visible: query = query.filter(visible=True) # Lastly... LIMIT if 'enable_num_entries_shown' in feed_config and True == feed_config[ 'enable_num_entries_shown'] and 'num_entries_shown' in feed_config: log.info("using config num_entires_shown") log.info(feed_config['num_entries_shown']) stream_entries += query[0:feed_config['num_entries_shown']] else: log.info("using 150") stream_entries += query[:count] log.debug("stream entries") log.debug(stream_entries) log.debug(django.db.connection.queries) for se in stream_entries: key = "%s_%d" % (str(stream_id), se.entry_id) stream_entries_cache[key] = se for e in entries: key = "%s_%d" % (str(stream_id), e.id) # Easy Access from templates if key in stream_entries_cache: e.stream_entry = stream_entries_cache[key] else: e.stream_entry = {} return entries
def manage_page(request, username, page_name): if request.user.username == username: log.info("Grabbing webpage") webpage = get_object_or_404(lifestream.models.Webpage, user=request.user, name=page_name) webpage_properties = patchouli_auth.preferences.getPageProperties(webpage) streams = [] for stream_id in webpage_properties['stream_ids']: stream = get_object_or_404(lifestream.models.Stream, user=request.user, id=stream_id) streams.append(stream) stream_config = StreamConfig(stream.config) feed_rows = stream.feed_set.all() stream_config.ensureFeedsConfig(feed_rows) feed_id_to_feed = {} for row in feed_rows: feed_id_to_feed[row.pk] = row stream.feeds = [] for feed in stream_config.config['feeds']: feed_row = feed_id_to_feed[feed['url_hash']] stream.feeds.append({'url': feed_row.url, 'title': feed_row.title, 'pk': feed_row.pk, 'enabled': feed_row.enabled, 'disabled_reason': feed_row.disabled_reason, 'entries_visible_default': feed['entries_visible_default']}) plugins = [StreamEditorPlugin(log)] entries = lifestream.models.recent_entries(request.user, stream, 150, False) stream.entry_pair = entry_pair_for_entries(request, entries, plugins) for entry, entry_html in stream.entry_pair: log.info(entry.stream_entry) feed_model = lifestream.models.FeedForm() stream.url = "/u/%s/s/%s" % (username, stream.name) gravatarHash = hashlib.md5( django.utils.encoding.smart_str(request.user.email)).hexdigest() gravatar = "http://www.gravatar.com/avatar/%s.jpg?d=monsterid&s=80" % gravatarHash preferences = patchouli_auth.preferences.getPreferences(request.user) if webpage_properties['css_type'] == 'css_raw': css_raw_default = webpage_properties['css_value'] else: css_raw_default = '' if webpage_properties['css_type'] == 'css_url': css_url_default = webpage_properties['css_value'] else: css_url_default = 'http://' if webpage_properties['js_type'] == 'js_raw': js_raw_default = webpage_properties['js_value'] else: js_raw_default = '' if webpage_properties['js_type'] == 'js_url': js_url_default = webpage_properties['js_value'] else: js_url_default = 'http://' template_data = { #'feeds': feeds, 'css_raw_default': css_raw_default, 'css_url_default': css_url_default, 'page_props_s': str(webpage_properties), 'unused_feeds': [], 'form': feed_model, 'gravatar': gravatar, 'js_raw_default': js_raw_default, 'js_url_default': js_url_default, # Common 'lang_dir': 'LTR', 'page_lang': 'en', 'page_langs': lang.HTML_LANG, 'page_lang_desc': lang.HTML_LANG[webpage_properties['page_lang']], 'page_lang_dirs': lang.DIR_CHOICES, 'page_name': page_name, 'request': request, 'streams': streams, 'stream_id': stream.id, 'stream_config': stream_config, 'username': request.user.username, 'page_props': webpage_properties, 'preferences': preferences} if streams: template_data['first_stream'] = streams[0] [template_data.update(plugin.template_variables()) for plugin in plugins] return render_to_response('stream_editor.html', template_data, context_instance=django.template.RequestContext(request)) else: return django.http.HttpResponse(HACKING_MESSAGE, status=400)
def _recent_entries(user, stream, feed_id, count=150, only_visible=True): # Grab entries for each feed basedon the stream constraints per feed stream_id = stream.id stream_config = StreamConfig(stream.config) feed_ids = [] if 'ANY' == feed_id: for feed in stream_config.config['feeds']: feed_ids.append(feed['url_hash']) else: feed_ids.append(feed_id) entries = [] for a_feed_id in feed_ids: feed_config = stream_config.feed_config(a_feed_id) query = (Entry.objects.order_by('-last_published_date') .filter(feed__user=user, feed__url_hash=a_feed_id, # TODO .... Why constrain by Stream? We want to reuse feeds, right? # the cron somehow repairs the links betwee a feed and all the streams it lives in feed__streams__id = stream_id )) # Visible is special, we want to see hidden items in the editor if only_visible: query = query.filter(streamentry__visible=True) # Lastly... LIMIT if 'enable_num_entries_shown' in feed_config and True == feed_config['enable_num_entries_shown'] and 'num_entries_shown' in feed_config: entries += query[0:feed_config['num_entries_shown']] else: entries += query[:count] entries = sorted(entries, compare_entries) entries = entries[:count] log.info("Got back %d entries", len(entries)) # Performance tweak... We always also want to load all StreamEntry objects in one go. This isn't # hooked up as a straight ont-to-one so select_related can't be used on Entry stream_entries = [] for a_feed_id in feed_ids: feed_config = stream_config.feed_config(a_feed_id) query = (StreamEntry.objects.select_related() .order_by('-entry__last_published_date') .filter(entry__feed__url_hash=a_feed_id, stream__id=stream_id)) if only_visible: query = query.filter(visible=True) # Lastly... LIMIT if 'enable_num_entries_shown' in feed_config and True == feed_config['enable_num_entries_shown'] and 'num_entries_shown' in feed_config: log.info("using config num_entires_shown") log.info(feed_config['num_entries_shown']) stream_entries += query[0:feed_config['num_entries_shown']] else: log.info("using 150") stream_entries += query[:count] log.debug("stream entries") log.debug(stream_entries) log.debug(django.db.connection.queries) for se in stream_entries: key = "%s_%d" % (str(stream_id), se.entry_id) stream_entries_cache[key] = se for e in entries: key = "%s_%d" % (str(stream_id), e.id) # Easy Access from templates if key in stream_entries_cache: e.stream_entry = stream_entries_cache[key] else: e.stream_entry = {} return entries