def _parse_user_info(user):
return {
'user_info': {
'is_anonymous': json.encode(user.is_anonymous()),
'is_authenticated': json.encode(user.is_authenticated()),
'username': json.encode(user.username if user.is_authenticated() else 'Anonymous')
}
}
def process(self):
outline = opml.from_string(self.opml_xml)
self.clear_feeds()
folders = self.process_outline(outline)
UserSubscriptionFolders.objects.create(user=self.user, folders=json.encode(folders))
return folders
def delete_feed(request):
feed_id = int(request.POST['feed_id'])
user_sub = get_object_or_404(UserSubscription, user=request.user, feed=feed_id)
user_sub.delete()
MUserStory.objects(user_id=request.user.pk, feed_id=feed_id).delete()
def _find_feed_in_folders(old_folders):
new_folders = []
for k, folder in enumerate(old_folders):
if isinstance(folder, int):
if folder == feed_id:
logging.info(" ---> [%s] Delete folder: %s'th item: %s folders/feeds" % (
request.user, k, len(old_folders)
))
# folders.remove(folder)
else:
new_folders.append(folder)
elif isinstance(folder, dict):
for f_k, f_v in folder.items():
new_folders.append({f_k: _find_feed_in_folders(f_v)})
return new_folders
user_sub_folders_object = UserSubscriptionFolders.objects.get(user=request.user)
user_sub_folders = json.decode(user_sub_folders_object.folders)
user_sub_folders = _find_feed_in_folders(user_sub_folders)
user_sub_folders_object.folders = json.encode(user_sub_folders)
user_sub_folders_object.save()
return dict(code=1)
def delete_feed(self, feed_id, in_folder):
def _find_feed_in_folders(old_folders, folder_name='', multiples_found=False, deleted=False):
new_folders = []
for k, folder in enumerate(old_folders):
if isinstance(folder, int):
if (folder == feed_id and (
(folder_name != in_folder) or
(folder_name == in_folder and deleted))):
multiples_found = True
logging.info(" ---> [%s] Deleting feed, and a multiple has been found in '%s'" % (self.user, folder_name))
if folder == feed_id and folder_name == in_folder and not deleted:
logging.info(" ---> [%s] Delete feed: %s'th item: %s folders/feeds" % (
self.user, k, len(old_folders)
))
deleted = True
else:
new_folders.append(folder)
elif isinstance(folder, dict):
for f_k, f_v in folder.items():
nf, multiples_found, deleted = _find_feed_in_folders(f_v, f_k, multiples_found, deleted)
new_folders.append({f_k: nf})
return new_folders, multiples_found, deleted
user_sub_folders = json.decode(self.folders)
user_sub_folders, multiples_found, deleted = _find_feed_in_folders(user_sub_folders)
self.folders = json.encode(user_sub_folders)
self.save()
if not multiples_found and deleted:
user_sub = UserSubscription.objects.get(user=self.user, feed=feed_id)
user_sub.delete()
MUserStory.objects(user_id=self.user.pk, feed_id=feed_id).delete()
def process(self):
self.clear_feeds()
self.parse()
folders = defaultdict(list)
for item in self.feeds:
folders = self.process_item(item, folders)
# print dict(folders)
self.rearrange_folders(folders)
logging.info(" ---> [%s] Google Reader import: %s" % (self.user, self.subscription_folders))
UserSubscriptionFolders.objects.create(user=self.user,
folders=json.encode(self.subscription_folders))
def set_preference(request):
code = 1
preference_name = request.POST['preference']
preference_value = request.POST['value']
preferences = json.decode(request.user.profile.preferences)
preferences[preference_name] = preference_value
request.user.profile.preferences = json.encode(preferences)
request.user.profile.save()
response = dict(code=code)
return response
def set_view_setting(request):
code = 1
feed_id = request.POST['feed_id']
feed_view_setting = request.POST['feed_view_setting']
view_settings = json.decode(request.user.profile.view_settings)
view_settings[feed_id] = feed_view_setting
request.user.profile.view_settings = json.encode(view_settings)
request.user.profile.save()
response = dict(code=code)
return response
def add_url(request):
code = 0
url = request.POST['url']
folder = request.POST['folder']
feed = None
logging.info(" ---> [%s] Adding URL: %s (in %s)" % (request.user, url, folder))
if url:
url = urlnorm.normalize(url)
# See if it exists as a duplicate first
duplicate_feed = DuplicateFeed.objects.filter(duplicate_address=url)
if duplicate_feed:
feed = [duplicate_feed[0].feed]
else:
feed = Feed.objects.filter(feed_address=url)
if feed:
feed = feed[0]
else:
try:
feed = fetch_address_from_page(url)
except:
code = -2
message = "This feed has been added, but something went wrong"\
" when downloading it. Maybe the server's busy."
if not feed:
code = -1
message = "That URL does not point to an RSS feed or a website that has an RSS feed."
else:
us, _ = UserSubscription.objects.get_or_create(
feed=feed,
user=request.user,
defaults={'needs_unread_recalc': True}
)
code = 1
message = ""
user_sub_folders_object, created = UserSubscriptionFolders.objects.get_or_create(user=request.user,
defaults={'folders': '[]'}
)
if created:
user_sub_folders = []
else:
user_sub_folders = json.decode(user_sub_folders_object.folders)
user_sub_folders = _add_object_to_folder(feed.pk, folder, user_sub_folders)
user_sub_folders_object.folders = json.encode(user_sub_folders)
user_sub_folders_object.save()
return dict(code=code, message=message)
def save_popular_authors(self, feed_authors=None, lock=None):
if not feed_authors:
authors = defaultdict(int)
for story in MStory.objects(story_feed_id=self.pk).only('story_author_name'):
authors[story.story_author_name] += 1
feed_authors = sorted([(k, v) for k, v in authors.items() if k],
key=itemgetter(1),
reverse=True)[:20]
popular_authors = json.encode(feed_authors)
if len(popular_authors) < 1024:
self.popular_authors = popular_authors
self.save(lock=lock)
return
if len(feed_authors) > 1:
self.save_popular_authors(feed_authors=feed_authors[:-1], lock=lock)
def opml_upload(request):
xml_opml = None
message = "OK"
code = 1
payload = {}
if request.method == 'POST':
if 'file' in request.FILES:
logging.info(" ---> [%s] OPML Upload" % request.user)
file = request.FILES['file']
xml_opml = file.read()
opml_importer = OPMLImporter(xml_opml, request.user)
folders = opml_importer.process()
feeds = UserSubscription.objects.filter(user=request.user).values()
payload = dict(folders=folders, feeds=feeds)
else:
message = "Attach an .opml file."
code = -1
data = json.encode(dict(message=message, code=code, payload=payload))
return HttpResponse(data, mimetype='text/plain')
def add_folder(request):
folder = request.POST['folder']
parent_folder = request.POST['parent_folder']
logging.info(" ---> [%s] Adding Folder: %s (in %s)" % (request.user, folder, parent_folder))
if folder:
code = 1
message = ""
user_sub_folders_object, _ = UserSubscriptionFolders.objects.get_or_create(user=request.user)
if user_sub_folders_object.folders:
user_sub_folders = json.decode(user_sub_folders_object.folders)
else:
user_sub_folders = []
obj = {folder: []}
user_sub_folders = _add_object_to_folder(obj, parent_folder, user_sub_folders)
user_sub_folders_object.folders = json.encode(user_sub_folders)
user_sub_folders_object.save()
else:
code = -1
message = "Gotta write in a folder name."
return dict(code=code, message=message)
def merge_feeds(original_feed_id, duplicate_feed_id):
from apps.reader.models import UserSubscription, UserSubscriptionFolders, MUserStory
from apps.analyzer.models import MClassifierTitle, MClassifierAuthor, MClassifierFeed, MClassifierTag
try:
original_feed = Feed.objects.get(pk=original_feed_id)
duplicate_feed = Feed.objects.get(pk=duplicate_feed_id)
except Feed.DoesNotExist:
logging.info(" ***> Already deleted feed: %s" % duplicate_feed_id)
return
logging.info(" ---> Feed: [%s - %s] %s - %s" % (original_feed_id, duplicate_feed_id,
original_feed, original_feed.feed_link))
logging.info(" --> %s" % original_feed.feed_address)
logging.info(" --> %s" % duplicate_feed.feed_address)
user_subs = UserSubscription.objects.filter(feed=duplicate_feed)
for user_sub in user_subs:
# Rewrite feed in subscription folders
try:
user_sub_folders = UserSubscriptionFolders.objects.get(user=user_sub.user)
except Exception, e:
logging.info(" *** ---> UserSubscriptionFolders error: %s" % e)
continue
# Switch to original feed for the user subscription
logging.info(" ===> %s " % user_sub.user)
user_sub.feed = original_feed
user_sub.needs_unread_recalc = True
try:
user_sub.save()
folders = json.decode(user_sub_folders.folders)
folders = rewrite_folders(folders, original_feed, duplicate_feed)
user_sub_folders.folders = json.encode(folders)
user_sub_folders.save()
except (IntegrityError, OperationError):
logging.info(" !!!!> %s already subscribed" % user_sub.user)
user_sub.delete()
def delete_folder(self, folder_to_delete, in_folder, feed_ids_in_folder):
def _find_folder_in_folders(old_folders, folder_name, feeds_to_delete):
new_folders = []
for k, folder in enumerate(old_folders):
if isinstance(folder, int):
new_folders.append(folder)
if folder in feeds_to_delete:
feeds_to_delete.remove(folder)
elif isinstance(folder, dict):
for f_k, f_v in folder.items():
if f_k == folder_to_delete and folder_name == in_folder:
logging.info(" ---> [%s] Deleting folder '%s' in '%s': %s" % (self.user, f_k, folder_name, folder))
else:
nf, feeds_to_delete = _find_folder_in_folders(f_v, f_k, feeds_to_delete)
new_folders.append({f_k: nf})
return new_folders, feeds_to_delete
user_sub_folders = json.decode(self.folders)
user_sub_folders, feeds_to_delete = _find_folder_in_folders(user_sub_folders, '', feed_ids_in_folder)
self.folders = json.encode(user_sub_folders)
self.save()
UserSubscription.objects.filter(user=self.user, feed__in=feeds_to_delete).delete()
def save_feed_story_history_statistics(self, lock=None, current_counts=None):
"""
Fills in missing months between earlier occurances and now.
Save format: [('YYYY-MM, #), ...]
Example output: [(2010-12, 123), (2011-01, 146)]
"""
now = datetime.datetime.now()
min_year = now.year
total = 0
month_count = 0
if not current_counts:
current_counts = self.story_count_history and json.decode(self.story_count_history)
if not current_counts:
current_counts = []
# Count stories, aggregate by year and month. Map Reduce!
map_f = """
function() {
var date = (this.story_date.getFullYear()) + "-" + (this.story_date.getMonth()+1);
emit(date, 1);
}
"""
reduce_f = """
function(key, values) {
var total = 0;
for (var i=0; i < values.length; i++) {
total += values[i];
}
return total;
}
"""
dates = {}
res = MStory.objects(story_feed_id=self.pk).map_reduce(map_f, reduce_f)
for r in res:
dates[r.key] = r.value
# Add on to existing months, always amending up, never down. (Current month
# is guaranteed to be accurate, since trim_feeds won't delete it until after
# a month. Hacker News can have 1,000+ and still be counted.)
for current_month, current_count in current_counts:
if current_month not in dates or dates[current_month] < current_count:
dates[current_month] = current_count
year = int(re.findall(r"(\d{4})-\d{1,2}", current_month)[0])
if year < min_year:
min_year = year
# Assemble a list with 0's filled in for missing months,
# trimming left and right 0's.
months = []
start = False
for year in range(min_year, now.year+1):
for month in range(1, 12+1):
if datetime.datetime(year, month, 1) < now:
key = u'%s-%s' % (year, month)
if dates.get(key) or start:
start = True
months.append((key, dates.get(key, 0)))
total += dates.get(key, 0)
month_count += 1
self.story_count_history = json.encode(months)
if not total:
self.average_stories_per_month = 0
else:
self.average_stories_per_month = total / month_count
self.save(lock)
