if isinstance(lsum.summary, basestring) and len(lsum.summary) == 0: lsum.summary = "" if url_parser.image and not url_m.image: url_m.image = url_parser.image url_m.save() try: filtr = url_parser.find_collection(lsum, filters) if filtr and filtr.xpath is not None and len(filtr.xpath.strip()) == 0: filtr.xpath = None filtr.save() if filtr and filtr.xpath is not None: lsum.summary = url_parser.extract_link_xpath(filtr.xpath) lsum.collection_id = filtr.to_collection_id logger.info("new collection : %d" % filtr.to_collection_id) except DeleteLinkException: logger.info("Link not saved, filtered") return try: if isinstance(lsum.summary, unicode) and len(lsum.summary) == 0: lsum.summary = "" lsum.tags = "" lsum.save() lsum.authors.add(author) except Exception, exc: logger.exception(exc) logger.info("Added new link for user %s" % user_id)
def index_url(link, user_id, link_at, author_name, source_name): """Entry point to store our link & linksum into the database""" if not isinstance(user_id, (list, tuple)): user_id = [int(user_id)] urls = find_urls(link) if not urls: logger.info(u"No link provided") return #loads our filters filters = Filter.objects.filter(Q(user__pk__in=user_id) | Q(user__isnull=True)) try: source = Source.objects.get(name__iexact=source_name) except Source.DoesNotExist: logger.info(u"source %s unknown" % source_name) return -1 try: author = Author.objects.get(name=author_name) except Author.DoesNotExist: logger.info(u"Created author '{0}'".format(author_name)) author, created = Author.objects.get_or_create(name=author_name, source=source) for url in urls: link_extractor = LinkExtractor(url) try: link_extractor.fetch_url_content() except index_exc.FetchException: logger.exception(u"Can't fetch link") continue try: link_extractor.extract() except Exception, exc: logger.warning(u"Can't extract url_content from %s" % url, exc_info=True) continue try: url_instance = create_url(link_extractor) except index_exc.UrlCreationException: logger.warning(u"can't create url for link {0}".format(url), exc_info=True) continue users_lsums = dict([(user, None) for user in user_id]) for user in user_id: try: links_numb = LinkSum.objects.get(url=url_instance, user__id=user) links_numb.recommanded += 1 links_numb.save() links_numb.authors.add(author) del users_lsums[user] logger.info(u"linksum already in database") except LinkSum.DoesNotExist: pass lsum = LinkSum(url=url_instance, collection_id=default_collection.pk, read=False) try: find_collection(lsum, filters) except index_exc.DeleteLinkException: logger.info(u"Link not saved, filtered") continue for user in users_lsums: try: lsum.user_id = user lsum.save() lsum.authors.add(author) lsum.sources.add(source) for tags in url_instance.tags.all(): lsum.tags.add(tags) except Exception, exc: logger.exception(exc) logger.info(u"Added new link for user {0}".format(user_id))