Exemple #1
0
    def eventtags(self):
        """ Return a list of tuples containing the event tag value
        (category value) in position 0 and the link to the related
        filter in position 1.

        The results are sorted by value (position 0).
        """

        categorized = IDirectoryCategorized(self)

        categories = dict()
        categories['cat1'] = categorized.keywords(categories=('cat1', ))
        categories['cat2'] = categorized.keywords(categories=('cat2', ))

        baseurl = self.get_parent().absolute_url() + '?filter=true&%s=%s'

        tags = list()

        for key in sorted(categories):
            for tag in sorted(categories[key]):
                if not tag:
                    continue

                tags.append((
                    tag.strip().replace(' ', ' '),
                    baseurl % (key, urllib.quote(tag.encode('utf-8')))
                ))

        return tags
Exemple #2
0
    def eventtags(self):
        """ Return a list of tuples containing the event tag value
        (category value) in position 0 and the link to the related
        filter in position 1.

        The results are sorted by value (position 0).
        """

        categorized = IDirectoryCategorized(self)

        categories = dict()
        categories['cat1'] = categorized.keywords(categories=('cat1', ))
        categories['cat2'] = categorized.keywords(categories=('cat2', ))

        baseurl = self.get_parent().absolute_url() + '?filter=true&%s=%s'

        tags = list()

        for key in sorted(categories):
            for tag in sorted(categories[key]):
                if not tag:
                    continue

                tags.append(
                    (tag.strip().replace(' ', ' '),
                     baseurl % (key, urllib.quote(tag.encode('utf-8')))))

        return tags
Exemple #3
0
def add_category_binds(fieldmap):
    listwrap = lambda val: ','.join(val) if val else ''
    listunwrap = lambda val: [v.strip() for v in val.split(',') if v.strip()]
    cattransform = lambda obj: IDirectoryCategorized(obj)

    for cat in CATEGORIES:
        fieldmap.bind_wrapper(cat, listwrap)
        fieldmap.bind_unwrapper(cat, listunwrap)
        fieldmap.bind_transformer(cat, cattransform)
Exemple #4
0
    def getDisplayValue(self, prop):
        """ Categories need to be flattened and joined sanely. The default
        practice of joining by space is not useful if there are spaces in the
        categories.

        """
        if prop in const.CATEGORIES:
            values = IDirectoryCategorized(self.context).keywords((prop, ))
            return ';'.join(v for v in values if v)
        return super(Placemark, self).getDisplayValue(prop)
Exemple #5
0
    def features(self):
        terms = utils.get_filter_terms(self.context, self.request)

        for feature in super(KMLFolderDocument, self).features:
            if not terms:
                yield feature
            else:
                item = IDirectoryCategorized(feature.context)

                if catalog.is_exact_match(item, terms):
                    yield feature
Exemple #6
0
    def test_export(self):
        xls = TempXLS()

        directory = self.add_directory()
        directory.title = u'test'
        directory.cat1 = u'Testkategorie 1'
        directory.cat2 = u'Testkategorie 2'

        item = self.add_item(directory, 'item1')
        item.description = u'description1'

        categorized = IDirectoryCategorized(item)
        categorized.cat1 = [u'Eins']
        categorized.cat2 = [u'Zwei']

        item = self.add_item(directory, 'item2')
        categorized = IDirectoryCategorized(item)
        categorized.cat1 = [u'One']
        categorized.cat2 = [u'Two']

        export_xls(directory, xls.file, 'de', False)

        wb = xls.load()
        ws = wb.sheet_by_index(0)

        self.assertEqual(ws.nrows, 3)

        self.assertEqual(ws.cell(1, 0).value, u'item1')
        self.assertEqual(ws.cell(1, 1).value, u'description1')
        self.assertEqual(ws.cell(1, 2).value, u'Eins')
        self.assertEqual(ws.cell(1, 3).value, u'Zwei')

        self.assertEqual(ws.cell(2, 0).value, u'item2')
        self.assertEqual(ws.cell(2, 1).value, u'')
        self.assertEqual(ws.cell(2, 2).value, u'One')
        self.assertEqual(ws.cell(2, 3).value, u'Two')
Exemple #7
0
    def test_export(self):
        xls = TempXLS()

        directory = self.add_directory()
        directory.title = u'test'
        directory.cat1 = u'Testkategorie 1'
        directory.cat2 = u'Testkategorie 2'

        item = self.add_item(directory, 'item1')
        item.description = u'description1'

        categorized = IDirectoryCategorized(item)
        categorized.cat1 = [u'Eins']
        categorized.cat2 = [u'Zwei']

        item = self.add_item(directory, 'item2')
        categorized = IDirectoryCategorized(item)
        categorized.cat1 = [u'One']
        categorized.cat2 = [u'Two']

        export_xls(directory, xls.file, 'de', False)

        wb = xls.load()
        ws = wb.sheet_by_index(0)

        self.assertEqual(ws.nrows, 3)

        self.assertEqual(ws.cell(1, 0).value, u'item1')
        self.assertEqual(ws.cell(1, 1).value, u'description1')
        self.assertEqual(ws.cell(1, 2).value, u'Eins')
        self.assertEqual(ws.cell(1, 3).value, u'Zwei')

        self.assertEqual(ws.cell(2, 0).value, u'item2')
        self.assertEqual(ws.cell(2, 1).value, u'')
        self.assertEqual(ws.cell(2, 2).value, u'One')
        self.assertEqual(ws.cell(2, 3).value, u'Two')
    def test_categories(self):
        directory = self.add_directory()
        directory.cat1 = 'One'
        directory.cat2 = 'Two'
        directory.cat3 = 'Three'

        item = self.add_item(directory)

        # only present after setting them on the behavior
        self.assertFalse(hasattr(aq_base(item), 'cat1'))
        self.assertFalse(hasattr(aq_base(item), 'cat2'))
        self.assertFalse(hasattr(aq_base(item), 'cat3'))
        self.assertFalse(hasattr(aq_base(item), 'cat4'))

        categorized = IDirectoryCategorized(item)
        self.assertTrue(hasattr(categorized, 'cat1'))
        self.assertTrue(hasattr(categorized, 'cat2'))
        self.assertTrue(hasattr(categorized, 'cat3'))
        self.assertTrue(hasattr(categorized, 'cat4'))

        categorized.cat1 = '1'
        categorized.cat2 = '2'
        categorized.cat3 = '3'
        categorized.cat4 = '4'  # Will be ignored

        self.assertTrue(hasattr(aq_base(item), 'cat1'))
        self.assertTrue(hasattr(aq_base(item), 'cat2'))
        self.assertTrue(hasattr(aq_base(item), 'cat3'))
        self.assertTrue(hasattr(aq_base(item), 'cat4'))

        categories = categorized.categories()
        self.assertEqual(len(categories), 3)

        item_used = [c[0] for c in categories]
        dir_used = directory.used_categories()
        self.assertEqual(item_used, dir_used)

        item_labels = [c[1] for c in categories]
        dir_labels = directory.labels().values()
        self.assertEqual(item_labels, dir_labels)

        values = [c[2] for c in categories]
        self.assertEqual(values, ['1', '2', '3'])

        # Set the category to contain multiple keywords and use only
        # one category
        categorized.cat1 = ['tag one', 'tag two']
        directory.cat2 = None
        directory.cat3 = None

        categories = categorized.categories()

        self.assertEqual(len(categories), 1)
        self.assertEqual(categories[0][2], ['tag one', 'tag two'])

        # Set more categories and see if the keywords function correctly
        # returns all keywords flattened, even if invisible

        categorized.cat1 = ['Ready', 'Or']
        categorized.cat2 = 'Not'
        categorized.cat3 = ['Here', 'I']
        categorized.cat4 = 'Come'

        keywords = categorized.keywords()
        self.assertEqual(keywords, ['Ready', 'Or', 'Not', 'Here', 'I', 'Come'])
Exemple #9
0
    def test_categories(self):
        directory = self.add_directory()
        directory.cat1 = 'One'
        directory.cat2 = 'Two'
        directory.cat3 = 'Three'

        item = self.add_item(directory)

        # only present after setting them on the behavior
        self.assertFalse(hasattr(aq_base(item), 'cat1'))
        self.assertFalse(hasattr(aq_base(item), 'cat2'))
        self.assertFalse(hasattr(aq_base(item), 'cat3'))
        self.assertFalse(hasattr(aq_base(item), 'cat4'))

        categorized = IDirectoryCategorized(item)
        self.assertTrue(hasattr(categorized, 'cat1'))
        self.assertTrue(hasattr(categorized, 'cat2'))
        self.assertTrue(hasattr(categorized, 'cat3'))
        self.assertTrue(hasattr(categorized, 'cat4'))

        categorized.cat1 = '1'
        categorized.cat2 = '2'
        categorized.cat3 = '3'
        categorized.cat4 = '4'  # Will be ignored

        self.assertTrue(hasattr(aq_base(item), 'cat1'))
        self.assertTrue(hasattr(aq_base(item), 'cat2'))
        self.assertTrue(hasattr(aq_base(item), 'cat3'))
        self.assertTrue(hasattr(aq_base(item), 'cat4'))

        categories = categorized.categories()
        self.assertEqual(len(categories), 3)

        item_used = [c[0] for c in categories]
        dir_used = directory.used_categories()
        self.assertEqual(item_used, dir_used)

        item_labels = [c[1] for c in categories]
        dir_labels = directory.labels().values()
        self.assertEqual(item_labels, dir_labels)

        values = [c[2] for c in categories]
        self.assertEqual(values, ['1', '2', '3'])

        # Set the category to contain multiple keywords and use only
        # one category
        categorized.cat1 = ['tag one', 'tag two']
        directory.cat2 = None
        directory.cat3 = None

        categories = categorized.categories()

        self.assertEqual(len(categories), 1)
        self.assertEqual(categories[0][2], ['tag one', 'tag two'])

        # Set more categories and see if the keywords function correctly
        # returns all keywords flattened, even if invisible

        categorized.cat1 = ['Ready', 'Or']
        categorized.cat2 = 'Not'
        categorized.cat3 = ['Here', 'I']
        categorized.cat4 = 'Come'

        keywords = categorized.keywords()
        self.assertEqual(keywords, ['Ready', 'Or', 'Not', 'Here', 'I', 'Come'])
Exemple #10
0
    def _fetch_one(self,
                   source,
                   function,
                   limit=None,
                   reimport=False,
                   source_ids=[],
                   autoremove=False):
        imported = []
        len_deleted = 0

        try:
            events = sorted(function(),
                            key=lambda e: (e['last_update'], e['source_id']))
        except NoImportDataException:
            log.info('no data received for %s' % source)
            return imported, len_deleted

        existing = self.grouped_existing_events(source)

        # Autoremove externally deleted events
        if autoremove:
            new_ids = [event['source_id'] for event in events]
            old_ids = existing.keys()
            delta = list(set(old_ids) - set(new_ids))
            if limit is not None:
                delta = delta[:limit]
            for source_id in delta:
                # source id's are not necessarily unique
                for event_id in existing[source_id]:
                    log.info('Deleting %s' % (event_id))
                    self.context.manage_delObjects(event_id)
                    len_deleted += 1

        if len(events) == 0:
            return imported, len_deleted

        fetch_ids = set(event['fetch_id'] for event in events)
        assert len(fetch_ids) == 1, """
            Each event needs a fetch_id which describes the id of the
            whole fetch process and is therefore the same for all events
            in a single fetch. See seantis.dir.events.source.guidle:events
        """

        self.annotation_key = hashlib.sha1(list(fetch_ids)[0]).hexdigest()
        last_update = self.get_last_update_time()
        last_update_in_run = datetime.min.replace(tzinfo=pytz.timezone('utc'))

        if not last_update:
            log.info('initial import')
            changed_offers_only = False
        elif reimport:
            log.info('reimport everything')
            changed_offers_only = False
        else:
            # log.info('importing updates since {}'.format(last_update))
            changed_offers_only = True

        total = len(events) if not limit else limit

        workflowTool = getToolByName(self.context, 'portal_workflow')

        categories = dict(cat1=set(), cat2=set())

        limit_reached_id = None

        for ix, event in enumerate(events):

            if limit_reached_id and limit_reached_id != event['source_id']:
                break

            if source_ids and event['source_id'] not in source_ids:
                continue

            assert 'last_update' in event, """
                Each event needs a last_update datetime info which is used
                to determine if any changes were done. This is used for
                importing only changed events.
            """

            if last_update_in_run < event['last_update']:
                last_update_in_run = event['last_update']

            if changed_offers_only and event['source_id'] in existing:
                if event['last_update'] <= last_update:
                    continue

            # keep a set of all categories for the suggestions
            for cat in categories:
                if cat not in event:
                    event[cat] = set()
                categories[cat] |= event[cat]

            # stop at limit
            if limit and (len(imported) + 1) >= limit and not limit_reached_id:
                log.info('reached limit of %i events' % limit)
                # don't quit right away, all events of the same source_id
                # need to be imported first since they have the same
                # update_time
                limit_reached_id = event['source_id']

            # flush to disk every 500th event to keep memory usage low
            if len(imported) != 0 and len(imported) % 500 == 0:
                transaction.savepoint(True)

            log.info('importing %i/%i %s @ %s' %
                     ((len(imported) + 1), total, event['title'],
                      event['start'].strftime('%d.%m.%Y %H:%M')))

            event['source'] = source

            # If the existing event has been hidden, we keep it hidden
            hide_event = False
            if event['source_id'] in existing:
                for event_id in existing[event['source_id']]:
                    review_state = self.context.get(event_id).review_state
                    hide_event |= review_state == 'hidden'

            # source id's are not necessarily unique as a single external
            # event might have to be represented as more than one event in
            # seantis.dir.events - therefore updating is done through
            # deleting first, adding second
            if event['source_id'] in existing:
                for event_id in existing[event['source_id']]:
                    self.context.manage_delObjects(event_id)
                del existing[event['source_id']]

            # image and attachments are downloaded
            downloads = {
                'image': NamedImage,
                'attachment_1': NamedFile,
                'attachment_2': NamedFile
            }

            def allow_download(download, url):

                if download != 'image':
                    return True

                # whitelist the images that are known to work
                # not working is *.bmp. We could convert but I'd rather
                # force people to use a sane format
                return url.lower().endswith(
                    ('png', 'jpg', 'jpeg', '@@images/image'))

            for download, method in downloads.items():
                url = event.get(download)
                name = download + '_name'

                if not url or not allow_download(download, url):
                    event[download] = None
                else:
                    try:
                        event[download] = method(self.download(url))
                        if name in event:
                            event[download].filename = event[name]
                    except HTTPError:
                        event[download] = None

                if name in event:
                    del event[name]

            # latitude and longitude are set through the interface
            lat, lon = event.get('latitude'), event.get('longitude')
            if 'latitude' in event:
                del event['latitude']
            if 'longitude' in event:
                del event['longitude']

            # so are categories
            cats = map(event.get, ('cat1', 'cat2'))
            del event['cat1']
            del event['cat2']

            assert 'cat3' not in event and 'cat4' not in event, """
                unsupported categories
            """

            obj = createContentInContainer(self.context,
                                           'seantis.dir.events.item',
                                           checkConstraints=False,
                                           **event)

            # set coordinates now
            if lat and lon:
                try:
                    IWriteGeoreferenced(obj).setGeoInterface(
                        'Point', map(float, (lon, lat)))
                except ValueError:
                    pass

            # followed by the categories
            IDirectoryCategorized(obj).cat1 = list(cats[0])
            IDirectoryCategorized(obj).cat2 = list(cats[1])

            workflowTool.doActionFor(obj, 'submit')
            workflowTool.doActionFor(obj, 'publish')

            for download in downloads:
                getattr(obj, download)

            alsoProvides(obj, IExternalEvent)

            if hide_event:
                workflowTool.doActionFor(obj, 'hide')

            imported.append(obj)

        self.set_last_update_time(last_update_in_run)

        # add categories to suggestions
        for category in categories:
            key = '%s_suggestions' % category
            existing = getattr(self.context, key)
            existing = set(existing) if existing is not None else set()
            new = categories[category] | existing

            setattr(self.context, key, sorted(new))

            diff = categories[category] - existing
            if len(diff):
                log.info('added to %s %s' % (category, diff))

        # log.info('committing events for %s' % source)
        transaction.commit()

        return imported, len_deleted