def eventtags(self):
""" Return a list of tuples containing the event tag value
(category value) in position 0 and the link to the related
filter in position 1.
The results are sorted by value (position 0).
"""
categorized = IDirectoryCategorized(self)
categories = dict()
categories['cat1'] = categorized.keywords(categories=('cat1', ))
categories['cat2'] = categorized.keywords(categories=('cat2', ))
baseurl = self.get_parent().absolute_url() + '?filter=true&%s=%s'
tags = list()
for key in sorted(categories):
for tag in sorted(categories[key]):
if not tag:
continue
tags.append((
tag.strip().replace(' ', ' '),
baseurl % (key, urllib.quote(tag.encode('utf-8')))
))
return tags
def eventtags(self):
""" Return a list of tuples containing the event tag value
(category value) in position 0 and the link to the related
filter in position 1.
The results are sorted by value (position 0).
"""
categorized = IDirectoryCategorized(self)
categories = dict()
categories['cat1'] = categorized.keywords(categories=('cat1', ))
categories['cat2'] = categorized.keywords(categories=('cat2', ))
baseurl = self.get_parent().absolute_url() + '?filter=true&%s=%s'
tags = list()
for key in sorted(categories):
for tag in sorted(categories[key]):
if not tag:
continue
tags.append(
(tag.strip().replace(' ', ' '),
baseurl % (key, urllib.quote(tag.encode('utf-8')))))
return tags
def add_category_binds(fieldmap):
listwrap = lambda val: ','.join(val) if val else ''
listunwrap = lambda val: [v.strip() for v in val.split(',') if v.strip()]
cattransform = lambda obj: IDirectoryCategorized(obj)
for cat in CATEGORIES:
fieldmap.bind_wrapper(cat, listwrap)
fieldmap.bind_unwrapper(cat, listunwrap)
fieldmap.bind_transformer(cat, cattransform)
def getDisplayValue(self, prop):
""" Categories need to be flattened and joined sanely. The default
practice of joining by space is not useful if there are spaces in the
categories.
"""
if prop in const.CATEGORIES:
values = IDirectoryCategorized(self.context).keywords((prop, ))
return ';'.join(v for v in values if v)
return super(Placemark, self).getDisplayValue(prop)
def features(self):
terms = utils.get_filter_terms(self.context, self.request)
for feature in super(KMLFolderDocument, self).features:
if not terms:
yield feature
else:
item = IDirectoryCategorized(feature.context)
if catalog.is_exact_match(item, terms):
yield feature
def test_export(self):
xls = TempXLS()
directory = self.add_directory()
directory.title = u'test'
directory.cat1 = u'Testkategorie 1'
directory.cat2 = u'Testkategorie 2'
item = self.add_item(directory, 'item1')
item.description = u'description1'
categorized = IDirectoryCategorized(item)
categorized.cat1 = [u'Eins']
categorized.cat2 = [u'Zwei']
item = self.add_item(directory, 'item2')
categorized = IDirectoryCategorized(item)
categorized.cat1 = [u'One']
categorized.cat2 = [u'Two']
export_xls(directory, xls.file, 'de', False)
wb = xls.load()
ws = wb.sheet_by_index(0)
self.assertEqual(ws.nrows, 3)
self.assertEqual(ws.cell(1, 0).value, u'item1')
self.assertEqual(ws.cell(1, 1).value, u'description1')
self.assertEqual(ws.cell(1, 2).value, u'Eins')
self.assertEqual(ws.cell(1, 3).value, u'Zwei')
self.assertEqual(ws.cell(2, 0).value, u'item2')
self.assertEqual(ws.cell(2, 1).value, u'')
self.assertEqual(ws.cell(2, 2).value, u'One')
self.assertEqual(ws.cell(2, 3).value, u'Two')
def test_export(self):
xls = TempXLS()
directory = self.add_directory()
directory.title = u'test'
directory.cat1 = u'Testkategorie 1'
directory.cat2 = u'Testkategorie 2'
item = self.add_item(directory, 'item1')
item.description = u'description1'
categorized = IDirectoryCategorized(item)
categorized.cat1 = [u'Eins']
categorized.cat2 = [u'Zwei']
item = self.add_item(directory, 'item2')
categorized = IDirectoryCategorized(item)
categorized.cat1 = [u'One']
categorized.cat2 = [u'Two']
export_xls(directory, xls.file, 'de', False)
wb = xls.load()
ws = wb.sheet_by_index(0)
self.assertEqual(ws.nrows, 3)
self.assertEqual(ws.cell(1, 0).value, u'item1')
self.assertEqual(ws.cell(1, 1).value, u'description1')
self.assertEqual(ws.cell(1, 2).value, u'Eins')
self.assertEqual(ws.cell(1, 3).value, u'Zwei')
self.assertEqual(ws.cell(2, 0).value, u'item2')
self.assertEqual(ws.cell(2, 1).value, u'')
self.assertEqual(ws.cell(2, 2).value, u'One')
self.assertEqual(ws.cell(2, 3).value, u'Two')
def test_categories(self):
directory = self.add_directory()
directory.cat1 = 'One'
directory.cat2 = 'Two'
directory.cat3 = 'Three'
item = self.add_item(directory)
# only present after setting them on the behavior
self.assertFalse(hasattr(aq_base(item), 'cat1'))
self.assertFalse(hasattr(aq_base(item), 'cat2'))
self.assertFalse(hasattr(aq_base(item), 'cat3'))
self.assertFalse(hasattr(aq_base(item), 'cat4'))
categorized = IDirectoryCategorized(item)
self.assertTrue(hasattr(categorized, 'cat1'))
self.assertTrue(hasattr(categorized, 'cat2'))
self.assertTrue(hasattr(categorized, 'cat3'))
self.assertTrue(hasattr(categorized, 'cat4'))
categorized.cat1 = '1'
categorized.cat2 = '2'
categorized.cat3 = '3'
categorized.cat4 = '4' # Will be ignored
self.assertTrue(hasattr(aq_base(item), 'cat1'))
self.assertTrue(hasattr(aq_base(item), 'cat2'))
self.assertTrue(hasattr(aq_base(item), 'cat3'))
self.assertTrue(hasattr(aq_base(item), 'cat4'))
categories = categorized.categories()
self.assertEqual(len(categories), 3)
item_used = [c[0] for c in categories]
dir_used = directory.used_categories()
self.assertEqual(item_used, dir_used)
item_labels = [c[1] for c in categories]
dir_labels = directory.labels().values()
self.assertEqual(item_labels, dir_labels)
values = [c[2] for c in categories]
self.assertEqual(values, ['1', '2', '3'])
# Set the category to contain multiple keywords and use only
# one category
categorized.cat1 = ['tag one', 'tag two']
directory.cat2 = None
directory.cat3 = None
categories = categorized.categories()
self.assertEqual(len(categories), 1)
self.assertEqual(categories[0][2], ['tag one', 'tag two'])
# Set more categories and see if the keywords function correctly
# returns all keywords flattened, even if invisible
categorized.cat1 = ['Ready', 'Or']
categorized.cat2 = 'Not'
categorized.cat3 = ['Here', 'I']
categorized.cat4 = 'Come'
keywords = categorized.keywords()
self.assertEqual(keywords, ['Ready', 'Or', 'Not', 'Here', 'I', 'Come'])
def test_categories(self):
directory = self.add_directory()
directory.cat1 = 'One'
directory.cat2 = 'Two'
directory.cat3 = 'Three'
item = self.add_item(directory)
# only present after setting them on the behavior
self.assertFalse(hasattr(aq_base(item), 'cat1'))
self.assertFalse(hasattr(aq_base(item), 'cat2'))
self.assertFalse(hasattr(aq_base(item), 'cat3'))
self.assertFalse(hasattr(aq_base(item), 'cat4'))
categorized = IDirectoryCategorized(item)
self.assertTrue(hasattr(categorized, 'cat1'))
self.assertTrue(hasattr(categorized, 'cat2'))
self.assertTrue(hasattr(categorized, 'cat3'))
self.assertTrue(hasattr(categorized, 'cat4'))
categorized.cat1 = '1'
categorized.cat2 = '2'
categorized.cat3 = '3'
categorized.cat4 = '4' # Will be ignored
self.assertTrue(hasattr(aq_base(item), 'cat1'))
self.assertTrue(hasattr(aq_base(item), 'cat2'))
self.assertTrue(hasattr(aq_base(item), 'cat3'))
self.assertTrue(hasattr(aq_base(item), 'cat4'))
categories = categorized.categories()
self.assertEqual(len(categories), 3)
item_used = [c[0] for c in categories]
dir_used = directory.used_categories()
self.assertEqual(item_used, dir_used)
item_labels = [c[1] for c in categories]
dir_labels = directory.labels().values()
self.assertEqual(item_labels, dir_labels)
values = [c[2] for c in categories]
self.assertEqual(values, ['1', '2', '3'])
# Set the category to contain multiple keywords and use only
# one category
categorized.cat1 = ['tag one', 'tag two']
directory.cat2 = None
directory.cat3 = None
categories = categorized.categories()
self.assertEqual(len(categories), 1)
self.assertEqual(categories[0][2], ['tag one', 'tag two'])
# Set more categories and see if the keywords function correctly
# returns all keywords flattened, even if invisible
categorized.cat1 = ['Ready', 'Or']
categorized.cat2 = 'Not'
categorized.cat3 = ['Here', 'I']
categorized.cat4 = 'Come'
keywords = categorized.keywords()
self.assertEqual(keywords, ['Ready', 'Or', 'Not', 'Here', 'I', 'Come'])
def _fetch_one(self,
source,
function,
limit=None,
reimport=False,
source_ids=[],
autoremove=False):
imported = []
len_deleted = 0
try:
events = sorted(function(),
key=lambda e: (e['last_update'], e['source_id']))
except NoImportDataException:
log.info('no data received for %s' % source)
return imported, len_deleted
existing = self.grouped_existing_events(source)
# Autoremove externally deleted events
if autoremove:
new_ids = [event['source_id'] for event in events]
old_ids = existing.keys()
delta = list(set(old_ids) - set(new_ids))
if limit is not None:
delta = delta[:limit]
for source_id in delta:
# source id's are not necessarily unique
for event_id in existing[source_id]:
log.info('Deleting %s' % (event_id))
self.context.manage_delObjects(event_id)
len_deleted += 1
if len(events) == 0:
return imported, len_deleted
fetch_ids = set(event['fetch_id'] for event in events)
assert len(fetch_ids) == 1, """
Each event needs a fetch_id which describes the id of the
whole fetch process and is therefore the same for all events
in a single fetch. See seantis.dir.events.source.guidle:events
"""
self.annotation_key = hashlib.sha1(list(fetch_ids)[0]).hexdigest()
last_update = self.get_last_update_time()
last_update_in_run = datetime.min.replace(tzinfo=pytz.timezone('utc'))
if not last_update:
log.info('initial import')
changed_offers_only = False
elif reimport:
log.info('reimport everything')
changed_offers_only = False
else:
# log.info('importing updates since {}'.format(last_update))
changed_offers_only = True
total = len(events) if not limit else limit
workflowTool = getToolByName(self.context, 'portal_workflow')
categories = dict(cat1=set(), cat2=set())
limit_reached_id = None
for ix, event in enumerate(events):
if limit_reached_id and limit_reached_id != event['source_id']:
break
if source_ids and event['source_id'] not in source_ids:
continue
assert 'last_update' in event, """
Each event needs a last_update datetime info which is used
to determine if any changes were done. This is used for
importing only changed events.
"""
if last_update_in_run < event['last_update']:
last_update_in_run = event['last_update']
if changed_offers_only and event['source_id'] in existing:
if event['last_update'] <= last_update:
continue
# keep a set of all categories for the suggestions
for cat in categories:
if cat not in event:
event[cat] = set()
categories[cat] |= event[cat]
# stop at limit
if limit and (len(imported) + 1) >= limit and not limit_reached_id:
log.info('reached limit of %i events' % limit)
# don't quit right away, all events of the same source_id
# need to be imported first since they have the same
# update_time
limit_reached_id = event['source_id']
# flush to disk every 500th event to keep memory usage low
if len(imported) != 0 and len(imported) % 500 == 0:
transaction.savepoint(True)
log.info('importing %i/%i %s @ %s' %
((len(imported) + 1), total, event['title'],
event['start'].strftime('%d.%m.%Y %H:%M')))
event['source'] = source
# If the existing event has been hidden, we keep it hidden
hide_event = False
if event['source_id'] in existing:
for event_id in existing[event['source_id']]:
review_state = self.context.get(event_id).review_state
hide_event |= review_state == 'hidden'
# source id's are not necessarily unique as a single external
# event might have to be represented as more than one event in
# seantis.dir.events - therefore updating is done through
# deleting first, adding second
if event['source_id'] in existing:
for event_id in existing[event['source_id']]:
self.context.manage_delObjects(event_id)
del existing[event['source_id']]
# image and attachments are downloaded
downloads = {
'image': NamedImage,
'attachment_1': NamedFile,
'attachment_2': NamedFile
}
def allow_download(download, url):
if download != 'image':
return True
# whitelist the images that are known to work
# not working is *.bmp. We could convert but I'd rather
# force people to use a sane format
return url.lower().endswith(
('png', 'jpg', 'jpeg', '@@images/image'))
for download, method in downloads.items():
url = event.get(download)
name = download + '_name'
if not url or not allow_download(download, url):
event[download] = None
else:
try:
event[download] = method(self.download(url))
if name in event:
event[download].filename = event[name]
except HTTPError:
event[download] = None
if name in event:
del event[name]
# latitude and longitude are set through the interface
lat, lon = event.get('latitude'), event.get('longitude')
if 'latitude' in event:
del event['latitude']
if 'longitude' in event:
del event['longitude']
# so are categories
cats = map(event.get, ('cat1', 'cat2'))
del event['cat1']
del event['cat2']
assert 'cat3' not in event and 'cat4' not in event, """
unsupported categories
"""
obj = createContentInContainer(self.context,
'seantis.dir.events.item',
checkConstraints=False,
**event)
# set coordinates now
if lat and lon:
try:
IWriteGeoreferenced(obj).setGeoInterface(
'Point', map(float, (lon, lat)))
except ValueError:
pass
# followed by the categories
IDirectoryCategorized(obj).cat1 = list(cats[0])
IDirectoryCategorized(obj).cat2 = list(cats[1])
workflowTool.doActionFor(obj, 'submit')
workflowTool.doActionFor(obj, 'publish')
for download in downloads:
getattr(obj, download)
alsoProvides(obj, IExternalEvent)
if hide_event:
workflowTool.doActionFor(obj, 'hide')
imported.append(obj)
self.set_last_update_time(last_update_in_run)
# add categories to suggestions
for category in categories:
key = '%s_suggestions' % category
existing = getattr(self.context, key)
existing = set(existing) if existing is not None else set()
new = categories[category] | existing
setattr(self.context, key, sorted(new))
diff = categories[category] - existing
if len(diff):
log.info('added to %s %s' % (category, diff))
# log.info('committing events for %s' % source)
transaction.commit()
return imported, len_deleted
