def test_importer_values(self): try: importer = ExternalEventImporter(self.directory) string_values = [ 'title', 'short_description', 'long_description', 'locality', 'street', 'housenumber', 'zipcode', 'town', 'location_url', 'event_url', 'organizer', 'contact_name', 'contact_email', 'contact_phone', 'prices', 'registration', 'source_id', 'fetch_id' ] now = default_now().replace(microsecond=0) then = now + timedelta(days=10) event = {s: s for s in string_values} event['last_update'] = now event['start'] = then event['end'] = then + timedelta(hours=1) event['timezone'] = default_timezone() event['whole_day'] = False event['recurrence'] = 'RRULE:FREQ=DAILY;COUNT=2' event['cat1'] = set(['c1', 'c2']) event['cat2'] = set(['c3', 'c4']) event['longitude'] = 7.8673189 event['latitude'] = 46.6859853 # :TODO: test image and attachement download # event['image'] = # event['attachment_1'] = # event['attachment_2'] = imports, deleted = importer.fetch_one('source', lambda: [event]) imported = self.catalog.query() self.assertEquals(imports, 1) self.assertEquals(len(imported), 1) imported = imported[0].getObject() for s in string_values: self.assertTrue(s in vars(imported)) self.assertTrue(vars(imported)[s] == s) self.assertEquals(imported.start, now + timedelta(days=10)) self.assertEquals(imported.end, now + timedelta(days=10, hours=1)) self.assertEquals(imported.recurrence, 'RRULE:FREQ=DAILY;COUNT=2') self.assertFalse(imported.whole_day) self.assertTrue('c1' in imported.cat1) self.assertTrue('c2' in imported.cat1) self.assertTrue('c3' in imported.cat2) self.assertTrue('c4' in imported.cat2) self.assertEquals( IGeoreferenced(imported).coordinates, [7.8673189, 46.6859853]) finally: # Clean up (transaction has been commited) self.cleanup_after_fetch_one()
def test_importer_keep_hidden(self): try: # Import event importer = ExternalEventImporter(self.directory) event = self.create_fetch_entry(source_id='s', fetch_id='f') imports, deleted = importer.fetch_one('source', lambda: [event]) self.assertEquals(imports, 1) # Hide event brains = self.catalog.catalog( object_provides=IExternalEvent.__identifier__) self.assertEquals(len(brains), 1) hidden = brains[0].getObject() hidden.hide() # Re-import event imports, deleted = importer.fetch_one('source', lambda: [event]) self.assertEquals(imports, 0) imports, deleted = importer.fetch_one('source', lambda: [event], reimport=True) self.assertEquals(imports, 1) brains = self.catalog.catalog( object_provides=IExternalEvent.__identifier__) self.assertEquals(len(brains), 1) event = brains[0].getObject() self.assertTrue( event.modification_date != hidden.modification_date) self.assertEquals(event.review_state, 'hidden') finally: # Clean up (transaction has been commited) self.cleanup_after_fetch_one()
def test_importer_update_category_suggestions(self): try: importer = ExternalEventImporter(self.directory) events = [] fetch = lambda: events events.append(self.create_fetch_entry( source_id='1', fetch_id='1', cat1=set(['cat1-1']), cat2=set(['cat2-1']) )) events.append(self.create_fetch_entry( source_id='2', fetch_id='1', cat1=set(['cat1-2', 'cat1-4']), cat2=set(['cat2-1']) )) events.append(self.create_fetch_entry( source_id='3', fetch_id='1', cat1=set(), cat2=set(['cat2-1', 'cat2-2', 'cat2-3']) )) imports, deleted = importer.fetch_one('source', fetch) self.assertEquals(imports, 3) self.assertTrue('cat1-1' in self.directory.cat1_suggestions) self.assertTrue('cat1-2' in self.directory.cat1_suggestions) self.assertTrue('cat1-4' in self.directory.cat1_suggestions) self.assertTrue('cat2-1' in self.directory.cat2_suggestions) self.assertTrue('cat2-2' in self.directory.cat2_suggestions) self.assertTrue('cat2-3' in self.directory.cat2_suggestions) finally: # Clean up (transaction has been commited) self.cleanup_after_fetch_one()
def test_importer_existing_events(self): importer = ExternalEventImporter(self.directory) sources = [('s1', 'id1'), ('s2', 'id1'), ('s1', 'id1'), ('s2', 'id2'), ('', 'id2'), (None, 'id1'), ('s1', ''), ('s2', None)] for source in sources: args = {} if source[0] is not None: args = {'source': source[0], 'source_id': source[1]} event = self.create_event(**args) event.submit() event.publish() if source is not None: alsoProvides(event, IExternalEvent) event.reindexObject() self.assertEquals(importer.grouped_existing_events(None), {}) self.assertEquals(importer.grouped_existing_events(''), {}) self.assertEquals(len(importer.grouped_existing_events('s1')), 1) self.assertEquals(len(importer.grouped_existing_events('s1')['id1']), 2) self.assertEquals(len(importer.grouped_existing_events('s2')), 2) self.assertEquals(len(importer.grouped_existing_events('s2')['id1']), 1) self.assertEquals(len(importer.grouped_existing_events('s2')['id2']), 1) self.assertEquals(importer.grouped_existing_events('s3'), {})
def test_importer_existing_events(self): importer = ExternalEventImporter(self.directory) sources = [('s1', 'id1'), ('s2', 'id1'), ('s1', 'id1'), ('s2', 'id2'), ('', 'id2'), (None, 'id1'), ('s1', ''), ('s2', None)] for source in sources: args = {} if source[0] is not None: args = { 'source': source[0], 'source_id': source[1] } event = self.create_event(**args) event.submit() event.publish() if source is not None: alsoProvides(event, IExternalEvent) event.reindexObject() self.assertEquals(importer.grouped_existing_events(None), {}) self.assertEquals(importer.grouped_existing_events(''), {}) self.assertEquals(len(importer.grouped_existing_events('s1')), 1) self.assertEquals( len(importer.grouped_existing_events('s1')['id1']), 2 ) self.assertEquals(len(importer.grouped_existing_events('s2')), 2) self.assertEquals( len(importer.grouped_existing_events('s2')['id1']), 1 ) self.assertEquals( len(importer.grouped_existing_events('s2')['id2']), 1 ) self.assertEquals(importer.grouped_existing_events('s3'), {})
def test_importer_fetch_one(self): try: importer = ExternalEventImporter(self.directory) events = [] fetch = lambda: events from_ids = lambda ids: [ self.create_fetch_entry(source_id=id, fetch_id='f') for id in ids ] # Simple import ids = [ 'event1', 'event2', 'event3', 'event4', 'event5', 'event6', 'event7', 'event8' ] events = from_ids(ids[:4]) imports, deleted = importer.fetch_one('source', fetch) self.assertEquals(imports, 4) imported = [i.getObject().source_id for i in self.catalog.query()] self.assertEquals(ids[:4], imported) # Import with limit events = from_ids(ids[4:]) imports, deleted = importer.fetch_one('source', fetch, limit=2) self.assertEquals(imports, 2) self.assertEquals(len(self.catalog.query()), 6) imports, deleted = importer.fetch_one('source', fetch) self.assertEquals(imports, 2) self.assertEquals(len(self.catalog.query()), 8) # Force reimport imports, deleted = importer.fetch_one('source', fetch) self.assertEquals(imports, 0) self.assertEquals(len(self.catalog.query()), 8) imports, deleted = importer.fetch_one('source', fetch, reimport=True) self.assertEquals(imports, 4) self.assertEquals(len(self.catalog.query()), 8) # Reimport updated events events = from_ids(ids) imports, deleted = importer.fetch_one('source', fetch) self.assertEquals(imports, 8) self.assertEquals(len(self.catalog.query()), 8) # Test import of given source IDs only events = from_ids(ids) imports, deleted = importer.fetch_one('source', fetch, source_ids=ids[2:6]) self.assertEquals(imports, 4) self.assertEquals(len(self.catalog.query()), 8) finally: # Clean up (transaction has been commited) self.cleanup_after_fetch_one()
def test_importer_fetch_one(self): try: importer = ExternalEventImporter(self.directory) events = [] fetch = lambda: events from_ids = lambda ids: [self.create_fetch_entry(source_id=id, fetch_id='f') for id in ids] # Simple import ids = ['event1', 'event2', 'event3', 'event4', 'event5', 'event6', 'event7', 'event8'] events = from_ids(ids[:4]) imports, deleted = importer.fetch_one('source', fetch) self.assertEquals(imports, 4) imported = [i.getObject().source_id for i in self.catalog.query()] self.assertEquals(ids[:4], imported) # Import with limit events = from_ids(ids[4:]) imports, deleted = importer.fetch_one('source', fetch, limit=2) self.assertEquals(imports, 2) self.assertEquals(len(self.catalog.query()), 6) imports, deleted = importer.fetch_one('source', fetch) self.assertEquals(imports, 2) self.assertEquals(len(self.catalog.query()), 8) # Force reimport imports, deleted = importer.fetch_one('source', fetch) self.assertEquals(imports, 0) self.assertEquals(len(self.catalog.query()), 8) imports, deleted = importer.fetch_one('source', fetch, reimport=True) self.assertEquals(imports, 4) self.assertEquals(len(self.catalog.query()), 8) # Reimport updated events events = from_ids(ids) imports, deleted = importer.fetch_one('source', fetch) self.assertEquals(imports, 8) self.assertEquals(len(self.catalog.query()), 8) # Test import of given source IDs only events = from_ids(ids) imports, deleted = importer.fetch_one( 'source', fetch, source_ids=ids[2:6] ) self.assertEquals(imports, 4) self.assertEquals(len(self.catalog.query()), 8) finally: # Clean up (transaction has been commited) self.cleanup_after_fetch_one()
def test_importer_switch_indexing(self): importer = ExternalEventImporter(self.directory) event = self.create_event() event.submit() event.publish() reindex_directory(self.directory) self.assertEquals(len(self.catalog.ix_published.index), 1) importer.disable_indexing() event = self.create_event() event.submit() event.publish() reindex_directory(self.directory) self.assertEquals(len(self.catalog.ix_published.index), 1) importer.enable_indexing() reindex_directory(self.directory) self.assertEquals(len(self.catalog.ix_published.index), 2)
def test_importer_autoremove(self): try: importer = ExternalEventImporter(self.directory) events = [] fetch = lambda: events # First import events.append( self.create_fetch_entry(source_id='source_id_1', fetch_id='fetch_id')) events.append( self.create_fetch_entry(source_id='source_id_1', fetch_id='fetch_id')) events.append( self.create_fetch_entry(source_id='source_id_2', fetch_id='fetch_id')) imports, deleted = importer.fetch_one('source', fetch) self.assertEquals(imports, 3) self.assertEquals(deleted, 0) self.assertEquals(len(self.directory.keys()), 3) # Second import events.pop(0) events.pop(0) events.append( self.create_fetch_entry(source_id='source_id_3', fetch_id='fetch_id')) imports, deleted = importer.fetch_one('source', fetch, autoremove=True) self.assertEquals(imports, 1) self.assertEquals(deleted, 2) ids = [item.source_id for id, item in self.directory.objectItems()] self.assertEquals(len(ids), 2) self.assertTrue('source_id_1' not in ids) finally: # Clean up (transaction has been commited) self.cleanup_after_fetch_one()
def test_importer_export_imported(self): try: # Import event importer = ExternalEventImporter(self.directory) event = self.create_fetch_entry(source_id='s', fetch_id='f') imports, deleted = importer.fetch_one('source', lambda: [event]) self.assertEquals(imports, 1) # Add own event event = self.create_event(start=datetime.today().replace(second=0)) event.submit() event.publish() reindex_directory(self.directory) # Export events events = [idx for idx in enumerate(self.catalog.export())] self.assertEquals(len(events), 1) finally: # Clean up (transaction has been commited) self.cleanup_after_fetch_one()
def test_importer_autoremove(self): try: importer = ExternalEventImporter(self.directory) events = [] fetch = lambda: events # First import events.append(self.create_fetch_entry(source_id='source_id_1', fetch_id='fetch_id')) events.append(self.create_fetch_entry(source_id='source_id_1', fetch_id='fetch_id')) events.append(self.create_fetch_entry(source_id='source_id_2', fetch_id='fetch_id')) imports, deleted = importer.fetch_one('source', fetch) self.assertEquals(imports, 3) self.assertEquals(deleted, 0) self.assertEquals(len(self.directory.keys()), 3) # Second import events.pop(0) events.pop(0) events.append(self.create_fetch_entry(source_id='source_id_3', fetch_id='fetch_id')) imports, deleted = importer.fetch_one('source', fetch, autoremove=True) self.assertEquals(imports, 1) self.assertEquals(deleted, 2) ids = [item.source_id for id, item in self.directory.objectItems()] self.assertEquals(len(ids), 2) self.assertTrue('source_id_1' not in ids) finally: # Clean up (transaction has been commited) self.cleanup_after_fetch_one()
def test_importer_keep_hidden(self): try: # Import event importer = ExternalEventImporter(self.directory) event = self.create_fetch_entry(source_id='s', fetch_id='f') imports, deleted = importer.fetch_one('source', lambda: [event]) self.assertEquals(imports, 1) # Hide event brains = self.catalog.catalog( object_provides=IExternalEvent.__identifier__ ) self.assertEquals(len(brains), 1) hidden = brains[0].getObject() hidden.hide() # Re-import event imports, deleted = importer.fetch_one('source', lambda: [event]) self.assertEquals(imports, 0) imports, deleted = importer.fetch_one( 'source', lambda: [event], reimport=True ) self.assertEquals(imports, 1) brains = self.catalog.catalog( object_provides=IExternalEvent.__identifier__ ) self.assertEquals(len(brains), 1) event = brains[0].getObject() self.assertTrue( event.modification_date != hidden.modification_date ) self.assertEquals(event.review_state, 'hidden') finally: # Clean up (transaction has been commited) self.cleanup_after_fetch_one()
def test_importer_update_category_suggestions(self): try: importer = ExternalEventImporter(self.directory) events = [] fetch = lambda: events events.append( self.create_fetch_entry(source_id='1', fetch_id='1', cat1=set(['cat1-1']), cat2=set(['cat2-1']))) events.append( self.create_fetch_entry(source_id='2', fetch_id='1', cat1=set(['cat1-2', 'cat1-4']), cat2=set(['cat2-1']))) events.append( self.create_fetch_entry(source_id='3', fetch_id='1', cat1=set(), cat2=set( ['cat2-1', 'cat2-2', 'cat2-3']))) imports, deleted = importer.fetch_one('source', fetch) self.assertEquals(imports, 3) self.assertTrue('cat1-1' in self.directory.cat1_suggestions) self.assertTrue('cat1-2' in self.directory.cat1_suggestions) self.assertTrue('cat1-4' in self.directory.cat1_suggestions) self.assertTrue('cat2-1' in self.directory.cat2_suggestions) self.assertTrue('cat2-2' in self.directory.cat2_suggestions) self.assertTrue('cat2-3' in self.directory.cat2_suggestions) finally: # Clean up (transaction has been commited) self.cleanup_after_fetch_one()
def test_importer_update_time(self): importer = ExternalEventImporter(self.directory) set_time = lambda t: importer.set_last_update_time(t) get_time = lambda: importer.get_last_update_time() # No key self.assertRaises(AttributeError, get_time) self.assertRaises(AttributeError, set_time, default_now()) self.assertRaises(AttributeError, get_time) importer.annotation_key = 'key' # Wrong dates self.assertRaises(AssertionError, set_time, None) self.assertRaises(AssertionError, set_time, 25) self.assertRaises(AssertionError, set_time, datetime.today()) # Ok update = default_now() importer.set_last_update_time(update) last_update = importer.get_last_update_time() self.assertEquals(last_update, update.replace(microsecond=0))
def test_importer_sources(self): self.create_guidle_source(enabled=True) self.create_guidle_source(enabled=False) self.assertEqual(len(ExternalEventImporter(self.directory).sources()), 1)
def test_importer_values(self): try: importer = ExternalEventImporter(self.directory) string_values = [ 'title', 'short_description', 'long_description', 'locality', 'street', 'housenumber', 'zipcode', 'town', 'location_url', 'event_url', 'organizer', 'contact_name', 'contact_email', 'contact_phone', 'prices', 'registration', 'source_id', 'fetch_id' ] now = default_now().replace( year=2015, month=1, day=1, microsecond=0 ) then = now + timedelta(days=10) event = {s: s for s in string_values} event['last_update'] = now event['start'] = then event['end'] = then + timedelta(hours=1) event['timezone'] = default_timezone() event['whole_day'] = False event['recurrence'] = 'RRULE:FREQ=DAILY;COUNT=2' event['cat1'] = set(['c1', 'c2']) event['cat2'] = set(['c3', 'c4']) event['longitude'] = 7.8673189 event['latitude'] = 46.6859853 # :TODO: test image and attachement download # event['image'] = # event['attachment_1'] = # event['attachment_2'] = imports, deleted = importer.fetch_one('source', lambda: [event]) imported = self.catalog.query() self.assertEquals(imports, 1) self.assertEquals(len(imported), 1) imported = imported[0].getObject() for s in string_values: self.assertTrue(s in vars(imported)) self.assertTrue(vars(imported)[s] == s) self.assertEquals(imported.start, now + timedelta(days=10)) self.assertEquals(imported.end, now + timedelta(days=10, hours=1)) self.assertEquals(imported.recurrence, 'RRULE:FREQ=DAILY;COUNT=2') self.assertFalse(imported.whole_day) self.assertTrue('c1' in imported.cat1) self.assertTrue('c2' in imported.cat1) self.assertTrue('c3' in imported.cat2) self.assertTrue('c4' in imported.cat2) self.assertEquals( IGeoreferenced(imported).coordinates, [7.8673189, 46.6859853] ) finally: # Clean up (transaction has been commited) self.cleanup_after_fetch_one()