def test_it_works_for_datasets(self): obj = DatasetFactory() task = process_dataset.delay(obj) response = self.client.get('/s/task/{}/'.format(task.id)) self.assertContains(response, 'Processing dataset {}'.format(obj))
def test_it_filters_on_aggregator(self): scheduler = AggregatorSchedulerFactory() DatasetSchedulerFactory(object_id=DatasetFactory(name='NOT ME').pk) response = self.client.get( '/s/scheduler/?model=aggregator&pk={}'.format(scheduler.object_id)) self.assertContains(response, scheduler.content_object.name) self.assertNotContains(response, 'NOT ME')
def test_it_logs_using_redis_logger(self): obj = DatasetFactory() loggy = MagicMock() with patch('webui.scheduler.tasks.get_redis_logger', return_value=loggy): process_dataset.delay(obj) self.assert_(loggy.info.called)
def test_it_filters_on_source(self): scheduler = DatasetSchedulerFactory() DatasetSchedulerFactory(object_id=DatasetFactory(name='NOT ME').pk) source = scheduler.content_object.source response = self.client.get('/s/scheduler/?model=source&pk={}'.format( source.pk)) self.assertContains(response, scheduler.content_object.name) self.assertNotContains(response, 'NOT ME')
def test_it_filters_on_dataset(self): scheduler = DatasetSchedulerFactory() DatasetSchedulerFactory(object_id=DatasetFactory(name='NOT ME').pk) dataset = scheduler.content_object response = self.client.get('/s/scheduler/?model=dataset&pk={}'.format( dataset.pk)) self.assertContains(response, dataset.name) self.assertNotContains(response, 'NOT ME')
def test_dataset_subtasks_log_using_the_same_redis_key(self): source = SourceFactory(name='boo name') source.datasets.add(DatasetFactory( name='wow name' )) loggy = MagicMock() with patch('webui.scheduler.tasks.get_redis_logger', return_value=loggy) as get_redis_logger: task = process_source.delay(source) #TODO[vad]: this is broken! assert_calls is not a magick method get_redis_logger.assert_calls([task.id, task.id])
class DatasetSchedulerFactory(SchedulerFactory): """ factory for creating a Scheduler executed on a dataset """ object_id = factory.LazyAttribute(lambda x: DatasetFactory().pk) content_type = factory.LazyAttribute( lambda x: ContentType.objects.get_for_model(Dataset))
def setUp(self): self.dataset = Dataset.objects.all()[0] self.update_url = self.dataset.get_absolute_url() + 'edit/' self.post_data = DatasetFactory.attributes()
def setUp(self): self.source = Source.objects.all()[0] self.create_url = self.source.get_absolute_url() + 'dataset/create/' self.post_data = DatasetFactory.attributes()
def handle(self, *args, **options): """ entry point """ from django.contrib.sites.models import Site site = Site.objects.get() site.name = 'controller' site.domain = 'localhost:8001' site.save() AdminFactory() trentinocultura = SourceFactory( name='trentinocultura', description='description description', scraper_name='trentinocultura', scraper_api_key='', ) DatasetFactory( source=trentinocultura, name='Trentinocultura agenda', url='http://www.trentinocultura.net/asp_cat/main.asp?IDProspettiva' '=35&SearchType=AGENDA_SEARCH&Pag=%d&TipoVista=AGENDA&cmd=new', description='Eventi in Trentino', download='scraper:trentinocultura:trentinocultura', curator='Federico Scrinzi', license='All rights reserved', other_meta='{}', bounding_box='10.3817591116112,45.6730626059259,' '12.4775685651704,47.0917759206089', ) ingiro_local = SourceFactory( name='in-giro (locale)', description='i dati dello scraper di in-giro, ma in locale (file ' 'webui.scheduler.tests.data/in-giro.zip) utile per' 'test, ma anche per boh, altro? =)', ) DatasetFactory( source=ingiro_local, name='eventi-e-poi-ingiro', url='http://in-giro.net', description='Eventi e POI presi da in-giro', download='http://testserver/in-giro.zip', curator='Ciccio Pasticcio', license='All rights reserved', other_meta='{}', ) bgg_source = SourceFactory( name='BoardGameGeek (test)', description='pochi dati per testare il matching su silk' ) bgt_source = SourceFactory( name='BoardGameTournament (test)', description='pochi dati per testare il matching su silk', ) bgg_dataset = DatasetFactory( source=bgg_source, name='boardgamegeek-games', url='http://boardgamegeek.com', description='Lista di boardgames presi da boardgamegeek', download='https://dl.dropbox.com/u/3435878/boardgamegeek.csv', curator='Stefano Parmesan', license='All rights reserved', other_meta='{}', ) bgt_dataset = DatasetFactory( source=bgt_source, name='boardgametournament-games', url='http://boardgametournament.com', description='Lista di boardgames presi da boardgametournament', download='https://dl.dropbox.com/u/3435878/' 'boardgametournament.csv', curator='Stefano Parmesan', license='All rights reserved', other_meta='{}', ) DatasetFactory( source=bgt_source, name='boardgametournament-games-xls', url='http://boardgametournament.com', description='Lista di boardgames presi da boardgametournament', download='https://dl.dropbox.com/u/3435878/' 'boardgametournament.xls', curator='Stefano Parmesan', license='All rights reserved', encoding="utf8", other_meta='{}', ) DatasetFactory( source=bgt_source, name='boardgametournament-games-xlsx', url='http://boardgametournament.com', description='Lista di boardgames presi da boardgametournament', download='https://dl.dropbox.com/u/3435878/' 'boardgametournament.xlsx', curator='Stefano Parmesan', license='All rights reserved', encoding="utf8", other_meta='{}', ) with open(get_test_file('boardgamegeek_refine_rules.json')) as fin: rule = ''.join(fin.readlines()) bgg_archiveitem = ArchiveItemFactory( dataset=bgg_dataset, file_target='boardgamegeek.csv', file_hash='ea6ee15e9b052171db4f96743aa11425', rule=RuleFactory( hash="ea6ee15e9b052171db4f96743aa11425", rule=rule, ) ) with open(get_test_file('boardgametournament_refine_rules.json')) \ as fin: rule = ''.join(fin.readlines()) bgt_archiveitem = ArchiveItemFactory( dataset=bgt_dataset, file_target='boardgametournament.csv', file_hash='be864f716b6a7716f3b1c2254f4f5eea', rule=RuleFactory( hash="be864f716b6a7716f3b1c2254f4f5eea", rule=rule, ) ) with open(get_test_file('boardgames_aggregator_silk_rules.xml')) \ as fin: rule = ''.join(fin.readlines()) aggregator = AggregatorFactory( name='BoardGames', description='Un dataset di giochi da tavolo', silk_rule=rule, entity_type='{}BoardGame'.format( settings.TRIPLE_DATABASE['PREFIXES']['sdv1'] ), vertex_selector="g.V('type', 'sd$BoardGame')%limit.id.fill(m)", ) for archiveitem in (bgg_archiveitem, bgt_archiveitem): AggregatorArchiveItem.objects.create( aggregator=aggregator, archiveitem=archiveitem, ) osm_source = SourceFactory( name='OSM (test)', description='pochi dati per testare lo slicer' ) osm_dataset = DatasetFactory( source=osm_source, name='osm-dataset', url='http://openstreetmap.org', download='https://dl.dropbox.com/u/781790/osm-10nodes.csv', curator='Davide setti', license='CC PUCCI', ) with open(get_test_file('osm-refine-rules.json')) as fin: rule = ''.join(fin.readlines()) osm_archiveitem = ArchiveItemFactory( dataset=osm_dataset, file_target='osm-10nodes.csv', file_hash='e6f4a5c5f5fe12765f7b3ca04ab7a82d', rule=RuleFactory( hash="e6f4a5c5f5fe12765f7b3ca04ab7a82d", rule=rule, ) ) poi_aggregator = AggregatorFactory( name='POI', description='POI aggregator', entity_type=settings.TRIPLE_DATABASE['PREFIXES']['sdv1'] + 'POI', vertex_selector="g.V('type', 'sd$POI')%limit.id.fill(m)" ) AggregatorArchiveItem.objects.create( aggregator=poi_aggregator, archiveitem=osm_archiveitem, ) DatasetFactory( source=osm_source, name='Aeroporti', url='http://dati.trentino.it', description='Aeroporti del trentino, file SHP', download='http://testserver/aeroporti_tn.zip', curator='Federico Scrinzi', license='Open Data', other_meta='{}', ) strange_source = SourceFactory( name='Strange or malformed (test)', description='pochi dati con valori strani tipo None', ) DatasetFactory( source=strange_source, name='strange symbols', url='http://testserver/', description='Some strange symbols', download='http://testserver/strangesymbols.csv', curator='Federico Scrinzi', license='Open Data', other_meta='{}', ) DatasetFactory( source=strange_source, name='looks like ascii', url='http://testserver/', description="file that looks like ascii but it's UTF8", download='http://testserver/lookslikeascii.csv', curator='Federico Scrinzi', license='Open Data', other_meta='{}', )