def test_it_works_for_datasets(self):
obj = DatasetFactory()
task = process_dataset.delay(obj)
response = self.client.get('/s/task/{}/'.format(task.id))
self.assertContains(response, 'Processing dataset {}'.format(obj))
def test_it_filters_on_aggregator(self):
scheduler = AggregatorSchedulerFactory()
DatasetSchedulerFactory(object_id=DatasetFactory(name='NOT ME').pk)
response = self.client.get(
'/s/scheduler/?model=aggregator&pk={}'.format(scheduler.object_id))
self.assertContains(response, scheduler.content_object.name)
self.assertNotContains(response, 'NOT ME')
def test_it_logs_using_redis_logger(self):
obj = DatasetFactory()
loggy = MagicMock()
with patch('webui.scheduler.tasks.get_redis_logger',
return_value=loggy):
process_dataset.delay(obj)
self.assert_(loggy.info.called)
def test_it_filters_on_source(self):
scheduler = DatasetSchedulerFactory()
DatasetSchedulerFactory(object_id=DatasetFactory(name='NOT ME').pk)
source = scheduler.content_object.source
response = self.client.get('/s/scheduler/?model=source&pk={}'.format(
source.pk))
self.assertContains(response, scheduler.content_object.name)
self.assertNotContains(response, 'NOT ME')
def test_it_filters_on_dataset(self):
scheduler = DatasetSchedulerFactory()
DatasetSchedulerFactory(object_id=DatasetFactory(name='NOT ME').pk)
dataset = scheduler.content_object
response = self.client.get('/s/scheduler/?model=dataset&pk={}'.format(
dataset.pk))
self.assertContains(response, dataset.name)
self.assertNotContains(response, 'NOT ME')
def test_dataset_subtasks_log_using_the_same_redis_key(self):
source = SourceFactory(name='boo name')
source.datasets.add(DatasetFactory(
name='wow name'
))
loggy = MagicMock()
with patch('webui.scheduler.tasks.get_redis_logger',
return_value=loggy) as get_redis_logger:
task = process_source.delay(source)
#TODO[vad]: this is broken! assert_calls is not a magick method
get_redis_logger.assert_calls([task.id, task.id])
class DatasetSchedulerFactory(SchedulerFactory):
""" factory for creating a Scheduler executed on a dataset
"""
object_id = factory.LazyAttribute(lambda x: DatasetFactory().pk)
content_type = factory.LazyAttribute(
lambda x: ContentType.objects.get_for_model(Dataset))
def setUp(self):
self.dataset = Dataset.objects.all()[0]
self.update_url = self.dataset.get_absolute_url() + 'edit/'
self.post_data = DatasetFactory.attributes()
def setUp(self):
self.source = Source.objects.all()[0]
self.create_url = self.source.get_absolute_url() + 'dataset/create/'
self.post_data = DatasetFactory.attributes()
def setUp(self):
self.dataset = Dataset.objects.all()[0]
self.update_url = self.dataset.get_absolute_url() + 'edit/'
self.post_data = DatasetFactory.attributes()
def setUp(self):
self.source = Source.objects.all()[0]
self.create_url = self.source.get_absolute_url() + 'dataset/create/'
self.post_data = DatasetFactory.attributes()
def handle(self, *args, **options):
"""
entry point
"""
from django.contrib.sites.models import Site
site = Site.objects.get()
site.name = 'controller'
site.domain = 'localhost:8001'
site.save()
AdminFactory()
trentinocultura = SourceFactory(
name='trentinocultura',
description='description description',
scraper_name='trentinocultura',
scraper_api_key='',
)
DatasetFactory(
source=trentinocultura,
name='Trentinocultura agenda',
url='http://www.trentinocultura.net/asp_cat/main.asp?IDProspettiva'
'=35&SearchType=AGENDA_SEARCH&Pag=%d&TipoVista=AGENDA&cmd=new',
description='Eventi in Trentino',
download='scraper:trentinocultura:trentinocultura',
curator='Federico Scrinzi',
license='All rights reserved',
other_meta='{}',
bounding_box='10.3817591116112,45.6730626059259,'
'12.4775685651704,47.0917759206089',
)
ingiro_local = SourceFactory(
name='in-giro (locale)',
description='i dati dello scraper di in-giro, ma in locale (file '
'webui.scheduler.tests.data/in-giro.zip) utile per'
'test, ma anche per boh, altro? =)',
)
DatasetFactory(
source=ingiro_local,
name='eventi-e-poi-ingiro',
url='http://in-giro.net',
description='Eventi e POI presi da in-giro',
download='http://testserver/in-giro.zip',
curator='Ciccio Pasticcio',
license='All rights reserved',
other_meta='{}',
)
bgg_source = SourceFactory(
name='BoardGameGeek (test)',
description='pochi dati per testare il matching su silk'
)
bgt_source = SourceFactory(
name='BoardGameTournament (test)',
description='pochi dati per testare il matching su silk',
)
bgg_dataset = DatasetFactory(
source=bgg_source,
name='boardgamegeek-games',
url='http://boardgamegeek.com',
description='Lista di boardgames presi da boardgamegeek',
download='https://dl.dropbox.com/u/3435878/boardgamegeek.csv',
curator='Stefano Parmesan',
license='All rights reserved',
other_meta='{}',
)
bgt_dataset = DatasetFactory(
source=bgt_source,
name='boardgametournament-games',
url='http://boardgametournament.com',
description='Lista di boardgames presi da boardgametournament',
download='https://dl.dropbox.com/u/3435878/'
'boardgametournament.csv',
curator='Stefano Parmesan',
license='All rights reserved',
other_meta='{}',
)
DatasetFactory(
source=bgt_source,
name='boardgametournament-games-xls',
url='http://boardgametournament.com',
description='Lista di boardgames presi da boardgametournament',
download='https://dl.dropbox.com/u/3435878/'
'boardgametournament.xls',
curator='Stefano Parmesan',
license='All rights reserved',
encoding="utf8",
other_meta='{}',
)
DatasetFactory(
source=bgt_source,
name='boardgametournament-games-xlsx',
url='http://boardgametournament.com',
description='Lista di boardgames presi da boardgametournament',
download='https://dl.dropbox.com/u/3435878/'
'boardgametournament.xlsx',
curator='Stefano Parmesan',
license='All rights reserved',
encoding="utf8",
other_meta='{}',
)
with open(get_test_file('boardgamegeek_refine_rules.json')) as fin:
rule = ''.join(fin.readlines())
bgg_archiveitem = ArchiveItemFactory(
dataset=bgg_dataset,
file_target='boardgamegeek.csv',
file_hash='ea6ee15e9b052171db4f96743aa11425',
rule=RuleFactory(
hash="ea6ee15e9b052171db4f96743aa11425",
rule=rule,
)
)
with open(get_test_file('boardgametournament_refine_rules.json')) \
as fin:
rule = ''.join(fin.readlines())
bgt_archiveitem = ArchiveItemFactory(
dataset=bgt_dataset,
file_target='boardgametournament.csv',
file_hash='be864f716b6a7716f3b1c2254f4f5eea',
rule=RuleFactory(
hash="be864f716b6a7716f3b1c2254f4f5eea",
rule=rule,
)
)
with open(get_test_file('boardgames_aggregator_silk_rules.xml')) \
as fin:
rule = ''.join(fin.readlines())
aggregator = AggregatorFactory(
name='BoardGames',
description='Un dataset di giochi da tavolo',
silk_rule=rule,
entity_type='{}BoardGame'.format(
settings.TRIPLE_DATABASE['PREFIXES']['sdv1']
),
vertex_selector="g.V('type', 'sd$BoardGame')%limit.id.fill(m)",
)
for archiveitem in (bgg_archiveitem, bgt_archiveitem):
AggregatorArchiveItem.objects.create(
aggregator=aggregator,
archiveitem=archiveitem,
)
osm_source = SourceFactory(
name='OSM (test)',
description='pochi dati per testare lo slicer'
)
osm_dataset = DatasetFactory(
source=osm_source,
name='osm-dataset',
url='http://openstreetmap.org',
download='https://dl.dropbox.com/u/781790/osm-10nodes.csv',
curator='Davide setti',
license='CC PUCCI',
)
with open(get_test_file('osm-refine-rules.json')) as fin:
rule = ''.join(fin.readlines())
osm_archiveitem = ArchiveItemFactory(
dataset=osm_dataset,
file_target='osm-10nodes.csv',
file_hash='e6f4a5c5f5fe12765f7b3ca04ab7a82d',
rule=RuleFactory(
hash="e6f4a5c5f5fe12765f7b3ca04ab7a82d",
rule=rule,
)
)
poi_aggregator = AggregatorFactory(
name='POI',
description='POI aggregator',
entity_type=settings.TRIPLE_DATABASE['PREFIXES']['sdv1'] + 'POI',
vertex_selector="g.V('type', 'sd$POI')%limit.id.fill(m)"
)
AggregatorArchiveItem.objects.create(
aggregator=poi_aggregator,
archiveitem=osm_archiveitem,
)
DatasetFactory(
source=osm_source,
name='Aeroporti',
url='http://dati.trentino.it',
description='Aeroporti del trentino, file SHP',
download='http://testserver/aeroporti_tn.zip',
curator='Federico Scrinzi',
license='Open Data',
other_meta='{}',
)
strange_source = SourceFactory(
name='Strange or malformed (test)',
description='pochi dati con valori strani tipo None',
)
DatasetFactory(
source=strange_source,
name='strange symbols',
url='http://testserver/',
description='Some strange symbols',
download='http://testserver/strangesymbols.csv',
curator='Federico Scrinzi',
license='Open Data',
other_meta='{}',
)
DatasetFactory(
source=strange_source,
name='looks like ascii',
url='http://testserver/',
description="file that looks like ascii but it's UTF8",
download='http://testserver/lookslikeascii.csv',
curator='Federico Scrinzi',
license='Open Data',
other_meta='{}',
)