def _get_downloaded_file(filename):
""" given a test filename, return the file_meta dictionary needed by
the workflow. Used for tests.
"""
if filename:
from util import get_test_file
csv_file = get_test_file(filename, 'scheduler')
magic_type = util.MS.file(csv_file)
return {
'out_file': csv_file,
'file_name': filename,
'file_size': os.path.getsize(csv_file),
'content_type': 'text/csv',
'md5sum': md5_for_file(csv_file),
'magic_type': util.magic_to_mime(magic_type),
}
print
print '*' * 80
print "WARNING"
print '*' * 80
print "When testing the workflow, you should set what to do with " \
"each source you're dealing with. Check " \
"webui.scheduler.test_helpers.py"
print '*' * 80
print
return {}
def _get_downloaded_file(filename):
""" given a test filename, return the file_meta dictionary needed by
the workflow. Used for tests.
"""
if filename:
from util import get_test_file
csv_file = get_test_file(
filename, 'scheduler'
)
magic_type = util.MS.file(csv_file)
return {
'out_file': csv_file,
'file_name': filename,
'file_size': os.path.getsize(csv_file),
'content_type': 'text/csv',
'md5sum': md5_for_file(csv_file),
'magic_type': util.magic_to_mime(magic_type),
}
print
print '*' * 80
print "WARNING"
print '*' * 80
print "When testing the workflow, you should set what to do with " \
"each source you're dealing with. Check " \
"webui.scheduler.test_helpers.py"
print '*' * 80
print
return {}
def rdflib_graph_parse_mock(self, source=None, *args, **kwargs):
from util import get_test_file
if source.startswith('http'):
local_source = re.sub('\W+', '_', source)
source = get_test_file(local_source, app_name='cnmain')
return rdflib_graph_parse_mock._original(self, source, *args, **kwargs)
def rdflib_graph_parse_mock(self, source=None, *args, **kwargs):
from util import get_test_file
if source.startswith('http'):
local_source = re.sub('\W+', '_', source)
source = get_test_file(local_source, app_name='cnmain')
return rdflib_graph_parse_mock._original(self, source, *args, **kwargs)
def _get_test_file(filename, app_name='cnmain'):
""" utility function around get_test_file """
from util import get_test_file
return get_test_file(filename, app_name)
def _get_test_file(filename, app_name='cnmain'):
""" utility function around get_test_file """
from util import get_test_file
return get_test_file(filename, app_name)
def handle(self, *args, **options):
"""
entry point
"""
from django.contrib.sites.models import Site
site = Site.objects.get()
site.name = 'controller'
site.domain = 'localhost:8001'
site.save()
AdminFactory()
trentinocultura = SourceFactory(
name='trentinocultura',
description='description description',
scraper_name='trentinocultura',
scraper_api_key='',
)
DatasetFactory(
source=trentinocultura,
name='Trentinocultura agenda',
url='http://www.trentinocultura.net/asp_cat/main.asp?IDProspettiva'
'=35&SearchType=AGENDA_SEARCH&Pag=%d&TipoVista=AGENDA&cmd=new',
description='Eventi in Trentino',
download='scraper:trentinocultura:trentinocultura',
curator='Federico Scrinzi',
license='All rights reserved',
other_meta='{}',
bounding_box='10.3817591116112,45.6730626059259,'
'12.4775685651704,47.0917759206089',
)
ingiro_local = SourceFactory(
name='in-giro (locale)',
description='i dati dello scraper di in-giro, ma in locale (file '
'webui.scheduler.tests.data/in-giro.zip) utile per'
'test, ma anche per boh, altro? =)',
)
DatasetFactory(
source=ingiro_local,
name='eventi-e-poi-ingiro',
url='http://in-giro.net',
description='Eventi e POI presi da in-giro',
download='http://testserver/in-giro.zip',
curator='Ciccio Pasticcio',
license='All rights reserved',
other_meta='{}',
)
bgg_source = SourceFactory(
name='BoardGameGeek (test)',
description='pochi dati per testare il matching su silk'
)
bgt_source = SourceFactory(
name='BoardGameTournament (test)',
description='pochi dati per testare il matching su silk',
)
bgg_dataset = DatasetFactory(
source=bgg_source,
name='boardgamegeek-games',
url='http://boardgamegeek.com',
description='Lista di boardgames presi da boardgamegeek',
download='https://dl.dropbox.com/u/3435878/boardgamegeek.csv',
curator='Stefano Parmesan',
license='All rights reserved',
other_meta='{}',
)
bgt_dataset = DatasetFactory(
source=bgt_source,
name='boardgametournament-games',
url='http://boardgametournament.com',
description='Lista di boardgames presi da boardgametournament',
download='https://dl.dropbox.com/u/3435878/'
'boardgametournament.csv',
curator='Stefano Parmesan',
license='All rights reserved',
other_meta='{}',
)
DatasetFactory(
source=bgt_source,
name='boardgametournament-games-xls',
url='http://boardgametournament.com',
description='Lista di boardgames presi da boardgametournament',
download='https://dl.dropbox.com/u/3435878/'
'boardgametournament.xls',
curator='Stefano Parmesan',
license='All rights reserved',
encoding="utf8",
other_meta='{}',
)
DatasetFactory(
source=bgt_source,
name='boardgametournament-games-xlsx',
url='http://boardgametournament.com',
description='Lista di boardgames presi da boardgametournament',
download='https://dl.dropbox.com/u/3435878/'
'boardgametournament.xlsx',
curator='Stefano Parmesan',
license='All rights reserved',
encoding="utf8",
other_meta='{}',
)
with open(get_test_file('boardgamegeek_refine_rules.json')) as fin:
rule = ''.join(fin.readlines())
bgg_archiveitem = ArchiveItemFactory(
dataset=bgg_dataset,
file_target='boardgamegeek.csv',
file_hash='ea6ee15e9b052171db4f96743aa11425',
rule=RuleFactory(
hash="ea6ee15e9b052171db4f96743aa11425",
rule=rule,
)
)
with open(get_test_file('boardgametournament_refine_rules.json')) \
as fin:
rule = ''.join(fin.readlines())
bgt_archiveitem = ArchiveItemFactory(
dataset=bgt_dataset,
file_target='boardgametournament.csv',
file_hash='be864f716b6a7716f3b1c2254f4f5eea',
rule=RuleFactory(
hash="be864f716b6a7716f3b1c2254f4f5eea",
rule=rule,
)
)
with open(get_test_file('boardgames_aggregator_silk_rules.xml')) \
as fin:
rule = ''.join(fin.readlines())
aggregator = AggregatorFactory(
name='BoardGames',
description='Un dataset di giochi da tavolo',
silk_rule=rule,
entity_type='{}BoardGame'.format(
settings.TRIPLE_DATABASE['PREFIXES']['sdv1']
),
vertex_selector="g.V('type', 'sd$BoardGame')%limit.id.fill(m)",
)
for archiveitem in (bgg_archiveitem, bgt_archiveitem):
AggregatorArchiveItem.objects.create(
aggregator=aggregator,
archiveitem=archiveitem,
)
osm_source = SourceFactory(
name='OSM (test)',
description='pochi dati per testare lo slicer'
)
osm_dataset = DatasetFactory(
source=osm_source,
name='osm-dataset',
url='http://openstreetmap.org',
download='https://dl.dropbox.com/u/781790/osm-10nodes.csv',
curator='Davide setti',
license='CC PUCCI',
)
with open(get_test_file('osm-refine-rules.json')) as fin:
rule = ''.join(fin.readlines())
osm_archiveitem = ArchiveItemFactory(
dataset=osm_dataset,
file_target='osm-10nodes.csv',
file_hash='e6f4a5c5f5fe12765f7b3ca04ab7a82d',
rule=RuleFactory(
hash="e6f4a5c5f5fe12765f7b3ca04ab7a82d",
rule=rule,
)
)
poi_aggregator = AggregatorFactory(
name='POI',
description='POI aggregator',
entity_type=settings.TRIPLE_DATABASE['PREFIXES']['sdv1'] + 'POI',
vertex_selector="g.V('type', 'sd$POI')%limit.id.fill(m)"
)
AggregatorArchiveItem.objects.create(
aggregator=poi_aggregator,
archiveitem=osm_archiveitem,
)
DatasetFactory(
source=osm_source,
name='Aeroporti',
url='http://dati.trentino.it',
description='Aeroporti del trentino, file SHP',
download='http://testserver/aeroporti_tn.zip',
curator='Federico Scrinzi',
license='Open Data',
other_meta='{}',
)
strange_source = SourceFactory(
name='Strange or malformed (test)',
description='pochi dati con valori strani tipo None',
)
DatasetFactory(
source=strange_source,
name='strange symbols',
url='http://testserver/',
description='Some strange symbols',
download='http://testserver/strangesymbols.csv',
curator='Federico Scrinzi',
license='Open Data',
other_meta='{}',
)
DatasetFactory(
source=strange_source,
name='looks like ascii',
url='http://testserver/',
description="file that looks like ascii but it's UTF8",
download='http://testserver/lookslikeascii.csv',
curator='Federico Scrinzi',
license='Open Data',
other_meta='{}',
)