def index(ticket):
ticket['state'] = 'populating_index'
ticket.save()
# Make sure the parsed content is in the cache
download_cache_directory = config['download_cache_directory']
in_path = os.path.join(download_cache_directory, ticket['data_json'])
if not os.path.exists(in_path):
ticket.fail('Parsed content for %s not found' % in_path)
return
data = open(in_path).read()
if len(data) < 1:
raise Exception('The parsed data in this ticket is empty.' )
# TODO check for metadata section to update collection from this?
owner = bibserver.dao.Account.get(ticket['owner'])
importer = Importer(owner=owner)
collection = {
'label': ticket['collection'],
'collection': util.slugify(ticket['collection']),
'description': ticket.get('description'),
'source': ticket['source_url'],
'format': ticket['format'],
'license': ticket.get('license', u"Not specified"),
}
collection, records = importer.upload(open(in_path), collection)
ticket['state'] = 'done'
ticket.save()
def test_upload(self):
owner = dao.Account(id='testaccount1')
owner.save()
i = Importer(owner=owner)
data = open('test/data/sample.bibtex.bibjson')
collection_in = {'label': u'My Test Collection'}
coll, records = i.upload(data, collection_in)
assert coll.id
assert owner.collections[0].id == coll.id, owner.collections
assert len(records) == 1, records
recid = records[0]['_id']
out = bibserver.dao.Record.get(recid)
assert out["year"] == '2008', out
assert out['collection'] == coll['collection']
# now try uploading exactly the same data again
data = open('test/data/sample.bibtex.bibjson')
newcoll, records = i.upload(data, collection_in)
# still should have only one collection
assert len(owner.collections) == 1
assert newcoll.id == coll.id
assert len(records) == 1
assert records[0]['collection'] == coll['collection']
# still should have only one record in it
recs_for_collection = dao.Record.query('collection:"' +
coll['collection'] + '"')
assert recs_for_collection['hits']['total'] == 1, recs_for_collection
def test_upload(self):
owner = dao.Account(id='testaccount1')
owner.save()
i = Importer(owner=owner)
bibtex = open('test/data/sample.bibtex')
format_ = 'bibtex'
collection_in = {
'label': u'My Test Collection'
}
coll, records = i.upload(bibtex, format_, collection_in)
assert coll.id
assert owner.collections[0].id == coll.id, owner.collections
assert len(records) == 1, records
recid = records[0]['id']
out = bibserver.dao.Record.get(recid)
assert out["year"] == '2008', out
assert out['collection'] == coll['id']
# now try uploading exactly the same data again
bibtex = open('test/data/sample.bibtex')
newcoll, records = i.upload(bibtex, format_, collection_in)
# still should have only one collection
assert len(owner.collections) == 1
assert newcoll.id == coll.id
assert len(records) == 1
assert records[0]['collection'] == coll.id
# still should have only one record in it
recs_for_collection = dao.Record.query('collection:"' + coll.id + '"')
assert recs_for_collection['hits']['total'] == 1, recs_for_collection
def index(ticket):
ticket['state'] = 'populating_index'
ticket.save()
# Make sure the parsed content is in the cache
download_cache_directory = config['download_cache_directory']
in_path = os.path.join(download_cache_directory, ticket['data_json'])
if not os.path.exists(in_path):
ticket.fail('Parsed content for %s not found' % in_path)
return
data = open(in_path).read()
if len(data) < 1:
raise Exception('The parsed data in this ticket is empty.')
# TODO check for metadata section to update collection from this?
owner = bibserver.dao.Account.get(ticket['owner'])
importer = Importer(owner=owner)
collection = {
'label': ticket['collection'],
'collection': util.slugify(ticket['collection']),
'description': ticket.get('description'),
'source': ticket['source_url'],
'format': ticket['format'],
'license': ticket.get('license', u"Not specified"),
}
collection, records = importer.upload(open(in_path), collection)
ticket['state'] = 'done'
ticket.save()
def test_upload(self):
i = Importer()
bibtex = open('test/data/sample.bibtex').read()
pkg = {
'format': 'bibtex',
'data': bibtex
}
records = i.upload(pkg)
recid = records[1]['id']
out = bibserver.dao.Record.get(recid)
assert out['year'] == '2008', out
def test_bulkupload(self):
i = Importer()
colls = open('test/data/bulk_upload.json').read()
toupload = json.loads(colls)
# fix upload collection relative to test
toupload["collections"][0]["source"] = 'file://' + os.path.join(os.path.dirname(__file__), toupload["collections"][0]["source"])
toupload["collections"][1]["source"] = 'file://' + os.path.join(os.path.dirname(__file__), toupload["collections"][1]["source"])
# do the bulk upload
records = i.bulk_upload(toupload)
assert records == True, records
def test_bulkupload(self):
owner = dao.Account(id='testaccount2')
owner.save()
i = Importer(owner=owner)
colls = open('test/data/bulk_upload.json').read()
toupload = json.loads(colls)
# fix upload collection relative to test
toupload["collections"][0]["source"] = 'file://' + os.path.join(os.path.dirname(__file__), toupload["collections"][0]["source"])
toupload["collections"][1]["source"] = 'file://' + os.path.join(os.path.dirname(__file__), toupload["collections"][1]["source"])
# TODO: a better test
# do the bulk upload
records = i.bulk_upload(toupload)
assert records == True, records
