def post(self):
if not auth.collection.create(current_user, None):
abort(401)
try:
if not request.values.get('collection',None):
flash('You need to provide a collection name.')
return redirect('/upload')
if not request.values.get('source',None):
if not request.files.get('upfile',None):
if not request.json:
flash('You need to provide a source URL or an upload file.')
return redirect('/upload')
collection = request.values.get('collection')
format=request.values.get('format')
if request.files.get('upfile'):
fileobj = request.files.get('upfile')
if not format:
format = bibserver.importer.findformat(fileobj.filename)
else:
if not format:
format = bibserver.importer.findformat( request.values.get("source").strip('"') )
ticket = bibserver.ingest.IngestTicket(owner=current_user.id,
source_url=request.values.get("source"),
format=format,
collection=request.values.get('collection'),
description=request.values.get('description'),
)
# Allow only parsing
only_parse = request.values.get('only_parse')
if only_parse:
ticket['only_parse'] = True
license = request.values.get('license')
if license: ticket['license'] = license
# If the user is uploading a file, update the ticket with the 'downloaded' file
# And correct source
if request.files.get('upfile'):
data = fileobj.read()
ticket['data_md5'] = bibserver.ingest.store_data_in_cache(data)
ticket['source_url'] = config.get('SITE_URL','') + '/ticket/%s/data' % ticket.id
ticket['state'] = 'downloaded'
# if user is sending JSON, update the ticket with the received JSON
if request.json:
data = request.json
ticket['data_md5'] = bibserver.ingest.store_data_in_cache(json.dumps(data))
ticket['source_url'] = config.get('SITE_URL','') + '/ticket/%s/data' % ticket.id
ticket['state'] = 'downloaded'
ticket.save()
except Exception, inst:
msg = str(inst)
if app.debug or app.config['TESTING']:
raise
flash('error: ' + msg)
return render_template('upload.html')
def init():
for d in ('download_cache_directory', 'parserscrapers_plugin_directory'):
dd = config.get(d)
if not os.path.exists(dd):
os.mkdir(dd)
# Scan for available parser/scraper plugins
parserscrapers_plugin_directory = config.get('parserscrapers_plugin_directory')
if not parserscrapers_plugin_directory:
sys.stderr.write('Error: parserscrapers_plugin_directory config entry not found\n')
plugins = scan_parserscrapers(parserscrapers_plugin_directory)
if plugins:
for ps in plugins:
PLUGINS[ps['format']] = ps
filename = os.path.join(config.get('parserscrapers_plugin_directory'), 'plugins.json')
open(filename, 'w').write(json.dumps(PLUGINS))
def init():
for d in ('download_cache_directory', 'parserscrapers_plugin_directory'):
dd = config.get(d)
if not os.path.exists(dd):
os.mkdir(dd)
# Scan for available parser/scraper plugins
parserscrapers_plugin_directory = config.get(
'parserscrapers_plugin_directory')
if not parserscrapers_plugin_directory:
sys.stderr.write(
'Error: parserscrapers_plugin_directory config entry not found\n')
plugins = scan_parserscrapers(parserscrapers_plugin_directory)
if plugins:
for ps in plugins:
PLUGINS[ps['format']] = ps
filename = os.path.join(config.get('parserscrapers_plugin_directory'),
'plugins.json')
open(filename, 'w').write(json.dumps(PLUGINS))
def post(self):
if not auth.collection.create(current_user, None):
abort(401)
try:
if not request.values.get('collection', None):
flash('You need to provide a collection name.')
return redirect('/upload')
if not request.values.get('source', None):
if not request.files.get('upfile', None):
if not request.json:
flash(
'You need to provide a source URL or an upload file.'
)
return redirect('/upload')
collection = request.values.get('collection')
format = request.values.get('format')
if request.files.get('upfile'):
fileobj = request.files.get('upfile')
if not format:
format = bibserver.importer.findformat(fileobj.filename)
else:
if not format:
format = bibserver.importer.findformat(
request.values.get("source").strip('"'))
ticket = bibserver.ingest.IngestTicket(
owner=current_user.id,
source_url=request.values.get("source"),
format=format,
collection=request.values.get('collection'),
description=request.values.get('description'),
)
# Allow only parsing
only_parse = request.values.get('only_parse')
if only_parse:
ticket['only_parse'] = True
license = request.values.get('license')
if license: ticket['license'] = license
# If the user is uploading a file, update the ticket with the 'downloaded' file
# And correct source
if request.files.get('upfile'):
data = fileobj.read()
ticket['data_md5'] = bibserver.ingest.store_data_in_cache(data)
ticket['source_url'] = config.get(
'SITE_URL', '') + '/ticket/%s/data' % ticket.id
ticket['state'] = 'downloaded'
# if user is sending JSON, update the ticket with the received JSON
if request.json:
data = request.json
ticket['data_md5'] = bibserver.ingest.store_data_in_cache(
json.dumps(data))
ticket['source_url'] = config.get(
'SITE_URL', '') + '/ticket/%s/data' % ticket.id
ticket['state'] = 'downloaded'
ticket.save()
except Exception, inst:
msg = str(inst)
if app.debug or app.config['TESTING']:
raise
flash('error: ' + msg)
return render_template('upload.html')
def get_plugins():
filename = os.path.join(config.get('parserscrapers_plugin_directory'), 'plugins.json')
return json.loads(open(filename).read())
def get_plugins():
filename = os.path.join(config.get('parserscrapers_plugin_directory'),
'plugins.json')
return json.loads(open(filename).read())
