def harvest_aps(from_param, until_param, perpage):
"""
Performs a request to APS API servers retrieving JSON.
"""
page = 1
last_page = 1
conn = APS_connect(from_param, until_param, page, perpage)
if not conn:
write_message("Fatal Error: Cannot reach APS servers. Aborting.")
raise APSHarvesterConnectionError("Cannot connect to APS servers")
if conn.headers['link']:
links = conn.headers['link'].split(",")
for l in links:
if l.find('rel="next"') > 0:
next_page = int(re.search('(?<=(page=))\w+', l).group(0))
if l.find('rel="last"') > 0:
last_page = int(re.search('(?<=(page=))\w+', l).group(0))
# Fetch first page of data
data = json.loads(conn.next())
write_message("Data received from APS: \n%s" % (data,), verbose=5)
records = []
for d in data:
records.append(APSRecord(None, d["doi"], last_modified=d['last_modified_at']))
# Check for more pages
if last_page > 1:
for pagenum in range(next_page, last_page+1):
conn = APS_connect(from_param, until_param, pagenum, perpage)
data = json.loads(conn.next())
write_message("Data received from APS: \n%s" % (data,), verbose=5)
for d in data:
records.append(APSRecord(None, d["doi"], last_modified=d['last_modified_at']))
return records
def get_redirection_data(label):
"""
Returns all information about a given redirection identified by label.
@param label: the label identifying the redirection
@type label: string
@returns: a dictionary with the following keys:
* label: the label
* plugin: the name of the plugin
* parameters: the parameters that are passed to the plugin
(deserialized from JSON)
* creation_date: datetime object on when the redirection was first
created.
* modification_date: datetime object on when the redirection was
last modified.
@rtype: dict
@raises ValueError: in case the label does not exist.
"""
res = run_sql(
"SELECT label, plugin, parameters, creation_date, modification_date FROM goto WHERE label=%s",
(label, ))
if res:
return {
'label': res[0][0],
'plugin': CFG_GOTO_PLUGINS[res[0][1]],
'parameters': json_unicode_to_utf8(json.loads(res[0][2])),
'creation_date': res[0][3],
'modification_date': res[0][4]
}
else:
raise ValueError("%s label does not exist" % label)
def get_kb_mappings_embedded_json(kb_name="",
key="",
value="",
match_type="s",
limit=None):
"""Get leftside/rightside mappings from kb kb_name formatted as json dict.
The rightside is actually considered as a json string and hence embedded
within the final result.
If key given, give only those with left side (mapFrom) = key.
If value given, give only those with right side (mapTo) = value.
@param kb_name: the name of the kb
@param key: include only lines matching this on left side in the results
@param value: include only lines matching this on right side in the results
@param match_type: s = substring match, e = exact match
@param limit: maximum number of results to return (are ALL if set to None)
@return a list of mappings
"""
mappings = get_kb_mappings(kb_name, key, value, match_type)
ret = []
if limit is None:
limit = len(mappings)
for m in mappings[:limit]:
label = m['value'] or m['key']
value = m['key'] or m['value']
ret.append({'label': label, 'value': json.loads(value)})
return json.dumps(ret)
def test_insert_via_curl(self):
"""batchuploader - robotupload insert via CLI curl"""
curl_input_file = os.path.join(CFG_TMPDIR, 'curl_test.xml')
open(curl_input_file, "w").write(self.marcxml)
try:
result = run_shell_command(
'/usr/bin/curl -T %s %s -A %s -H "Content-Type: application/marcxml+xml"',
[
curl_input_file, self.nonce_url,
make_user_agent_string('BatchUploader')
])[1]
self.failUnless("[INFO]" in result)
current_task = get_last_taskid()
run_shell_command("%s/bibupload %%s" % CFG_BINDIR,
[str(current_task)])
results = json.loads(
open(self.callback_result_path).read())
self.failUnless('results' in results,
'"%s" did not contained [INFO]' % result)
self.assertEqual(len(results['results']), 1)
self.assertEqual(results['nonce'], "1234")
self.failUnless(results['results'][0]['success'])
self.failUnless(results['results'][0]['recid'] > 0)
self.failUnless(
"""<subfield code="a">Doe, John</subfield>"""
in results['results'][0]['marcxml'],
results['results'][0]['marcxml'])
finally:
os.remove(curl_input_file)
def test_legacy_insert_via_curl(self):
"""batchuploader - robotupload legacy insert via CLI curl"""
curl_input_file = os.path.join(CFG_TMPDIR, "curl_test.xml")
open(curl_input_file, "w").write(self.marcxml)
try:
## curl -F '[email protected]' -F 'mode=-i' [-F 'callback_url=http://...'] [-F 'nonce=1234'] http://cds.cern.ch/batchuploader/robotupload -A invenio_webupload
code, result, err = run_shell_command(
"/usr/bin/curl -v -F file=@%s -F 'mode=-i' -F callback_url=%s -F nonce=1234 %s -A %s",
[curl_input_file, self.callback_url, self.legacy_url, make_user_agent_string("BatchUploader")],
)
self.failUnless("[INFO]" in result, "[INFO] not find in results: %s, %s" % (result, err))
current_task = get_last_taskid()
run_shell_command("%s/bibupload %%s" % CFG_BINDIR, [str(current_task)])
results = json.loads(open(self.callback_result_path).read())
self.failUnless("results" in results, '"%s" did not contained [INFO]' % result)
self.assertEqual(len(results["results"]), 1)
self.assertEqual(results["nonce"], "1234")
self.failUnless(results["results"][0]["success"])
self.failUnless(results["results"][0]["recid"] > 0)
self.failUnless(
"""<subfield code="a">Doe, John</subfield>""" in results["results"][0]["marcxml"],
results["results"][0]["marcxml"],
)
finally:
os.remove(curl_input_file)
def test_legacy_insert_via_curl(self):
"""batchuploader - robotupload legacy insert via CLI curl"""
curl_input_file = os.path.join(CFG_TMPDIR, 'curl_test.xml')
open(curl_input_file, "w").write(self.marcxml)
try:
## curl -F '[email protected]' -F 'mode=-i' [-F 'callback_url=http://...'] [-F 'nonce=1234'] http://cds.cern.ch/batchuploader/robotupload -A invenio_webupload
code, result, err = run_shell_command(
"/usr/bin/curl -v -F file=@%s -F 'mode=-i' -F callback_url=%s -F nonce=1234 %s -A %s",
[
curl_input_file, self.callback_url,
self.legacy_url,
make_user_agent_string('BatchUploader')
])
self.failUnless(
"[INFO]" in result,
'[INFO] not find in results: %s, %s' % (result, err))
current_task = get_last_taskid()
run_shell_command("%s/bibupload %%s" % CFG_BINDIR,
[str(current_task)])
results = json.loads(
open(self.callback_result_path).read())
self.failUnless('results' in results,
'"%s" did not contained [INFO]' % result)
self.assertEqual(len(results['results']), 1)
self.assertEqual(results['nonce'], "1234")
self.failUnless(results['results'][0]['success'])
self.failUnless(results['results'][0]['recid'] > 0)
self.failUnless(
"""<subfield code="a">Doe, John</subfield>"""
in results['results'][0]['marcxml'],
results['results'][0]['marcxml'])
finally:
os.remove(curl_input_file)
def get_redirection_data(label):
"""
Returns all information about a given redirection identified by label.
@param label: the label identifying the redirection
@type label: string
@returns: a dictionary with the following keys:
* label: the label
* plugin: the name of the plugin
* parameters: the parameters that are passed to the plugin
(deserialized from JSON)
* creation_date: datetime object on when the redirection was first
created.
* modification_date: datetime object on when the redirection was
last modified.
@rtype: dict
@raises ValueError: in case the label does not exist.
"""
res = run_sql("SELECT label, plugin, parameters, creation_date, modification_date FROM goto WHERE label=%s", (label, ))
if res:
return {'label': res[0][0],
'plugin': CFG_GOTO_PLUGINS[res[0][1]],
'parameters': json_unicode_to_utf8(json.loads(res[0][2])),
'creation_date': res[0][3],
'modification_date': res[0][4]}
else:
raise ValueError("%s label does not exist" % label)
def get_keywords_from_drupal():
"""
Retrieve keywords from Drupal feed
"""
from invenio.websubmit_functions.file_cacher import Cache
import json
import urllib2
# Drupal's feed
DRUPAL_FEED = "http://home.web.cern.ch/api/tags-json-feed"
def get_drupal_data():
data = json.load(urllib2.urlopen(DRUPAL_FEED))
data = map(lambda x: str(x['name'].encode('utf-8')), data['tags'])
return json.dumps(data)
try:
cached = Cache('keywords.json', expiration=5)
if(cached.expired()):
cached.write(get_drupal_data())
data = cached.read()
except:
data = get_drupal_data()
return [str(x.encode('utf-8')) for x in json.loads(data)]
def get_json_parameters_from_cli(option, dummy_opt_str, value,
dummy_parser):
try:
option.parameters = json_unicode_to_utf8(json.loads(value))
except Exception, err:
raise optparse.OptionValueError(
"Cannot parse as a valid JSON serialization the provided parameters: %s. %s"
% (value, err))
def manage(self, req, form):
""" Web interface for the management of the info space """
uid = getUid(req)
argd = wash_urlargd(form, {'ln': (str, CFG_SITE_LANG)})
# If it is an Ajax request, extract any JSON data.
ajax_request = False
if 'jsondata' in form:
json_data = json.loads(str(form['jsondata']))
json_data = json_unicode_to_utf8(json_data)
ajax_request = True
json_response = {}
# Authorization.
user_info = collect_user_info(req)
if user_info['email'] == 'guest':
# User is not logged in.
if not ajax_request:
# Do not display the introductory recID selection box to guest
# users (as it used to be with v0.99.0):
dummy_auth_code, auth_message = acc_authorize_action(
req, 'runinfomanager')
referer = '/info'
return page_not_authorized(req=req,
referer=referer,
text=auth_message)
else:
# Session has most likely timed out.
json_response.update({'status': "timeout"})
return json.dumps(json_response)
# Handle request.
if not ajax_request:
body, errors, warnings = perform_request_init_info_interface()
title = 'Info Space Manager'
return page(title=title,
body=body,
errors=errors,
warnings=warnings,
uid=uid,
language=argd['ln'],
req=req)
else:
# Handle AJAX request.
if json_data["action"] == "listFiles":
json_response.update(
perform_request_edit_file(json_data["filename"]))
try:
return json.dumps(json_response)
except UnicodeDecodeError:
# Error decoding, the file can be a pdf, image or any kind
# of file non-editable
return json.dumps({"status": "error_file_not_readable"})
if json_data["action"] == "saveContent":
return json.dumps(
perform_request_save_file(json_data["filename"],
json_data["filecontent"]))
def manage(self, req, form):
""" Web interface for the management of the info space """
uid = getUid(req)
argd = wash_urlargd(form, {'ln': (str, CFG_SITE_LANG)})
# If it is an Ajax request, extract any JSON data.
ajax_request = False
if 'jsondata' in form:
json_data = json.loads(str(form['jsondata']))
json_data = json_unicode_to_utf8(json_data)
ajax_request = True
json_response = {}
# Authorization.
user_info = collect_user_info(req)
if user_info['email'] == 'guest':
# User is not logged in.
if not ajax_request:
# Do not display the introductory recID selection box to guest
# users (as it used to be with v0.99.0):
dummy_auth_code, auth_message = acc_authorize_action(req,
'runinfomanager')
referer = '/info'
return page_not_authorized(req=req, referer=referer,
text=auth_message)
else:
# Session has most likely timed out.
json_response.update({'status': "timeout"})
return json.dumps(json_response)
# Handle request.
if not ajax_request:
body, errors, warnings = perform_request_init_info_interface()
title = 'Info Space Manager'
return page(title=title,
body=body,
errors=errors,
warnings=warnings,
uid=uid,
language=argd['ln'],
req=req)
else:
# Handle AJAX request.
if json_data["action"] == "listFiles":
json_response.update(perform_request_edit_file(json_data["filename"]))
try:
return json.dumps(json_response)
except UnicodeDecodeError:
# Error decoding, the file can be a pdf, image or any kind
# of file non-editable
return json.dumps({"status": "error_file_not_readable"})
if json_data["action"] == "saveContent":
return json.dumps(perform_request_save_file(json_data["filename"],
json_data["filecontent"]))
def json_req_profiler(self, req, form):
if "ajaxProfile" in form:
profiler = cProfile.Profile()
return_val = profiler.runcall(func, self, req, form)
results = cStringIO.StringIO()
stats = pstats.Stats(profiler, stream=results)
stats.sort_stats('cumulative')
stats.print_stats(100)
json_in = json.loads(str(form['jsondata']))
# Deunicode all strings (Invenio doesn't have unicode
# support).
json_in = json_unicode_to_utf8(json_in)
json_data = json.loads(return_val)
json_data.update({"profilerStats": "<pre style='overflow: scroll'>" + json_in['requestType'] + results.getvalue() + "</pre>"})
return json.dumps(json_data)
else:
return func(self, req, form)
def json_req_profiler(self, req, form):
if "ajaxProfile" in form:
profiler = cProfile.Profile()
return_val = profiler.runcall(func, self, req, form)
results = cStringIO.StringIO()
stats = pstats.Stats(profiler, stream=results)
stats.sort_stats('cumulative')
stats.print_stats(100)
json_in = json.loads(str(form['jsondata']))
# Deunicode all strings (Invenio doesn't have unicode
# support).
json_in = json_unicode_to_utf8(json_in)
json_data = json.loads(return_val)
json_data.update({"profilerStats": "<pre style='overflow: scroll'>" + json_in['requestType'] + results.getvalue() + "</pre>"})
return json.dumps(json_data)
else:
return func(self, req, form)
def __extract_attribute(self, req):
"""
Load from the request the given assertion, extract all the attribute
to properly login the user, and verify that the data are actually
both well formed and signed correctly.
"""
from invenio.webinterface_handler import wash_urlargd
args = wash_urlargd(
req.form, {
'assertion': (str, ''),
'robot': (str, ''),
'digest': (str, ''),
'login_method': (str, '')
})
assertion = args['assertion']
digest = args['digest']
robot = args['robot']
login_method = args['login_method']
shared_key = load_robot_keys().get(login_method, {}).get(robot)
if shared_key is None:
raise InvenioWebAccessExternalAuthError(
"A key does not exist for robot: %s, login_method: %s" %
(robot, login_method))
if not self.verify(shared_key, assertion, digest):
raise InvenioWebAccessExternalAuthError(
"The provided assertion does not validate against the digest %s for robot %s"
% (repr(digest), repr(robot)))
if self.use_zlib:
try:
## Workaround to Perl implementation that does not add
## any padding to the base64 encoding.
needed_pad = (4 - len(assertion) % 4) % 4
assertion += needed_pad * '='
assertion = decompress(base64.urlsafe_b64decode(assertion))
except:
raise InvenioWebAccessExternalAuthError(
"The provided assertion is corrupted")
data = json_unicode_to_utf8(json.loads(assertion))
if not isinstance(data, dict):
raise InvenioWebAccessExternalAuthError(
"The provided assertion is invalid")
timeout = data[self.timeout_attribute_name]
if timeout < time.time():
raise InvenioWebAccessExternalAuthError(
"The provided assertion is expired")
userip = data.get(self.userip_attribute_name)
if not self.check_user_ip or (normalize_ip(
userip, self.check_user_ip) == normalize_ip(
req.remote_ip, self.check_user_ip)):
return data
else:
raise InvenioWebAccessExternalAuthError(
"The provided assertion has been issued for a different IP address (%s instead of %s)"
% (userip, req.remote_ip))
def harvest_aps(from_param, until_param, perpage):
"""
Performs a request to APS API servers retrieving JSON.
"""
page = 1
last_page = 1
conn = APS_connect(from_param, until_param, page, perpage)
if not conn:
write_message("Fatal Error: Cannot reach APS servers. Aborting.")
raise APSHarvesterConnectionError("Cannot connect to APS servers")
if conn.headers['link']:
links = conn.headers['link'].split(",")
for l in links:
if l.find('rel="next"') > 0:
next_page = int(re.search(r'(?<=(page=))\w+', l).group(0))
if l.find('rel="last"') > 0:
last_page = int(re.search(r'(?<=(page=))\w+', l).group(0))
# Fetch first page of data
data = json.loads(conn.next())
write_message("Data received from APS: \n%s" % (data, ), verbose=5)
records = []
for d in data:
records.append(
APSRecord(None,
d["doi"],
last_modified=d['metadata_last_modified_at']))
# Check for more pages
if last_page > 1:
for pagenum in range(next_page, last_page + 1):
conn = APS_connect(from_param, until_param, pagenum, perpage)
data = json.loads(conn.next())
write_message("Data received from APS: \n%s" % (data, ), verbose=5)
for d in data:
records.append(
APSRecord(None,
d["doi"],
last_modified=d['metadata_last_modified_at']))
return records
def test_insert_with_callback(self):
"""batchuploader - robotupload insert with callback"""
result = urllib2.urlopen(self.req_callback).read()
self.failUnless("[INFO]" in result, '"%s" did not contained [INFO]' % result)
current_task = get_last_taskid()
run_shell_command("%s/bibupload %%s" % CFG_BINDIR, [str(current_task)])
results = json.loads(open(self.callback_result_path).read())
self.failUnless('results' in results)
self.assertEqual(len(results['results']), 1)
self.failUnless(results['results'][0]['success'])
self.failUnless(results['results'][0]['recid'] > 0)
self.failUnless("""<subfield code="a">Doe, John</subfield>""" in results['results'][0]['marcxml'], results['results'][0]['marcxml'])
def templates(self, req, form):
"""handle a edit/templates request"""
uid = getUid(req)
argd = wash_urlargd(form, {'ln': (str, CFG_SITE_LANG)})
# If it is an Ajax request, extract any JSON data.
ajax_request = False
if form.has_key('jsondata'):
json_data = json.loads(str(form['jsondata']))
# Deunicode all strings (Invenio doesn't have unicode
# support).
json_data = json_unicode_to_utf8(json_data)
ajax_request = True
json_response = {'resultCode': 0}
# Authorization.
user_info = collect_user_info(req)
if user_info['email'] == 'guest':
# User is not logged in.
if not ajax_request:
# Do not display the introductory recID selection box to guest
# users (as it used to be with v0.99.0):
dummy_auth_code, auth_message = acc_authorize_action(
req, 'runbibedit')
referer = '/edit'
return page_not_authorized(req=req,
referer=referer,
text=auth_message,
navtrail=navtrail)
else:
# Session has most likely timed out.
json_response.update({'resultCode': 100})
return json.dumps(json_response)
# Handle request.
if not ajax_request:
# Show BibEdit template management start page.
body, errors, warnings = perform_request_init_template_interface()
title = 'Record Editor Template Manager'
return page(title=title,
body=body,
errors=errors,
warnings=warnings,
uid=uid,
language=argd['ln'],
navtrail=navtrail_bibedit,
lastupdated=__lastupdated__,
req=req,
body_css_classes=['bibedit'])
else:
# Handle AJAX request.
json_response.update(
perform_request_ajax_template_interface(json_data))
return json.dumps(json_response)
def test_insert_with_callback(self):
"""batchuploader - robotupload insert with callback"""
result = urllib2.urlopen(self.req_callback).read()
self.failUnless("[INFO]" in result, '"%s" did not contained [INFO]' % result)
current_task = get_last_taskid()
run_shell_command("%s/bibupload %%s" % CFG_BINDIR, [str(current_task)])
results = json.loads(open(self.callback_result_path).read())
self.failUnless('results' in results)
self.assertEqual(len(results['results']), 1)
self.failUnless(results['results'][0]['success'])
self.failUnless(results['results'][0]['recid'] > 0)
self.failUnless("""<subfield code="a">Doe, John</subfield>""" in results['results'][0]['marcxml'], results['results'][0]['marcxml'])
def templates(self, req, form):
"""handle a edit/templates request"""
uid = getUid(req)
argd = wash_urlargd(form, {'ln': (str, CFG_SITE_LANG)})
# If it is an Ajax request, extract any JSON data.
ajax_request = False
if form.has_key('jsondata'):
json_data = json.loads(str(form['jsondata']))
# Deunicode all strings (Invenio doesn't have unicode
# support).
json_data = json_unicode_to_utf8(json_data)
ajax_request = True
json_response = {'resultCode': 0}
# Authorization.
user_info = collect_user_info(req)
if user_info['email'] == 'guest':
# User is not logged in.
if not ajax_request:
# Do not display the introductory recID selection box to guest
# users (as it used to be with v0.99.0):
dummy_auth_code, auth_message = acc_authorize_action(req,
'runbibedit')
referer = '/edit'
return page_not_authorized(req=req, referer=referer,
text=auth_message, navtrail=navtrail)
else:
# Session has most likely timed out.
json_response.update({'resultCode': 100})
return json.dumps(json_response)
# Handle request.
if not ajax_request:
# Show BibEdit template management start page.
body, errors, warnings = perform_request_init_template_interface()
title = 'Record Editor Template Manager'
return page(title = title,
body = body,
errors = errors,
warnings = warnings,
uid = uid,
language = argd['ln'],
navtrail = navtrail_bibedit,
lastupdated = __lastupdated__,
req = req,
body_css_classes = ['bibedit'])
else:
# Handle AJAX request.
json_response.update(perform_request_ajax_template_interface(json_data))
return json.dumps(json_response)
def test_insert_with_oracle(self):
"""batchuploader - robotupload insert with oracle special treatment"""
import os
if os.path.exists('/opt/invenio/var/log/invenio.err'):
os.remove('/opt/invenio/var/log/invenio.err')
result = urllib2.urlopen(self.req_oracle).read()
self.failUnless("[INFO]" in result, '"%s" did not contained "[INFO]"' % result)
current_task = get_last_taskid()
run_shell_command("%s/bibupload %%s" % CFG_BINDIR, [str(current_task)])
results = json.loads(open(self.callback_result_path).read())
self.failUnless('results' in results, '"%s" did not contained "results" key' % results)
self.assertEqual(len(results['results']), 1)
self.failUnless(results['results'][0]['success'])
self.failUnless(results['results'][0]['recid'] > 0)
self.failUnless("""<subfield code="a">Doe, John</subfield>""" in results['results'][0]['marcxml'], results['results'][0]['marcxml'])
def test_insert_with_oracle(self):
"""batchuploader - robotupload insert with oracle special treatment"""
import os
if os.path.exists('/opt/invenio/var/log/invenio.err'):
os.remove('/opt/invenio/var/log/invenio.err')
result = urllib2.urlopen(self.req_oracle).read()
self.failUnless("[INFO]" in result, '"%s" did not contained "[INFO]"' % result)
current_task = get_last_taskid()
run_shell_command("%s/bibupload %%s" % CFG_BINDIR, [str(current_task)])
results = json.loads(open(self.callback_result_path).read())
self.failUnless('results' in results, '"%s" did not contained "results" key' % results)
self.assertEqual(len(results['results']), 1)
self.failUnless(results['results'][0]['success'])
self.failUnless(results['results'][0]['recid'] > 0)
self.failUnless("""<subfield code="a">Doe, John</subfield>""" in results['results'][0]['marcxml'], results['results'][0]['marcxml'])
def test_insert_with_nonce(self):
"""batchuploader - robotupload insert with nonce"""
result = urllib2.urlopen(self.req_nonce).read()
self.failUnless("[INFO]" in result, '"%s" did not contained "[INFO]"' % result)
current_task = get_last_taskid()
run_shell_command("%s/bibupload %%s" % CFG_BINDIR, [str(current_task)])
results = json.loads(open(self.callback_result_path).read())
self.failUnless("results" in results, '"%s" did not contained "results" key' % results)
self.assertEqual(len(results["results"]), 1)
self.assertEqual(results["nonce"], "1234")
self.failUnless(results["results"][0]["success"])
self.failUnless(results["results"][0]["recid"] > 0)
self.failUnless(
"""<subfield code="a">Doe, John</subfield>""" in results["results"][0]["marcxml"],
results["results"][0]["marcxml"],
)
def test_insert_via_curl(self):
"""batchuploader - robotupload insert via CLI curl"""
curl_input_file = os.path.join(CFG_TMPDIR, 'curl_test.xml')
open(curl_input_file, "w").write(self.marcxml)
try:
result = run_shell_command('/usr/bin/curl -T %s %s -A %s -H "Content-Type: application/marcxml+xml"', [curl_input_file, self.nonce_url, make_user_agent_string('BatchUploader')])[1]
self.failUnless("[INFO]" in result)
current_task = get_last_taskid()
run_shell_command("%s/bibupload %%s" % CFG_BINDIR, [str(current_task)])
results = json.loads(open(self.callback_result_path).read())
self.failUnless('results' in results, '"%s" did not contained [INFO]' % result)
self.assertEqual(len(results['results']), 1)
self.assertEqual(results['nonce'], "1234")
self.failUnless(results['results'][0]['success'])
self.failUnless(results['results'][0]['recid'] > 0)
self.failUnless("""<subfield code="a">Doe, John</subfield>""" in results['results'][0]['marcxml'], results['results'][0]['marcxml'])
finally:
os.remove(curl_input_file)
def __extract_attribute(self, req):
"""
Load from the request the given assertion, extract all the attribute
to properly login the user, and verify that the data are actually
both well formed and signed correctly.
"""
from invenio.webinterface_handler import wash_urlargd
args = wash_urlargd(req.form, {
'assertion': (str, ''),
'robot': (str, ''),
'digest': (str, ''),
'login_method': (str, '')})
assertion = args['assertion']
digest = args['digest']
robot = args['robot']
login_method = args['login_method']
shared_key = load_robot_keys().get(login_method, {}).get(robot)
if shared_key is None:
raise InvenioWebAccessExternalAuthError("A key does not exist for robot: %s, login_method: %s" % (robot, login_method))
if not self.verify(shared_key, assertion, digest):
raise InvenioWebAccessExternalAuthError("The provided assertion does not validate against the digest %s for robot %s" % (repr(digest), repr(robot)))
if self.use_zlib:
try:
## Workaround to Perl implementation that does not add
## any padding to the base64 encoding.
needed_pad = (4 - len(assertion) % 4) % 4
assertion += needed_pad * '='
assertion = decompress(base64.urlsafe_b64decode(assertion))
except:
raise InvenioWebAccessExternalAuthError("The provided assertion is corrupted")
data = json_unicode_to_utf8(json.loads(assertion))
if not isinstance(data, dict):
raise InvenioWebAccessExternalAuthError("The provided assertion is invalid")
timeout = data[self.timeout_attribute_name]
if timeout < time.time():
raise InvenioWebAccessExternalAuthError("The provided assertion is expired")
userip = data.get(self.userip_attribute_name)
if not self.check_user_ip or (normalize_ip(userip, self.check_user_ip) == normalize_ip(req.remote_ip, self.check_user_ip)):
return data
else:
raise InvenioWebAccessExternalAuthError("The provided assertion has been issued for a different IP address (%s instead of %s)" % (userip, req.remote_ip))
def get_kb_mappings_embedded_json(kb_name="", key="", value="", match_type="s", limit=None):
"""Get leftside/rightside mappings from kb kb_name formatted as json dict.
The rightside is actually considered as a json string and hence embedded
within the final result.
If key given, give only those with left side (mapFrom) = key.
If value given, give only those with right side (mapTo) = value.
@param kb_name: the name of the kb
@param key: include only lines matching this on left side in the results
@param value: include only lines matching this on right side in the results
@param match_type: s = substring match, e = exact match
@param limit: maximum number of results to return (are ALL if set to None)
@return a list of mappings
"""
mappings = get_kb_mappings(kb_name, key, value, match_type)
ret = []
if limit is None:
limit = len(mappings)
for m in mappings[:limit]:
label = m['value'] or m['key']
value = m['key'] or m['value']
ret.append({'label': label, 'value': json.loads(value)})
return json.dumps(ret)
def index(self, req, form):
"""Handle all BibEdit requests.
The responsibilities of this functions is:
* JSON decoding and encoding.
* Redirection, if necessary.
* Authorization.
* Calling the appropriate function from the engine.
"""
uid = getUid(req)
argd = wash_urlargd(form, {'ln': (str, CFG_SITE_LANG)})
# Abort if the simplejson module isn't available
if not CFG_JSON_AVAILABLE:
title = 'Record Editor'
body = '''Sorry, the record editor cannot operate when the
`simplejson' module is not installed. Please see the INSTALL
file.'''
return page(title = title,
body = body,
errors = [],
warnings = [],
uid = uid,
language = argd['ln'],
navtrail = navtrail,
lastupdated = __lastupdated__,
req = req,
body_css_classes = ['bibedit'])
# If it is an Ajax request, extract any JSON data.
ajax_request, recid = False, None
if form.has_key('jsondata'):
json_data = json.loads(str(form['jsondata']))
# Deunicode all strings (Invenio doesn't have unicode
# support).
json_data = json_unicode_to_utf8(json_data)
ajax_request = True
if json_data.has_key('recID'):
recid = json_data['recID']
json_response = {'resultCode': 0, 'ID': json_data['ID']}
# Authorization.
user_info = collect_user_info(req)
if user_info['email'] == 'guest':
# User is not logged in.
if not ajax_request:
# Do not display the introductory recID selection box to guest
# users (as it used to be with v0.99.0):
dummy_auth_code, auth_message = acc_authorize_action(req,
'runbibedit')
referer = '/edit/'
if self.recid:
referer = '/%s/%s/edit/' % (CFG_SITE_RECORD, self.recid)
return page_not_authorized(req=req, referer=referer,
text=auth_message, navtrail=navtrail)
else:
# Session has most likely timed out.
json_response.update({'resultCode': 100})
return json.dumps(json_response)
elif self.recid:
# Handle RESTful calls from logged in users by redirecting to
# generic URL.
redirect_to_url(req, '%s/%s/edit/#state=edit&recid=%s&recrev=%s' % (
CFG_SITE_SECURE_URL, CFG_SITE_RECORD, self.recid, ""))
elif recid is not None:
json_response.update({'recID': recid})
if json_data['requestType'] == "getRecord":
# Authorize access to record.
if not user_can_edit_record_collection(req, recid):
json_response.update({'resultCode': 101})
return json.dumps(json_response)
# Handle request.
if not ajax_request:
# Show BibEdit start page.
body, errors, warnings = perform_request_init(uid, argd['ln'], req, __lastupdated__)
title = 'Record Editor'
return page(title = title,
body = body,
errors = errors,
warnings = warnings,
uid = uid,
language = argd['ln'],
navtrail = navtrail,
lastupdated = __lastupdated__,
req = req,
body_css_classes = ['bibedit'])
else:
# Handle AJAX request.
json_response.update(perform_request_ajax(req, recid, uid,
json_data))
return json.dumps(json_response)
def task_submit_check_options():
""" Checks the tasks arguments for validity
"""
#----------------#
# General Checks #
#----------------#
## FFMPEG CONFIGURATION ##
## The status of ffmpeg should be checked before a task is submitted
## There is a minimum configuration that ffmpeg must be compiled with
## See bibencode_utils and bibencode_config
config = check_ffmpeg_configuration()
if config:
## Prints missing configuration
string = ''
for item in config:
string += ('\t' + item + '\n')
write_message(
"FFmpeg options are missing. Please recompile and add:\n" + string
)
return False
## MODE ##
## Check if the mode is a valid
if _topt('mode') is None:
write_message('You have to specify a mode using \'-m MODE\'')
return False
if _topt('mode') not in CFG_BIBENCODE_VALID_MODES:
write_message('%s is not a valid mode. Use one of %s'
% (_topt('mode'), CFG_BIBENCODE_VALID_MODES))
return False
## INPUT ##
## Check if the input file is given and if it exists
## You should allways use an absolute path to the file
if _topt('mode') in ('encode', 'extract', 'meta', 'batch'):
if _topt('input') is None:
write_message('You must specify an input file using \'-i FILE\'')
return False
else:
if not os.path.exists(_topt('input')):
print("The file %s does not exist" % _topt('input'))
return False
## OUTPUT ##
## Check if the output file is given and if it exists
## You should allways use an absolute path to the file
if _topt('mode') in ('encode', 'extract', 'meta'):
if _topt('output') is None:
write_message('No output file is given. Please specify with'
' \'-o NAME\''
)
return False
#---------------#
# Encoding Mode #
#---------------#
if _topt('mode') == 'encode':
## PROFILE ## Check for a valid profile if this is given
if _topt('profile_name') is not None:
if _topt('profile_name') not in get_encoding_profiles():
write_message('%s not found in %s' %
(_topt('profile_name'),
CFG_BIBENCODE_PROFILES_ENCODING)
)
return False
## If the profile exists
else:
pass
## AUDIOCODEC ##
## Checks if the audiocodec is one of the predefined
if _topt('acodec') is not None:
if _topt('acodec') not in CFG_BIBENCODE_FFMPEG_VALID_ACODECS:
write_message(
'%s is not a valid audiocodec.\nAvailable codecs: %s'
% (_topt('acodec'), CFG_BIBENCODE_FFMPEG_VALID_ACODECS)
)
return False
## VIDEOCODEC ## Checks if the videocodec is one of the predefined
if _topt('vcodec') is not None:
if _topt('vcodec') not in CFG_BIBENCODE_FFMPEG_VALID_VCODECS:
write_message(
'%s is not a valid videocodec.\nAvailable codecs: %s'
% (_topt('vcodec'), CFG_BIBENCODE_FFMPEG_VALID_VCODECS)
)
return False
## SIZE ##
## Checks if the size is either WxH or an FFMPEG preset
if _topt('size') is not None:
if not CFG_BIBENCODE_FFMPEG_RE_VALID_SIZE.match(_topt('size')):
if _topt('size') not in CFG_BIBENCODE_FFMPEG_VALID_SIZES:
write_message(
'%s is not a valid frame size.\nEither use the'
' \'WxH\' notation or one of these values:\n%s'
% (_topt('size'), CFG_BIBENCODE_FFMPEG_VALID_SIZES)
)
return False
## Check if both a size and vertical or horizontal resolution
if (_topt('width') or _topt('height')) and _topt('size'):
write_message('Options \'width\' and \'height\' can not be '
'combined with \'resolution\'')
return False
## PASSES ##
## If a number of passes is given, it should be either 1 oder 2.
## You could do an infinite number of passes with ffmpeg,
## But it will almost never make a difference above 2 passes.
## So, we currently only support 2 passes.
if _topt('passes') is not None:
if _topt('passes') not in (1, 2):
write_message('The number of passes must be either 1 or 2')
return False
else:
task_set_option('passes', 1)
## BITRATE ##
## Check if the given bitrate is either 1000 sth. or 1000k sth.
if _topt('abitrate') is not None:
pass
if _topt('vbitrate') is not None:
pass
#-----------------#
# Extraction Mode #
#-----------------#
elif _topt('mode') == 'extract':
## PROFILE ##
## If a profile is given, check its validity
if _topt('profile_name') is not None:
if _topt('profile_name') not in get_extract_profiles():
write_message('%s not found in %s' %
(_topt('profile_name'),
CFG_BIBENCODE_PROFILES_EXTRACT)
)
return False
## If the profile exists
else:
pass
## You cannot give both a number and specific positions
## !!! Think about allowing both -> First extract by number,
## !!! then additionally the specific positions
if (
((_topt('numberof') is not None) and
(_topt('positions') is not None))
or
((_topt('numberof') is None) and
(_topt('positions') is None))
):
write_message('Please specify either a number of frames to '
'take or specific positions')
return False
## SIZE ##
## Checks if the size is either WxH or an FFMPEG specific value
if _topt('size') is not None:
if not CFG_BIBENCODE_FFMPEG_RE_VALID_SIZE.match(_topt('size')):
if _topt('size') not in CFG_BIBENCODE_FFMPEG_VALID_SIZES:
write_message(
'%s is not a valid frame size.\nEither use the'
'\'WxH\' notation or one of these valus:\n%s'
% (_topt('size'), CFG_BIBENCODE_FFMPEG_VALID_SIZES)
)
return False
#---------------#
# Metadata Mode #
#---------------#
elif _topt('mode') == 'meta':
## You have to give exactly one meta suboption
if not _xor(_topt('meta_input'),
_topt('meta_dump')):
write_message("You can either dump or write metadata")
return False
## METADATA INPUT ##
if _topt('meta_input') is not None:
## Check if this is either a filename (that should exist)
## or if this a jsonic metadata notation
if os.path.exists(_topt('meta_input')):
pass
else:
try:
metadict = json.loads(_topt('meta_input'))
task_set_option('meta_input', metadict)
except ValueError:
write_message('The value %s of the \'--meta\' parameter is '
'neither a valid filename nor a jsonic dict'
% _topt('meta_input'))
return False
#------------#
# Batch Mode #
#------------#
elif _topt('mode') == 'batch':
if _topt('collection') and _topt('search'):
write_message('You can either use \'search\' or \'collection\'')
return False
elif _topt('collection'):
template = json_decode_file(_topt('input'))
print('\n')
print("#---------------------------------------------#")
print("# YOU ARE ABOUT TO UPDATE A WHOLE COLLECTION #")
print("#---------------------------------------------#")
print('\n')
print('The selected template file contains:')
pprint(template)
print('\n')
elif _topt('search'):
template = json_decode_file(_topt('input'))
message = "# YOU ARE ABOUT TO UPDATE RECORDS MATCHING '%s' #" % _topt('search')
print('\n')
print("#" + "-"*(len(message)-2) + "#")
print(message)
print("#" + "-"*(len(message)-2) + "#")
print('\n')
print('The selected template file contains:')
pprint(template)
print('\n')
#-------------#
# Daemon Mode #
#-------------#
elif _topt('mode') == 'daemon':
task_set_task_param('task_specific_name', 'daemon')
## You can either give none or both folders, but not only one
if _xor(_topt('new_job_folder'), _topt('old_job_folder')):
write_message('When specifying folders for the daemon mode, you '
'have to specify both the folder for the new jobs '
'and the old ones')
return False
## If every check went fine
return True
def index(self, req, form):
"""Handle all requests"""
uid = getUid(req)
argd = wash_urlargd(form, {'ln' : (str, CFG_SITE_LANG),
'state' : (str, '')})
ln = argd['ln']
state = argd['state']
_ = gettext_set_language(ln)
# Abort if the simplejson module isn't available
if not CFG_JSON_AVAILABLE:
title = 'Authorlist Manager'
body = '''Sorry, the record editor cannot operate when the
`simplejson' module is not installed. Please see the INSTALL
file.'''
return page(title = title,
body = body,
errors = [],
warnings = [],
uid = uid,
language = ln,
navtrail = navtrail,
lastupdated = __lastupdated__,
req = req)
# Extract additional JSON data from form
if 'options' in form:
options = json.loads(str(form['options']))
# Deunicode all strings (Invenio doesn't have unicode
# support).
options = json_unicode_to_utf8(options)
# Authorization.
not_authorized = authorlist_engine.user_authorization(req, ln)
if not_authorized:
return not_authorized
# User is authorized, let's handle different states
# if no state parameter, load the main page
if state == '':
return page(title = _('Author List Manager'),
metaheaderadd = authorlist_templates.index_header(),
body = authorlist_templates.body(),
errors = [],
warnings = [],
uid = uid,
language = ln,
navtrail = navtrail,
lastupdated = __lastupdated__,
req = req)
elif state == 'itemize':
data = authorlist_db.itemize(uid)
req.content_type = 'application/json'
req.write(json.dumps(data))
# open paremeter set? initialize a Authorlist instance
elif state == 'open':
# if 'id' in url, check if user has right to modify this paper
try:
received = wash_urlargd(form, {'id': (str, None)})
paper_id = received['id']
if authorlist_engine.check_user_rights(uid, paper_id):
return page(title = _('Author List Manager'),
metaheaderadd = authorlist_templates.list_header(),
body = authorlist_templates.body(),
errors = [],
warnings = [],
uid = uid,
language = ln,
navtrail = navtrail,
lastupdated = __lastupdated__,
req = req)
else:
# no rights to modify this paper
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
except:
# redirect to the main page if weird stuff happens
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
# On load state we will answer with the JSON encoded data of the passed
# paper id. Should usually not be directly surfed by the user.
elif state == 'load':
try:
received = wash_urlargd(form, {'id': (str, None)})
paper_id = received['id']
data = authorlist_db.load(paper_id)
req.content_type = 'application/json'
req.write(json.dumps(data))
except:
# redirect to the main page if weird stuff happens
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
# The save state saves the send data in the database using the passed
# paper id. Responds with a JSON object containing the id of the paper
# as saved in the database. Should usually not be surfed directly by the
# user
elif state == 'save':
try:
received = wash_urlargd(form, {'id': (str, None),
'data': (str, '')})
paper_id = received['id']
in_data = json.loads(received['data'])
out_data = authorlist_db.save(paper_id, uid, in_data)
req.content_type = 'application/json'
req.write(json.dumps(out_data))
except:
# redirect to the main page if something weird happens
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
# Clones the paper with the given id in the database and responds with a
# JSON object containing the id of the clone. Should usually not surfed
# directly by the user.
elif state == 'clone':
try:
received = wash_urlargd(form, {'id': (str, None)})
paper_id = received['id']
data = authorlist_db.clone(paper_id, uid)
req.content_type = 'application/json'
req.write(json.dumps(data))
except:
# redirect to the main page if something weird happens
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
# Transform the sent data into the format passed in the URL using a
# authorlist_engine converter. Reponds with the MIME type of the
# converter and offers it as a download (content-disposition header).
elif state == 'export':
try:
received = wash_urlargd(form, {'format': (str, None),
'data': (str, '')})
data_format = received['format']
data = received['data']
converter = authorlist_engine.Converters.get(data_format)
attachement = 'attachement; filename="%s"' % converter.FILE_NAME
req.headers_out['Content-Type'] = converter.CONTENT_TYPE
req.headers_out['Content-Disposition'] = attachement
#redirect_to_url(req, authorlist_engine.dumps(data, converter))
req.write(authorlist_engine.dumps(data, converter))
except:
# throw exception if something weird happens
return sys.exc_info()
elif state == 'delete':
try:
received = wash_urlargd(form, {'id': (str, None)})
paper_id = received['id']
data = authorlist_db.delete(paper_id)
req.content_type = 'application/json'
req.write(json.dumps(data))
except:
# redirect to the main page if something weird happens
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
elif state == 'import':
try:
received = wash_urlargd(form, {'importid': (str, None)})
recID = received['importid']
data = authorlist_engine.retrieve_data_from_record(recID)
req.content_type = 'application/json'
req.write(json.dumps(data))
except:
# redirect to the main page if something weird happens
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
elif state == 'importxml':
try:
received = wash_urlargd(form, {'xmlfile': (Field, None)})
xml_string = received['xmlfile'].value
import_data = authorlist_engine.retrieve_data_from_xml(xml_string)
req.content_type = 'application/json'
req.write(json.dumps(import_data))
except:
# redirect to the main page if something weird happens
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
# No state given, just go to the main page.
else:
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
def index(self, req, form):
"""Handle all BibMerge requests.
The responsibilities of this functions are:
* JSON decoding and encoding.
* Redirection, if necessary.
* Authorization.
* Calling the appropriate function from the engine.
"""
# If it is an Ajax request, extract any JSON data.
ajax_request, recid1, recid2 = False, None, None
argd = wash_urlargd(form, {'ln': (str, CFG_SITE_LANG)})
if form.has_key('jsondata'):
json_data = json.loads(str(form['jsondata']))
# Deunicode all strings (Invenio doesn't have unicode
# support).
json_data = json_unicode_to_utf8(json_data)
ajax_request = True
json_response = {}
if json_data.has_key('recID1'):
recid1 = json_data['recID1']
if json_data.has_key('recID2'):
recid2 = json_data['recID2']
# Authorization.
user_info = collect_user_info(req)
if user_info['email'] == 'guest':
# User is not logged in.
if not ajax_request:
# Do not display the introductory recID selection box to guest
# users (as it used to be with v0.99.0):
auth_code, auth_message = acc_authorize_action(req, 'runbibmerge')
referer = '/merge/'
return page_not_authorized(req=req, referer=referer,
text=auth_message, navtrail=navtrail)
else:
# Session has most likely timed out.
json_response.update({'resultCode': 1,
'resultText': 'Error: Not logged in'})
return json.dumps(json_response)
elif self.recid:
# Handle RESTful call by storing recid and redirecting to
# generic URL.
redirect_to_url(req, '%s/%s/merge/' % (CFG_SITE_SECURE_URL, CFG_SITE_RECORD) )
if recid1 is not None:
# Authorize access to record 1.
auth_code, auth_message = acc_authorize_action(req, 'runbibmerge',
collection=guess_primary_collection_of_a_record(recid1))
if auth_code != 0:
json_response.update({'resultCode': 1, 'resultText': 'No access to record %s' % recid1})
return json.dumps(json_response)
if recid2 is not None:
# Authorize access to record 2.
auth_code, auth_message = acc_authorize_action(req, 'runbibmerge',
collection=guess_primary_collection_of_a_record(recid2))
if auth_code != 0:
json_response.update({'resultCode': 1, 'resultText': 'No access to record %s' % recid2})
return json.dumps(json_response)
# Handle request.
uid = getUid(req)
if not ajax_request:
# Show BibEdit start page.
body, errors, warnings = perform_request_init()
metaheaderadd = """<script type="text/javascript" src="%(site)s/js/json2.js"></script>
<script type="text/javascript" src="%(site)s/js/bibmerge_engine.js"></script>""" % {'site': CFG_SITE_SECURE_URL}
title = 'Record Merger'
return page(title = title,
metaheaderadd = metaheaderadd,
body = body,
errors = errors,
warnings = warnings,
uid = uid,
language = argd['ln'],
navtrail = navtrail,
lastupdated = __lastupdated__,
req = req)
else:
# Handle AJAX request.
json_response = perform_request_ajax(req, uid, json_data)
return json.dumps(json_response)
def task_submit_elaborate_specific_parameter(key, value, opts, args):
""" Given the string key it checks it's meaning, eventually using the
value. Usually it fills some key in the options dict.
It must return True if it has elaborated the key, False, if it doesn't
know that key.
eg:
if key in ('-n', '--number'):
self.options['number'] = value
return True
return False
"""
## A dictionary used for mapping CLI parameters to task_option keys+-
parameter_mapping = {
'-p': 'profile_name',
'-i': 'input',
'--input': 'input',
'-o': 'output',
'--output': 'output',
'-m': 'mode',
'--mode': 'mode',
'--acodec': 'acodec',
'--vcodec': 'vcodec',
'--abitrate': 'abitrate',
'--vbitrate': 'vbitrate',
'--resolution': 'size',
'--passes': 'passes',
'--special': 'special',
'--specialfirst': 'specialfirst',
'--specialsecond': 'specialsecond',
'--width': 'width',
'--height': 'height',
'--aspect': 'aspect',
'--number': 'numberof',
'--positions': 'positions',
'-D': 'meta_dump',
'-W': 'meta_input',
'--dump': 'meta_dump',
'--write': 'meta_input',
'--newjobfolder': 'new_job_folder',
'--oldjobfolder': 'old_job_folder',
'--recid': 'recid',
'--collection': 'collection',
'--search': 'search'
}
## PASSES ##
## Transform 'passes' to integer
if key in ('--passes', ):
try:
value = int(value)
except ValueError:
write_message('Value of \'--passes\' must be an integer')
return False
## HEIGHT, WIDTH ##
if key in ('--height', '--width'):
try:
value = int(value)
except ValueError:
write_message('Value of \'--height\' or \'--width\''
' must be an integer')
return False
## META MODE ##
## Transform meta mode values to boolean
if key in ('-D', '--dump'):
if not value in ("ffprobe", "mediainfo", "pbcore"):
write_message(
"Unknown dumping format, must be 'ffprobe', 'mediainfo' or 'pbcore'"
)
return False
if key in ('--substitute', ):
value = True
## Transform the 'positions' parameter into a list
if key in ('--positions', ):
try:
parsed = json.loads(value)
if type(parsed) is not type(list()):
write_message('Value of \'--positions\' must be a json list')
return False
else:
value = parsed
except ValueError:
write_message('Value of \'--positions\' must be a json list')
return False
## NUMBEROF ##
## Transform 'number' to integer
if key in ('--number'):
try:
value = int(value)
except ValueError:
write_message('Value of \'--number\' must be an integer')
return False
## ASPECT ##
if key in ('--aspect'):
try:
xasp, yasp = str(value).split(':')
xasp = float(xasp)
yasp = float(yasp)
value = xasp / yasp
except:
write_message('Value of \'--aspect\' must be in \'4:3\' format')
return False
## RECID ##
if key in ('--recid'):
try:
value = int(value)
except ValueError:
write_message('Value of \'--recid\' must be an integer')
return False
## GENERAL MAPPING ##
## For all general or other parameters just use the mapping dictionary
if key in parameter_mapping:
task_set_option(parameter_mapping[key], value)
return True
return False
def index(self, req, form):
"""Handle all requests"""
uid = getUid(req)
argd = wash_urlargd(form, {
'ln': (str, CFG_SITE_LANG),
'state': (str, '')
})
ln = argd['ln']
state = argd['state']
_ = gettext_set_language(ln)
# Abort if the simplejson module isn't available
if not CFG_JSON_AVAILABLE:
title = 'Authorlist Manager'
body = '''Sorry, the record editor cannot operate when the
`simplejson' module is not installed. Please see the INSTALL
file.'''
return page(title=title,
body=body,
errors=[],
warnings=[],
uid=uid,
language=ln,
navtrail=navtrail,
lastupdated=__lastupdated__,
req=req)
# Extract additional JSON data from form
if 'options' in form:
options = json.loads(str(form['options']))
# Deunicode all strings (Invenio doesn't have unicode
# support).
options = json_unicode_to_utf8(options)
# Authorization.
not_authorized = authorlist_engine.user_authorization(req, ln)
if not_authorized:
return not_authorized
# User is authorized, let's handle different states
# if no state parameter, load the main page
if state == '':
return page(title=_('Author List Manager'),
metaheaderadd=authorlist_templates.index_header(),
body=authorlist_templates.body(),
errors=[],
warnings=[],
uid=uid,
language=ln,
navtrail=navtrail,
lastupdated=__lastupdated__,
req=req)
elif state == 'itemize':
data = authorlist_db.itemize(uid)
req.content_type = 'application/json'
req.write(json.dumps(data))
# open paremeter set? initialize a Authorlist instance
elif state == 'open':
# if 'id' in url, check if user has right to modify this paper
try:
received = wash_urlargd(form, {'id': (str, None)})
paper_id = received['id']
if authorlist_engine.check_user_rights(uid, paper_id):
return page(
title=_('Author List Manager'),
metaheaderadd=authorlist_templates.list_header(),
body=authorlist_templates.body(),
errors=[],
warnings=[],
uid=uid,
language=ln,
navtrail=navtrail,
lastupdated=__lastupdated__,
req=req)
else:
# no rights to modify this paper
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
except:
# redirect to the main page if weird stuff happens
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
# On load state we will answer with the JSON encoded data of the passed
# paper id. Should usually not be directly surfed by the user.
elif state == 'load':
try:
received = wash_urlargd(form, {'id': (str, None)})
paper_id = received['id']
data = authorlist_db.load(paper_id)
req.content_type = 'application/json'
req.write(json.dumps(data))
except:
# redirect to the main page if weird stuff happens
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
# The save state saves the send data in the database using the passed
# paper id. Responds with a JSON object containing the id of the paper
# as saved in the database. Should usually not be surfed directly by the
# user
elif state == 'save':
try:
received = wash_urlargd(form, {
'id': (str, None),
'data': (str, '')
})
paper_id = received['id']
in_data = json.loads(received['data'])
out_data = authorlist_db.save(paper_id, uid, in_data)
req.content_type = 'application/json'
req.write(json.dumps(out_data))
except:
# redirect to the main page if something weird happens
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
# Clones the paper with the given id in the database and responds with a
# JSON object containing the id of the clone. Should usually not surfed
# directly by the user.
elif state == 'clone':
try:
received = wash_urlargd(form, {'id': (str, None)})
paper_id = received['id']
data = authorlist_db.clone(paper_id, uid)
req.content_type = 'application/json'
req.write(json.dumps(data))
except:
# redirect to the main page if something weird happens
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
# Transform the sent data into the format passed in the URL using a
# authorlist_engine converter. Reponds with the MIME type of the
# converter and offers it as a download (content-disposition header).
elif state == 'export':
try:
received = wash_urlargd(form, {
'format': (str, None),
'data': (str, '')
})
data_format = received['format']
data = received['data']
converter = authorlist_engine.Converters.get(data_format)
attachement = 'attachement; filename="%s"' % converter.FILE_NAME
req.headers_out['Content-Type'] = converter.CONTENT_TYPE
req.headers_out['Content-Disposition'] = attachement
#redirect_to_url(req, authorlist_engine.dumps(data, converter))
req.write(authorlist_engine.dumps(data, converter))
except:
# throw exception if something weird happens
return sys.exc_info()
elif state == 'delete':
try:
received = wash_urlargd(form, {'id': (str, None)})
paper_id = received['id']
data = authorlist_db.delete(paper_id)
req.content_type = 'application/json'
req.write(json.dumps(data))
except:
# redirect to the main page if something weird happens
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
elif state == 'import':
try:
received = wash_urlargd(form, {'importid': (str, None)})
recID = received['importid']
data = authorlist_engine.retrieve_data_from_record(recID)
req.content_type = 'application/json'
req.write(json.dumps(data))
except:
# redirect to the main page if something weird happens
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
elif state == 'importxml':
try:
received = wash_urlargd(form, {'xmlfile': (Field, None)})
xml_string = received['xmlfile'].value
import_data = authorlist_engine.retrieve_data_from_xml(
xml_string)
req.content_type = 'application/json'
req.write(json.dumps(import_data))
except:
# redirect to the main page if something weird happens
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
# No state given, just go to the main page.
else:
redirect_to_url(req, '%s/authorlist/' % (CFG_SITE_URL))
def _process_json_request(self, form, req):
"""Takes care about the json requests."""
argd = wash_urlargd(form, {self._JSON_DATA_KEY: (str, "")})
# load json data
json_data_string = argd[self._JSON_DATA_KEY]
json_data_unicode = json.loads(json_data_string)
json_data = json_unicode_to_utf8(json_data_unicode)
language = json_data["language"]
search_criteria = json_data["searchCriteria"]
output_tags = json_data["outputTags"]
output_tags = output_tags.split(",")
output_tags = [tag.strip() for tag in output_tags]
action_type = json_data["actionType"]
current_record_id = json_data["currentRecordID"]
commands = json_data["commands"]
output_format = json_data["outputFormat"]
page_to_display = json_data["pageToDisplay"]
collection = json_data["collection"]
compute_modifications = json_data["compute_modifications"]
checked_records = json_data["checked_records"]
json_response = {}
if action_type == self._action_types.test_search:
json_response.update(
multi_edit_engine.perform_request_test_search(
search_criteria,
[],
output_format,
page_to_display,
language,
output_tags,
collection,
req=req,
checked_records=checked_records,
)
)
json_response["display_info_box"] = 1
json_response["info_html"] = ""
return json.dumps(json_response)
elif action_type == self._action_types.display_detailed_record:
json_response.update(
multi_edit_engine.perform_request_detailed_record(current_record_id, [], output_format, language)
)
return json.dumps(json_response)
elif action_type == self._action_types.preview_results:
commands_list, upload_mode, tag_list = self._create_commands_list(commands)
json_response = {}
json_response.update(
multi_edit_engine.perform_request_test_search(
search_criteria,
commands_list,
output_format,
page_to_display,
language,
output_tags,
collection,
compute_modifications,
upload_mode,
req,
checked_records,
)
)
return json.dumps(json_response)
elif action_type == self._action_types.display_detailed_result:
commands_list, upload_mode, tag_list = self._create_commands_list(commands)
json_response.update(
multi_edit_engine.perform_request_detailed_record(
current_record_id, commands_list, output_format, language
)
)
return json.dumps(json_response)
elif action_type == self._action_types.submit_changes:
commands_list, upload_mode, tag_list = self._create_commands_list(commands)
json_response.update(
multi_edit_engine.perform_request_submit_changes(
search_criteria, commands_list, language, upload_mode, tag_list, collection, req, checked_records
)
)
return json.dumps(json_response)
# In case we obtain wrong action type we return empty page.
return " "
def task_submit_check_options():
""" Checks the tasks arguments for validity
"""
#----------------#
# General Checks #
#----------------#
## FFMPEG CONFIGURATION ##
## The status of ffmpeg should be checked before a task is submitted
## There is a minimum configuration that ffmpeg must be compiled with
## See bibencode_utils and bibencode_config
config = check_ffmpeg_configuration()
if config:
## Prints missing configuration
string = ''
for item in config:
string += ('\t' + item + '\n')
write_message(
"FFmpeg options are missing. Please recompile and add:\n" + string)
return False
## MODE ##
## Check if the mode is a valid
if _topt('mode') is None:
write_message('You have to specify a mode using \'-m MODE\'')
return False
if _topt('mode') not in CFG_BIBENCODE_VALID_MODES:
write_message('%s is not a valid mode. Use one of %s' %
(_topt('mode'), CFG_BIBENCODE_VALID_MODES))
return False
## INPUT ##
## Check if the input file is given and if it exists
## You should allways use an absolute path to the file
if _topt('mode') in ('encode', 'extract', 'meta', 'batch'):
if _topt('input') is None:
write_message('You must specify an input file using \'-i FILE\'')
return False
else:
if not os.path.exists(_topt('input')):
print("The file %s does not exist" % _topt('input'))
return False
## OUTPUT ##
## Check if the output file is given and if it exists
## You should allways use an absolute path to the file
if _topt('mode') in ('encode', 'extract', 'meta'):
if _topt('output') is None:
write_message('No output file is given. Please specify with'
' \'-o NAME\'')
return False
#---------------#
# Encoding Mode #
#---------------#
if _topt('mode') == 'encode':
## PROFILE ## Check for a valid profile if this is given
if _topt('profile_name') is not None:
if _topt('profile_name') not in get_encoding_profiles():
write_message(
'%s not found in %s' %
(_topt('profile_name'), CFG_BIBENCODE_PROFILES_ENCODING))
return False
## If the profile exists
else:
pass
## AUDIOCODEC ##
## Checks if the audiocodec is one of the predefined
if _topt('acodec') is not None:
if _topt('acodec') not in CFG_BIBENCODE_FFMPEG_VALID_ACODECS:
write_message(
'%s is not a valid audiocodec.\nAvailable codecs: %s' %
(_topt('acodec'), CFG_BIBENCODE_FFMPEG_VALID_ACODECS))
return False
## VIDEOCODEC ## Checks if the videocodec is one of the predefined
if _topt('vcodec') is not None:
if _topt('vcodec') not in CFG_BIBENCODE_FFMPEG_VALID_VCODECS:
write_message(
'%s is not a valid videocodec.\nAvailable codecs: %s' %
(_topt('vcodec'), CFG_BIBENCODE_FFMPEG_VALID_VCODECS))
return False
## SIZE ##
## Checks if the size is either WxH or an FFMPEG preset
if _topt('size') is not None:
if not CFG_BIBENCODE_FFMPEG_RE_VALID_SIZE.match(_topt('size')):
if _topt('size') not in CFG_BIBENCODE_FFMPEG_VALID_SIZES:
write_message(
'%s is not a valid frame size.\nEither use the'
' \'WxH\' notation or one of these values:\n%s' %
(_topt('size'), CFG_BIBENCODE_FFMPEG_VALID_SIZES))
return False
## Check if both a size and vertical or horizontal resolution
if (_topt('width') or _topt('height')) and _topt('size'):
write_message('Options \'width\' and \'height\' can not be '
'combined with \'resolution\'')
return False
## PASSES ##
## If a number of passes is given, it should be either 1 oder 2.
## You could do an infinite number of passes with ffmpeg,
## But it will almost never make a difference above 2 passes.
## So, we currently only support 2 passes.
if _topt('passes') is not None:
if _topt('passes') not in (1, 2):
write_message('The number of passes must be either 1 or 2')
return False
else:
task_set_option('passes', 1)
## BITRATE ##
## Check if the given bitrate is either 1000 sth. or 1000k sth.
if _topt('abitrate') is not None:
pass
if _topt('vbitrate') is not None:
pass
#-----------------#
# Extraction Mode #
#-----------------#
elif _topt('mode') == 'extract':
## PROFILE ##
## If a profile is given, check its validity
if _topt('profile_name') is not None:
if _topt('profile_name') not in get_extract_profiles():
write_message(
'%s not found in %s' %
(_topt('profile_name'), CFG_BIBENCODE_PROFILES_EXTRACT))
return False
## If the profile exists
else:
pass
## You cannot give both a number and specific positions
## !!! Think about allowing both -> First extract by number,
## !!! then additionally the specific positions
if (((_topt('numberof') is not None) and
(_topt('positions') is not None)) or
((_topt('numberof') is None) and (_topt('positions') is None))):
write_message('Please specify either a number of frames to '
'take or specific positions')
return False
## SIZE ##
## Checks if the size is either WxH or an FFMPEG specific value
if _topt('size') is not None:
if not CFG_BIBENCODE_FFMPEG_RE_VALID_SIZE.match(_topt('size')):
if _topt('size') not in CFG_BIBENCODE_FFMPEG_VALID_SIZES:
write_message(
'%s is not a valid frame size.\nEither use the'
'\'WxH\' notation or one of these valus:\n%s' %
(_topt('size'), CFG_BIBENCODE_FFMPEG_VALID_SIZES))
return False
#---------------#
# Metadata Mode #
#---------------#
elif _topt('mode') == 'meta':
## You have to give exactly one meta suboption
if not _xor(_topt('meta_input'), _topt('meta_dump')):
write_message("You can either dump or write metadata")
return False
## METADATA INPUT ##
if _topt('meta_input') is not None:
## Check if this is either a filename (that should exist)
## or if this a jsonic metadata notation
if os.path.exists(_topt('meta_input')):
pass
else:
try:
metadict = json.loads(_topt('meta_input'))
task_set_option('meta_input', metadict)
except ValueError:
write_message(
'The value %s of the \'--meta\' parameter is '
'neither a valid filename nor a jsonic dict' %
_topt('meta_input'))
return False
#------------#
# Batch Mode #
#------------#
elif _topt('mode') == 'batch':
if _topt('collection') and _topt('search'):
write_message('You can either use \'search\' or \'collection\'')
return False
elif _topt('collection'):
template = json_decode_file(_topt('input'))
print('\n')
print("#---------------------------------------------#")
print("# YOU ARE ABOUT TO UPDATE A WHOLE COLLECTION #")
print("#---------------------------------------------#")
print('\n')
print('The selected template file contains:')
pprint(template)
print('\n')
elif _topt('search'):
template = json_decode_file(_topt('input'))
message = "# YOU ARE ABOUT TO UPDATE RECORDS MATCHING '%s' #" % _topt(
'search')
print('\n')
print("#" + "-" * (len(message) - 2) + "#")
print(message)
print("#" + "-" * (len(message) - 2) + "#")
print('\n')
print('The selected template file contains:')
pprint(template)
print('\n')
#-------------#
# Daemon Mode #
#-------------#
elif _topt('mode') == 'daemon':
task_set_task_param('task_specific_name', 'daemon')
## You can either give none or both folders, but not only one
if _xor(_topt('new_job_folder'), _topt('old_job_folder')):
write_message('When specifying folders for the daemon mode, you '
'have to specify both the folder for the new jobs '
'and the old ones')
return False
## If every check went fine
return True
def get_json_parameters_from_cli(dummy_option, dummy_opt_str, value, parser):
try:
setattr(parser.values, 'parameters', json_unicode_to_utf8(json.loads(value)))
except Exception, err:
raise optparse.OptionValueError("Cannot parse as a valid JSON serialization the provided parameters: %s. %s" % (value, err))
def index(self, req, form):
"""Handle all BibMerge requests.
The responsibilities of this functions are:
* JSON decoding and encoding.
* Redirection, if necessary.
* Authorization.
* Calling the appropriate function from the engine.
"""
# If it is an Ajax request, extract any JSON data.
ajax_request, recid1, recid2 = False, None, None
argd = wash_urlargd(form, {'ln': (str, CFG_SITE_LANG)})
if form.has_key('jsondata'):
json_data = json.loads(str(form['jsondata']))
# Deunicode all strings (Invenio doesn't have unicode
# support).
json_data = json_unicode_to_utf8(json_data)
ajax_request = True
json_response = {}
try:
if json_data.has_key('recID1'):
recid1 = int(json_data['recID1'])
json_data['recID1'] = recid1
if json_data.has_key('recID2'):
if json_data.get('record2Mode') == "recid":
recid2 = int(json_data['recID2'])
json_data['recID2'] = recid2
except ValueError:
json_response.update({
'resultCode': 1,
'resultText': 'Invalid record ID!'
})
return json.dumps(json_response)
if json_data.has_key("duplicate"):
if json_data.get('record2Mode') == "recid":
json_data["duplicate"] = int(json_data["duplicate"])
# Authorization.
user_info = collect_user_info(req)
if user_info['email'] == 'guest':
# User is not logged in.
if not ajax_request:
# Do not display the introductory recID selection box to guest
# users (as it used to be with v0.99.0):
auth_code, auth_message = acc_authorize_action(
req, 'runbibmerge')
referer = '/merge/'
return page_not_authorized(req=req,
referer=referer,
text=auth_message,
navtrail=navtrail)
else:
# Session has most likely timed out.
json_response.update({
'resultCode': 1,
'resultText': 'Error: Not logged in'
})
return json.dumps(json_response)
elif self.recid:
# Handle RESTful call by storing recid and redirecting to
# generic URL.
redirect_to_url(
req, '%s/%s/merge/' % (CFG_SITE_SECURE_URL, CFG_SITE_RECORD))
if recid1 is not None:
# Authorize access to record 1.
auth_code, auth_message = acc_authorize_action(
req,
'runbibmerge',
collection=guess_primary_collection_of_a_record(recid1))
if auth_code != 0:
json_response.update({
'resultCode':
1,
'resultText':
'No access to record %s' % recid1
})
return json.dumps(json_response)
if recid2 is not None:
# Authorize access to record 2.
auth_code, auth_message = acc_authorize_action(
req,
'runbibmerge',
collection=guess_primary_collection_of_a_record(recid2))
if auth_code != 0:
json_response.update({
'resultCode':
1,
'resultText':
'No access to record %s' % recid2
})
return json.dumps(json_response)
# Handle request.
uid = getUid(req)
if not ajax_request:
# Show BibEdit start page.
body, errors, warnings = perform_request_init()
scripts = ["json2.js", "bibmerge_engine.js"]
metaheaderadd = ""
for script in scripts:
metaheaderadd += '<script type="text/javascript" src="%s/%s"></script>' % (
CFG_SITE_URL, auto_version_url("js/" + script))
return page(title='Record Merger',
metaheaderadd=metaheaderadd,
body=body,
errors=errors,
warnings=warnings,
uid=uid,
language=argd['ln'],
navtrail=navtrail,
lastupdated=__lastupdated__,
req=req)
else:
# Handle AJAX request.
json_response = perform_request_ajax(req, uid, json_data)
return json.dumps(json_response)
def _process_json_request(self, form, req):
"""Takes care about the json requests."""
argd = wash_urlargd(form, {
self._JSON_DATA_KEY: (str, ""),
})
# load json data
json_data_string = argd[self._JSON_DATA_KEY]
json_data_unicode = json.loads(json_data_string)
json_data = json_unicode_to_utf8(json_data_unicode)
language = json_data["language"]
search_criteria = json_data["searchCriteria"]
output_tags = json_data["outputTags"]
output_tags = output_tags.split(',')
output_tags = [tag.strip() for tag in output_tags]
action_type = json_data["actionType"]
current_record_id = json_data["currentRecordID"]
commands = json_data["commands"]
output_format = json_data["outputFormat"]
page_to_display = json_data["pageToDisplay"]
collection = json_data["collection"]
compute_modifications = json_data["compute_modifications"]
checked_records = json_data["checked_records"]
json_response = {}
if action_type == self._action_types.test_search:
json_response.update(multi_edit_engine.perform_request_test_search(
search_criteria,
[],
output_format,
page_to_display,
language,
output_tags,
collection,
checked_records))
json_response['display_info_box'] = 1
json_response['info_html'] = ""
return json.dumps(json_response)
elif action_type == self._action_types.display_detailed_record:
json_response.update(multi_edit_engine.perform_request_detailed_record(
current_record_id,
[],
output_format,
language))
return json.dumps(json_response)
elif action_type == self._action_types.preview_results:
commands_list, upload_mode, tag_list = self._create_commands_list(commands)
json_response = {}
json_response.update(multi_edit_engine.perform_request_test_search(
search_criteria,
commands_list,
output_format,
page_to_display,
language,
output_tags,
collection,
compute_modifications,
upload_mode,
checked_records))
return json.dumps(json_response)
elif action_type == self._action_types.display_detailed_result:
commands_list, upload_mode, tag_list = self._create_commands_list(commands)
json_response.update(multi_edit_engine.perform_request_detailed_record(
current_record_id,
commands_list,
output_format,
language))
return json.dumps(json_response)
elif action_type == self._action_types.submit_changes:
commands_list, upload_mode, tag_list = self._create_commands_list(commands)
json_response.update(multi_edit_engine.perform_request_submit_changes(search_criteria, commands_list, language, upload_mode, tag_list, collection, req, checked_records))
return json.dumps(json_response)
# In case we obtain wrong action type we return empty page.
return " "
def parse_arg(argument_str, arg_name):
try:
return encode(json.loads(argument_str))
except ValueError:
raise RulesParseError(rule_name, "Invalid value in argument '%s'" %
arg_name)
def parse_arg(argument_str, arg_name):
try:
return encode(json.loads(argument_str))
except ValueError:
raise RulesParseError(rule_name,
"Invalid value in argument '%s'" % arg_name)
def index(self, req, form):
"""Handle all BibEdit requests.
The responsibilities of this functions is:
* JSON decoding and encoding.
* Redirection, if necessary.
* Authorization.
* Calling the appropriate function from the engine.
"""
uid = getUid(req)
argd = wash_urlargd(form, {'ln': (str, CFG_SITE_LANG)})
# If it is an Ajax request, extract any JSON data.
ajax_request, recid = False, None
if form.has_key('jsondata'):
json_data = json.loads(str(form['jsondata']))
# Deunicode all strings (Invenio doesn't have unicode
# support).
json_data = json_unicode_to_utf8(json_data)
ajax_request = True
if json_data.has_key('recID'):
recid = json_data['recID']
json_response = {'resultCode': 0, 'ID': json_data['ID']}
# Authorization.
user_info = collect_user_info(req)
if user_info['email'] == 'guest':
# User is not logged in.
if not ajax_request:
# Do not display the introductory recID selection box to guest
# users (as it used to be with v0.99.0):
auth_code, auth_message = acc_authorize_action(
req, 'runbibedit')
referer = '/edit/'
if self.recid:
referer = '/%s/%s/edit/' % (CFG_SITE_RECORD, self.recid)
return page_not_authorized(req=req,
referer=referer,
text=auth_message,
navtrail=navtrail)
else:
# Session has most likely timed out.
json_response.update({'resultCode': 100})
return json.dumps(json_response)
elif self.recid:
# Handle redirects from /record/<record id>/edit
# generic URL.
redirect_to_url(
req, '%s/%s/edit/#state=edit&recid=%s&recrev=%s' %
(CFG_SITE_SECURE_URL, CFG_SITE_RECORD, self.recid, ""))
elif recid is not None:
json_response.update({'recID': recid})
if json_data['requestType'] == "getRecord":
# Authorize access to record.
if not user_can_edit_record_collection(req, recid):
json_response.update({'resultCode': 101})
return json.dumps(json_response)
# Handle request.
if not ajax_request:
# Show BibEdit start page.
body, errors, warnings = perform_request_init(
uid, argd['ln'], req, __lastupdated__)
title = 'Record Editor'
return page(title=title,
body=body,
errors=errors,
warnings=warnings,
uid=uid,
language=argd['ln'],
navtrail=navtrail,
lastupdated=__lastupdated__,
req=req,
body_css_classes=['bibedit'])
else:
# Handle AJAX request.
json_response.update(
perform_request_ajax(req, recid, uid, json_data))
return json.dumps(json_response)
def task_submit_elaborate_specific_parameter(key, value, opts, args):
""" Given the string key it checks it's meaning, eventually using the
value. Usually it fills some key in the options dict.
It must return True if it has elaborated the key, False, if it doesn't
know that key.
eg:
if key in ('-n', '--number'):
self.options['number'] = value
return True
return False
"""
## A dictionary used for mapping CLI parameters to task_option keys+-
parameter_mapping = {
'-p': 'profile_name',
'-i': 'input',
'--input': 'input',
'-o': 'output',
'--output': 'output',
'-m': 'mode',
'--mode': 'mode',
'--acodec': 'acodec',
'--vcodec': 'vcodec',
'--abitrate': 'abitrate',
'--vbitrate': 'vbitrate',
'--resolution': 'size',
'--passes': 'passes',
'--special': 'special',
'--specialfirst': 'specialfirst',
'--specialsecond': 'specialsecond',
'--width': 'width',
'--height': 'height',
'--aspect': 'aspect',
'--number': 'numberof',
'--positions': 'positions',
'-D': 'meta_dump',
'-W': 'meta_input',
'--dump': 'meta_dump',
'--write': 'meta_input',
'--newjobfolder': 'new_job_folder',
'--oldjobfolder': 'old_job_folder',
'--recid': 'recid',
'--collection': 'collection',
'--search': 'search'
}
## PASSES ##
## Transform 'passes' to integer
if key in ('--passes', ):
try:
value = int(value)
except ValueError:
write_message('Value of \'--passes\' must be an integer')
return False
## HEIGHT, WIDTH ##
if key in ('--height', '--width'):
try:
value = int(value)
except ValueError:
write_message('Value of \'--height\' or \'--width\''
' must be an integer')
return False
## META MODE ##
## Transform meta mode values to boolean
if key in ('-D', '--dump'):
if not value in ("ffprobe", "mediainfo", "pbcore"):
write_message("Unknown dumping format, must be 'ffprobe', 'mediainfo' or 'pbcore'")
return False
if key in ('--substitute', ):
value = True
## Transform the 'positions' parameter into a list
if key in ('--positions',):
try:
parsed = json.loads(value)
if type(parsed) is not type(list()):
write_message(
'Value of \'--positions\' must be a json list'
)
return False
else:
value = parsed
except ValueError:
write_message(
'Value of \'--positions\' must be a json list'
)
return False
## NUMBEROF ##
## Transform 'number' to integer
if key in ('--number'):
try:
value = int(value)
except ValueError:
write_message('Value of \'--number\' must be an integer')
return False
## ASPECT ##
if key in ('--aspect'):
try:
xasp, yasp = str(value).split(':')
xasp = float(xasp)
yasp = float(yasp)
value = xasp / yasp
except:
write_message('Value of \'--aspect\' must be in \'4:3\' format')
return False
## RECID ##
if key in ('--recid'):
try:
value = int(value)
except ValueError:
write_message('Value of \'--recid\' must be an integer')
return False
## GENERAL MAPPING ##
## For all general or other parameters just use the mapping dictionary
if key in parameter_mapping:
task_set_option(parameter_mapping[key], value)
return True
return False
