def addinput(self, project, inputtemplate, contents, **kwargs):
"""Add an input file to the CLAM service. Explictly providing the contents as a string. This is not suitable for large files as the contents are kept in memory! Use ``addinputfile()`` instead for large files.
project - the ID of the project you want to add the file to.
inputtemplate - The input template you want to use to add this file (InputTemplate instance)
contents - The contents for the file to add (string)
Keyword arguments:
* filename - the filename on the server (mandatory!)
* metadata - A metadata object.
* metafile - A metadata file (filename)
Any other keyword arguments will be passed as metadata and matched with the input template's parameters.
Example::
client.addinput("myproject", "someinputtemplate", "This is a test.", filename="test.txt")
With metadata, assuming such metadata parameters are defined::
client.addinput("myproject", "someinputtemplate", "This is a test.", filename="test.txt", parameter1="blah", parameterX=3.5))
"""
if isinstance( inputtemplate, str) or (sys.version < '3' and isinstance( inputtemplate, unicode)): #pylint: disable=undefined-variable
data = self.get(project) #causes an extra query to server
inputtemplate = data.inputtemplate(inputtemplate)
elif not isinstance(inputtemplate, clam.common.data.InputTemplate):
raise Exception("inputtemplate must be instance of InputTemplate. Get from CLAMData.inputtemplate(id)")
if 'filename' in kwargs:
filename = self.getinputfilename(inputtemplate, kwargs['filename'])
else:
raise Exception("No filename provided!")
data = {"contents": contents, 'inputtemplate': inputtemplate.id}
for key, value in kwargs.items():
if key == 'filename':
pass #nothing to do
elif key == 'metadata':
assert isinstance(value, clam.common.data.CLAMMetaData)
data['metadata'] = value.xml()
elif key == 'metafile':
data['metafile'] = open(value,'r')
else:
data[key] = value
requestparams = self.initrequest(data)
r = requests.post(self.url + project + '/input/' + filename,**requestparams)
if r.status_code == 400:
raise clam.common.data.BadRequest()
elif r.status_code == 401:
raise clam.common.data.AuthRequired()
elif r.status_code == 403:
if r.text[0] == '<':
#XML response
return self._parseupload(r.text)
else:
raise clam.common.data.PermissionDenied(r.text)
elif r.status_code == 404:
raise clam.common.data.NotFound(r.text)
elif r.status_code == 500:
raise clam.common.data.ServerError(r.text)
elif r.status_code == 405:
raise clam.common.data.ServerError("Server returned 405: Method not allowed for POST on " + self.url + project + '/input/' + filename)
elif r.status_code == 408:
raise clam.common.data.TimeOut()
elif not (r.status_code >= 200 and r.status_code <= 299):
raise Exception("An error occured, return code " + str(r.status_code))
return self._parseupload(r.text)
def addinputfile(self, project, inputtemplate, sourcefile, **kwargs):
"""Add/upload an input file to the CLAM service. Supports proper file upload streaming.
project - the ID of the project you want to add the file to.
inputtemplate - The input template you want to use to add this file (InputTemplate instance)
sourcefile - The file you want to add: string containing a filename (or instance of ``file``)
Keyword arguments (optional but recommended!):
* ``filename`` - the filename on the server (will be same as sourcefile if not specified)
* ``metadata`` - A metadata object.
* ``metafile`` - A metadata file (filename)
Any other keyword arguments will be passed as metadata and matched with the input template's parameters.
Example::
client.addinputfile("myproject", "someinputtemplate", "/path/to/local/file")
With metadata, assuming such metadata parameters are defined::
client.addinputfile("myproject", "someinputtemplate", "/path/to/local/file", parameter1="blah", parameterX=3.5)
"""
if isinstance( inputtemplate, str) or (sys.version < '3' and isinstance( inputtemplate, unicode)): #pylint: disable=undefined-variable
data = self.get(project) #causes an extra query to server
inputtemplate = data.inputtemplate(inputtemplate)
elif not isinstance(inputtemplate, clam.common.data.InputTemplate):
raise Exception("inputtemplate must be instance of InputTemplate. Get from CLAMData.inputtemplate(id)")
if not isinstance(sourcefile, IOBase):
sourcefile = open(sourcefile,'rb')
if 'filename' in kwargs:
filename = self.getinputfilename(inputtemplate, kwargs['filename'])
else:
filename = self.getinputfilename(inputtemplate, os.path.basename(sourcefile.name) )
data = {"file": (filename,sourcefile,inputtemplate.formatclass.mimetype), 'inputtemplate': inputtemplate.id}
for key, value in kwargs.items():
if key == 'filename':
pass #nothing to do
elif key == 'metadata':
assert isinstance(value, clam.common.data.CLAMMetaData)
data['metadata'] = value.xml()
elif key == 'metafile':
data['metafile'] = open(value,'rb')
else:
data[key] = value
requestparams = self.initrequest(data)
if 'auth'in requestparams:
#TODO: streaming support doesn't work with authentication unfortunately, disabling streaming for now:
del data['file']
requestparams['data'] = data
requestparams['files'] = [('file', (filename,sourcefile, inputtemplate.formatclass.mimetype))]
if 'metafile' in kwargs:
del data['metafile']
requestparams['files'].append(('metafile',('.'+ filename + '.METADATA', open(kwargs['metafile'],'rb'), 'text/xml')))
else:
#streaming support
encodeddata = MultipartEncoder(fields=requestparams['data']) #from requests-toolbelt, necessary for streaming support
requestparams['data'] = encodeddata
requestparams['headers']['Content-Type'] = encodeddata.content_type
r = requests.post(self.url + project + '/input/' + filename,**requestparams)
sourcefile.close()
if r.status_code == 400:
raise clam.common.data.BadRequest()
elif r.status_code == 401:
raise clam.common.data.AuthRequired()
elif r.status_code == 403:
if r.text[0] == '<':
#XML response
return self._parseupload(r.text)
else:
raise clam.common.data.PermissionDenied(r.text)
elif r.status_code == 404:
raise clam.common.data.NotFound(r.text)
elif r.status_code == 500:
raise clam.common.data.ServerError(r.text)
elif r.status_code == 405:
raise clam.common.data.ServerError("Server returned 405: Method not allowed for POST on " + self.url + project + '/input/' + filename)
elif r.status_code == 408:
raise clam.common.data.TimeOut()
elif not (r.status_code >= 200 and r.status_code <= 299):
raise Exception("An error occured, return code " + str(r.status_code))
return self._parseupload(r.text)
def addinput(self, project, inputtemplate, contents, **kwargs):
"""Add an input file to the CLAM service. Explictly providing the contents as a string. This is not suitable for large files as the contents are kept in memory! Use ``addinputfile()`` instead for large files.
project - the ID of the project you want to add the file to.
inputtemplate - The input template you want to use to add this file (InputTemplate instance)
contents - The contents for the file to add (string)
Keyword arguments:
* filename - the filename on the server (mandatory!)
* metadata - A metadata object.
* metafile - A metadata file (filename)
Any other keyword arguments will be passed as metadata and matched with the input template's parameters.
Example::
client.addinput("myproject", "someinputtemplate", "This is a test.", filename="test.txt")
With metadata, assuming such metadata parameters are defined::
client.addinput("myproject", "someinputtemplate", "This is a test.", filename="test.txt", parameter1="blah", parameterX=3.5))
"""
if isinstance(inputtemplate, str): #pylint: disable=undefined-variable
data = self.get(project) #causes an extra query to server
inputtemplate = data.inputtemplate(inputtemplate)
elif not isinstance(inputtemplate, clam.common.data.InputTemplate):
raise Exception(
"inputtemplate must be instance of InputTemplate. Get from CLAMData.inputtemplate(id)"
)
if 'filename' in kwargs:
filename = self.getinputfilename(inputtemplate, kwargs['filename'])
else:
raise Exception("No filename provided!")
data = {"contents": contents, 'inputtemplate': inputtemplate.id}
for key, value in kwargs.items():
if key == 'filename':
pass #nothing to do
elif key == 'metadata':
assert isinstance(value, clam.common.data.CLAMMetaData)
data['metadata'] = value.xml()
elif key == 'metafile':
data['metafile'] = open(value, 'r')
else:
data[key] = value
requestparams = self.initrequest(data)
r = requests.post(self.url + project + '/input/' + filename,
**requestparams)
if r.status_code == 400:
raise clam.common.data.BadRequest()
elif r.status_code == 401:
raise clam.common.data.AuthRequired()
elif r.status_code == 403:
if r.text[0] == '<':
#XML response
return self._parseupload(r.text)
else:
raise clam.common.data.PermissionDenied(r.text)
elif r.status_code == 404:
raise clam.common.data.NotFound(r.text)
elif r.status_code == 500:
raise clam.common.data.ServerError(r.text)
elif r.status_code == 405:
raise clam.common.data.ServerError(
"Server returned 405: Method not allowed for POST on " +
self.url + project + '/input/' + filename)
elif r.status_code == 408:
raise clam.common.data.TimeOut()
elif not (r.status_code >= 200 and r.status_code <= 299):
raise Exception("An error occured, return code " +
str(r.status_code))
return self._parseupload(r.text)
def addinputfile(self, project, inputtemplate, sourcefile, **kwargs):
"""Add/upload an input file to the CLAM service. Supports proper file upload streaming.
project - the ID of the project you want to add the file to.
inputtemplate - The input template you want to use to add this file (InputTemplate instance)
sourcefile - The file you want to add: string containing a filename (or instance of ``file``)
Keyword arguments (optional but recommended!):
* ``filename`` - the filename on the server (will be same as sourcefile if not specified)
* ``metadata`` - A metadata object.
* ``metafile`` - A metadata file (filename)
Any other keyword arguments will be passed as metadata and matched with the input template's parameters.
Example::
client.addinputfile("myproject", "someinputtemplate", "/path/to/local/file")
With metadata, assuming such metadata parameters are defined::
client.addinputfile("myproject", "someinputtemplate", "/path/to/local/file", parameter1="blah", parameterX=3.5)
"""
if isinstance(inputtemplate, str): #pylint: disable=undefined-variable
data = self.get(project) #causes an extra query to server
inputtemplate = data.inputtemplate(inputtemplate)
elif not isinstance(inputtemplate, clam.common.data.InputTemplate):
raise Exception(
"inputtemplate must be instance of InputTemplate. Get from CLAMData.inputtemplate(id)"
)
if not isinstance(sourcefile, IOBase):
sourcefile = open(sourcefile, 'rb')
if 'filename' in kwargs:
filename = self.getinputfilename(inputtemplate,
kwargs['filename'])
else:
filename = self.getinputfilename(
inputtemplate, os.path.basename(sourcefile.name))
data = {
"file": (filename, sourcefile, inputtemplate.formatclass.mimetype),
'inputtemplate': inputtemplate.id
}
for key, value in kwargs.items():
if key == 'filename':
pass #nothing to do
elif key == 'metadata':
assert isinstance(value, clam.common.data.CLAMMetaData)
data['metadata'] = value.xml()
elif key == 'metafile':
data['metafile'] = open(value, 'rb')
else:
data[key] = value
requestparams = self.initrequest(data)
if 'auth' in requestparams:
#TODO: streaming support doesn't work with authentication unfortunately, disabling streaming for now:
del data['file']
requestparams['data'] = data
requestparams['files'] = [('file',
(filename, sourcefile,
inputtemplate.formatclass.mimetype))]
if 'metafile' in kwargs:
del data['metafile']
requestparams['files'].append(
('metafile', ('.' + filename + '.METADATA',
open(kwargs['metafile'], 'rb'), 'text/xml')))
else:
#streaming support
encodeddata = MultipartEncoder(
fields=requestparams['data']
) #from requests-toolbelt, necessary for streaming support
requestparams['data'] = encodeddata
requestparams['headers']['Content-Type'] = encodeddata.content_type
r = requests.post(self.url + project + '/input/' + filename,
**requestparams)
sourcefile.close()
if r.status_code == 400:
raise clam.common.data.BadRequest()
elif r.status_code == 401:
raise clam.common.data.AuthRequired()
elif r.status_code == 403:
if r.text[0] == '<':
#XML response
return self._parseupload(r.text)
else:
raise clam.common.data.PermissionDenied(r.text)
elif r.status_code == 404:
raise clam.common.data.NotFound(r.text)
elif r.status_code == 500:
raise clam.common.data.ServerError(r.text)
elif r.status_code == 405:
raise clam.common.data.ServerError(
"Server returned 405: Method not allowed for POST on " +
self.url + project + '/input/' + filename)
elif r.status_code == 408:
raise clam.common.data.TimeOut()
elif not (r.status_code >= 200 and r.status_code <= 299):
raise Exception("An error occured, return code " +
str(r.status_code))
return self._parseupload(r.text)
#the latter two arguments are required for authenticated webservices, they can be omitted otherwise
#if you use SSL (https) and SSL verification fails, you can pass a verify= parameter with the path to your certificate of certificate authority bundle
clamclient = clam.common.client.CLAMClient(
"https://webservices-lst.science.ru.nl/valkuil", args.username,
args.password)
project = "valkuil" + str(random.getrandbits(64))
clamclient.create(project)
data = clamclient.get(project)
for inputfile in glob.glob(os.path.join(args.inputdir, '*.folia.xml')):
print("Uploading " + os.path.basename(inputfile), file=sys.stderr)
clamclient.addinputfile(project, data.inputtemplate("foliainput"),
inputfile)
data = clamclient.start(project, sensitivity=0.75)
#Always check for parameter errors! Don't just assume everything went well! Use startsafe() instead of start
#to simply raise exceptions on parameter errors.
if data.errors:
print("An error occured: " + data.errormsg, file=sys.stderr)
for parametergroup, paramlist in data.parameters:
for parameter in paramlist:
if parameter.error:
print("Error in parameter " + parameter.id + ": " +
parameter.error,
file=sys.stderr)
clamclient.delete(
def add_file_to_clam(localfilename):
clamclient.addinputfile(project, data.inputtemplate(inputtemplate), localfilename, encoding='utf-8')
