def addinput(self, project, inputtemplate, contents, **kwargs): """Add an input file to the CLAM service. Explictly providing the contents as a string. This is not suitable for large files as the contents are kept in memory! Use ``addinputfile()`` instead for large files. project - the ID of the project you want to add the file to. inputtemplate - The input template you want to use to add this file (InputTemplate instance) contents - The contents for the file to add (string) Keyword arguments: * filename - the filename on the server (mandatory!) * metadata - A metadata object. * metafile - A metadata file (filename) Any other keyword arguments will be passed as metadata and matched with the input template's parameters. Example:: client.addinput("myproject", "someinputtemplate", "This is a test.", filename="test.txt") With metadata, assuming such metadata parameters are defined:: client.addinput("myproject", "someinputtemplate", "This is a test.", filename="test.txt", parameter1="blah", parameterX=3.5)) """ if isinstance( inputtemplate, str) or (sys.version < '3' and isinstance( inputtemplate, unicode)): #pylint: disable=undefined-variable data = self.get(project) #causes an extra query to server inputtemplate = data.inputtemplate(inputtemplate) elif not isinstance(inputtemplate, clam.common.data.InputTemplate): raise Exception("inputtemplate must be instance of InputTemplate. Get from CLAMData.inputtemplate(id)") if 'filename' in kwargs: filename = self.getinputfilename(inputtemplate, kwargs['filename']) else: raise Exception("No filename provided!") data = {"contents": contents, 'inputtemplate': inputtemplate.id} for key, value in kwargs.items(): if key == 'filename': pass #nothing to do elif key == 'metadata': assert isinstance(value, clam.common.data.CLAMMetaData) data['metadata'] = value.xml() elif key == 'metafile': data['metafile'] = open(value,'r') else: data[key] = value requestparams = self.initrequest(data) r = requests.post(self.url + project + '/input/' + filename,**requestparams) if r.status_code == 400: raise clam.common.data.BadRequest() elif r.status_code == 401: raise clam.common.data.AuthRequired() elif r.status_code == 403: if r.text[0] == '<': #XML response return self._parseupload(r.text) else: raise clam.common.data.PermissionDenied(r.text) elif r.status_code == 404: raise clam.common.data.NotFound(r.text) elif r.status_code == 500: raise clam.common.data.ServerError(r.text) elif r.status_code == 405: raise clam.common.data.ServerError("Server returned 405: Method not allowed for POST on " + self.url + project + '/input/' + filename) elif r.status_code == 408: raise clam.common.data.TimeOut() elif not (r.status_code >= 200 and r.status_code <= 299): raise Exception("An error occured, return code " + str(r.status_code)) return self._parseupload(r.text)
def addinputfile(self, project, inputtemplate, sourcefile, **kwargs): """Add/upload an input file to the CLAM service. Supports proper file upload streaming. project - the ID of the project you want to add the file to. inputtemplate - The input template you want to use to add this file (InputTemplate instance) sourcefile - The file you want to add: string containing a filename (or instance of ``file``) Keyword arguments (optional but recommended!): * ``filename`` - the filename on the server (will be same as sourcefile if not specified) * ``metadata`` - A metadata object. * ``metafile`` - A metadata file (filename) Any other keyword arguments will be passed as metadata and matched with the input template's parameters. Example:: client.addinputfile("myproject", "someinputtemplate", "/path/to/local/file") With metadata, assuming such metadata parameters are defined:: client.addinputfile("myproject", "someinputtemplate", "/path/to/local/file", parameter1="blah", parameterX=3.5) """ if isinstance( inputtemplate, str) or (sys.version < '3' and isinstance( inputtemplate, unicode)): #pylint: disable=undefined-variable data = self.get(project) #causes an extra query to server inputtemplate = data.inputtemplate(inputtemplate) elif not isinstance(inputtemplate, clam.common.data.InputTemplate): raise Exception("inputtemplate must be instance of InputTemplate. Get from CLAMData.inputtemplate(id)") if not isinstance(sourcefile, IOBase): sourcefile = open(sourcefile,'rb') if 'filename' in kwargs: filename = self.getinputfilename(inputtemplate, kwargs['filename']) else: filename = self.getinputfilename(inputtemplate, os.path.basename(sourcefile.name) ) data = {"file": (filename,sourcefile,inputtemplate.formatclass.mimetype), 'inputtemplate': inputtemplate.id} for key, value in kwargs.items(): if key == 'filename': pass #nothing to do elif key == 'metadata': assert isinstance(value, clam.common.data.CLAMMetaData) data['metadata'] = value.xml() elif key == 'metafile': data['metafile'] = open(value,'rb') else: data[key] = value requestparams = self.initrequest(data) if 'auth'in requestparams: #TODO: streaming support doesn't work with authentication unfortunately, disabling streaming for now: del data['file'] requestparams['data'] = data requestparams['files'] = [('file', (filename,sourcefile, inputtemplate.formatclass.mimetype))] if 'metafile' in kwargs: del data['metafile'] requestparams['files'].append(('metafile',('.'+ filename + '.METADATA', open(kwargs['metafile'],'rb'), 'text/xml'))) else: #streaming support encodeddata = MultipartEncoder(fields=requestparams['data']) #from requests-toolbelt, necessary for streaming support requestparams['data'] = encodeddata requestparams['headers']['Content-Type'] = encodeddata.content_type r = requests.post(self.url + project + '/input/' + filename,**requestparams) sourcefile.close() if r.status_code == 400: raise clam.common.data.BadRequest() elif r.status_code == 401: raise clam.common.data.AuthRequired() elif r.status_code == 403: if r.text[0] == '<': #XML response return self._parseupload(r.text) else: raise clam.common.data.PermissionDenied(r.text) elif r.status_code == 404: raise clam.common.data.NotFound(r.text) elif r.status_code == 500: raise clam.common.data.ServerError(r.text) elif r.status_code == 405: raise clam.common.data.ServerError("Server returned 405: Method not allowed for POST on " + self.url + project + '/input/' + filename) elif r.status_code == 408: raise clam.common.data.TimeOut() elif not (r.status_code >= 200 and r.status_code <= 299): raise Exception("An error occured, return code " + str(r.status_code)) return self._parseupload(r.text)
def addinput(self, project, inputtemplate, contents, **kwargs): """Add an input file to the CLAM service. Explictly providing the contents as a string. This is not suitable for large files as the contents are kept in memory! Use ``addinputfile()`` instead for large files. project - the ID of the project you want to add the file to. inputtemplate - The input template you want to use to add this file (InputTemplate instance) contents - The contents for the file to add (string) Keyword arguments: * filename - the filename on the server (mandatory!) * metadata - A metadata object. * metafile - A metadata file (filename) Any other keyword arguments will be passed as metadata and matched with the input template's parameters. Example:: client.addinput("myproject", "someinputtemplate", "This is a test.", filename="test.txt") With metadata, assuming such metadata parameters are defined:: client.addinput("myproject", "someinputtemplate", "This is a test.", filename="test.txt", parameter1="blah", parameterX=3.5)) """ if isinstance(inputtemplate, str): #pylint: disable=undefined-variable data = self.get(project) #causes an extra query to server inputtemplate = data.inputtemplate(inputtemplate) elif not isinstance(inputtemplate, clam.common.data.InputTemplate): raise Exception( "inputtemplate must be instance of InputTemplate. Get from CLAMData.inputtemplate(id)" ) if 'filename' in kwargs: filename = self.getinputfilename(inputtemplate, kwargs['filename']) else: raise Exception("No filename provided!") data = {"contents": contents, 'inputtemplate': inputtemplate.id} for key, value in kwargs.items(): if key == 'filename': pass #nothing to do elif key == 'metadata': assert isinstance(value, clam.common.data.CLAMMetaData) data['metadata'] = value.xml() elif key == 'metafile': data['metafile'] = open(value, 'r') else: data[key] = value requestparams = self.initrequest(data) r = requests.post(self.url + project + '/input/' + filename, **requestparams) if r.status_code == 400: raise clam.common.data.BadRequest() elif r.status_code == 401: raise clam.common.data.AuthRequired() elif r.status_code == 403: if r.text[0] == '<': #XML response return self._parseupload(r.text) else: raise clam.common.data.PermissionDenied(r.text) elif r.status_code == 404: raise clam.common.data.NotFound(r.text) elif r.status_code == 500: raise clam.common.data.ServerError(r.text) elif r.status_code == 405: raise clam.common.data.ServerError( "Server returned 405: Method not allowed for POST on " + self.url + project + '/input/' + filename) elif r.status_code == 408: raise clam.common.data.TimeOut() elif not (r.status_code >= 200 and r.status_code <= 299): raise Exception("An error occured, return code " + str(r.status_code)) return self._parseupload(r.text)
def addinputfile(self, project, inputtemplate, sourcefile, **kwargs): """Add/upload an input file to the CLAM service. Supports proper file upload streaming. project - the ID of the project you want to add the file to. inputtemplate - The input template you want to use to add this file (InputTemplate instance) sourcefile - The file you want to add: string containing a filename (or instance of ``file``) Keyword arguments (optional but recommended!): * ``filename`` - the filename on the server (will be same as sourcefile if not specified) * ``metadata`` - A metadata object. * ``metafile`` - A metadata file (filename) Any other keyword arguments will be passed as metadata and matched with the input template's parameters. Example:: client.addinputfile("myproject", "someinputtemplate", "/path/to/local/file") With metadata, assuming such metadata parameters are defined:: client.addinputfile("myproject", "someinputtemplate", "/path/to/local/file", parameter1="blah", parameterX=3.5) """ if isinstance(inputtemplate, str): #pylint: disable=undefined-variable data = self.get(project) #causes an extra query to server inputtemplate = data.inputtemplate(inputtemplate) elif not isinstance(inputtemplate, clam.common.data.InputTemplate): raise Exception( "inputtemplate must be instance of InputTemplate. Get from CLAMData.inputtemplate(id)" ) if not isinstance(sourcefile, IOBase): sourcefile = open(sourcefile, 'rb') if 'filename' in kwargs: filename = self.getinputfilename(inputtemplate, kwargs['filename']) else: filename = self.getinputfilename( inputtemplate, os.path.basename(sourcefile.name)) data = { "file": (filename, sourcefile, inputtemplate.formatclass.mimetype), 'inputtemplate': inputtemplate.id } for key, value in kwargs.items(): if key == 'filename': pass #nothing to do elif key == 'metadata': assert isinstance(value, clam.common.data.CLAMMetaData) data['metadata'] = value.xml() elif key == 'metafile': data['metafile'] = open(value, 'rb') else: data[key] = value requestparams = self.initrequest(data) if 'auth' in requestparams: #TODO: streaming support doesn't work with authentication unfortunately, disabling streaming for now: del data['file'] requestparams['data'] = data requestparams['files'] = [('file', (filename, sourcefile, inputtemplate.formatclass.mimetype))] if 'metafile' in kwargs: del data['metafile'] requestparams['files'].append( ('metafile', ('.' + filename + '.METADATA', open(kwargs['metafile'], 'rb'), 'text/xml'))) else: #streaming support encodeddata = MultipartEncoder( fields=requestparams['data'] ) #from requests-toolbelt, necessary for streaming support requestparams['data'] = encodeddata requestparams['headers']['Content-Type'] = encodeddata.content_type r = requests.post(self.url + project + '/input/' + filename, **requestparams) sourcefile.close() if r.status_code == 400: raise clam.common.data.BadRequest() elif r.status_code == 401: raise clam.common.data.AuthRequired() elif r.status_code == 403: if r.text[0] == '<': #XML response return self._parseupload(r.text) else: raise clam.common.data.PermissionDenied(r.text) elif r.status_code == 404: raise clam.common.data.NotFound(r.text) elif r.status_code == 500: raise clam.common.data.ServerError(r.text) elif r.status_code == 405: raise clam.common.data.ServerError( "Server returned 405: Method not allowed for POST on " + self.url + project + '/input/' + filename) elif r.status_code == 408: raise clam.common.data.TimeOut() elif not (r.status_code >= 200 and r.status_code <= 299): raise Exception("An error occured, return code " + str(r.status_code)) return self._parseupload(r.text)
#the latter two arguments are required for authenticated webservices, they can be omitted otherwise #if you use SSL (https) and SSL verification fails, you can pass a verify= parameter with the path to your certificate of certificate authority bundle clamclient = clam.common.client.CLAMClient( "https://webservices-lst.science.ru.nl/valkuil", args.username, args.password) project = "valkuil" + str(random.getrandbits(64)) clamclient.create(project) data = clamclient.get(project) for inputfile in glob.glob(os.path.join(args.inputdir, '*.folia.xml')): print("Uploading " + os.path.basename(inputfile), file=sys.stderr) clamclient.addinputfile(project, data.inputtemplate("foliainput"), inputfile) data = clamclient.start(project, sensitivity=0.75) #Always check for parameter errors! Don't just assume everything went well! Use startsafe() instead of start #to simply raise exceptions on parameter errors. if data.errors: print("An error occured: " + data.errormsg, file=sys.stderr) for parametergroup, paramlist in data.parameters: for parameter in paramlist: if parameter.error: print("Error in parameter " + parameter.id + ": " + parameter.error, file=sys.stderr) clamclient.delete(
def add_file_to_clam(localfilename): clamclient.addinputfile(project, data.inputtemplate(inputtemplate), localfilename, encoding='utf-8')