def post(self, query): i = self.institution logging.debug('posting data to %s' % i.url) logging.debug('---- request ----') logging.debug(query) garbage, path = splittype(i.url) host, selector = splithost(path) h = HTTPSConnection(host, timeout=60) # Discover requires a particular ordering of headers, so send the # request step by step. h.putrequest('POST', selector, skip_host=True, skip_accept_encoding=True) h.putheader('Content-Type', 'application/x-ofx') h.putheader('Host', host) h.putheader('Content-Length', len(query)) h.putheader('Connection', 'Keep-Alive') if self.accept: h.putheader('Accept', self.accept) if self.user_agent: h.putheader('User-Agent', self.user_agent) h.endheaders(query.encode()) res = h.getresponse() response = res.read().decode('ascii', 'ignore') logging.debug('---- response ----') logging.debug(res.__dict__) logging.debug(response) res.close() return response
def make_http_request(url, data=None, files=None, certificate=None): # type: (str, Optional[Envelope], Optional[Attachments], str) -> Tuple[int, bytes] if data is None: data = {} if files is None: files = {} schema, netloc, url, params, query, fragments = urlparse(url) logger.debug("Sending POST request to {0}".format(url)) body, boundary = encode_multipart_formdata(data, files) if certificate: ssl_context = ssl.create_default_context(cafile=certificate) else: ssl_context = ssl._create_default_https_context() connection = HTTPSConnection(netloc, context=ssl_context) connection.connect() connection.putrequest("POST", url) connection.putheader("Content-Type", "multipart/form-data; boundary={0}".format(boundary)) connection.putheader("Content-Length", str(len(body))) connection.endheaders() connection.send(body) r = connection.getresponse() return r.status, r.read()
def _do_post(self, query, extra_headers=[]): """ Do a POST to the Institution. :param query: Body content to POST (OFX Query) :type query: str :param extra_headers: Extra headers to send with the request, as a list of (Name, Value) header 2-tuples. :type extra_headers: list :return: 2-tuple of (HTTPResponse, str response body) :rtype: tuple """ i = self.institution logging.debug('posting data to %s' % i.url) garbage, path = splittype(i.url) host, selector = splithost(path) try: h = HTTPSConnection(host, timeout=60) h.connect() except ssl.SSLError as ex: if (ex.reason == "UNSUPPORTED_PROTOCOL"): h = HTTPSConnection(host, timeout=60, context=ssl.SSLContext(ssl.PROTOCOL_TLSv1)) h.connect() else: raise # Discover requires a particular ordering of headers, so send the # request step by step. h.putrequest('POST', selector, skip_host=True, skip_accept_encoding=True) headers = [('Content-Type', 'application/x-ofx'), ('Host', host), ('Content-Length', len(query)), ('Connection', 'Keep-Alive')] if self.accept: headers.append(('Accept', self.accept)) if self.user_agent: headers.append(('User-Agent', self.user_agent)) for ehname, ehval in extra_headers: headers.append((ehname, ehval)) logging.debug('---- request headers ----') for hname, hval in headers: logging.debug('%s: %s', hname, hval) h.putheader(hname, hval) logging.debug('---- request body (query) ----') logging.debug(query) h.endheaders(query.encode()) res = h.getresponse() response = res.read().decode('ascii', 'ignore') logging.debug('---- response ----') logging.debug(res.__dict__) logging.debug('Headers: %s', res.getheaders()) logging.debug(response) res.close() return res, response
def doheadrequest(self, timeout=TIMEOUT): """ Sends a HEAD request to the database node. The header returned by the database node contains some information on statistics. This information is stored in the headers object of the request instance. """ self.headers = {} url = self.baseurl + self.querypath urlobj = urlsplit(url) if urlobj.scheme == 'https': if self.verifyhttps: conn = HTTPSConnection(urlobj.netloc, timeout=timeout) else: conn = HTTPSConnection( urlobj.netloc, timeout=timeout, context=ssl._create_unverified_context()) else: conn = HTTPConnection(urlobj.netloc, timeout=timeout) conn.putrequest("HEAD", urlobj.path + "?" + urlobj.query) conn.endheaders() try: res = conn.getresponse() except socket.timeout as e: self.status = 408 self.reason = "Socket timeout" raise TimeOutError self.status = res.status self.reason = res.reason if res.status == 200: headers = res.getheaders() elif res.status == 204: headers = [("vamdc-count-species", 0), ("vamdc-count-states", 0), ("vamdc-truncated", 0), ("vamdc-count-molecules", 0), ("vamdc-count-sources", 0), ("vamdc-approx-size", 0), ("vamdc-count-radiative", 0), ("vamdc-count-atoms", 0)] elif res.status == 408: print("TIMEOUT") headers = [("vamdc-count-species", 0), ("vamdc-count-states", 0), ("vamdc-truncated", 0), ("vamdc-count-molecules", 0), ("vamdc-count-sources", 0), ("vamdc-approx-size", 0), ("vamdc-count-radiative", 0), ("vamdc-count-atoms", 0)] else: print("STATUS: %d" % res.status) headers = [("vamdc-count-species", 0), ("vamdc-count-states", 0), ("vamdc-truncated", 0), ("vamdc-count-molecules", 0), ("vamdc-count-sources", 0), ("vamdc-approx-size", 0), ("vamdc-count-radiative", 0), ("vamdc-count-atoms", 0)] for key, value in headers: self.headers[key] = value
def _do_post(self, query, extra_headers=[]): """ Do a POST to the Institution. :param query: Body content to POST (OFX Query) :type query: str :param extra_headers: Extra headers to send with the request, as a list of (Name, Value) header 2-tuples. :type extra_headers: list :return: 2-tuple of (HTTPResponse, str response body) :rtype: tuple """ i = self.institution logging.debug('posting data to %s' % i.url) garbage, path = splittype(i.url) host, selector = splithost(path) try: h = HTTPSConnection(host, timeout=60) h.connect() except ssl.SSLError as ex: if (ex.reason == "UNSUPPORTED_PROTOCOL"): h = HTTPSConnection(host, timeout=60, context=ssl.SSLContext(ssl.PROTOCOL_TLSv1)) h.connect() else: raise # Discover requires a particular ordering of headers, so send the # request step by step. h.putrequest('POST', selector, skip_host=True, skip_accept_encoding=True) headers = [ ('Content-Type', 'application/x-ofx'), ('Host', host), ('Content-Length', len(query)), ('Connection', 'Keep-Alive') ] if self.accept: headers.append(('Accept', self.accept)) if self.user_agent: headers.append(('User-Agent', self.user_agent)) for ehname, ehval in extra_headers: headers.append((ehname, ehval)) logging.debug('---- request headers ----') for hname, hval in headers: logging.debug('%s: %s', hname, hval) h.putheader(hname, hval) logging.debug('---- request body (query) ----') logging.debug(query) h.endheaders(query.encode()) res = h.getresponse() response = res.read().decode('ascii', 'ignore') logging.debug('---- response ----') logging.debug(res.__dict__) logging.debug('Headers: %s', res.getheaders()) logging.debug(response) res.close() return res, response
def fetch_file(self, server, path): "Fetch file using httplib module." print("downloading https://%s%s" % (server, path)) req = HTTPSConnection(server) req.putrequest('GET', path) req.putheader('Host', server) req.putheader('Accept', 'text/svg') req.endheaders() r1 = req.getresponse() data = r1.read().decode('utf-8') req.close() return data
def fetchFile(self, server, path): "Fetch file using httplib module." print("downloading http://%s%s" % (server, path)) req = HTTPSConnection(server) req.putrequest('GET', path) req.putheader('Host', server) req.putheader('Accept', 'text/svg') req.endheaders() r1 = req.getresponse() data = r1.read().decode('utf-8') req.close() return data
def endheaders(self, *args, **kwargs): if self._proxy_auth is None: self._proxy_auth = self._encode_auth() HTTPSConnection.endheaders(self, *args, **kwargs)
def dorequest(self, timeout=TIMEOUT, HttpMethod="POST", parsexsams=True): """ Sends the request to the database node and returns a result.Result instance. The request uses 'POST' requests by default. If the request fails or if stated in the parameter 'HttpMethod', 'GET' requests will be performed. The returned result will be parsed by default and the model defined in 'specmodel' will be populated by default (parseexams = True). """ self.xml = None #self.get_xml(self.Source.Requesturl) url = self.baseurl + self.querypath urlobj = urlsplit(url) if urlobj.scheme == 'https': if self.verifyhttps: conn = HTTPSConnection(urlobj.netloc, timeout=timeout) else: conn = HTTPSConnection( urlobj.netloc, timeout=timeout, context=ssl._create_unverified_context()) else: conn = HTTPConnection(urlobj.netloc, timeout=timeout) conn.putrequest(HttpMethod, urlobj.path + "?" + urlobj.query) conn.endheaders() try: res = conn.getresponse() except socket.timeout: # error handling has to be included self.status = 408 self.reason = "Socket timeout" raise TimeOutError self.status = res.status self.reason = res.reason if not parsexsams: if res.status == 200: result = r.Result() result.Content = res.read() elif res.status == 400 and HttpMethod == 'POST': # Try to use http-method: GET result = self.dorequest(HttpMethod='GET', parsexsams=parsexsams) else: result = None else: if res.status == 200: self.xml = res.read() result = r.Result() result.Xml = self.xml result.populate_model() elif res.status == 400 and HttpMethod == 'POST': # Try to use http-method: GET result = self.dorequest(HttpMethod='GET', parsexsams=parsexsams) else: result = None return result
class HessianProxy(object): def __init__(self, service_uri, credentials=None, key_file=None, cert_file=None, timeout=10, buffer_size=65535, error_factory=lambda x: x, overload=False): self._headers = list() self._headers.append(('User-Agent', 'mustaine/' + __version__,)) self._headers.append(('Content-Type', 'application/x-hessian',)) if sys.version_info < (2,6): warn('HessianProxy timeout not enforceable before Python 2.6', RuntimeWarning, stacklevel=2) timeout = {} else: timeout = {'timeout': timeout} self._uri = urlparse(service_uri) if self._uri.scheme == 'http': self._client = HTTPConnection(self._uri.hostname, self._uri.port or 80, strict=True, **timeout) elif self._uri.scheme == 'https': self._client = HTTPSConnection(self._uri.hostname, self._uri.port or 443, key_file=key_file, cert_file=cert_file, strict=True, **timeout) else: raise NotImplementedError("HessianProxy only supports http:// and https:// URIs") # autofill credentials if they were passed via url instead of kwargs if (self._uri.username and self._uri.password) and not credentials: credentials = (self._uri.username, self._uri.password) if credentials: auth = 'Basic ' + base64.b64encode(':'.join(credentials)) self._headers.append(('Authorization', auth)) self._buffer_size = buffer_size self._error_factory = error_factory self._overload = overload self._parser = Parser() class __RemoteMethod(object): # dark magic for autoloading methods def __init__(self, caller, method): self.__caller = caller self.__method = method def __call__(self, *args): return self.__caller(self.__method, args) def __getattr__(self, method): return self.__RemoteMethod(self, method) def __repr__(self): return "<mustaine.client.HessianProxy(\"%s\")>" % (self._uri.geturl(),) def __str__(self): return self.__repr__() def __call__(self, method, args): try: self._client.putrequest('POST', self._uri.path) for header in self._headers: self._client.putheader(*header) request = encode_object(Call(method, args, overload=self._overload)) self._client.putheader("Content-Length", str(len(request))) self._client.endheaders() self._client.send(str(request)) response = self._client.getresponse() if response.status != 200: raise ProtocolError(self._uri.geturl(), response.status, response.reason) length = response.getheader('Content-Length', -1) if length == '0': raise ProtocolError(self._uri.geturl(), 'FATAL:', 'Server sent zero-length response') reply = self._parser.parse_stream(BufferedReader(response, buffer_size=self._buffer_size)) self._client.close() if isinstance(reply.value, Fault): raise self._error_factory(reply.value) else: return reply.value except: self._client.close() raise
def endheaders(self, *args, **kwargs): if self._proxy_auth is None: self._proxy_auth = self._encode_auth() HTTPSConnection.endheaders(self, *args, **kwargs)
class HessianProxy(object): def __init__( self, service_uri, credentials=None, key_file=None, cert_file=None, timeout=10, buffer_size=65535, error_factory=lambda x: x, overload=False, ): self._headers = list() self._headers.append(("User-Agent", "mustaine/" + __version__)) self._headers.append(("Content-Type", "application/x-hessian")) if sys.version_info < (2, 6): warn("HessianProxy timeout not enforceable before Python 2.6", RuntimeWarning, stacklevel=2) timeout = {} else: timeout = {"timeout": timeout} self._uri = urlparse(service_uri) if self._uri.scheme == "http": self._client = HTTPConnection(self._uri.hostname, self._uri.port or 80, strict=True, **timeout) elif self._uri.scheme == "https": self._client = HTTPSConnection( self._uri.hostname, self._uri.port or 443, key_file=key_file, cert_file=cert_file, strict=True, **timeout ) else: raise NotImplementedError("HessianProxy only supports http:// and https:// URIs") # autofill credentials if they were passed via url instead of kwargs if (self._uri.username and self._uri.password) and not credentials: credentials = (self._uri.username, self._uri.password) if credentials: auth = "Basic " + base64.b64encode(":".join(credentials)) self._headers.append(("Authorization", auth)) self._buffer_size = buffer_size self._error_factory = error_factory self._overload = overload self._parser = Parser() class __RemoteMethod(object): # dark magic for autoloading methods def __init__(self, caller, method): self.__caller = caller self.__method = method def __call__(self, *args): return self.__caller(self.__method, args) def __getattr__(self, method): return self.__RemoteMethod(self, method) def __repr__(self): return '<mustaine.client.HessianProxy("%s")>' % (self._uri.geturl(),) def __str__(self): return self.__repr__() def __call__(self, method, args): try: self._client.putrequest("POST", self._uri.path) for header in self._headers: self._client.putheader(*header) request = encode_object(Call(method, args, overload=self._overload)) self._client.putheader("Content-Length", str(len(request)).encode("utf8")) self._client.endheaders() self._client.send(request) response = self._client.getresponse() if response.status != 200: raise ProtocolError(self._uri.geturl(), response.status, response.reason) length = response.getheader("Content-Length", -1) if length == "0": raise ProtocolError(self._uri.geturl(), "FATAL:", "Server sent zero-length response") reply = self._parser.parse_stream(BufferedReader(response, buffer_size=self._buffer_size)) self._client.close() if isinstance(reply.value, Fault): raise self._error_factory(reply.value) else: return reply.value except: self._client.close() raise
class ClamAVFileUploadHandler(FileUploadHandler): chunk_size = CHUNK_SIZE skip_av_check = False def new_file(self, *args, **kwargs): super().new_file(*args, **kwargs) extension = pathlib.Path(self.file_name).suffix if extension in CLAM_AV_IGNORE_EXTENSIONS: self.skip_av_check = True return if CLAM_USE_HTTP: self.av_conn = HTTPConnection(host=CLAM_AV_DOMAIN, ) else: self.av_conn = HTTPSConnection( # noqa S309 host=CLAM_AV_DOMAIN, port=443, ) credentials = b64encode( bytes( f"{CLAM_AV_USERNAME}:{CLAM_AV_PASSWORD}", encoding="utf8", )).decode("ascii") try: self.av_conn.connect() self.av_conn.putrequest("POST", CLAM_PATH) self.av_conn.putheader("Content-Type", self.content_type) self.av_conn.putheader("Authorization", f"Basic {credentials}") self.av_conn.putheader("Transfer-encoding", "chunked") self.av_conn.endheaders() except Exception as ex: logger.error("Error connecting to ClamAV service", exc_info=True) raise AntiVirusServiceErrorException(ex) def receive_data_chunk(self, raw_data, start): if not self.skip_av_check: self.av_conn.send(hex(len(raw_data))[2:].encode("utf-8")) self.av_conn.send(b"\r\n") self.av_conn.send(raw_data) self.av_conn.send(b"\r\n") return raw_data def file_complete(self, file_size): if self.skip_av_check: return None self.av_conn.send(b"0\r\n\r\n") resp = self.av_conn.getresponse() response_content = resp.read() scanned_file = ScannedFile() if resp.status != 200: scanned_file.av_passed = False scanned_file.av_reason = "Non 200 response from AV server" scanned_file.save() raise AntiVirusServiceErrorException( f"Non 200 response from anti virus service, content: {response_content}" ) else: json_response = json.loads(response_content) if "malware" not in json_response: scanned_file.av_passed = False scanned_file.av_reason = "Malformed response from AV server" scanned_file.save() raise MalformedAntiVirusResponseException() if json_response["malware"]: scanned_file.av_passed = False scanned_file.av_reason = json_response["reason"] scanned_file.save() logger.error(f"Malware found in user uploaded file " f"'{self.file_name}', exiting upload process") else: scanned_file.av_passed = True scanned_file.save() # We are using 'content_type_extra' as the a means of making # the results available to following file handlers # TODO - put in a PR to Django project to allow file_complete # to return objects and not break out of file handler loop if not hasattr(self.content_type_extra, "clam_av_results"): self.content_type_extra["clam_av_results"] = [] self.content_type_extra["clam_av_results"].append({ "file_name": self.file_name, "av_passed": scanned_file.av_passed, "scanned_at": scanned_file.scanned_at, }) return None
def upload_document(path, progress_tracker=None, file_id=0, post_url='/upload_file3'): """ :param path: :param progress_tracker: :param file_id: 0 if uploading a new document, not 0 if repplacing an existing document. :param post_url: :return: """ mr = None if progress_tracker: mr = Wrap(progress_tracker) else: mr = MultiRead() mr.add_field('file_id', str(file_id)) # 0 == It's a new document, > 0 == overwrite # We store the filename here 'cos the one encoded in the file part # must be ASCII (IETF, RFC 2183, section 2.3 : Current [RFC 2045] grammar # restricts parameter values (and hence Content-Disposition filenames) # to US-ASCII. We recognize the great desirability of allowing # arbitrary character sets in filenames, but it is beyond the # scope of this document to define the necessary mechanisms. mr.add_field('encoding_safe_filename', os.path.split(path)[-1]) mr.add_file_part(path) mr.close_parts() host, port = extract_host_port( configuration.get("DownloadSite", "base_url")) mainlog.debug( u"Upload to {}:{}{} (determined from DownloadSite/base_url : {})". format(host, port, post_url, configuration.get("DownloadSite", "base_url"))) if configuration.get("DownloadSite", "base_url").startswith('https'): h = HTTPSConnection(host, port) else: h = HTTPConnection(host, port) h.putrequest('POST', post_url) h.putheader('content-type', mr.content_type()) h.putheader('content-length', str(mr.total_size())) h.putheader('x-filesize', str(mr.total_size())) h.endheaders() mr.open() h.send(mr) mr.close() server_response = h.getresponse() if server_response.status == OK: server_response.getheaders( ) # Skip headers (is this really necessary ?) t = server_response.read() file_id = int(t) mainlog.debug("Successfully uploaded {} bytes".format(mr.total_size())) h.close() return file_id else: raise Exception( "Unable to upload, server response status was {}".format( server_response.status))