def post(self, query):
i = self.institution
logging.debug('posting data to %s' % i.url)
logging.debug('---- request ----')
logging.debug(query)
garbage, path = splittype(i.url)
host, selector = splithost(path)
h = HTTPSConnection(host, timeout=60)
# Discover requires a particular ordering of headers, so send the
# request step by step.
h.putrequest('POST',
selector,
skip_host=True,
skip_accept_encoding=True)
h.putheader('Content-Type', 'application/x-ofx')
h.putheader('Host', host)
h.putheader('Content-Length', len(query))
h.putheader('Connection', 'Keep-Alive')
if self.accept:
h.putheader('Accept', self.accept)
if self.user_agent:
h.putheader('User-Agent', self.user_agent)
h.endheaders(query.encode())
res = h.getresponse()
response = res.read().decode('ascii', 'ignore')
logging.debug('---- response ----')
logging.debug(res.__dict__)
logging.debug(response)
res.close()
return response
def make_http_request(url, data=None, files=None, certificate=None):
# type: (str, Optional[Envelope], Optional[Attachments], str) -> Tuple[int, bytes]
if data is None:
data = {}
if files is None:
files = {}
schema, netloc, url, params, query, fragments = urlparse(url)
logger.debug("Sending POST request to {0}".format(url))
body, boundary = encode_multipart_formdata(data, files)
if certificate:
ssl_context = ssl.create_default_context(cafile=certificate)
else:
ssl_context = ssl._create_default_https_context()
connection = HTTPSConnection(netloc, context=ssl_context)
connection.connect()
connection.putrequest("POST", url)
connection.putheader("Content-Type",
"multipart/form-data; boundary={0}".format(boundary))
connection.putheader("Content-Length", str(len(body)))
connection.endheaders()
connection.send(body)
r = connection.getresponse()
return r.status, r.read()
def _do_post(self, query, extra_headers=[]):
"""
Do a POST to the Institution.
:param query: Body content to POST (OFX Query)
:type query: str
:param extra_headers: Extra headers to send with the request, as a list
of (Name, Value) header 2-tuples.
:type extra_headers: list
:return: 2-tuple of (HTTPResponse, str response body)
:rtype: tuple
"""
i = self.institution
logging.debug('posting data to %s' % i.url)
garbage, path = splittype(i.url)
host, selector = splithost(path)
try:
h = HTTPSConnection(host, timeout=60)
h.connect()
except ssl.SSLError as ex:
if (ex.reason == "UNSUPPORTED_PROTOCOL"):
h = HTTPSConnection(host,
timeout=60,
context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
h.connect()
else:
raise
# Discover requires a particular ordering of headers, so send the
# request step by step.
h.putrequest('POST',
selector,
skip_host=True,
skip_accept_encoding=True)
headers = [('Content-Type', 'application/x-ofx'), ('Host', host),
('Content-Length', len(query)),
('Connection', 'Keep-Alive')]
if self.accept:
headers.append(('Accept', self.accept))
if self.user_agent:
headers.append(('User-Agent', self.user_agent))
for ehname, ehval in extra_headers:
headers.append((ehname, ehval))
logging.debug('---- request headers ----')
for hname, hval in headers:
logging.debug('%s: %s', hname, hval)
h.putheader(hname, hval)
logging.debug('---- request body (query) ----')
logging.debug(query)
h.endheaders(query.encode())
res = h.getresponse()
response = res.read().decode('ascii', 'ignore')
logging.debug('---- response ----')
logging.debug(res.__dict__)
logging.debug('Headers: %s', res.getheaders())
logging.debug(response)
res.close()
return res, response
def doheadrequest(self, timeout=TIMEOUT):
"""
Sends a HEAD request to the database node. The header returned by the database node contains some
information on statistics. This information is stored in the headers object of the request instance.
"""
self.headers = {}
url = self.baseurl + self.querypath
urlobj = urlsplit(url)
if urlobj.scheme == 'https':
if self.verifyhttps:
conn = HTTPSConnection(urlobj.netloc, timeout=timeout)
else:
conn = HTTPSConnection(
urlobj.netloc,
timeout=timeout,
context=ssl._create_unverified_context())
else:
conn = HTTPConnection(urlobj.netloc, timeout=timeout)
conn.putrequest("HEAD", urlobj.path + "?" + urlobj.query)
conn.endheaders()
try:
res = conn.getresponse()
except socket.timeout as e:
self.status = 408
self.reason = "Socket timeout"
raise TimeOutError
self.status = res.status
self.reason = res.reason
if res.status == 200:
headers = res.getheaders()
elif res.status == 204:
headers = [("vamdc-count-species", 0), ("vamdc-count-states", 0),
("vamdc-truncated", 0), ("vamdc-count-molecules", 0),
("vamdc-count-sources", 0), ("vamdc-approx-size", 0),
("vamdc-count-radiative", 0), ("vamdc-count-atoms", 0)]
elif res.status == 408:
print("TIMEOUT")
headers = [("vamdc-count-species", 0), ("vamdc-count-states", 0),
("vamdc-truncated", 0), ("vamdc-count-molecules", 0),
("vamdc-count-sources", 0), ("vamdc-approx-size", 0),
("vamdc-count-radiative", 0), ("vamdc-count-atoms", 0)]
else:
print("STATUS: %d" % res.status)
headers = [("vamdc-count-species", 0), ("vamdc-count-states", 0),
("vamdc-truncated", 0), ("vamdc-count-molecules", 0),
("vamdc-count-sources", 0), ("vamdc-approx-size", 0),
("vamdc-count-radiative", 0), ("vamdc-count-atoms", 0)]
for key, value in headers:
self.headers[key] = value
def _do_post(self, query, extra_headers=[]):
"""
Do a POST to the Institution.
:param query: Body content to POST (OFX Query)
:type query: str
:param extra_headers: Extra headers to send with the request, as a list
of (Name, Value) header 2-tuples.
:type extra_headers: list
:return: 2-tuple of (HTTPResponse, str response body)
:rtype: tuple
"""
i = self.institution
logging.debug('posting data to %s' % i.url)
garbage, path = splittype(i.url)
host, selector = splithost(path)
try:
h = HTTPSConnection(host, timeout=60)
h.connect()
except ssl.SSLError as ex:
if (ex.reason == "UNSUPPORTED_PROTOCOL"):
h = HTTPSConnection(host, timeout=60, context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
h.connect()
else:
raise
# Discover requires a particular ordering of headers, so send the
# request step by step.
h.putrequest('POST', selector, skip_host=True,
skip_accept_encoding=True)
headers = [
('Content-Type', 'application/x-ofx'),
('Host', host),
('Content-Length', len(query)),
('Connection', 'Keep-Alive')
]
if self.accept:
headers.append(('Accept', self.accept))
if self.user_agent:
headers.append(('User-Agent', self.user_agent))
for ehname, ehval in extra_headers:
headers.append((ehname, ehval))
logging.debug('---- request headers ----')
for hname, hval in headers:
logging.debug('%s: %s', hname, hval)
h.putheader(hname, hval)
logging.debug('---- request body (query) ----')
logging.debug(query)
h.endheaders(query.encode())
res = h.getresponse()
response = res.read().decode('ascii', 'ignore')
logging.debug('---- response ----')
logging.debug(res.__dict__)
logging.debug('Headers: %s', res.getheaders())
logging.debug(response)
res.close()
return res, response
def fetch_file(self, server, path):
"Fetch file using httplib module."
print("downloading https://%s%s" % (server, path))
req = HTTPSConnection(server)
req.putrequest('GET', path)
req.putheader('Host', server)
req.putheader('Accept', 'text/svg')
req.endheaders()
r1 = req.getresponse()
data = r1.read().decode('utf-8')
req.close()
return data
def fetchFile(self, server, path):
"Fetch file using httplib module."
print("downloading http://%s%s" % (server, path))
req = HTTPSConnection(server)
req.putrequest('GET', path)
req.putheader('Host', server)
req.putheader('Accept', 'text/svg')
req.endheaders()
r1 = req.getresponse()
data = r1.read().decode('utf-8')
req.close()
return data
def dorequest(self, timeout=TIMEOUT, HttpMethod="POST", parsexsams=True):
"""
Sends the request to the database node and returns a result.Result instance. The
request uses 'POST' requests by default. If the request fails or if stated in the parameter 'HttpMethod',
'GET' requests will be performed.
The returned result will be parsed by default and the model defined in 'specmodel' will be populated by default
(parseexams = True).
"""
self.xml = None
#self.get_xml(self.Source.Requesturl)
url = self.baseurl + self.querypath
urlobj = urlsplit(url)
if urlobj.scheme == 'https':
if self.verifyhttps:
conn = HTTPSConnection(urlobj.netloc, timeout=timeout)
else:
conn = HTTPSConnection(
urlobj.netloc,
timeout=timeout,
context=ssl._create_unverified_context())
else:
conn = HTTPConnection(urlobj.netloc, timeout=timeout)
conn.putrequest(HttpMethod, urlobj.path + "?" + urlobj.query)
conn.endheaders()
try:
res = conn.getresponse()
except socket.timeout:
# error handling has to be included
self.status = 408
self.reason = "Socket timeout"
raise TimeOutError
self.status = res.status
self.reason = res.reason
if not parsexsams:
if res.status == 200:
result = r.Result()
result.Content = res.read()
elif res.status == 400 and HttpMethod == 'POST':
# Try to use http-method: GET
result = self.dorequest(HttpMethod='GET',
parsexsams=parsexsams)
else:
result = None
else:
if res.status == 200:
self.xml = res.read()
result = r.Result()
result.Xml = self.xml
result.populate_model()
elif res.status == 400 and HttpMethod == 'POST':
# Try to use http-method: GET
result = self.dorequest(HttpMethod='GET',
parsexsams=parsexsams)
else:
result = None
return result
class HessianProxy(object):
def __init__(self, service_uri, credentials=None, key_file=None, cert_file=None, timeout=10, buffer_size=65535, error_factory=lambda x: x, overload=False):
self._headers = list()
self._headers.append(('User-Agent', 'mustaine/' + __version__,))
self._headers.append(('Content-Type', 'application/x-hessian',))
if sys.version_info < (2,6):
warn('HessianProxy timeout not enforceable before Python 2.6', RuntimeWarning, stacklevel=2)
timeout = {}
else:
timeout = {'timeout': timeout}
self._uri = urlparse(service_uri)
if self._uri.scheme == 'http':
self._client = HTTPConnection(self._uri.hostname,
self._uri.port or 80,
strict=True,
**timeout)
elif self._uri.scheme == 'https':
self._client = HTTPSConnection(self._uri.hostname,
self._uri.port or 443,
key_file=key_file,
cert_file=cert_file,
strict=True,
**timeout)
else:
raise NotImplementedError("HessianProxy only supports http:// and https:// URIs")
# autofill credentials if they were passed via url instead of kwargs
if (self._uri.username and self._uri.password) and not credentials:
credentials = (self._uri.username, self._uri.password)
if credentials:
auth = 'Basic ' + base64.b64encode(':'.join(credentials))
self._headers.append(('Authorization', auth))
self._buffer_size = buffer_size
self._error_factory = error_factory
self._overload = overload
self._parser = Parser()
class __RemoteMethod(object):
# dark magic for autoloading methods
def __init__(self, caller, method):
self.__caller = caller
self.__method = method
def __call__(self, *args):
return self.__caller(self.__method, args)
def __getattr__(self, method):
return self.__RemoteMethod(self, method)
def __repr__(self):
return "<mustaine.client.HessianProxy(\"%s\")>" % (self._uri.geturl(),)
def __str__(self):
return self.__repr__()
def __call__(self, method, args):
try:
self._client.putrequest('POST', self._uri.path)
for header in self._headers:
self._client.putheader(*header)
request = encode_object(Call(method, args, overload=self._overload))
self._client.putheader("Content-Length", str(len(request)))
self._client.endheaders()
self._client.send(str(request))
response = self._client.getresponse()
if response.status != 200:
raise ProtocolError(self._uri.geturl(), response.status, response.reason)
length = response.getheader('Content-Length', -1)
if length == '0':
raise ProtocolError(self._uri.geturl(), 'FATAL:', 'Server sent zero-length response')
reply = self._parser.parse_stream(BufferedReader(response, buffer_size=self._buffer_size))
self._client.close()
if isinstance(reply.value, Fault):
raise self._error_factory(reply.value)
else:
return reply.value
except:
self._client.close()
raise
class HessianProxy(object):
def __init__(
self,
service_uri,
credentials=None,
key_file=None,
cert_file=None,
timeout=10,
buffer_size=65535,
error_factory=lambda x: x,
overload=False,
):
self._headers = list()
self._headers.append(("User-Agent", "mustaine/" + __version__))
self._headers.append(("Content-Type", "application/x-hessian"))
if sys.version_info < (2, 6):
warn("HessianProxy timeout not enforceable before Python 2.6", RuntimeWarning, stacklevel=2)
timeout = {}
else:
timeout = {"timeout": timeout}
self._uri = urlparse(service_uri)
if self._uri.scheme == "http":
self._client = HTTPConnection(self._uri.hostname, self._uri.port or 80, strict=True, **timeout)
elif self._uri.scheme == "https":
self._client = HTTPSConnection(
self._uri.hostname,
self._uri.port or 443,
key_file=key_file,
cert_file=cert_file,
strict=True,
**timeout
)
else:
raise NotImplementedError("HessianProxy only supports http:// and https:// URIs")
# autofill credentials if they were passed via url instead of kwargs
if (self._uri.username and self._uri.password) and not credentials:
credentials = (self._uri.username, self._uri.password)
if credentials:
auth = "Basic " + base64.b64encode(":".join(credentials))
self._headers.append(("Authorization", auth))
self._buffer_size = buffer_size
self._error_factory = error_factory
self._overload = overload
self._parser = Parser()
class __RemoteMethod(object):
# dark magic for autoloading methods
def __init__(self, caller, method):
self.__caller = caller
self.__method = method
def __call__(self, *args):
return self.__caller(self.__method, args)
def __getattr__(self, method):
return self.__RemoteMethod(self, method)
def __repr__(self):
return '<mustaine.client.HessianProxy("%s")>' % (self._uri.geturl(),)
def __str__(self):
return self.__repr__()
def __call__(self, method, args):
try:
self._client.putrequest("POST", self._uri.path)
for header in self._headers:
self._client.putheader(*header)
request = encode_object(Call(method, args, overload=self._overload))
self._client.putheader("Content-Length", str(len(request)).encode("utf8"))
self._client.endheaders()
self._client.send(request)
response = self._client.getresponse()
if response.status != 200:
raise ProtocolError(self._uri.geturl(), response.status, response.reason)
length = response.getheader("Content-Length", -1)
if length == "0":
raise ProtocolError(self._uri.geturl(), "FATAL:", "Server sent zero-length response")
reply = self._parser.parse_stream(BufferedReader(response, buffer_size=self._buffer_size))
self._client.close()
if isinstance(reply.value, Fault):
raise self._error_factory(reply.value)
else:
return reply.value
except:
self._client.close()
raise
class ClamAVFileUploadHandler(FileUploadHandler):
chunk_size = CHUNK_SIZE
skip_av_check = False
def new_file(self, *args, **kwargs):
super().new_file(*args, **kwargs)
extension = pathlib.Path(self.file_name).suffix
if extension in CLAM_AV_IGNORE_EXTENSIONS:
self.skip_av_check = True
return
if CLAM_USE_HTTP:
self.av_conn = HTTPConnection(host=CLAM_AV_DOMAIN, )
else:
self.av_conn = HTTPSConnection( # noqa S309
host=CLAM_AV_DOMAIN,
port=443,
)
credentials = b64encode(
bytes(
f"{CLAM_AV_USERNAME}:{CLAM_AV_PASSWORD}",
encoding="utf8",
)).decode("ascii")
try:
self.av_conn.connect()
self.av_conn.putrequest("POST", CLAM_PATH)
self.av_conn.putheader("Content-Type", self.content_type)
self.av_conn.putheader("Authorization", f"Basic {credentials}")
self.av_conn.putheader("Transfer-encoding", "chunked")
self.av_conn.endheaders()
except Exception as ex:
logger.error("Error connecting to ClamAV service", exc_info=True)
raise AntiVirusServiceErrorException(ex)
def receive_data_chunk(self, raw_data, start):
if not self.skip_av_check:
self.av_conn.send(hex(len(raw_data))[2:].encode("utf-8"))
self.av_conn.send(b"\r\n")
self.av_conn.send(raw_data)
self.av_conn.send(b"\r\n")
return raw_data
def file_complete(self, file_size):
if self.skip_av_check:
return None
self.av_conn.send(b"0\r\n\r\n")
resp = self.av_conn.getresponse()
response_content = resp.read()
scanned_file = ScannedFile()
if resp.status != 200:
scanned_file.av_passed = False
scanned_file.av_reason = "Non 200 response from AV server"
scanned_file.save()
raise AntiVirusServiceErrorException(
f"Non 200 response from anti virus service, content: {response_content}"
)
else:
json_response = json.loads(response_content)
if "malware" not in json_response:
scanned_file.av_passed = False
scanned_file.av_reason = "Malformed response from AV server"
scanned_file.save()
raise MalformedAntiVirusResponseException()
if json_response["malware"]:
scanned_file.av_passed = False
scanned_file.av_reason = json_response["reason"]
scanned_file.save()
logger.error(f"Malware found in user uploaded file "
f"'{self.file_name}', exiting upload process")
else:
scanned_file.av_passed = True
scanned_file.save()
# We are using 'content_type_extra' as the a means of making
# the results available to following file handlers
# TODO - put in a PR to Django project to allow file_complete
# to return objects and not break out of file handler loop
if not hasattr(self.content_type_extra, "clam_av_results"):
self.content_type_extra["clam_av_results"] = []
self.content_type_extra["clam_av_results"].append({
"file_name":
self.file_name,
"av_passed":
scanned_file.av_passed,
"scanned_at":
scanned_file.scanned_at,
})
return None
def upload_document(path,
progress_tracker=None,
file_id=0,
post_url='/upload_file3'):
"""
:param path:
:param progress_tracker:
:param file_id: 0 if uploading a new document, not 0 if repplacing an existing document.
:param post_url:
:return:
"""
mr = None
if progress_tracker:
mr = Wrap(progress_tracker)
else:
mr = MultiRead()
mr.add_field('file_id',
str(file_id)) # 0 == It's a new document, > 0 == overwrite
# We store the filename here 'cos the one encoded in the file part
# must be ASCII (IETF, RFC 2183, section 2.3 : Current [RFC 2045] grammar
# restricts parameter values (and hence Content-Disposition filenames)
# to US-ASCII. We recognize the great desirability of allowing
# arbitrary character sets in filenames, but it is beyond the
# scope of this document to define the necessary mechanisms.
mr.add_field('encoding_safe_filename', os.path.split(path)[-1])
mr.add_file_part(path)
mr.close_parts()
host, port = extract_host_port(
configuration.get("DownloadSite", "base_url"))
mainlog.debug(
u"Upload to {}:{}{} (determined from DownloadSite/base_url : {})".
format(host, port, post_url,
configuration.get("DownloadSite", "base_url")))
if configuration.get("DownloadSite", "base_url").startswith('https'):
h = HTTPSConnection(host, port)
else:
h = HTTPConnection(host, port)
h.putrequest('POST', post_url)
h.putheader('content-type', mr.content_type())
h.putheader('content-length', str(mr.total_size()))
h.putheader('x-filesize', str(mr.total_size()))
h.endheaders()
mr.open()
h.send(mr)
mr.close()
server_response = h.getresponse()
if server_response.status == OK:
server_response.getheaders(
) # Skip headers (is this really necessary ?)
t = server_response.read()
file_id = int(t)
mainlog.debug("Successfully uploaded {} bytes".format(mr.total_size()))
h.close()
return file_id
else:
raise Exception(
"Unable to upload, server response status was {}".format(
server_response.status))
