def fetch_contact_page(password):
""" Gets the HTML of the Jet City Improv fortress contact page.
Arguments:
password {[string]} -- Password to enter Fortress. Be careful: NEVER hardcode or log this value.
"""
contact_info_url = "www.jetcityimprov.org/fortress/phone-numbers-and-emails/"
postpass_host = "www.jetcityimprov.org/wp-login.php?action=postpass"
host = "www.jetcityimprov.org"
body = {"body": {"post_password": password, "Submit": "Enter"}}
response_html = ""
ssl_context = ssl.create_default_context(
) # To make sure we're connecting securely
body_bytes = bytes(json.dumps(body), encoding='utf8')
https = HTTPSConnection(host, context=ssl_context)
https.connect()
https.request("POST", "/wp-login.php?action=postpass", body=body_bytes)
resp = https.getresponse()
result = resp.read()
response_html = bytes.decode(result, "utf-8")
return response_html
def connect(self):
"""
Override the connect() function to intercept calls to certain
host/ports.
If no app at host/port has been registered for interception then
a normal HTTPSConnection is made.
"""
if debuglevel:
sys.stderr.write('connect: %s, %s\n' % (
self.host,
self.port,
))
try:
(app, script_name) = self.get_app(self.host, self.port)
if app:
if debuglevel:
sys.stderr.write('INTERCEPTING call to %s:%s\n' % \
(self.host, self.port,))
self.sock = FakeWSGISocket(app, self.host, self.port,
script_name)
else:
HTTPSConnection.connect(self)
except Exception as e:
if debuglevel: # intercept & print out tracebacks
traceback.print_exc()
raise
def connect(self):
"""
Override the connect() function to intercept calls to certain
host/ports.
If no app at host/port has been registered for interception then
a normal HTTPSConnection is made.
"""
if debuglevel:
sys.stderr.write('connect: %s, %s\n' % (self.host, self.port,))
try:
(app, script_name) = self.get_app(self.host, self.port)
if app:
if debuglevel:
sys.stderr.write('INTERCEPTING call to %s:%s\n' %
(self.host, self.port,))
self.sock = wsgi_fake_socket(app, self.host, self.port,
script_name, self.environ)
else:
HTTPSConnection.connect(self)
except Exception:
if debuglevel: # intercept & print out tracebacks
traceback.print_exc()
raise
def make_http_request(url, data=None, files=None, certificate=None):
# type: (str, Optional[Envelope], Optional[Attachments], str) -> Tuple[int, bytes]
if data is None:
data = {}
if files is None:
files = {}
schema, netloc, url, params, query, fragments = urlparse(url)
logger.debug("Sending POST request to {0}".format(url))
body, boundary = encode_multipart_formdata(data, files)
if certificate:
ssl_context = ssl.create_default_context(cafile=certificate)
else:
ssl_context = ssl._create_default_https_context()
connection = HTTPSConnection(netloc, context=ssl_context)
connection.connect()
connection.putrequest("POST", url)
connection.putheader("Content-Type",
"multipart/form-data; boundary={0}".format(boundary))
connection.putheader("Content-Length", str(len(body)))
connection.endheaders()
connection.send(body)
r = connection.getresponse()
return r.status, r.read()
class Transport_HTTP(Transport):
def __init__(self, hostname, ssl_context=None):
try:
socket.gethostbyname(hostname.split(':')[0])
except socket.gaierror:
raise RuntimeError(f'Unable to resolve hostname: {hostname}')
if ssl_context is None:
self.conn = HTTPConnection(hostname, timeout=60)
else:
self.conn = HTTPSConnection(hostname, context=ssl_context, timeout=60)
try:
print(f'++++ Connecting to {hostname}++++')
self.conn.connect()
except Exception as err:
raise RuntimeError('Connection Failure : ' + str(err))
self.headers = {'Content-type': 'application/x-www-form-urlencoded','Accept': 'text/plain'}
def _send_post_request(self, path, data):
try:
self.conn.request('POST', path, tobytes(data), self.headers)
response = self.conn.getresponse()
if response.status == 200:
return response.read().decode('latin-1')
except Exception as err:
raise RuntimeError('Connection Failure : ' + str(err))
raise RuntimeError('Server responded with error code ' + str(response.status))
async def send_data(self, ep_name, data):
return self._send_post_request('/' + ep_name, data)
class GoogleDictionary():
def __init__(self):
self.host = 'translate.google.com'
self.connection = HTTPSConnection(self.host)
self.empty_comma = re.compile(r',(?=,)')
def lookup(self, lookedup_word, src_language='ru', dst_language='en'):
secret_token = get_current_token(lookedup_word)
url = self.format_lookup_url(lookedup_word, secret_token, src_language, dst_language)
self.connection.request("GET", url)
response = self.connection.getresponse()
response_content = response.read().decode('utf8')
#json_obj = json.loads(self.empty_comma.subn('', response_content)[0].replace(u'\xA0', u' ').replace('[,', '[1,'))
response_content = response_content.replace(',,', ',"",');
response_content = response_content.replace(',,', ',"",');
response_content = response_content.replace('[,', '["",');
response_content = response_content.replace(',]', ',""]');
response_content = response_content.replace('\xA0', ' ')
#fixed_content = self.empty_comma.subn('', response_content)[0].replace('\xA0', ' ')
return json.loads(response_content)
def format_lookup_url(self, word, secret_token, src_language, dst_language):
url = '/translate_a/single?client=t&sl={0}&tl={1}&hl=en&dt=at&dt=bd&dt=ex&' \
'dt=ld&dt=md&dt=qca&dt=rw&dt=rm&dt=ss&dt=t&ie=UTF-8&oe=UTF-8&otf=2&' \
'rom=1&ssel=3&tsel=3&kc=1&tk={2}&q={3}'.format(src_language, dst_language, secret_token, quote(word))
return url
def unpack(self, json_obj):
window_content = json_obj[0]
if len(json_obj) > 1:
article_by_pos = json_obj[1]
defs = {}
for article in article_by_pos:
pos = article[0]
definition = article[1]
defs[pos] = definition
else:
defs = { 'unk', window_content[0][0] }
return defs
def get_pronunciation_url(self, word, language):
secret_token = get_current_token(word)
url_sound ="https://translate.google.com/translate_tts?ie=UTF-8&client=t&tk={0}&tl={1}&q={2}". \
format(secret_token, language, quote(word))
return url_sound
def get_sound_file(self, word, language):
url_sound = self.get_pronunciation_url(word, language)
try:
self.connection.request("GET", url_sound)
except ImproperConnectionState: # reconnect and try again
self.connection.close()
self.connection.connect()
self.connection.request("GET", url_sound)
response = self.connection.getresponse()
content_type = response.headers['Content-Type']
return response.read(), content_type
def _do_post(self, query, extra_headers=[]):
"""
Do a POST to the Institution.
:param query: Body content to POST (OFX Query)
:type query: str
:param extra_headers: Extra headers to send with the request, as a list
of (Name, Value) header 2-tuples.
:type extra_headers: list
:return: 2-tuple of (HTTPResponse, str response body)
:rtype: tuple
"""
i = self.institution
logging.debug('posting data to %s' % i.url)
garbage, path = splittype(i.url)
host, selector = splithost(path)
try:
h = HTTPSConnection(host, timeout=60)
h.connect()
except ssl.SSLError as ex:
if (ex.reason == "UNSUPPORTED_PROTOCOL"):
h = HTTPSConnection(host,
timeout=60,
context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
h.connect()
else:
raise
# Discover requires a particular ordering of headers, so send the
# request step by step.
h.putrequest('POST',
selector,
skip_host=True,
skip_accept_encoding=True)
headers = [('Content-Type', 'application/x-ofx'), ('Host', host),
('Content-Length', len(query)),
('Connection', 'Keep-Alive')]
if self.accept:
headers.append(('Accept', self.accept))
if self.user_agent:
headers.append(('User-Agent', self.user_agent))
for ehname, ehval in extra_headers:
headers.append((ehname, ehval))
logging.debug('---- request headers ----')
for hname, hval in headers:
logging.debug('%s: %s', hname, hval)
h.putheader(hname, hval)
logging.debug('---- request body (query) ----')
logging.debug(query)
h.endheaders(query.encode())
res = h.getresponse()
response = res.read().decode('ascii', 'ignore')
logging.debug('---- response ----')
logging.debug(res.__dict__)
logging.debug('Headers: %s', res.getheaders())
logging.debug(response)
res.close()
return res, response
class Telegram:
def __init__(self, token):
url = 'https://api.telegram.org/bot'
self.__url = urlparse('{}{}/'.format(url, token))
self.__web = HTTPSConnection(self.__url.netloc)
def __enter__(self):
return self
def __send_req(self, target, method='GET'):
self.__web.connect()
self.__web.request(method, urljoin(self.__url.path, target))
def get_botname(self):
'Return: json data.'
self.__send_req('getMe')
data = Request(self.__web)
return data.data
def get_resp(self, limit=20):
'Return: request object.'
self.__send_req('getUpdates?' + 'limit={}'.format(limit))
return Request(self.__web)
def msg_send(self, chat_id, message):
'Return: request data.'
data = urlencode({'chat_id': chat_id, 'text': message})
self.__send_req('sendMessage?' + data, 'POST')
data = Request(self.__web)
return data.data
def msg_delete(self, chat_id, message_id):
'Return: request data.'
data = urlencode({'chat_id': chat_id, 'message_id': message_id})
self.__send_req('deleteMessage?' + data, 'POST')
data = Request(self.__web)
return data.data
def __call__(self):
'Call get_resp()'
return self.get_resp()
def __del__(self):
self.__web.close()
def __exit__(self, *exec_info):
self.__del__()
def _do_post(self, query, extra_headers=[]):
"""
Do a POST to the Institution.
:param query: Body content to POST (OFX Query)
:type query: str
:param extra_headers: Extra headers to send with the request, as a list
of (Name, Value) header 2-tuples.
:type extra_headers: list
:return: 2-tuple of (HTTPResponse, str response body)
:rtype: tuple
"""
i = self.institution
logging.debug('posting data to %s' % i.url)
garbage, path = splittype(i.url)
host, selector = splithost(path)
try:
h = HTTPSConnection(host, timeout=60)
h.connect()
except ssl.SSLError as ex:
if (ex.reason == "UNSUPPORTED_PROTOCOL"):
h = HTTPSConnection(host, timeout=60, context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
h.connect()
else:
raise
# Discover requires a particular ordering of headers, so send the
# request step by step.
h.putrequest('POST', selector, skip_host=True,
skip_accept_encoding=True)
headers = [
('Content-Type', 'application/x-ofx'),
('Host', host),
('Content-Length', len(query)),
('Connection', 'Keep-Alive')
]
if self.accept:
headers.append(('Accept', self.accept))
if self.user_agent:
headers.append(('User-Agent', self.user_agent))
for ehname, ehval in extra_headers:
headers.append((ehname, ehval))
logging.debug('---- request headers ----')
for hname, hval in headers:
logging.debug('%s: %s', hname, hval)
h.putheader(hname, hval)
logging.debug('---- request body (query) ----')
logging.debug(query)
h.endheaders(query.encode())
res = h.getresponse()
response = res.read().decode('ascii', 'ignore')
logging.debug('---- response ----')
logging.debug(res.__dict__)
logging.debug('Headers: %s', res.getheaders())
logging.debug(response)
res.close()
return res, response
class FrankaAPI: #
def __init__(self, hostname, user, password): #构造函数，进行类变量初始化，输入主机名，账户名，密码
self._hostname = hostname #主机名
self._user = user #web网页账户
self._password = password #密码
def __enter__(self):
self._client = HTTPSConnection(
self._hostname, context=ssl._create_unverified_context())
self._client.connect() #链接
self._client.request('POST',
'/admin/api/login',
body=json.dumps({
'login':
self._user,
'password':
encode_password(self._user, self._password)
}),
headers={'content-type': 'application/json'})
self._token = self._client.getresponse().read().decode('utf8')
#print(self._token)
return self
def __exit__(self, type, value, traceback):
self._client.close()
def start_task(self, task):
self._client.request('POST',
'/desk/api/execution',
body='id=%s' % task,
headers={
'content-type':
'application/x-www-form-urlencoded',
'Cookie': 'authorization=%s' % self._token
})
return self._client.getresponse().read()
def open_brakes(self): #打开机械臂锁
self._client.request('POST',
'/desk/api/robot/open-brakes',
headers={
'content-type':
'application/x-www-form-urlencoded',
'Cookie': 'authorization=%s' % self._token
})
return self._client.getresponse().read()
def close_brakes(self): #关闭机械臂锁
self._client.request('POST',
'/desk/api/robot/close-brakes',
headers={
'content-type':
'application/x-www-form-urlencoded',
'Cookie': 'authorization=%s' % self._token
})
return self._client.getresponse().read()
class MyHttpCtxManagerClass:
def __init__(self, host):
self.host = host
self.connection = HTTPSConnection(self.host)
def __enter__(self):
self.connection.connect()
return self.connection
def __exit__(self, *args):
self.connection.close()
def check_updated_certs(_address, _port, certhashlist, newhash=None, timeout=config.default_timeout, connect_timeout=config.connect_timeout, traversefunc=None):
update_list = []
if None in [_address, _port]:
logging.error("address or port empty")
return None
addr, _port = url_to_ipv6(_address, _port)
cont = default_sslcont()
con = HTTPSConnection(addr, _port, context=cont, timeout=connect_timeout)
try:
con.connect()
except (ConnectionRefusedError, socket.timeout):
if not traversefunc:
logging.warning("Connection failed")
return None
# try_traverse does not work here, scnreqest creates loop
con.sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
con.sock.bind(('', 0))
traversefunc(("", con.sock.getsockname()[1]))
con.sock.settimeout(connect_timeout)
for count in range(0, config.traverse_retries):
try:
con.sock.connect((addr, _port))
break
except Exception:
pass
else:
logging.warning("traversal failed")
return None
con.sock.settimeout(timeout)
oldhash = dhash(ssl.DER_cert_to_PEM_cert(con.sock.getpeercert(True)).strip().rstrip())
if newhash and newhash != oldhash:
return None
oldsslcont = con.sock.context
for _hash, _security in certhashlist:
con.request("POST", "/usebroken/{hash}".format(hash=_hash), headers=cert_update_header)
con.sock = con.sock.unwrap()
con.sock = cont.wrap_socket(con.sock, server_side=False)
con.sock.do_handshake()
brokensslcert = ssl.DER_cert_to_PEM_cert(con.sock.getpeercert(True)).strip().rstrip()
con.sock = con.sock.unwrap()
# without next line the connection would be unencrypted now
con.sock = oldsslcont.wrap_socket(con.sock, server_side=False)
# con.sock.do_handshake()
ret = con.getresponse()
if ret.status != 200:
logging.info("checking cert failed, code: %s, reason: %s", ret.status, ret.reason)
continue
if con.sock and oldhash != dhash(ssl.DER_cert_to_PEM_cert(con.sock.getpeercert(True)).strip().rstrip()):
logging.error("certificate switch detected, stop checking")
break
if dhash(brokensslcert) == _hash:
update_list.append((_hash, _security))
con.close()
return update_list
def connect(self):
if not self.__ca_file:
HTTPSConnection.connect(self)
else:
HTTPConnection.connect(self)
if self.__ca_file == HTTPSConfigurableConnection.IGNORE:
self.sock = ssl.wrap_socket(self.sock, cert_reqs=ssl.CERT_NONE)
else:
self.sock = ssl.wrap_socket(self.sock,
ca_certs=self.__ca_file,
cert_reqs=ssl.CERT_REQUIRED)
def connect(self):
"""
Override the connect() function to intercept calls to certain
host/ports.
If no app at host/port has been registered for interception then
a normal HTTPSConnection is made.
"""
if debuglevel:
sys.stderr.write('connect: %s, %s\n' % (
self.host,
self.port,
))
try:
(app, script_name) = self.get_app(self.host, self.port)
if app:
if debuglevel:
sys.stderr.write('INTERCEPTING call to %s:%s\n' % (
self.host,
self.port,
))
self.sock = wsgi_fake_socket(app,
self.host,
self.port,
script_name,
https=True)
else:
try:
import ssl
if not hasattr(self, 'key_file'):
self.key_file = None
if not hasattr(self, 'cert_file'):
self.cert_file = None
if not hasattr(self, '_context'):
try:
self._context = ssl.create_default_context()
except AttributeError:
self._context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
self._context.options |= ssl.OP_NO_SSLv2
if not hasattr(self, 'check_hostname'):
self._check_hostname = (
self._context.verify_mode != ssl.CERT_NONE)
else:
self._check_hostname = self.check_hostname
except (ImportError, AttributeError):
pass
HTTPSConnection.connect(self)
except Exception:
if debuglevel: # intercept & print out tracebacks
traceback.print_exc()
raise
def connect(self):
"""
Overridden connect() method to wrap the socket using an SSLSocket,
and check the server certificates.
"""
try:
self.sock = socket.create_connection((self.host, self.port))
except AttributeError:
HTTPSConnection.connect(self)
if not _have_ssl:
# No SSL available - insecure connection
print("WARNING: No SSL support - connection may be insecure")
elif self.ca_certs:
# Wrap the socket in an SSLSocket, and tell it to validate
# the server certificates. Note that this does not check that
# the certificate's host matches, so we must do that ourselves.
self.sock = ssl.wrap_socket(self.sock,
ca_certs=self.ca_certs,
cert_reqs=ssl.CERT_REQUIRED,
ssl_version=ssl.PROTOCOL_TLSv1)
cert = self.sock.getpeercert()
cert_hosts = []
host_valid = False
if "subject" in cert:
for x in cert["subject"]:
if x[0][0] == "commonName":
cert_hosts.append(x[0][1])
if "subjectAltName" in cert:
for x in cert["subjectAltName"]:
if x[0] == "dns":
cert_hosts.append(x[1])
for cert_host in cert_hosts:
if self.host.startswith(cert_host):
host_valid = True
if not host_valid:
raise ssl.SSLError("Host name '%s' doesn't match "\
"certificate host %s"\
% (self.host, str(cert_hosts)))
else:
# No CA certificates supplied, so can't validate the server
# certificates, but we still wrap the socket in an SSLSocket
# so that all data is encrypted.
self.sock = ssl.wrap_socket(self.sock,
ca_certs=None,
cert_reqs=ssl.CERT_NONE,
ssl_version=ssl.PROTOCOL_TLSv1)
def connect(self):
"""
Overridden connect() method to wrap the socket using an SSLSocket,
and check the server certificates.
"""
try:
self.sock = socket.create_connection((self.host, self.port))
except AttributeError:
HTTPSConnection.connect(self)
if not _have_ssl:
# No SSL available - insecure connection
print("WARNING: No SSL support - connection may be insecure")
elif self.ca_certs:
# Wrap the socket in an SSLSocket, and tell it to validate
# the server certificates. Note that this does not check that
# the certificate's host matches, so we must do that ourselves.
self.sock = ssl.wrap_socket(self.sock,
ca_certs = self.ca_certs,
cert_reqs = ssl.CERT_REQUIRED,
ssl_version = ssl.PROTOCOL_TLSv1)
cert = self.sock.getpeercert()
cert_hosts = []
host_valid = False
if "subject" in cert:
for x in cert["subject"]:
if x[0][0] == "commonName":
cert_hosts.append(x[0][1])
if "subjectAltName" in cert:
for x in cert["subjectAltName"]:
if x[0] == "dns":
cert_hosts.append(x[1])
for cert_host in cert_hosts:
if self.host.startswith(cert_host):
host_valid = True
if not host_valid:
raise ssl.SSLError("Host name '%s' doesn't match "\
"certificate host %s"\
% (self.host, str(cert_hosts)))
else:
# No CA certificates supplied, so can't validate the server
# certificates, but we still wrap the socket in an SSLSocket
# so that all data is encrypted.
self.sock = ssl.wrap_socket(self.sock,
ca_certs = None,
cert_reqs = ssl.CERT_NONE,
ssl_version = ssl.PROTOCOL_TLSv1)
def collect(self):
connection = HTTPSConnection(self.ip_api_host)
try:
connection.connect()
connection.request(GET_METHOD, self.ip_api_url)
response = connection.getresponse()
if response.code == 200:
return CollectingResult(
data=response.readline().decode(ENCODING))
else:
return CollectingResult(sucess=False, data="0.0.0.0")
except Exception as e:
return CollectingResult(sucess=False, data=e.args[1])
finally:
connection.close()
def netloc_has_https(netloc):
port = 443
if ':' in netloc:
after_colon = netloc.split(':')[-1]
if after_colon.isdigit():
port = int(after_colon)
https_conn = HTTPSConnection(host=netloc, timeout=1, port=port)
try:
https_conn.connect()
https_conn.close()
return True
except Exception as e:
print(netloc, " doesn't have https!")
return False, e.args[1]
def create_channel(self):
global channel_center, server_dict
self.manager.maker_in()
host = server_dict[self.group][self.server]['ip']
handle = HTTPSConnection(host)
handle._http_vsn = 10
handle._http_vsn_str = 'HTTP/1.0'
try:
handle.connect()
except KeyboardInterrupt:
pass
except:
printer.critical(format_exc())
else:
channel_center.put_channel(self.channel, self.group, handle)
self.manager.maker_out()
def _new_conn(self):
"""
Return a fresh :class:`httplib.HTTPSConnection`.
"""
self.num_connections += 1
log.info("Starting new HTTPS connection (%d): %s"
% (self.num_connections, self.host))
actual_host = self.host
actual_port = self.port
if self.proxy is not None:
actual_host = self.proxy.host
actual_port = self.proxy.port
if not ssl: # Platform-specific: Python compiled without +ssl
if not HTTPSConnection or HTTPSConnection is object:
raise SSLError("Can't connect to HTTPS URL because the SSL "
"module is not available.")
connection = HTTPSConnection(host=actual_host,
port=actual_port,
strict=self.strict)
else:
connection = VerifiedHTTPSConnection(host=actual_host,
port=actual_port,
strict=self.strict)
connection.set_cert(key_file=self.key_file, cert_file=self.cert_file,
cert_reqs=self.cert_reqs, ca_certs=self.ca_certs,
assert_hostname=self.assert_hostname,
assert_fingerprint=self.assert_fingerprint)
connection.ssl_version = self.ssl_version
if self.proxy is not None:
# Python 2.7+
try:
set_tunnel = connection.set_tunnel
except AttributeError: # Platform-specific: Python 2.6
set_tunnel = connection._set_tunnel
set_tunnel(self.host, self.port, self.proxy_headers)
# Establish tunnel connection early, because otherwise httplib
# would improperly set Host: header to proxy's IP:port.
connection.connect()
return connection
def _new_conn(self):
"""
Return a fresh :class:`httplib.HTTPSConnection`.
"""
self.num_connections += 1
log.info("Starting new HTTPS connection (%d): %s"
% (self.num_connections, self.host))
actual_host = self.host
actual_port = self.port
if self.proxy is not None:
actual_host = self.proxy.host
actual_port = self.proxy.port
if not ssl: # Platform-specific: Python compiled without +ssl
if not HTTPSConnection or HTTPSConnection is object:
raise SSLError("Can't connect to HTTPS URL because the SSL "
"module is not available.")
connection = HTTPSConnection(host=actual_host,
port=actual_port,
strict=self.strict)
else:
connection = VerifiedHTTPSConnection(host=actual_host,
port=actual_port,
strict=self.strict)
connection.set_cert(key_file=self.key_file, cert_file=self.cert_file,
cert_reqs=self.cert_reqs, ca_certs=self.ca_certs)
connection.ssl_version = self.ssl_version
if self.proxy is not None:
# Python 2.7+
try:
set_tunnel = connection.set_tunnel
# Python 2.6
except AttributeError:
set_tunnel = connection._set_tunnel
set_tunnel(self.host, self.port, self.proxy_headers)
# Establish tunnel connection early, because otherwise httplib
# would improperly set Host: header to proxy's IP:port.
connection.connect()
return connection
def connect(self):
"""
Override the connect() function to intercept calls to certain
host/ports.
If no app at host/port has been registered for interception then
a normal HTTPSConnection is made.
"""
if debuglevel:
sys.stderr.write('connect: %s, %s\n' % (self.host, self.port,))
try:
(app, script_name) = self.get_app(self.host, self.port)
if app:
if debuglevel:
sys.stderr.write('INTERCEPTING call to %s:%s\n' %
(self.host, self.port,))
self.sock = wsgi_fake_socket(app, self.host, self.port,
script_name, https=True)
else:
try:
import ssl
if not hasattr(self, 'key_file'):
self.key_file = None
if not hasattr(self, 'cert_file'):
self.cert_file = None
if not hasattr(self, '_context'):
try:
self._context = ssl.create_default_context()
except AttributeError:
self._context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
self._context.options |= ssl.OP_NO_SSLv2
if not hasattr(self, 'check_hostname'):
self._check_hostname = (self._context.verify_mode
!= ssl.CERT_NONE)
else:
self._check_hostname = self.check_hostname
except (ImportError, AttributeError):
pass
HTTPSConnection.connect(self)
except Exception:
if debuglevel: # intercept & print out tracebacks
traceback.print_exc()
raise
def connect(self) -> None:
if not self.__ca_file:
HTTPSConnection.connect(self)
else:
HTTPConnection.connect(self)
# TODO: Use SSLContext.wrap_socket() instead of the deprecated ssl.wrap_socket()!
# See https://docs.python.org/3/library/ssl.html#socket-creation
if self.__ca_file == HTTPSConfigurableConnection.IGNORE:
self.sock = ssl.wrap_socket( # pylint: disable=deprecated-method
self.sock,
cert_reqs=ssl.CERT_NONE,
)
else:
self.sock = ssl.wrap_socket( # pylint: disable=deprecated-method
self.sock,
ca_certs=self.__ca_file,
cert_reqs=ssl.CERT_REQUIRED,
)
def __make_request(self, url, params):
parse_result = urlparse(url)
hostname = parse_result.netloc
context = ssl.SSLContext()
conn = HTTPSConnection(hostname, None, context=context)
conn.connect()
body = urlencode(params)
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': len(body),
}
request_url = f"{parse_result.path}?{parse_result.query}"
try:
conn.request("POST", request_url, body, headers)
response = conn.getresponse()
data = response.read()
finally:
conn.close()
return data
def connect(self, srvhost, srvport):
self.con = ssl._create_unverified_context(protocol=ssl.PROTOCOL_TLSv1, cafile=ca_crt)
from http.client import HTTPSConnection
c = HTTPSConnection(srvhost, port=srvport, context=self.con)
try:
c.connect()
except ConnectionRefusedError as e:
raise ConnectException("Cannot connect to 'https://%s:%d/'. %s" % (srvhost, srvport, e))
except:
raise ConnectException("Connect to 'https://%s:%d/' failed. Unknown error." % (srvhost, srvport))
finally:
c.close()
try:
self.proxy = ServerProxy("https://%s:%d/" % (srvhost, srvport),
context=self.con)
except:
raise ConnectException(
"Connect to crtyptoTank server at 'https://%s:%d/' failed. Unknown error." % (srvhost, srvport)
)
def pyget(host, req, headers={}, timeout=None):
while True:
handler = HTTPSConnection(host, timeout=timeout)
handler._http_vsn = 10
handler._http_vsn_str = 'HTTP/1.0'
try:
handler.connect()
except (sock_timeout, TimeoutError):
handler.close()
continue
except:
print_exc()
continue
break
#--------------------------------------------------------
try:
handler.request('GET', req, headers=headers)
except:
handler.close()
print_exc()
return None
try:
ack = handler.getresponse()
body = ack.read()
except:
handler.close()
print_exc()
return None
#--------------------------------------------------------
handler.close()
key_val = {}
key_val['body'] = body
key_val['head'] = str(ack.msg)
key_val['status'] = ack.status
return key_val
class Transport_HTTP(Transport):
def __init__(self, hostname, ssl_context=None):
try:
socket.gethostbyname(hostname.split(':')[0])
except socket.gaierror:
raise RuntimeError("Unable to resolve hostname :" + hostname)
if ssl_context is None:
self.conn = HTTPConnection(hostname, timeout=45)
else:
self.conn = HTTPSConnection(hostname,
context=ssl_context,
timeout=45)
try:
print("Connecting to " + hostname)
self.conn.connect()
except Exception as err:
raise RuntimeError("Connection Failure : " + str(err))
self.headers = {
"Content-type": "application/x-www-form-urlencoded",
"Accept": "text/plain"
}
def _send_post_request(self, path, data):
try:
self.conn.request("POST", path, tobytes(data), self.headers)
response = self.conn.getresponse()
if response.status == 200:
return response.read().decode('latin-1')
except Exception as err:
raise RuntimeError("Connection Failure : " + str(err))
raise RuntimeError("Server responded with error code " +
str(response.status))
def send_data(self, ep_name, data):
return self._send_post_request('/' + ep_name, data)
class twse(object):
def __init__(self):
self.connect()
def orgnize_data(self, raw_data):
data = []
for entry in raw_data:
a_day = {}
invalid = False
for i in range(9):
if entry[i] == '--':
invalid = True
break
if invalid:
continue
# entry[0]: date
seg = entry[0].split('/')
a_day['date'] = '%d_%s_%s' % (int(seg[0]) + 1911, seg[1], seg[2])
a_day['capacity'] = int(entry[1].replace(',', ''))
a_day['turnover'] = int(entry[1].replace(',', ''))
a_day['open'] = float(entry[3].replace(',', ''))
a_day['high'] = float(entry[3].replace(',', ''))
a_day['low'] = float(entry[3].replace(',', ''))
a_day['close'] = float(entry[3].replace(',', ''))
a_day['change'] = float(0.0 if entry[7].replace(',', '') == 'X0.00' else entry[7].replace(',', ''))
a_day['transaction'] = int(entry[8].replace(',', ''))
data.append(a_day)
return data
def connect(self):
if hasattr(self, 'conn'):
if self.conn is not None:
try:
self.close()
except:
pass
while True:
self.conn = HTTPSConnection(TWSE_HOST, timeout=10)
try:
self.conn.connect()
except Exception as e:
print('Error:', e.__class__, ' occurs')
time.sleep(20.0)
continue
break
def get_month(self, id, year, month):
url = '%s?date=%d%02d01&stockNo=%s' %\
(QUERY_URL, year, month, id)
# print(url)
retry = 0
while True:
try:
self.conn.request('GET', url)
resp = self.conn.getresponse()
if resp.status == 200:
break
else:
if retry >= 10:
print('Fatal Error: retry > 10')
return [], 'over_retry'
print(resp.status, resp.reason)
retry += 1
print('retry %d 20 seconds later' % retry)
time.sleep(20.0)
self.connect()
except Exception as e:
print('error:', e.__class__, ' occurs')
traceback.print_exc()
time.sleep(40.0)
self.connect()
continue
# headers = resp.getheaders()
# for header in headers:
# print(header)
datastr = resp.read().decode('utf-8')
# print(datastr)
try:
data = json.loads(datastr)
except json.decoder.JSONDecodeError:
print('decode error occurs. rawdata string: %s' % datastr)
self.connect()
return self.get_month(id, year, month)
if data['stat'] == 'OK':
# print(data['fields'])
return self.orgnize_data(data['data']), 'OK'
else:
print('error: %s' % data['stat'])
return [], data['stat']
def close(self):
self.conn.close()
class Client(object):
"""
Provides methods to access the taskforce http service. These are basically
for convenience in clients, and are particularly useful when using
Unix domain sockets (thanks to Erik van Zijst for the nice approach --
https://pypi.python.org/pypi/uhttplib).
Parameters:
address - The address to listen on, defaults to "httpd.def_address".
This may be specified as "[host][:port]" for TCP, or
as "path" to select a Udom service (path must contain
at least one "/" character).
use_ssl - If None (default) the connection will not use SSL.
If False, SSL will be used but the certificate will
not be verified.
If True, SSL will be used and the server must have a
valid certificate (assumes python >= 2.7.9)
timeout - The timeout in seconds (float) for query I/O.
log - A 'logging' object to log errors and activity.
"""
def __init__(self, address=None, use_ssl=None, timeout=5, log=None):
if log:
self.log = log
else:
self.log = logging.getLogger(__name__)
self.log.addHandler(logging.NullHandler())
if address:
self.address = address
else:
self.address = httpd.def_address
if self.address.find('/') >= 0:
if use_ssl is None:
self.http = udomHTTPConnection(self.address, timeout)
else:
ssl_params = self._build_params(use_ssl, timeout)
self.http = udomHTTPSConnection(self.address, **ssl_params)
else:
port = None
m = re.match(r'^(.*):(.*)$', self.address)
if m:
self.log.debug("Matched host '%s', port '%s'", m.group(1),
m.group(2))
host = m.group(1)
try:
port = int(m.group(2))
except:
raise HttpError(
code=500,
content_type='text/plain',
content="TCP listen port must be an integer")
else:
host = self.address
self.log.debug("No match, proceding with host '%s'", host)
if use_ssl is None:
if not port:
port = httpd.def_port
self.log.debug("Connecting to host '%s', port '%s'", host,
port)
self.http = HTTPConnection(host, port, timeout=timeout)
else:
if not port:
port = httpd.def_sslport
ssl_params = self._build_params(use_ssl, timeout)
self.http = HTTPSConnection(host, port, **ssl_params)
self.log.debug("Connecting via ssl to host '%s', port '%s'",
host, port)
self.address = "%s:%d" % (host, port)
self.http.connect()
self.sock = self.http.sock
self.lastpath = None
self.log.info("HTTP connected via %s", self.http.sock)
if use_ssl and hasattr(self.http.sock, 'cipher'): # pragma: no cover
self.log.debug("Cipher: %s", self.http.sock.cipher())
def _build_params(self, use_ssl, timeout):
ssl_params = {'timeout': timeout}
try:
ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
except AttributeError: # pragma: no cover
self.log.info("No ssl.SSLContext(), assuming older python")
return ssl_params
if use_ssl is False:
ctx.verify_mode = ssl.CERT_NONE
else: # pragma: no cover
ctx.verify_mode = ssl.CERT_REQUIRED
if 'OP_NO_SSLv2' in ssl.__dict__:
ctx.options |= ssl.OP_NO_SSLv2
else: # pragma: no cover
self.log.info(
"Implementation does not offer ssl.OP_NO_SSLv2 which may allow less secure connections"
)
if 'OP_NO_SSLv3' in ssl.__dict__:
ctx.options |= ssl.OP_NO_SSLv3
else: # pragma: no cover
self.log.info(
"Implementation does not offer ssl.OP_NO_SSLv3 which may allow less secure connections"
)
ssl_params['context'] = ctx
return ssl_params
def get(self, path, query=None):
"""
Issue a GET request. If specfied, "query" should be a dict of name/value
pairs. Names should be normal identifiers which start with an alpha
followed by alnums or underscores. The values are url-encoded and
become the "query" part of the request header (ie, the part after
the '?' in the URI).
The result is the tuple:
(code, content, content_type)
If the request is unsuccessful returning code 400 or higher, the
http.HttpError exception is raised.
"""
self.lastpath = path
if query is not None:
self.lastpath += '?' + urlencode(query)
self.http.request('GET', self.lastpath)
resp = self.http.getresponse()
ctype = resp.getheader('Content-Type')
data = resp.read().decode('utf-8')
self.log.debug("Request '%s' status %d, %s length %d", self.lastpath,
resp.status, ctype, len(data))
if resp.status < 400:
return (resp.status, data, ctype)
else:
raise HttpError(code=resp.status, content_type=ctype, content=data)
def getmap(self, path, query=None):
"""
Performs a GET request where the response content type is required to be
"application/json" and the content is a JSON-encoded data structure.
The decoded structure is returned.
"""
code, data, ctype = self.get(path, query)
if ctype != 'application/json':
self.log.error("Expecting JSON from GET of '%s', got '%s'",
self.lastpath, ctype)
raise HttpError(code=400,
content_type='text/plain',
content='Remote returned invalid content type: ' +
ctype)
try:
result = json.loads(data)
except Exception as e: # pragma: no cover
self.log.error("Could not load JSON content from GET %r -- %s",
self.lastpath, e)
raise HttpError(code=400,
content_type='text/plain',
content='Could not load JSON content')
return result
def post(self, path, valuemap=None, query=None):
"""
Performs a POST request. "valuemap" is a dict sent as "application/x-www-form-urlencoded".
"query" is as for get(). Return is same as get().
"""
self.lastpath = path
if query is not None:
self.lastpath += '?' + urlencode(query)
if valuemap:
self.http.request(
'POST', self.lastpath, urlencode(valuemap),
{"Content-type": "application/x-www-form-urlencoded"})
else:
self.http.request('POST', self.lastpath, '')
resp = self.http.getresponse()
ctype = resp.getheader('Content-Type')
data = resp.read().decode('utf-8')
self.log.debug("Request '%s' status %d, %s length %d", self.lastpath,
resp.status, ctype, len(data))
if resp.status < 400:
return (resp.status, data, ctype)
else:
raise HttpError(code=resp.status, content_type=ctype, content=data)
def postmap(self, path, valuemap=None, query=None):
"""
Performs a POST request as per post() but the response content type
is required to be "application/json" and is processed as with getmap().
"""
code, data, ctype = self.post(path, valuemap, query)
if ctype != 'application/json':
self.log.error("Expecting JSON from POST of '%s', got '%s'",
self.lastpath, ctype)
raise HttpError(code=400,
content_type='text/plain',
content='Remote returned invalid content type: ' +
ctype)
try:
result = json.loads(data)
except Exception as e: # pragma: no cover
self.log.error("Could not load JSON content from POST %r -- %s",
self.lastpath, e)
raise HttpError(code=400,
content_type='text/plain',
content='Could not load JSON content')
return result
def request(self, method, url, *args):
"""
Pass-thru method to make this class behave a little like HTTPConnection
"""
return self.http.request(method, url, *args)
def getresponse(self):
"""
Pass-thru method to make this class behave a little like HTTPConnection
"""
resp = self.http.getresponse()
self.log.info("resp is %s", str(resp))
if resp.status < 400:
return resp
else:
errtext = resp.read()
content_type = resp.getheader('Content-Type', 'text/plain')
raise HttpError(code=resp.status,
content_type=content_type,
content=errtext)
# create HTTP basic authentication string, this consists of
# "username:password" base64 encoded
username = b"my_username" # enter your username
password = b"my_password" # enter your password
auth = base64.encodestring(b"%s:%s" % (username, password)).decode().strip()
headers = {
"Authorization": "Basic %s" % auth,
"Content-type": "application/json"
}
# message to send to server
client = HTTPSConnection("remotemanager.digi.com")
client.connect()
while 1:
message = """{"stream_id": "RGB",
"stream_type": "STRING",
"value": "%d,%d,%d"}""" % (randint(0, 255), randint(0, 255), randint(
0, 255))
print(message)
client.request("POST",
"https://remotemanager.digi.com/ws/v1/streams/history",
body=message,
headers=headers)
response = client.getresponse()
print(response.status, response.reason)
class HTTPGetter:
def __init__(self, baseUrl, maxPending=10, auth=""):
self.baseUrl = baseUrl
self.parsedBaseUrl = urlparse(baseUrl)
self.maxPending = maxPending
self.requests = []
self.pendingRequests = []
if self.parsedBaseUrl.scheme == "http":
self.httpConnection = HTTPConnection(self.parsedBaseUrl.netloc)
elif self.parsedBaseUrl.scheme == "https":
context = ssl.create_default_context()
self.httpConnection = HTTPSConnection(self.parsedBaseUrl.netloc,
context=context)
else:
raise UnsupportedURLScheme(self.parsedBaseUrl.scheme)
self.httpRequestHeaders = headers = {
'Host': self.parsedBaseUrl.netloc,
'Content-Length': 0,
'Connection': 'Keep-Alive',
'User-Agent': 'FlightGear terrasync.py'
}
if (auth and not auth.isspace()):
self.httpRequestHeaders['Authorization'] = 'Basic %s' % b64encode(
auth.encode("utf-8")).decode("ascii")
def assemblePath(self, httpGetCallback):
"""Return the path-on-server for the file to download.
Example: '/scenery/Airports/N/E/4/.dirindex'
"""
assert not self.parsedBaseUrl.path.endswith('/'), \
repr(self.parsedBaseUrl)
return self.parsedBaseUrl.path + str(httpGetCallback.src)
def assembleUrl(self, httpGetCallback):
"""Return the URL of the file to download."""
baseUrl = self.parsedBaseUrl.geturl()
assert not baseUrl.endswith('/'), repr(baseUrl)
return urljoin(baseUrl + '/', httpGetCallback.src.asRelative())
def doGet(self, httpGetCallback):
time.sleep(1.25) # throttle the rate
pathOnServer = self.assemblePath(httpGetCallback)
self.httpConnection.request("GET", pathOnServer, None,
self.httpRequestHeaders)
httpResponse = self.httpConnection.getresponse()
# 'httpResponse' is an http.client.HTTPResponse instance
return httpGetCallback.callback(self.assembleUrl(httpGetCallback),
httpResponse)
def get(self, httpGetCallback):
nbRetries = nbRetriesLeft = 5
while True:
try:
return self.doGet(httpGetCallback)
except HTTPException as exc:
if nbRetriesLeft == 0:
raise NetworkError(
"after {nbRetries} retries for URL {url}: {errMsg}".
format(nbRetries=nbRetries,
url=self.assembleUrl(httpGetCallback),
errMsg=exc)) from exc
# Try to reconnect
self.httpConnection.close()
time.sleep(1)
self.httpConnection.connect()
nbRetriesLeft -= 1
def my_http_ctx_manager_func(host):
connection = HTTPSConnection(host)
connection.connect()
yield connection
connection.close()
class Robot(_Robot):
def __init__(self,
fci_ip,
dynamic_rel=1.0,
user=None,
password=None,
repeat_on_error=True,
stop_at_python_signal=True):
super().__init__(fci_ip,
dynamic_rel=dynamic_rel,
repeat_on_error=repeat_on_error,
stop_at_python_signal=stop_at_python_signal)
self.hostname = fci_ip
self.user = user
self.password = password
self.client = None
self.token = None
@staticmethod
def _encode_password(user, password):
bs = ','.join([
str(b)
for b in hashlib.sha256((password + '#' + user +
'@franka').encode('utf-8')).digest()
])
return base64.encodebytes(bs.encode('utf-8')).decode('utf-8')
def __enter__(self):
self.client = HTTPSConnection(
self.hostname,
timeout=12,
context=ssl._create_unverified_context()) # [s]
self.client.connect()
self.client.request('POST',
'/admin/api/login',
body=json.dumps({
'login':
self.user,
'password':
self._encode_password(self.user, self.password)
}),
headers={'content-type': 'application/json'})
self.token = self.client.getresponse().read().decode('utf8')
return self
def __exit__(self, type, value, traceback):
self.client.close()
def start_task(self, task):
self.client.request('POST',
'/desk/api/execution',
body=f'id={task}',
headers={
'content-type':
'application/x-www-form-urlencoded',
'Cookie': f'authorization={self.token}'
})
return self.client.getresponse().read()
def unlock_brakes(self):
self.client.request('POST',
'/desk/api/robot/open-brakes',
headers={
'content-type':
'application/x-www-form-urlencoded',
'Cookie': f'authorization={self.token}'
})
return self.client.getresponse().read()
def lock_brakes(self):
self.client.request('POST',
'/desk/api/robot/close-brakes',
headers={
'content-type':
'application/x-www-form-urlencoded',
'Cookie': f'authorization={self.token}'
})
return self.client.getresponse().read()
def move_async(self, *args) -> Thread:
p = Thread(target=self.move, args=tuple(args), daemon=True)
p.start()
sleep(0.001) # Sleep one control cycle
return p
def get_gripper(self):
return _Gripper(self.fci_ip)
def connect(self):
_HTTPSConnection.connect(self)
self.sock.setsockopt(IPPROTO_TCP, TCP_NODELAY, 1)
class XMLManager(object):
def __init__(self, cls, db_name, db_user, db_passwd, db_host, db_port,
db_table, ddl_dir, enable_ssl):
self.cls = cls
if not db_name:
db_name = cls.__name__.lower()
self.db_name = db_name
self.db_user = db_user
self.db_passwd = db_passwd
self.db_host = db_host
self.db_port = db_port
self.db_table = db_table
self.ddl_dir = ddl_dir
self.s3 = None
self.converter = XMLConverter(self)
self.impl = getDOMImplementation()
self.doc = self.impl.createDocument(None, 'objects', None)
self.connection = None
self.enable_ssl = enable_ssl
self.auth_header = None
if self.db_user:
import base64
base64string = base64.encodestring(
'%s:%s' % (self.db_user, self.db_passwd))[:-1]
authheader = "Basic %s" % base64string
self.auth_header = authheader
def _connect(self):
if self.db_host:
if self.enable_ssl:
from http.client import HTTPSConnection as Connection
else:
from http.client import HTTPConnection as Connection
self.connection = Connection(self.db_host, self.db_port)
def _make_request(self, method, url, post_data=None, body=None):
"""
Make a request on this connection
"""
if not self.connection:
self._connect()
try:
self.connection.close()
except:
pass
self.connection.connect()
headers = {}
if self.auth_header:
headers["Authorization"] = self.auth_header
self.connection.request(method, url, body, headers)
resp = self.connection.getresponse()
return resp
def new_doc(self):
return self.impl.createDocument(None, 'objects', None)
def _object_lister(self, cls, doc):
for obj_node in doc.getElementsByTagName('object'):
if not cls:
class_name = obj_node.getAttribute('class')
cls = find_class(class_name)
id = obj_node.getAttribute('id')
obj = cls(id)
for prop_node in obj_node.getElementsByTagName('property'):
prop_name = prop_node.getAttribute('name')
prop = obj.find_property(prop_name)
if prop:
if hasattr(prop, 'item_type'):
value = self.get_list(prop_node, prop.item_type)
else:
value = self.decode_value(prop, prop_node)
value = prop.make_value_from_datastore(value)
setattr(obj, prop.name, value)
yield obj
def reset(self):
self._connect()
def get_doc(self):
return self.doc
def encode_value(self, prop, value):
return self.converter.encode_prop(prop, value)
def decode_value(self, prop, value):
return self.converter.decode_prop(prop, value)
def get_s3_connection(self):
if not self.s3:
self.s3 = boto.connect_s3(self.aws_access_key_id,
self.aws_secret_access_key)
return self.s3
def get_list(self, prop_node, item_type):
values = []
try:
items_node = prop_node.getElementsByTagName('items')[0]
except:
return []
for item_node in items_node.getElementsByTagName('item'):
value = self.converter.decode(item_type, item_node)
values.append(value)
return values
def get_object_from_doc(self, cls, id, doc):
obj_node = doc.getElementsByTagName('object')[0]
if not cls:
class_name = obj_node.getAttribute('class')
cls = find_class(class_name)
if not id:
id = obj_node.getAttribute('id')
obj = cls(id)
for prop_node in obj_node.getElementsByTagName('property'):
prop_name = prop_node.getAttribute('name')
prop = obj.find_property(prop_name)
value = self.decode_value(prop, prop_node)
value = prop.make_value_from_datastore(value)
if value != None:
try:
setattr(obj, prop.name, value)
except:
pass
return obj
def get_props_from_doc(self, cls, id, doc):
"""
Pull out the properties from this document
Returns the class, the properties in a hash, and the id if provided as a tuple
:return: (cls, props, id)
"""
obj_node = doc.getElementsByTagName('object')[0]
if not cls:
class_name = obj_node.getAttribute('class')
cls = find_class(class_name)
if not id:
id = obj_node.getAttribute('id')
props = {}
for prop_node in obj_node.getElementsByTagName('property'):
prop_name = prop_node.getAttribute('name')
prop = cls.find_property(prop_name)
value = self.decode_value(prop, prop_node)
value = prop.make_value_from_datastore(value)
if value != None:
props[prop.name] = value
return (cls, props, id)
def get_object(self, cls, id):
if not self.connection:
self._connect()
if not self.connection:
raise NotImplementedError(
"Can't query without a database connection")
url = "/%s/%s" % (self.db_name, id)
resp = self._make_request('GET', url)
if resp.status == 200:
doc = parse(resp)
else:
raise Exception("Error: %s" % resp.status)
return self.get_object_from_doc(cls, id, doc)
def query(self, cls, filters, limit=None, order_by=None):
if not self.connection:
self._connect()
if not self.connection:
raise NotImplementedError(
"Can't query without a database connection")
from urllib.parse import urlencode
query = str(self._build_query(cls, filters, limit, order_by))
if query:
url = "/%s?%s" % (self.db_name, urlencode({"query": query}))
else:
url = "/%s" % self.db_name
resp = self._make_request('GET', url)
if resp.status == 200:
doc = parse(resp)
else:
raise Exception("Error: %s" % resp.status)
return self._object_lister(cls, doc)
def _build_query(self, cls, filters, limit, order_by):
import types
if len(filters) > 4:
raise Exception('Too many filters, max is 4')
parts = []
properties = cls.properties(hidden=False)
for filter, value in filters:
name, op = filter.strip().split()
found = False
for property in properties:
if property.name == name:
found = True
if types.TypeType(value) == list:
filter_parts = []
for val in value:
val = self.encode_value(property, val)
filter_parts.append("'%s' %s '%s'" %
(name, op, val))
parts.append("[%s]" % " OR ".join(filter_parts))
else:
value = self.encode_value(property, value)
parts.append("['%s' %s '%s']" % (name, op, value))
if not found:
raise Exception('%s is not a valid field' % name)
if order_by:
if order_by.startswith("-"):
key = order_by[1:]
type = "desc"
else:
key = order_by
type = "asc"
parts.append("['%s' starts-with ''] sort '%s' %s" %
(key, key, type))
return ' intersection '.join(parts)
def query_gql(self, query_string, *args, **kwds):
raise NotImplementedError("GQL queries not supported in XML")
def save_list(self, doc, items, prop_node):
items_node = doc.createElement('items')
prop_node.appendChild(items_node)
for item in items:
item_node = doc.createElement('item')
items_node.appendChild(item_node)
if isinstance(item, Node):
item_node.appendChild(item)
else:
text_node = doc.createTextNode(item)
item_node.appendChild(text_node)
def save_object(self, obj, expected_value=None):
"""
Marshal the object and do a PUT
"""
doc = self.marshal_object(obj)
if obj.id:
url = "/%s/%s" % (self.db_name, obj.id)
else:
url = "/%s" % (self.db_name)
resp = self._make_request("PUT", url, body=doc.toxml())
new_obj = self.get_object_from_doc(obj.__class__, None, parse(resp))
obj.id = new_obj.id
for prop in obj.properties():
try:
propname = prop.name
except AttributeError:
propname = None
if propname:
value = getattr(new_obj, prop.name)
if value:
setattr(obj, prop.name, value)
return obj
def marshal_object(self, obj, doc=None):
if not doc:
doc = self.new_doc()
if not doc:
doc = self.doc
obj_node = doc.createElement('object')
if obj.id:
obj_node.setAttribute('id', obj.id)
obj_node.setAttribute(
'class',
'%s.%s' % (obj.__class__.__module__, obj.__class__.__name__))
root = doc.documentElement
root.appendChild(obj_node)
for property in obj.properties(hidden=False):
prop_node = doc.createElement('property')
prop_node.setAttribute('name', property.name)
prop_node.setAttribute('type', property.type_name)
value = property.get_value_for_datastore(obj)
if value is not None:
value = self.encode_value(property, value)
if isinstance(value, list):
self.save_list(doc, value, prop_node)
elif isinstance(value, Node):
prop_node.appendChild(value)
else:
text_node = doc.createTextNode(
str(value).encode("ascii", "ignore"))
prop_node.appendChild(text_node)
obj_node.appendChild(prop_node)
return doc
def unmarshal_object(self, fp, cls=None, id=None):
if isinstance(fp, str) or isinstance(fp, str):
doc = parseString(fp)
else:
doc = parse(fp)
return self.get_object_from_doc(cls, id, doc)
def unmarshal_props(self, fp, cls=None, id=None):
"""
Same as unmarshalling an object, except it returns
from "get_props_from_doc"
"""
if isinstance(fp, str) or isinstance(fp, str):
doc = parseString(fp)
else:
doc = parse(fp)
return self.get_props_from_doc(cls, id, doc)
def delete_object(self, obj):
url = "/%s/%s" % (self.db_name, obj.id)
return self._make_request("DELETE", url)
def set_key_value(self, obj, name, value):
self.domain.put_attributes(obj.id, {name: value}, replace=True)
def delete_key_value(self, obj, name):
self.domain.delete_attributes(obj.id, name)
def get_key_value(self, obj, name):
a = self.domain.get_attributes(obj.id, name)
if name in a:
return a[name]
else:
return None
def get_raw_item(self, obj):
return self.domain.get_item(obj.id)
def set_property(self, prop, obj, name, value):
pass
def get_property(self, prop, obj, name):
pass
def load_object(self, obj):
if not obj._loaded:
obj = obj.get_by_id(obj.id)
obj._loaded = True
return obj
def connect(self):
_HTTPSConnection.connect(self)
self.sock.setsockopt(IPPROTO_TCP, TCP_NODELAY, 1)
class ClamAVFileUploadHandler(FileUploadHandler):
chunk_size = CHUNK_SIZE
skip_av_check = False
def new_file(self, *args, **kwargs):
super().new_file(*args, **kwargs)
extension = pathlib.Path(self.file_name).suffix
if extension in CLAM_AV_IGNORE_EXTENSIONS:
self.skip_av_check = True
return
if CLAM_USE_HTTP:
self.av_conn = HTTPConnection(host=CLAM_AV_DOMAIN, )
else:
self.av_conn = HTTPSConnection( # noqa S309
host=CLAM_AV_DOMAIN,
port=443,
)
credentials = b64encode(
bytes(
f"{CLAM_AV_USERNAME}:{CLAM_AV_PASSWORD}",
encoding="utf8",
)).decode("ascii")
try:
self.av_conn.connect()
self.av_conn.putrequest("POST", CLAM_PATH)
self.av_conn.putheader("Content-Type", self.content_type)
self.av_conn.putheader("Authorization", f"Basic {credentials}")
self.av_conn.putheader("Transfer-encoding", "chunked")
self.av_conn.endheaders()
except Exception as ex:
logger.error("Error connecting to ClamAV service", exc_info=True)
raise AntiVirusServiceErrorException(ex)
def receive_data_chunk(self, raw_data, start):
if not self.skip_av_check:
self.av_conn.send(hex(len(raw_data))[2:].encode("utf-8"))
self.av_conn.send(b"\r\n")
self.av_conn.send(raw_data)
self.av_conn.send(b"\r\n")
return raw_data
def file_complete(self, file_size):
if self.skip_av_check:
return None
self.av_conn.send(b"0\r\n\r\n")
resp = self.av_conn.getresponse()
response_content = resp.read()
scanned_file = ScannedFile()
if resp.status != 200:
scanned_file.av_passed = False
scanned_file.av_reason = "Non 200 response from AV server"
scanned_file.save()
raise AntiVirusServiceErrorException(
f"Non 200 response from anti virus service, content: {response_content}"
)
else:
json_response = json.loads(response_content)
if "malware" not in json_response:
scanned_file.av_passed = False
scanned_file.av_reason = "Malformed response from AV server"
scanned_file.save()
raise MalformedAntiVirusResponseException()
if json_response["malware"]:
scanned_file.av_passed = False
scanned_file.av_reason = json_response["reason"]
scanned_file.save()
logger.error(f"Malware found in user uploaded file "
f"'{self.file_name}', exiting upload process")
else:
scanned_file.av_passed = True
scanned_file.save()
# We are using 'content_type_extra' as the a means of making
# the results available to following file handlers
# TODO - put in a PR to Django project to allow file_complete
# to return objects and not break out of file handler loop
if not hasattr(self.content_type_extra, "clam_av_results"):
self.content_type_extra["clam_av_results"] = []
self.content_type_extra["clam_av_results"].append({
"file_name":
self.file_name,
"av_passed":
scanned_file.av_passed,
"scanned_at":
scanned_file.scanned_at,
})
return None
def _connection(self):
conn = HTTPSConnection(self._server, self._port, context=self._context)
conn.connect()
return conn
class XMLManager(object):
def __init__(self, cls, db_name, db_user, db_passwd,
db_host, db_port, db_table, ddl_dir, enable_ssl):
self.cls = cls
if not db_name:
db_name = cls.__name__.lower()
self.db_name = db_name
self.db_user = db_user
self.db_passwd = db_passwd
self.db_host = db_host
self.db_port = db_port
self.db_table = db_table
self.ddl_dir = ddl_dir
self.s3 = None
self.converter = XMLConverter(self)
self.impl = getDOMImplementation()
self.doc = self.impl.createDocument(None, 'objects', None)
self.connection = None
self.enable_ssl = enable_ssl
self.auth_header = None
if self.db_user:
import base64
base64string = base64.encodestring('%s:%s' % (self.db_user, self.db_passwd))[:-1]
authheader = "Basic %s" % base64string
self.auth_header = authheader
def _connect(self):
if self.db_host:
if self.enable_ssl:
from http.client import HTTPSConnection as Connection
else:
from http.client import HTTPConnection as Connection
self.connection = Connection(self.db_host, self.db_port)
def _make_request(self, method, url, post_data=None, body=None):
"""
Make a request on this connection
"""
if not self.connection:
self._connect()
try:
self.connection.close()
except:
pass
self.connection.connect()
headers = {}
if self.auth_header:
headers["Authorization"] = self.auth_header
self.connection.request(method, url, body, headers)
resp = self.connection.getresponse()
return resp
def new_doc(self):
return self.impl.createDocument(None, 'objects', None)
def _object_lister(self, cls, doc):
for obj_node in doc.getElementsByTagName('object'):
if not cls:
class_name = obj_node.getAttribute('class')
cls = find_class(class_name)
id = obj_node.getAttribute('id')
obj = cls(id)
for prop_node in obj_node.getElementsByTagName('property'):
prop_name = prop_node.getAttribute('name')
prop = obj.find_property(prop_name)
if prop:
if hasattr(prop, 'item_type'):
value = self.get_list(prop_node, prop.item_type)
else:
value = self.decode_value(prop, prop_node)
value = prop.make_value_from_datastore(value)
setattr(obj, prop.name, value)
yield obj
def reset(self):
self._connect()
def get_doc(self):
return self.doc
def encode_value(self, prop, value):
return self.converter.encode_prop(prop, value)
def decode_value(self, prop, value):
return self.converter.decode_prop(prop, value)
def get_s3_connection(self):
if not self.s3:
self.s3 = boto.connect_s3(self.aws_access_key_id, self.aws_secret_access_key)
return self.s3
def get_list(self, prop_node, item_type):
values = []
try:
items_node = prop_node.getElementsByTagName('items')[0]
except:
return []
for item_node in items_node.getElementsByTagName('item'):
value = self.converter.decode(item_type, item_node)
values.append(value)
return values
def get_object_from_doc(self, cls, id, doc):
obj_node = doc.getElementsByTagName('object')[0]
if not cls:
class_name = obj_node.getAttribute('class')
cls = find_class(class_name)
if not id:
id = obj_node.getAttribute('id')
obj = cls(id)
for prop_node in obj_node.getElementsByTagName('property'):
prop_name = prop_node.getAttribute('name')
prop = obj.find_property(prop_name)
value = self.decode_value(prop, prop_node)
value = prop.make_value_from_datastore(value)
if value != None:
try:
setattr(obj, prop.name, value)
except:
pass
return obj
def get_props_from_doc(self, cls, id, doc):
"""
Pull out the properties from this document
Returns the class, the properties in a hash, and the id if provided as a tuple
:return: (cls, props, id)
"""
obj_node = doc.getElementsByTagName('object')[0]
if not cls:
class_name = obj_node.getAttribute('class')
cls = find_class(class_name)
if not id:
id = obj_node.getAttribute('id')
props = {}
for prop_node in obj_node.getElementsByTagName('property'):
prop_name = prop_node.getAttribute('name')
prop = cls.find_property(prop_name)
value = self.decode_value(prop, prop_node)
value = prop.make_value_from_datastore(value)
if value != None:
props[prop.name] = value
return (cls, props, id)
def get_object(self, cls, id):
if not self.connection:
self._connect()
if not self.connection:
raise NotImplementedError("Can't query without a database connection")
url = "/%s/%s" % (self.db_name, id)
resp = self._make_request('GET', url)
if resp.status == 200:
doc = parse(resp)
else:
raise Exception("Error: %s" % resp.status)
return self.get_object_from_doc(cls, id, doc)
def query(self, cls, filters, limit=None, order_by=None):
if not self.connection:
self._connect()
if not self.connection:
raise NotImplementedError("Can't query without a database connection")
from urllib.parse import urlencode
query = str(self._build_query(cls, filters, limit, order_by))
if query:
url = "/%s?%s" % (self.db_name, urlencode({"query": query}))
else:
url = "/%s" % self.db_name
resp = self._make_request('GET', url)
if resp.status == 200:
doc = parse(resp)
else:
raise Exception("Error: %s" % resp.status)
return self._object_lister(cls, doc)
def _build_query(self, cls, filters, limit, order_by):
import types
if len(filters) > 4:
raise Exception('Too many filters, max is 4')
parts = []
properties = cls.properties(hidden=False)
for filter, value in filters:
name, op = filter.strip().split()
found = False
for property in properties:
if property.name == name:
found = True
if types.TypeType(value) == list:
filter_parts = []
for val in value:
val = self.encode_value(property, val)
filter_parts.append("'%s' %s '%s'" % (name, op, val))
parts.append("[%s]" % " OR ".join(filter_parts))
else:
value = self.encode_value(property, value)
parts.append("['%s' %s '%s']" % (name, op, value))
if not found:
raise Exception('%s is not a valid field' % name)
if order_by:
if order_by.startswith("-"):
key = order_by[1:]
type = "desc"
else:
key = order_by
type = "asc"
parts.append("['%s' starts-with ''] sort '%s' %s" % (key, key, type))
return ' intersection '.join(parts)
def query_gql(self, query_string, *args, **kwds):
raise NotImplementedError("GQL queries not supported in XML")
def save_list(self, doc, items, prop_node):
items_node = doc.createElement('items')
prop_node.appendChild(items_node)
for item in items:
item_node = doc.createElement('item')
items_node.appendChild(item_node)
if isinstance(item, Node):
item_node.appendChild(item)
else:
text_node = doc.createTextNode(item)
item_node.appendChild(text_node)
def save_object(self, obj, expected_value=None):
"""
Marshal the object and do a PUT
"""
doc = self.marshal_object(obj)
if obj.id:
url = "/%s/%s" % (self.db_name, obj.id)
else:
url = "/%s" % (self.db_name)
resp = self._make_request("PUT", url, body=doc.toxml())
new_obj = self.get_object_from_doc(obj.__class__, None, parse(resp))
obj.id = new_obj.id
for prop in obj.properties():
try:
propname = prop.name
except AttributeError:
propname = None
if propname:
value = getattr(new_obj, prop.name)
if value:
setattr(obj, prop.name, value)
return obj
def marshal_object(self, obj, doc=None):
if not doc:
doc = self.new_doc()
if not doc:
doc = self.doc
obj_node = doc.createElement('object')
if obj.id:
obj_node.setAttribute('id', obj.id)
obj_node.setAttribute('class', '%s.%s' % (obj.__class__.__module__,
obj.__class__.__name__))
root = doc.documentElement
root.appendChild(obj_node)
for property in obj.properties(hidden=False):
prop_node = doc.createElement('property')
prop_node.setAttribute('name', property.name)
prop_node.setAttribute('type', property.type_name)
value = property.get_value_for_datastore(obj)
if value is not None:
value = self.encode_value(property, value)
if isinstance(value, list):
self.save_list(doc, value, prop_node)
elif isinstance(value, Node):
prop_node.appendChild(value)
else:
text_node = doc.createTextNode(str(value).encode("ascii", "ignore"))
prop_node.appendChild(text_node)
obj_node.appendChild(prop_node)
return doc
def unmarshal_object(self, fp, cls=None, id=None):
if isinstance(fp, str) or isinstance(fp, str):
doc = parseString(fp)
else:
doc = parse(fp)
return self.get_object_from_doc(cls, id, doc)
def unmarshal_props(self, fp, cls=None, id=None):
"""
Same as unmarshalling an object, except it returns
from "get_props_from_doc"
"""
if isinstance(fp, str) or isinstance(fp, str):
doc = parseString(fp)
else:
doc = parse(fp)
return self.get_props_from_doc(cls, id, doc)
def delete_object(self, obj):
url = "/%s/%s" % (self.db_name, obj.id)
return self._make_request("DELETE", url)
def set_key_value(self, obj, name, value):
self.domain.put_attributes(obj.id, {name : value}, replace=True)
def delete_key_value(self, obj, name):
self.domain.delete_attributes(obj.id, name)
def get_key_value(self, obj, name):
a = self.domain.get_attributes(obj.id, name)
if name in a:
return a[name]
else:
return None
def get_raw_item(self, obj):
return self.domain.get_item(obj.id)
def set_property(self, prop, obj, name, value):
pass
def get_property(self, prop, obj, name):
pass
def load_object(self, obj):
if not obj._loaded:
obj = obj.get_by_id(obj.id)
obj._loaded = True
return obj
def connect(self):
HTTPSConnection.connect(self)
try:
self.sock.settimeout(30)
except:
pass
class ssl_worker(pp_thread):
global pp_config
session_timeout = int(pp_config['thread_ssl_timeout'])
def __init__(self, key_val, manager, info = '', delay = 0):
pp_thread.__init__(self, info)
self.lock_close = Lock()
self.flag_closed= False
self.info = info
self.delay = delay
self.manager = manager
self.event_proc = Event()
self.arg = None
self.handler = None
self.host_ip = key_val['host_ip']
self.host_name = key_val['host_name']
self.group = key_val['group']
self.timeout = key_val['timeout'] if 'timeout' in key_val else None
def close(self):
if self.handler != None:
try:
self.handler.close()
except:
print_exc()
finally:
self.handler = None
def main(self):
if self.delay != 0 : sleep(self.delay)
while True:
self.handler = HTTPSConnection(self.host_ip, timeout = self.timeout)
self.handler._http_vsn = 10
self.handler._http_vsn_str = 'HTTP/1.0'
try:
self.handler.connect()
except TimeoutError:
self.close()
continue
except:
print_exc()
continue
break
self.manager.feedback('connected', self.group, self)
ev = self.event_proc.wait(self.session_timeout)
self.lock_close.acquire()
self.flag_closed = True
self.lock_close.release()
if self.flag_stop == True:
self.close()
return
if ev != True :
self.manager.feedback('timeout', self.group, self)
if self.arg == None:
self.close()
return
self.do_proc(self.arg)
def put(self, arg):
self.lock_close.acquire()
if self.flag_closed != True:
self.arg = arg
self.lock_close.release()
else:
self.lock_close.release()
return False
self.event_proc.set()
return True
def do_proc(self, arg): pass
def pyget(self, req, headers = {}):
try:
self.handler.request('GET', req, headers = headers)
except:
self.close()
self.manager.feedback('err_write', self.group, self)
print_exc()
return None
try:
ack = self.handler.getresponse()
body = ack.read()
except:
self.close()
self.manager.feedback('err_read', self.group, self)
print_exc()
return None
#--------------------------------------------------------
self.close()
self.manager.feedback('done', self.group, self)
key_val = {}
key_val['body'] = body
key_val['head'] = str(ack.msg)
key_val['status'] = ack.status
return key_val