def test_set_keylog_unset(tmpdir, context, ssl_server):
keylog = tmpdir / "sslkeylog.txt"
sslkeylog.set_keylog(str(keylog))
with closing(ssl_connect(context)) as s:
s.sendall(b"hello")
assert s.recv(1024) == b"hello"
data = keylog.read_text("utf-8").splitlines()
if sslkeylog.OPENSSL111:
assert len(data) == 10
else:
assert len(data) == 2
for line in data:
assert LOG_LINE_REGEX.search(line)
sslkeylog.set_keylog(None)
with closing(ssl_connect(context)) as s:
s.sendall(b"hello")
assert s.recv(1024) == b"hello"
data = keylog.read_text("utf-8").splitlines()
if sslkeylog.OPENSSL111:
assert len(data) == 10
else:
assert len(data) == 2
for line in data:
assert LOG_LINE_REGEX.search(line)
def main(iface, pcap_file, sslkeylogfile, port, cert, priv_key):
# Dump sslkeys
sslkeylog.set_keylog(sslkeylogfile)
sslkeylog.patch()
# Starts sniffing
sniffer = Sniffer(iface, pcap_file)
sniffer.start()
# Starts proxying
initiator = Initiator(cert, priv_key, port)
initiator.start()
import signal
def signal_handler(signal_number, frame):
sniffer.stop()
sniffer.join()
initiator.stop()
initiator.join()
exit()
signal.signal(signal.SIGINT, signal_handler)
signal.pause()
def test_login_ok():
config = configparser.ConfigParser()
config.read('config.ini')
user = config['ISTA']['user']
password = config['ISTA']['password']
sslkeylog.set_keylog("sslkeylog.txt")
ista = Ista(user, password)
out = ista.login_ista()
assert out is True
data = ista.get_readings()
assert data.value.mean() > 0
assert data.date.max() == pd.Timestamp('2020-01-08 00:00:00')
assert data.columns.values.tolist() == ['serial', 'date', 'value']
def test_set_keylog_callback(tmpdir, context, ssl_server):
keylog_callback = Mock()
sslkeylog.set_keylog(keylog_callback)
with closing(ssl_connect(context)) as s:
s.sendall(b"hello")
assert s.recv(1024) == b"hello"
if sslkeylog.OPENSSL111:
assert len(keylog_callback.call_args_list) == 10
else:
assert len(keylog_callback.call_args_list) == 2
for args, kwargs in keylog_callback.call_args_list:
assert isinstance(args[0], ssl.SSLSocket)
assert LOG_LINE_REGEX.search(args[1])
def connect_pub(pubName):
client = mqtt.Client(pubName)
client.on_publish = on_publish # callback in paho
# In order to decrypt TLS traffic on wireshark do:
# 1. execute pip install -r requirements.txt (sslkeylog library needs to be installed)
# 2. set SSLKEYLOGFILEASS2 environment variable to point to the file which will contain the TLS logging.
# 3. See the last part of the tutorial
# (https://jimshaver.net/2015/02/11/decrypting-tls-browser-traffic-with-wireshark-the-easy-way/)
# to tell Wireshark where the file pointed by SSLKEYLOGFILE is.
# e.g.(Windows:Powershell)>
# [Environment]::SetEnvironmentVariable("SSLKEYLOGFILEASS2", "C:\Users\Sane\proj\logs\sslkeylog.log", "User")
LOGGER.info(f"Logging TLS key in {os.environ['SSLKEYLOGFILEASS2']}")
sslkeylog.set_keylog(os.environ['SSLKEYLOGFILEASS2'])
# TLS
client.tls_set(shared.CA_FILE, tls_version=ssl.PROTOCOL_TLS)
client.connect(shared.BROKER, shared.PORT)
return client
def test_set_keylog_bio(tmpdir, context, ssl_server):
keylog = tmpdir / "sslkeylog.txt"
sslkeylog.set_keylog(str(keylog))
with closing(socket.create_connection(ADDRESS)) as sock:
incoming = ssl.MemoryBIO()
outgoing = ssl.MemoryBIO()
sslobj = context.wrap_bio(incoming,
outgoing,
server_side=False,
server_hostname=ADDRESS[0])
ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake)
time.sleep(2)
data = keylog.read_text("utf-8").splitlines()
if sslkeylog.OPENSSL111:
assert len(data) == 10
else:
assert len(data) == 2
for line in data:
assert LOG_LINE_REGEX.search(line)
def main():
parser = argparse.ArgumentParser(description="Gather high-frequency trade "
"data for a selected pair at a "
"cryptocurrency exchange "
"or gather an exchange-wide "
"low-frequency data and store them into "
"a database.")
group = parser.add_mutually_exclusive_group()
group.add_argument("-s",
"--stream",
help="where STREAM must be in the format "
"PAIR:EXCHANGE ")
group.add_argument("-m",
"--monitor",
help="where MONITOR is the exchange name to gather "
"its low-frequency data (i.e. list of traded pairs, "
" their parameters, etc )")
parser.add_argument("-d",
"--dbname",
help="where DBNAME is the name of "
"PostgreSQL database where the data is to be saved "
"(default: ob-analytics),",
default="ob-analytics")
parser.add_argument("-U",
"--user",
help="where USER is the name of "
"PostgreSQL database role to be used to save the data"
"(default: ob-analytics),",
default="ob-analytics")
parser.add_argument("-p",
"--port",
help="where PORT is the listening port "
"of the PostgreSQL server (default 5432)",
default="5432")
args = parser.parse_args()
sslkeylog.set_keylog("sslkeylog.txt")
if args.stream:
stream = args.stream.split(':')
if stream[1] == 'BITSTAMP':
ws_url = "wss://ws.bitstamp.net"
mh = BitstampMessageHandler
elif stream[1] == 'BITFINEX':
ws_url = "wss://api-pub.bitfinex.com/ws/2"
mh = BitfinexMessageHandler
else:
print('Exchange %s is not supported (yet)' % stream[1])
exitcode = 1
sys.exit(exitcode)
h = logging.handlers.RotatingFileHandler(
"./oba%s_%s.log" % (
'_'.join(stream),
args.dbname.upper(),
), 'a', 2**24, 20)
logging.basicConfig(format='%(asctime)s %(process)-6d %(name)s '
'%(levelname)-8s %(message)s',
handlers=[h])
logging.getLogger("obadiah").setLevel(logging.INFO)
logging.getLogger("websockets").setLevel(logging.INFO)
logger = logging.getLogger(__name__ + ".main")
task = asyncio.ensure_future(
capture(stream[1], stream[0], args.user, args.dbname, args.port,
ws_url, mh))
loop = asyncio.get_event_loop()
# loop.set_debug(True)
loop.add_signal_handler(
getattr(signal, 'SIGINT'),
functools.partial(lambda task: task.cancel(), task))
print("Press Ctrl-C to stop ...")
try:
exitcode = 0
asyncio.get_event_loop().run_until_complete(task)
except asyncio.CancelledError:
logger.info('Cancelled, exiting ...')
except Exception as e:
logger.exception(e)
exitcode = 1
sys.exit(exitcode)
elif args.monitor:
if args.monitor == 'BITFINEX':
h = logging.handlers.RotatingFileHandler(
"oba%s_%s.log" % (
args.monitor.upper(),
args.dbname.upper(),
), 'a', 2**24, 20)
logging.basicConfig(format='%(asctime)s %(process)-6d %(name)s '
'%(levelname)-8s %(message)s',
handlers=[h])
logging.getLogger(__name__.split('.')[0]).setLevel(logging.INFO)
logger = logging.getLogger(__name__ + ".main")
task = asyncio.ensure_future(monitor(args.user, args.dbname))
loop = asyncio.get_event_loop()
# loop.set_debug(True)
loop.add_signal_handler(
getattr(signal, 'SIGINT'),
functools.partial(lambda task: task.cancel(), task))
print("Press Ctrl-C to stop ...")
try:
exitcode = 0
asyncio.get_event_loop().run_until_complete(task)
except asyncio.CancelledError:
logger.info('Cancelled, exiting ...')
except Exception as e:
logger.exception(e)
exitcode = 1
else:
print('Exchange %s is not supported (yet)' % args.monitor)
exitcode = 1
sys.exit(exitcode)
else:
parser.print_usage()
import os
import sslkeylog
from fastapi import FastAPI, File, UploadFile
sslkeylog.set_keylog("sslkeylog.txt")
BASE_PATH = os.getenv("BASE_PATH", "/")
app = FastAPI()
@app.get(f"{BASE_PATH}")
async def home():
return {"hello": "world"}
@app.put(f"{BASE_PATH}uploadfile/", status_code=200)
async def create_upload_file(file: UploadFile = File(...)):
await file.read()
return
def test_login_no_ok():
sslkeylog.set_keylog("sslkeylog.txt")
ista = Ista("XXX", "XXX")
out = ista.login_ista()
assert out is False
def wrapsocket(sock, keyfile, certfile, ui, serverhostname=None):
"""Add SSL/TLS to a socket.
This is a glorified wrapper for ``ssl.wrap_socket()``. It makes sane
choices based on what security options are available.
In addition to the arguments supported by ``ssl.wrap_socket``, we allow
the following additional arguments:
* serverhostname - The expected hostname of the remote server. If the
server (and client) support SNI, this tells the server which certificate
to use.
"""
if not serverhostname:
raise error.Abort(_('serverhostname argument is required'))
if b'SSLKEYLOGFILE' in encoding.environ:
try:
import sslkeylog
sslkeylog.set_keylog(pycompat.fsdecode(
encoding.environ[b'SSLKEYLOGFILE']))
ui.warn(
b'sslkeylog enabled by SSLKEYLOGFILE environment variable\n')
except ImportError:
ui.warn(b'sslkeylog module missing, '
b'but SSLKEYLOGFILE set in environment\n')
for f in (keyfile, certfile):
if f and not os.path.exists(f):
raise error.Abort(
_('certificate file (%s) does not exist; cannot connect to %s')
% (f, pycompat.bytesurl(serverhostname)),
hint=_('restore missing file or fix references '
'in Mercurial config'))
settings = _hostsettings(ui, serverhostname)
# We can't use ssl.create_default_context() because it calls
# load_default_certs() unless CA arguments are passed to it. We want to
# have explicit control over CA loading because implicitly loading
# CAs may undermine the user's intent. For example, a user may define a CA
# bundle with a specific CA cert removed. If the system/default CA bundle
# is loaded and contains that removed CA, you've just undone the user's
# choice.
sslcontext = SSLContext(settings['protocol'])
# This is a no-op unless using modern ssl.
sslcontext.options |= settings['ctxoptions']
# This still works on our fake SSLContext.
sslcontext.verify_mode = settings['verifymode']
if settings['ciphers']:
try:
sslcontext.set_ciphers(pycompat.sysstr(settings['ciphers']))
except ssl.SSLError as e:
raise error.Abort(
_('could not set ciphers: %s')
% stringutil.forcebytestr(e.args[0]),
hint=_('change cipher string (%s) in config') %
settings['ciphers'])
if certfile is not None:
def password():
f = keyfile or certfile
return ui.getpass(_('passphrase for %s: ') % f, '')
sslcontext.load_cert_chain(certfile, keyfile, password)
if settings['cafile'] is not None:
try:
sslcontext.load_verify_locations(cafile=settings['cafile'])
except ssl.SSLError as e:
if len(e.args) == 1: # pypy has different SSLError args
msg = e.args[0]
else:
msg = e.args[1]
raise error.Abort(_('error loading CA file %s: %s') % (
settings['cafile'], stringutil.forcebytestr(msg)),
hint=_('file is empty or malformed?'))
caloaded = True
elif settings['allowloaddefaultcerts']:
# This is a no-op on old Python.
sslcontext.load_default_certs()
caloaded = True
else:
caloaded = False
try:
sslsocket = sslcontext.wrap_socket(sock, server_hostname=serverhostname)
except ssl.SSLError as e:
# If we're doing certificate verification and no CA certs are loaded,
# that is almost certainly the reason why verification failed. Provide
# a hint to the user.
# Only modern ssl module exposes SSLContext.get_ca_certs() so we can
# only show this warning if modern ssl is available.
# The exception handler is here to handle bugs around cert attributes:
# https://bugs.python.org/issue20916#msg213479. (See issues5313.)
# When the main 20916 bug occurs, 'sslcontext.get_ca_certs()' is a
# non-empty list, but the following conditional is otherwise True.
try:
if (caloaded and settings['verifymode'] == ssl.CERT_REQUIRED and
modernssl and not sslcontext.get_ca_certs()):
ui.warn(_('(an attempt was made to load CA certificates but '
'none were loaded; see '
'https://mercurial-scm.org/wiki/SecureConnections '
'for how to configure Mercurial to avoid this '
'error)\n'))
except ssl.SSLError:
pass
# Try to print more helpful error messages for known failures.
if util.safehasattr(e, 'reason'):
# This error occurs when the client and server don't share a
# common/supported SSL/TLS protocol. We've disabled SSLv2 and SSLv3
# outright. Hopefully the reason for this error is that we require
# TLS 1.1+ and the server only supports TLS 1.0. Whatever the
# reason, try to emit an actionable warning.
if e.reason == r'UNSUPPORTED_PROTOCOL':
# We attempted TLS 1.0+.
if settings['protocolui'] == 'tls1.0':
# We support more than just TLS 1.0+. If this happens,
# the likely scenario is either the client or the server
# is really old. (e.g. server doesn't support TLS 1.0+ or
# client doesn't support modern TLS versions introduced
# several years from when this comment was written).
if supportedprotocols != {'tls1.0'}:
ui.warn(_(
'(could not communicate with %s using security '
'protocols %s; if you are using a modern Mercurial '
'version, consider contacting the operator of this '
'server; see '
'https://mercurial-scm.org/wiki/SecureConnections '
'for more info)\n') % (
pycompat.bytesurl(serverhostname),
', '.join(sorted(supportedprotocols))))
else:
ui.warn(_(
'(could not communicate with %s using TLS 1.0; the '
'likely cause of this is the server no longer '
'supports TLS 1.0 because it has known security '
'vulnerabilities; see '
'https://mercurial-scm.org/wiki/SecureConnections '
'for more info)\n') %
pycompat.bytesurl(serverhostname))
else:
# We attempted TLS 1.1+. We can only get here if the client
# supports the configured protocol. So the likely reason is
# the client wants better security than the server can
# offer.
ui.warn(_(
'(could not negotiate a common security protocol (%s+) '
'with %s; the likely cause is Mercurial is configured '
'to be more secure than the server can support)\n') % (
settings['protocolui'],
pycompat.bytesurl(serverhostname)))
ui.warn(_('(consider contacting the operator of this '
'server and ask them to support modern TLS '
'protocol versions; or, set '
'hostsecurity.%s:minimumprotocol=tls1.0 to allow '
'use of legacy, less secure protocols when '
'communicating with this server)\n') %
pycompat.bytesurl(serverhostname))
ui.warn(_(
'(see https://mercurial-scm.org/wiki/SecureConnections '
'for more info)\n'))
elif (e.reason == r'CERTIFICATE_VERIFY_FAILED' and
pycompat.iswindows):
ui.warn(_('(the full certificate chain may not be available '
'locally; see "hg help debugssl")\n'))
raise
# check if wrap_socket failed silently because socket had been
# closed
# - see http://bugs.python.org/issue13721
if not sslsocket.cipher():
raise error.Abort(_('ssl connection failed'))
sslsocket._hgstate = {
'caloaded': caloaded,
'hostname': serverhostname,
'settings': settings,
'ui': ui,
}
return sslsocket
def main():
parser = argparse.ArgumentParser(description='%s version %.2f' %
(__prog_name__, __version__))
parser.add_argument(
'-l',
'--listen',
action='store',
metavar='<listen>',
dest='listen',
help='Address the relays will listen on. Default: 0.0.0.0',
default='0.0.0.0')
parser.add_argument('-r',
'--relay',
action='append',
nargs='+',
metavar='<relay>',
dest='relays',
help='''Create new relays.
Several relays can be created by repeating the paramter.
If the protocol is omitted, TCP will be assumed.
Format: [udp:|tcp:]lport:rhost:rport''',
required=True)
parser.add_argument(
'-s',
'--script',
action='store',
metavar='<script>',
dest='script',
type=argparse.FileType('r'),
help='''Python script implementing the handle_request() and
handle_response() functions (see example). They will be
called before forwarding traffic to the proxy, if specified.''',
default=False)
parser.add_argument('-p',
'--proxy',
action='store',
metavar='<proxy>',
dest='proxy',
help='''Proxy to forward all requests/responses to.
If omitted, traffic will only be printed to the console
(monitoring mode unless a script is specified).
Format: host:port''',
default=False)
parser.add_argument(
'-c',
'--cert',
action='store',
metavar='<cert>',
dest='cert',
type=argparse.FileType('r'),
help='Certificate file to use for SSL/TLS interception',
default=False)
parser.add_argument(
'-k',
'--key',
action='store',
metavar='<key>',
dest='key',
type=argparse.FileType('r'),
help='Private key file to use for SSL/TLS interception',
default=False)
parser.add_argument(
'-cc',
'--clientcert',
action='store',
metavar='<clientcert>',
dest='clientcert',
type=argparse.FileType('r'),
help='Client certificate file to use for connecting to server',
default=False)
parser.add_argument(
'-ck',
'--clientkey',
action='store',
metavar='<clientkey>',
dest='clientkey',
type=argparse.FileType('r'),
help='Client private key file to use for connecting to server',
default=False)
parser.add_argument('-t',
'--tlsver',
action='store',
metavar='<tls1|tls11|tls12|ssl2|ssl3>',
dest='tlsver',
help='Force SSL/TLS version',
default=False)
parser.add_argument(
'-sk',
'--sslkeylog',
action='store',
metavar='<ssl keylog file>',
dest='sslkeylog',
type=argparse.FileType('a'),
help='Dump SSL (pre-)master secrets to <ssl keylog file>',
default=False)
cfg = parser.parse_args()
cfg.prog_name = __prog_name__
relays = [item for sublist in cfg.relays for item in sublist]
cfg.relays = []
for r in relays:
r = r.split(':')
try:
if len(r) == 3:
cfg.relays.append(('tcp', int(r[0]), r[1], int(r[2])))
elif len(r) == 4 and r[0] in ['tcp', 'udp']:
cfg.relays.append((r[0], int(r[1]), r[2], int(r[3])))
else:
raise
if r[0] == 'udp' and cfg.listen.startswith('127.0.0'):
print(
color(
"[!] In UDP, it's not recommended to bind to 127.0.0.1. If you see errors, try to bind to your LAN IP address instead.",
1, 31))
except:
sys.exit('[!] error: Invalid relay specification, see help.')
if not (cfg.cert and cfg.key):
print(
color(
"[!] Server cert/key not provided, SSL/TLS interception will not be available. To generate certs, see provided script 'gen_certs.sh'.",
1, 31))
if not (cfg.clientcert and cfg.clientkey):
print("[i] Client cert/key not provided.")
# There is no point starting the local web server
# if we are not going to intercept the req/resp (monitor only).
if cfg.proxy:
start_ws()
else:
print(
color(
"[!] Interception disabled! %s will run in monitoring mode only."
% __prog_name__, 0, 31))
# If a script was specified, import it
if cfg.script:
try:
from imp import load_source
cfg.script_module = load_source(cfg.script.name, cfg.script.name)
except Exception as e:
print(color("[!] %s" % str(e), 1, 31))
sys.exit()
# If a ssl keylog file was specified, dump (pre-)master secrets
if cfg.sslkeylog:
try:
import sslkeylog
sslkeylog.set_keylog(cfg.sslkeylog)
except Exception as e:
print(color("[!] %s" % str(e), 1, 31))
sys.exit()
server_threads = []
for relay in cfg.relays:
server_threads.append(Thread(target=create_server, args=(relay, cfg)))
for t in server_threads:
t.setDaemon(True)
t.start()
time.sleep(.2)
while True:
try:
time.sleep(100)
except KeyboardInterrupt:
sys.exit("\rExiting...")
# stores image
newImagePath = self.image_full_path(newImageFileName + '.jpg')
with open(newImagePath, 'wb+') as newImageFile:
newImageFile.write(imageBytes)
successful = True
else:
successful = False
return successful
# In order to decrypt TLS traffic on wireshark do:
# 1. execute pip install -r requirements (sslkeylog library needs to be installed)
# 2. set SSLKEYLOGFILE environment variable to point to the file which will contain the TLS logging.
# 3. See the last part of the tutorial (https://jimshaver.net/2015/02/11/decrypting-tls-browser-traffic-with-wireshark-the-easy-way/) to tell Wireshark where the file pointed by SSLKEYLOGFILE is.
print(f"Logging TLS key in {os.environ['SSLKEYLOGFILE']}")
sslkeylog.set_keylog(os.environ['SSLKEYLOGFILE'])
ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
ssl_context.options |= (ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1
| ssl.OP_NO_COMPRESSION)
ssl_context.set_ciphers("ECDHE+AESGCM")
ssl_context.load_cert_chain(certfile="cert/example.crt",
keyfile="cert/example.key")
ssl_context.set_alpn_protocols(["h2"])
loop = asyncio.get_event_loop()
# Each client connection will create a new protocol instance
coro = loop.create_server(H2Protocol, '127.0.0.1', 8822, ssl=ssl_context)
server = loop.run_until_complete(coro)
