async def deliver_update_wrapper(self, session, response_body):
''' Handles update pings.
'''
# # Shit, I have a race condition somewhere.
# time.sleep(.01)
subscribed_ghid = Ghid.from_bytes(response_body[0:65])
notification_ghid = Ghid.from_bytes(response_body[65:130])
# for callback in self._subscriptions[subscribed_ghid]:
# callback(notification_ghid)
# Well this is a huge hack. But something about the event loop
# itself is f*****g up control flow and causing the world to hang
# here. May want to create a dedicated thread just for pushing
# updates? Like autoresponder, but autoupdater?
# TODO: fix this gross mess
def run_callbacks(subscribed_ghid, notification_ghid):
for callback in self._subscriptions[subscribed_ghid]:
callback(subscribed_ghid, notification_ghid)
worker = threading.Thread(
target=run_callbacks,
daemon=True,
args=(subscribed_ghid, notification_ghid),
name=_generate_threadnames('remoupd')[0],
)
worker.start()
return b'\x01'
def _unpack_object_def(self, data):
''' Deserializes an object from bytes.
General format:
version 1B int16 unsigned
address 65B ghid
author 65B ghid
private 1B bool
dynamic 1B bool
_legroom 1B int8 unsigned
api_id 65B bytes
is_link 1B bool
state ?B bytes (implicit length)
'''
try:
# version = data[0:1]
address = data[1:66]
author = data[66:131]
private = data[131:132]
dynamic = data[132:133]
_legroom = data[133:134]
api_id = ApiID.from_bytes(data[134:199])
is_link = data[199:200]
state = data[200:]
except Exception:
logger.error(
'Unable to unpack IPC object definition w/ traceback:\n'
''.join(traceback.format_exc())
)
raise
# Version stays unmodified (unused)
if address == bytes(65):
address = None
else:
address = Ghid.from_bytes(address)
if author == bytes(65):
author = None
else:
author = Ghid.from_bytes(author)
private = bool(int.from_bytes(private, 'big'))
dynamic = bool(int.from_bytes(dynamic, 'big'))
_legroom = int.from_bytes(_legroom, 'big')
if _legroom == 0:
_legroom = None
is_link = bool(int.from_bytes(is_link, 'big'))
# state also stays unmodified
return (address,
author,
state,
is_link,
api_id,
private,
dynamic,
_legroom)
async def share_ghid(self, connection, body):
''' Handles object share requests.
'''
ghid = Ghid.from_bytes(body[0:65])
origin = Ghid.from_bytes(body[65:130])
api_id = ApiID.from_bytes(body[130:195])
await self._hgxlink.handle_share(ghid, origin, api_id)
return b'\x01'
async def new_obj(self, connection, body):
''' Handles requests for new objects.
'''
(address, # Unused and set to None.
author, # Unused and set to None.
state,
is_link,
api_id,
private,
dynamic,
legroom) = self._unpack_object_def(body)
if is_link:
raise NotImplementedError('Linked objects are not yet supported.')
state = Ghid.from_bytes(state)
obj = await self._oracle.new_object(
gaoclass = _Dispatchable,
dispatch = self._dispatch,
ipc_protocol = self,
state = state,
dynamic = dynamic,
legroom = legroom,
api_id = api_id,
account = self._dispatch._account
)
# Add the endpoint as a listener.
await self._dispatch.register_object(connection, obj.ghid, private)
await self._dispatch.track_object(connection, obj.ghid)
return bytes(obj.ghid)
async def query_debindings(self, connection, body):
''' Handles debinding query requests.
'''
ghid = Ghid.from_bytes(body)
ghidlist = await self._librarian.debind_status(ghid)
parser = generate_ghidlist_parser()
return parser.pack(list(ghidlist))
async def list_debindings_wrapper(self, session, request_body):
''' Deserializes a publish request and forwards it to the
persister.
'''
ghid = Ghid.from_bytes(request_body)
ghidlist = self._bookie.debind_status(ghid)
parser = generate_ghidlist_parser()
return parser.pack(ghidlist)
async def new_ghid(self, connection, response, exc):
''' Handles responses to requests for new objects.
'''
if exc is not None:
raise exc
else:
return Ghid.from_bytes(response)
async def get_whoami(self, connection, response, exc):
''' Handles responses to whoami requests.
'''
if exc is not None:
raise exc
else:
ghid = Ghid.from_bytes(response)
return ghid
async def query_existence(self, connection, body):
''' Handle existence queries.
'''
ghid = Ghid.from_bytes(body)
if (await self._librarian.contains(ghid)):
return b'\x01'
else:
return b'\x00'
async def freeze_ghid(self, connection, response, exc):
''' Handles responses to object freezing requests.
'''
if exc is not None:
raise exc
else:
return Ghid.from_bytes(response)
async def query_wrapper(self, session, request_body):
''' Deserializes a publish request and forwards it to the
persister.
'''
ghid = Ghid.from_bytes(request_body)
if ghid in self._librarian:
return b'\x01'
else:
return b'\x00'
async def get_startup_obj(self, connection, response, exc):
''' Handle the response to retrieving startup obj ghids.
'''
if exc is not None:
raise exc
elif response == b'':
return None
else:
ghid = Ghid.from_bytes(response)
return ghid
def make_random_ghid():
''' Pseudorandom ghid generator. Uses a module-level check to ensure
uniqueness, without needing system entropy.
'''
ghid = None
while ghid in MADE_RANDOM_GHIDS:
algo = b'\x01'
digest = (random.getrandbits(512)).to_bytes(length=64, byteorder='big')
ghid = Ghid.from_bytes(algo + digest)
MADE_RANDOM_GHIDS.add(ghid)
return ghid
async def share_obj(self, connection, body):
''' Handles object share requests.
'''
ghid = Ghid.from_bytes(body[0:65])
recipient = Ghid.from_bytes(body[65:130])
# Instead of forbidding unregistered apps from sharing objects,
# go for it, but document that you will never be notified of a
# share success or failure without an app token.
requesting_token = self._dispatch.token_lookup(connection)
if requesting_token is None:
logger.info(
'CONN ' + str(connection) + ' is sharing ' + str(ghid) +
' with ' + str(recipient) + ' without defining an app ' +
'token, and therefore cannot be notified of share success ' +
'or failure.'
)
await self._rolodex.share_object(ghid, recipient, requesting_token)
return b'\x01'
def _unpack_object_def(self, data):
''' Deserializes an object from bytes.
'''
try:
version = data[0:1]
address = data[1:66]
author = data[66:131]
private = data[131:132]
dynamic = data[132:133]
_legroom = data[133:134]
api_id = data[134:199]
is_link = data[199:200]
state = data[200:]
except:
logger.error(
'Unable to unpack IPC object definition w/ traceback:\n'
''.join(traceback.format_exc()))
raise
# Version stays unmodified (unused)
if address == bytes(65):
address = None
else:
address = Ghid.from_bytes(address)
if author == bytes(65):
author = None
else:
author = Ghid.from_bytes(author)
private = bool(int.from_bytes(private, 'big'))
dynamic = bool(int.from_bytes(dynamic, 'big'))
_legroom = int.from_bytes(_legroom, 'big')
if _legroom == 0:
_legroom = None
if api_id == bytes(65):
api_id = None
is_link = bool(int.from_bytes(is_link, 'big'))
# state also stays unmodified
return (address, author, state, is_link, api_id, private, dynamic,
_legroom)
async def unsubscribe(self, connection, body):
''' Handle unsubscription requests.
'''
ghid = Ghid.from_bytes(body)
had_subscription = await self._postman.unsubscribe(connection, ghid)
if had_subscription:
return b'\x01'
# Still successful, but idempotent
else:
return b'\x00'
async def unsubscribe_wrapper(self, session, request_body):
''' Deserializes a publish request and forwards it to the
persister.
'''
ghid = Ghid.from_bytes(request_body)
callback = session._subscriptions[ghid]
unsubbed = self._postman.unsubscribe(ghid, callback)
del session._subscriptions[ghid]
if unsubbed:
return b'\x01'
else:
return b'\x00'
async def update_obj(self, connection, body):
''' Handles update object requests.
'''
logger.debug('Handling update request from ' + str(connection))
(ghid,
author, # Unused and set to None.
state,
is_link,
api_id, # Unused and set to None.
private, # TODO: use this.
dynamic, # Unused and set to None.
legroom # TODO: use this.
) = self._unpack_object_def(body)
if is_link:
raise NotImplementedError('Linked objects are not yet supported.')
state = Ghid.from_bytes(state)
obj = await self._oracle.get_object(
gaoclass = _Dispatchable,
ghid = ghid,
api_id = None, # Let _pull() apply this.
state = None, # Let _pull() apply this.
dispatch = self._dispatch,
ipc_protocol = self,
account = self._dispatch._account
)
obj.state = state
# Converting a private object to a public one
if self._dispatch.private_parent_lookup(ghid):
if not private:
await self._dispatch.make_public(ghid)
else:
if private:
raise CatOuttaBagError(
'Once made public, objects cannot be made private. They ' +
'have already been distributed to other applications.'
)
await obj.push()
# Schedule an update in the background.
make_background_future(
self._dispatch.distribute_update(
obj.ghid,
skip_conn = connection
)
)
return b'\x01'
def __init__(self, dispatch, ipc_core, author, dynamic, api_id, frozen, held, deleted, state, oracle, *args, **kwargs):
self.ghid = Ghid.from_bytes(b'\x01' + os.urandom(64))
self.author = author
self.dynamic = dynamic
self.api_id = api_id
# Don't forget that dispatchables assign state as a _DispatchableState
self.state = state[1]
self.frozen = frozen
self.held = held
self.deleted = deleted
self.ipccore = ipc_core
self.oracle = oracle
self.dispatch = dispatch
async def subscribe_wrapper(self, session, request_body):
''' Deserializes a publish request and forwards it to the
persister.
'''
ghid = Ghid.from_bytes(request_body)
def updater(subscribed_ghid,
notification_ghid,
call=session.send_subs_update):
call(subscribed_ghid, notification_ghid)
self._postman.subscribe(ghid, updater)
session._subscriptions[ghid] = updater
return b'\x01'
async def hold_obj(self, connection, body):
''' Handles object holding requests.
'''
ghid = Ghid.from_bytes(body)
obj = await self._oracle.get_object(
gaoclass = _Dispatchable,
ghid = ghid,
api_id = None, # Let _pull() apply this.
state = None, # Let _pull() apply this.
dispatch = self._dispatch,
ipc_protocol = self,
account = self._dispatch._account
)
await obj.hold()
return b'\x01'
def fingerprint(self):
''' The fingerprint! Use this for a sharing target.
'''
try:
fingerprint = self._cfg['user'].fingerprint
# May be undefined, in which case return None
if fingerprint is None:
return fingerprint
# Convert to a ghid if defined.
else:
return Ghid.from_str(fingerprint)
except Exception as exc:
raise ConfigError('Invalid configuration.') from exc
async def freeze_obj(self, connection, body):
''' Handles object freezing requests.
'''
ghid = Ghid.from_bytes(body)
obj = await self._oracle.get_object(
gaoclass = _Dispatchable,
ghid = ghid,
api_id = None, # Let _pull() apply this.
state = None, # Let _pull() apply this.
dispatch = self._dispatch,
ipc_protocol = self,
account = self._dispatch._account
)
frozen_address = await obj.freeze()
return bytes(frozen_address)
def user_id(self):
''' Gets the user_id from the config. Returns a ghid.
'''
try:
user_id = self._cfg['user'].user_id
# May be undefined, in which case return None
if user_id is None:
return user_id
# Convert to a ghid if defined.
else:
return Ghid.from_str(user_id)
except Exception as exc:
raise ConfigError('Invalid configuration.') from exc
async def update_ghid(self, connection, body):
''' Handles update object requests.
'''
(address,
author, # Will be unused and set to None
state,
is_link,
api_id, # Will be unused and set to None
private, # Will be unused and set to None
dynamic, # Will be unused and set to None
_legroom # Will be unused and set to None
) = self._unpack_object_def(body)
if is_link:
state = Ghid.from_bytes(state)
await self._hgxlink._pull_state(address, state)
return b'\x01'
async def delete_obj(self, connection, body):
''' Handles object deletion requests.
'''
ghid = Ghid.from_bytes(body)
obj = await self._oracle.get_object(
gaoclass = _Dispatchable,
ghid = ghid,
api_id = None, # Let _pull() apply this.
state = None, # Let _pull() apply this.
dispatch = self._dispatch,
ipc_protocol = self,
account = self._dispatch._account
)
await self._dispatch.untrack_object(connection, ghid)
# TODO: shift to a dispatch.delete_object method that can ignore this
# connection when the intevitable "deleted that object!" warning comes
# back
await obj.delete()
return b'\x01'
async def get_obj(self, connection, body):
''' Handles requests for an object. Server only.
'''
ghid = Ghid.from_bytes(body)
obj = await self._oracle.get_object(
gaoclass = _Dispatchable,
ghid = ghid,
api_id = None, # Let _pull() apply this.
state = None, # Let _pull() apply this.
dispatch = self._dispatch,
ipc_protocol = self,
account = self._dispatch._account
)
await self._dispatch.track_object(connection, obj.ghid)
if isinstance(obj.state, Ghid):
is_link = True
state = bytes(obj.state)
else:
is_link = False
state = obj.state
# For now, anyways.
# Note: need to add some kind of handling for legroom.
_legroom = None
# Not a big fan of how this works, seems inelegant to me
private = bool(self._dispatch.private_parent_lookup(obj.ghid))
return self._pack_object_def(
obj.ghid,
obj.author,
state,
is_link,
obj.api_id,
private,
obj.dynamic,
_legroom
)
async def subscription_update(self, connection, body):
''' Handles an incoming subscription update.
'''
subscribed_ghid = Ghid.from_bytes(body[0:65])
notification = body[65:]
try:
# Note that this handles postman scheduling as well.
ingested = await self._percore.ingest(
notification,
remotable = False,
skip_conn = connection
)
except AlreadyDebound as exc:
vomitus = exc.ghid
debindings = await self._librarian.debind_status(vomitus)
debinding_ghid = next(debinding for debinding in debindings)
logger.info(str(subscribed_ghid) +
' subscription update already debound: ' +
str(vomitus) + '. Pushing debinding upstream: ' +
str(debinding_ghid))
make_background_future(
self._salmonator.push(debinding_ghid)
)
else:
# But, if ingested, we need to notify the salmonator, so it can (if
# needed) also acquire the target
if ingested:
make_background_future(
self._salmonator.notify(subscribed_ghid)
)
return b'\x01'
def load(cls,
librarian,
oracle,
privateer,
user_id,
password,
_scrypt_hardness=None):
''' Loads a credential container from the <librarian>, with a
ghid of <user_id>, encrypted with scrypted <password>.
'''
# User_id resolves the "primary manifest", a dynamic object containing:
# <private key container dynamic ghid> 65b
# <private key container master secret> 53b
# <privateer persistent store dynamic ghid> 65b
# <privateer persistent store master secret> 53b
# <privateer quarantine store dynamic ghid> 65b
# <privateer quarantine master secret> 53b
# <secondary manifest dynamic ghid> 65b
# <secondary manifest master secret> 53b
# <random length, random fill padding>
# The primary manifest is encrypted via the privateer.ratchet_bootstrap
# process, using the inflated password as the master secret.
# The secondary manifest secret is maintained by the privateer. From
# there forwards, everything is business as usual.
logger.info(
'Recovering the primary manifest from the persistence subsystem.')
primary_manifest = librarian.summarize(user_id)
fingerprint = primary_manifest.author
logger.info('Expanding password using scrypt. Please be patient.')
primary_master = cls._password_expansion(fingerprint, password,
_scrypt_hardness)
del password
# Calculate the primary secret and then inject it into the temporary
# storage at privateer
cls._inject_secret(librarian,
privateer,
proxy=user_id,
master_secret=primary_master)
# We're done with the summary, so go ahead and overwrite this name
primary_manifest = oracle.get_object(gaoclass=_GAO, ghid=user_id)
logger.info(
'Password successfully expanded. Extracting the primary manifest.')
manifest = primary_manifest.extract_state()
logger.info('Verifying password. Please be patient.')
# Check the password so we can fail with a meaningful message!
# Should probably not hard-code the password validator length or summat
password_validator = manifest[0:32]
checker = cls._make_password_validator(secret=primary_master,
hardness=_scrypt_hardness)
if checker != password_validator:
logger.critical('Incorrect password.')
raise ValueError('Incorrect password.')
else:
logger.info(
'Password correct. Proceeding with manifest extraction.')
identity_ghid = Ghid.from_bytes(manifest[32:97])
identity_master = Secret.from_bytes(manifest[97:150])
persistent_ghid = Ghid.from_bytes(manifest[150:215])
persistent_master = Secret.from_bytes(manifest[215:268])
quarantine_ghid = Ghid.from_bytes(manifest[268:333])
quarantine_master = Secret.from_bytes(manifest[333:386])
secondary_manifest = Ghid.from_bytes(manifest[386:451])
secondary_master = Secret.from_bytes(manifest[451:504])
# Inject all the needed secrets.
cls._inject_secret(librarian,
privateer,
proxy=identity_ghid,
master_secret=identity_master)
cls._inject_secret(librarian,
privateer,
proxy=persistent_ghid,
master_secret=persistent_master)
cls._inject_secret(librarian,
privateer,
proxy=quarantine_ghid,
master_secret=quarantine_master)
cls._inject_secret(librarian,
privateer,
proxy=secondary_manifest,
master_secret=secondary_master)
logger.info('Manifest recovered. Retrieving private keys.')
identity_container = oracle.get_object(gaoclass=_GAODict,
ghid=identity_ghid)
identity = FirstParty._from_serialized(
identity_container.extract_state())
logger.info('Rebuilding credential.')
self = cls(identity=identity,
primary_master=primary_master,
identity_master=identity_master,
persistent_master=persistent_master,
quarantine_master=quarantine_master,
secondary_master=secondary_master)
if _scrypt_hardness:
self._scrypt_hardness = _scrypt_hardness
self.declare_primary(user_id, identity_ghid, persistent_ghid,
quarantine_ghid, secondary_manifest)
return self
async def subscribe(self, connection, body):
''' Handle subscription requests.
'''
ghid = Ghid.from_bytes(body)
await self._postman.subscribe(connection, ghid)
return b'\x01'
from golix import Ghid
# These are abnormal (don't use in production) inclusions
from golix._spec import _gidc, _geoc, _gobs, _gobd, _gdxx, _garq
from golix._spec import _asym_hand, _asym_ak, _asym_nk, _asym_else
from golix.utils import _dummy_signature
from golix.utils import _dummy_mac
from golix.utils import _dummy_asym
from golix.utils import _dummy_address
from golix.utils import _dummy_pubkey
# ###############################################
# Testing
# ###############################################
_dummy_ghid = Ghid(0, _dummy_address)
def run():
# GIDC test parsers
gidc_1 = {
'magic': b'GIDC',
'version': 2,
'cipher': 0,
'body': {
'signature_key': _dummy_pubkey,
'encryption_key': _dummy_pubkey,
'exchange_key': _dummy_pubkey,
},
'ghid': _dummy_ghid,
'signature': None
class _GaoDictBootstrap(dict):
# Just inject a class-level ghid.
ghid = Ghid.from_bytes(b'\x01' + bytes(64))
async def get_whoami(self, connection, body):
''' Handles whoami requests.
'''
ghid = Ghid.from_bytes(body)
self._hgxlink.whoami = ghid
return b''
async def load(cls, librarian, oracle, privateer, user_id, password,
_scrypt_hardness=None):
''' Loads a credential container from the <librarian>, with a
ghid of <user_id>, encrypted with scrypted <password>.
'''
# User_id resolves the "primary manifest", a dynamic object containing:
# <private key container dynamic ghid> 65b
# <private key container master secret> 53b
# <privateer persistent store dynamic ghid> 65b
# <privateer persistent store master secret> 53b
# <privateer quarantine store dynamic ghid> 65b
# <privateer quarantine master secret> 53b
# <secondary manifest dynamic ghid> 65b
# <secondary manifest master secret> 53b
# <random length, random fill padding>
# The primary manifest is encrypted via the privateer.ratchet_bootstrap
# process, using the inflated password as the master secret.
# The secondary manifest secret is maintained by the privateer. From
# there forwards, everything is business as usual.
logger.info(
'Recovering the primary manifest from the persistence subsystem.'
)
primary_manifest = await librarian.summarize(user_id)
fingerprint = primary_manifest.author
logger.info('Expanding password using scrypt. Please be patient.')
primary_master = cls._password_expansion(
fingerprint,
password,
_scrypt_hardness
)
del password
# Calculate the primary secret and then inject it into the temporary
# storage at privateer
cls._inject_secret(
librarian,
privateer,
proxy = user_id,
master_secret = primary_master
)
# We're done with the summary, so go ahead and overwrite this name
primary_manifest = oracle.get_object(
gaoclass = _GAO,
ghid = user_id
)
logger.info(
'Password successfully expanded. Extracting the primary manifest.'
)
manifest = primary_manifest.extract_state()
logger.info('Verifying password. Please be patient.')
# Check the password so we can fail with a meaningful message!
# Should probably not hard-code the password validator length or summat
password_validator = manifest[0:32]
checker = cls._make_password_validator(
secret = primary_master,
hardness = _scrypt_hardness
)
if checker != password_validator:
logger.critical('Incorrect password.')
raise ValueError('Incorrect password.')
else:
logger.info(
'Password correct. Proceeding with manifest extraction.'
)
identity_ghid = Ghid.from_bytes(manifest[32:97])
identity_master = Secret.from_bytes(manifest[97:150])
persistent_ghid = Ghid.from_bytes(manifest[150:215])
persistent_master = Secret.from_bytes(manifest[215:268])
quarantine_ghid = Ghid.from_bytes(manifest[268:333])
quarantine_master = Secret.from_bytes(manifest[333:386])
secondary_manifest = Ghid.from_bytes(manifest[386:451])
secondary_master = Secret.from_bytes(manifest[451:504])
# Inject all the needed secrets.
cls._inject_secret(
librarian,
privateer,
proxy = identity_ghid,
master_secret = identity_master
)
cls._inject_secret(
librarian,
privateer,
proxy = persistent_ghid,
master_secret = persistent_master
)
cls._inject_secret(
librarian,
privateer,
proxy = quarantine_ghid,
master_secret = quarantine_master
)
cls._inject_secret(
librarian,
privateer,
proxy = secondary_manifest,
master_secret = secondary_master
)
logger.info('Manifest recovered. Retrieving private keys.')
identity_container = oracle.get_object(
gaoclass = _GAODict,
ghid = identity_ghid
)
identity = FirstParty._from_serialized(
identity_container.extract_state()
)
logger.info('Rebuilding credential.')
self = cls(
identity = identity,
primary_master = primary_master,
identity_master = identity_master,
persistent_master = persistent_master,
quarantine_master = quarantine_master,
secondary_master = secondary_master
)
if _scrypt_hardness:
self._scrypt_hardness = _scrypt_hardness
self.declare_primary(
user_id,
identity_ghid,
persistent_ghid,
quarantine_ghid,
secondary_manifest
)
return self
async def sync_obj(self, connection, body):
''' Handles manual syncing requests. Server only.
'''
ghid = Ghid.from_bytes(body)
await self._salmonator.attempt_pull(ghid)
return b'\x01'
async def get_startup_obj(self, connection, body):
''' Handles requests for startup objects.
'''
ghid = Ghid.from_bytes(body)
self._hgxlink._startup_obj = ghid
return b'\x01'
async def delete_ghid(self, connection, body):
''' Handles object deletion requests.
'''
ghid = Ghid.from_bytes(body)
await self._hgxlink.handle_delete(ghid)
return b'\x01'
async def get(self, connection, body):
''' Handle get requests.
'''
ghid = Ghid.from_bytes(body)
return (await self._librarian.retrieve(ghid))
async def get_wrapper(self, session, request_body):
''' Deserializes a get request; forwards it to the persister.
'''
ghid = Ghid.from_bytes(request_body)
return self._librarian.retrieve(ghid)
async def register_startup_obj(self, connection, body):
''' Handles startup object registration. Server only.
'''
ghid = Ghid.from_bytes(body)
await self._dispatch.register_startup(connection, ghid)
return b'\x01'
async def discard_obj(self, connection, body):
''' Handles object discarding requests. Server only.
'''
ghid = Ghid.from_bytes(body)
await self._dispatch.untrack_object(connection, ghid)
return b'\x01'