def send_reload(config, user_id, volume_id, gateway_id): """ Generate and send a reload-request to a specific gateway. This method is used when updating an individual gateway (e.g. to change its driver, or host/port) Return 0 on success (HTTP 200) Return HTTP status code on error Return -1 on error """ gateway_cert = object_stub.load_gateway_cert(config, str(gateway_id)) if gateway_cert is None: raise MissingCertException("Missing gateway cert for '%s'" % gateway_id) url = 'http://%s:%s' % (gateway_cert.host, gateway_cert.port) msg = make_reload_request(config, user_id, volume_id, gateway_id=gateway_id) msgtxt = msg.SerializeToString() try: req = requests.post(url, data={"control-plane": msgtxt}) if req.status_code == 200: return 0 else: return req.status_code except requests.exceptions.Timeout: log.debug("Timed out on %s" % gateway_id) return -1 except requests.exceptions.RequestException, re: log.exception(re) return -1
def send_reload( config, user_id, volume_id, gateway_id ): """ Generate and send a reload-request to a specific gateway. This method is used when updating an individual gateway (e.g. to change its driver, or host/port) Return 0 on success (HTTP 200) Return HTTP status code on error Return -1 on error """ gateway_cert = object_stub.load_gateway_cert( config, str(gateway_id) ) if gateway_cert is None: raise MissingCertException("Missing gateway cert for '%s'" % gateway_id ) url = 'http://%s:%s' % (gateway_cert.host, gateway_cert.port) msg = make_reload_request( config, user_id, volume_id, gateway_id=gateway_id ) msgtxt = msg.SerializeToString() try: req = requests.post( url, data={"control-plane": msgtxt} ) if req.status_code == 200: return 0 else: return req.status_code except (OSError, IOError), ioe: return -1
def driver_reload( config, volume_name, gateway_name ): """ Download a gateway's driver and cache it locally. Looks in the cached cert directory for the gateway cert. This method is meant to be called immediately after a successful call to certs_reload(). Return True on success Raise an exception on failure """ cached_path = cache_cert_path( config, volume_name, gateway_name, "gateway-%s" % gateway_name ) gateway_cert = object_stub.load_gateway_cert( config, gateway_name, path=cached_path ) if gateway_cert is None: raise Exception("No such cert for gateway %s/%s" % (volume_name, gateway_name) ) ms_url = config['MS_url'] driver_downloader = config['helpers']['fetch_driver'] driver_hash = binascii.hexlify( gateway_cert.driver_hash ) driver_text = driver_fetch( ms_url, driver_hash, driver_downloader ) if driver_text is None: raise Exception("Failed to fetch driver %s from %s" % ( driver_hash, ms_url ) ) # save it cached_path = cache_cert_path( config, volume_name, gateway_name, "driver-%s" % driver_hash, suffix='' ) try: with open(cached_path, "w") as f: f.write( driver_text ) f.flush() except Exception, e: log.exception(e) raise
def gateway_certs_load_cached(config, cert_bundle, exclude=[]): """ Load and return the set of valid cached gateway certificates from the cert bundle. Don't load the ones in exclude """ ret = [] for block in cert_bundle.blocks: if block.block_id in exclude: continue gateway_cert = object_stub.load_gateway_cert(config, block.block_id) if gateway_cert is None: continue if is_gateway_cert_in_bundle(cert_bundle, gateway_cert): ret.append(gateway_cert) return ret
def gateway_certs_load_cached( config, cert_bundle, exclude=[] ): """ Load and return the set of valid cached gateway certificates from the cert bundle. Don't load the ones in exclude """ ret = [] for block in cert_bundle.blocks: if block.block_id in exclude: continue gateway_cert = object_stub.load_gateway_cert( config, block.block_id ) if gateway_cert is None: continue if is_gateway_cert_in_bundle( cert_bundle, gateway_cert ): ret.append( gateway_cert ) return ret
def driver_reload(config, volume_name, gateway_name): """ Download a gateway's driver and cache it locally. Looks in the cached cert directory for the gateway cert. This method is meant to be called immediately after a successful call to certs_reload(). Return True on success Raise an exception on failure """ cached_path = cache_cert_path(config, volume_name, gateway_name, "gateway-%s" % gateway_name) gateway_cert = object_stub.load_gateway_cert(config, gateway_name, path=cached_path) if gateway_cert is None: raise Exception("No such cert for gateway %s/%s" % (volume_name, gateway_name)) ms_url = config['MS_url'] driver_downloader = config['helpers']['fetch_driver'] driver_hash = binascii.hexlify(gateway_cert.driver_hash) driver_text = driver_fetch(ms_url, driver_hash, driver_downloader) if driver_text is None: raise Exception("Failed to fetch driver %s from %s" % (driver_hash, ms_url)) # save it cached_path = cache_cert_path(config, volume_name, gateway_name, "driver-%s" % driver_hash, suffix='') try: with open(cached_path, "w") as f: f.write(driver_text) f.flush() except Exception, e: log.exception(e) raise
def get_gateway_cert( config, gateway_name_or_id, check_cache=True ): """ Load a gateway certificate from disk. If it is not local, then go fetch it. Return the cert on success Raise an exception on error. """ gateway_cert = None if check_cache: gateway_cert = object_stub.load_gateway_cert( config, gateway_name_or_id ) if gateway_cert is None: downloader = config['helpers']['fetch_gateway_cert'] ms_url = config['MS_url'] gateway_cert = gateway_cert_fetch( ms_url, gateway_name_or_id, downloader ) if gateway_cert is None: raise Exception("Failed to obtain gateway certificate") return gateway_cert
def get_gateway_cert(config, gateway_name_or_id, check_cache=True): """ Load a gateway certificate from disk. If it is not local, then go fetch it. Return the cert on success Raise an exception on error. """ gateway_cert = None if check_cache: gateway_cert = object_stub.load_gateway_cert(config, gateway_name_or_id) if gateway_cert is None: downloader = config['helpers']['fetch_gateway_cert'] ms_url = config['MS_url'] gateway_cert = gateway_cert_fetch(ms_url, gateway_name_or_id, downloader) if gateway_cert is None: raise Exception("Failed to obtain gateway certificate") return gateway_cert
def broadcast_reload( config, user_id, volume_id, cert_bundle_version=None, volume_version=None, gateway_names=None ): """ Generate and broadcast a set of requests to all gateways that: * are write-capable * can receive writes * can coordinate writes. The message will have them synchronously reload their configurations. If gateway_names is given, then send to those gateways instead. Send it off and wait for their acknowledgements (or timeouts). This method is used when adding/removing gateways, and updating volume capability information. We'll need the volume private key. Return {"gateway_name": True|False|None} on success None indicates "unknown" """ import grequests logging.getLogger("requests").setLevel(logging.CRITICAL) logging.getLogger("grequests").setLevel(logging.CRITICAL) gateway_certs = None gateway_status = {} # sanity check--volume key is on file volume_cert = object_stub.load_volume_cert( config, str(volume_id) ) if volume_cert is None: raise MissingCertException("No volume cert for '%s'" % str(volume_id)) owner_cert = object_stub.load_user_cert( config, str(volume_cert.owner_id)) if owner_cert is None: raise MissingCertException("Missing user cert for %s, owner of volume '%s'" % (volume_cert.owner_id, volume_cert.name)) volume_pkey = storage.load_private_key( config, "user", owner_cert.email ) if volume_pkey is None: raise MissingKeyException("No volume key for owner '%s' of '%s'" % (owner_cert.email, volume_cert.name )) if gateway_names is None: writer_certs = list_volume_writers( config, volume_id ) coord_certs = list_volume_coordinators( config, volume_id ) recver_certs = list_gateways_by_type( config, volume_id, "RG" ) gateway_certs = writer_certs + coord_certs + recver_certs else: gateway_certs = [] for gateway_name in gateway_names: gateway_cert = object_stub.load_gateway_cert( config, gateway_name ) if gateway_cert is None: raise MissingCertException("No gateway cert for '%s'" % gateway_name ) gateway_certs.append( gateway_cert ) for gateway_cert in gateway_certs: gateway_status[gateway_cert.name] = None gateway_url_names = dict( [('http://%s:%s' % (cert.host, cert.port), cert.name) for cert in gateway_certs] ) urls = gateway_url_names.keys() msg = make_reload_request( config, user_id, volume_id, cert_bundle_version=cert_bundle_version, volume_version=volume_version ) if msg is None: raise Exception("BUG: failed to generate config-reload request") def req_exception(request, exception): log.info("Caught exception on broadcast to '%s'" % request.url) log.info( traceback.format_exception(type(exception), exception, None) ) gateway_name = gateway_url_names[request.url] gateway_status[gateway_name] = False msg_txt = msg.SerializeToString() reqs = [grequests.post(url, data={"control-plane": msg_txt}) for url in urls] # send all! iresps = grequests.imap( reqs, exception_handler=req_exception ) for resp in iresps: url = resp.url purl = urlparse.urlparse(url) hostname = purl.hostname port = purl.port gateway_name = gateway_url_names.get('http://%s:%s' % (hostname,port), None) if gateway_name is None: log.warn("Unknown URL '%s'" % url) if resp.status_code == 200: gateway_status[gateway_name] = True else: gateway_status[gateway_name] = False log.warn("HTTP %s on broadcast to '%s'" % (resp.status_code, gateway_name)) return gateway_status
def make_reload_request( config, user_id, volume_id, gateway_id=None, gateway_name=None, cert_bundle_version=None, volume_version=None ): """ Make a signed, serialized gateway-reload request. If gateway_id or gateway_name is not None, then the request will be destined to a particular gateway, and will be signed with the owner's private key. Otherwise, the request will be destined to all write/coordinate gateways in the volume, and will be signed with the volume owner's private key. Return the signed request. Raise on error. """ signing_key = None gateway_cert_version = None # need either volume key or gateway key if gateway_id is None and gateway_name is not None: gateway_id = object_stub.load_gateway_id( config, gateway_name ) if gateway_name is None and gateway_id is not None: gateway_name = object_stub.load_gateway_name( config, gateway_id ) if gateway_name is not None: # look up the gateway's cert--its version it must match gateway_cert_version gateway_cert = object_stub.load_gateway_cert( config, gateway_name ) if gateway_cert is None: raise MissingCertException("Missing gateway certificate for %s" % gateway_name ) assert volume_id == gateway_cert.volume_id, "Gateway '%s' is not in volume %s (but %s)" % (gateway_cert.name, volume_id, gateway_cert.volume_id) gateway_cert_version = gateway_cert.version # look up the owner's user user_cert = object_stub.load_user_cert( config, str(gateway_cert.owner_id) ) if user_cert is None: raise MissingCertException("Missing user certificate for %s, owner of '%s'" % (gateway_cert.owner_id, gateway_cert.name)) # look up the user's private key, to sign with that user_pkey = storage.load_private_key( config, "user", user_cert.email ) if user_pkey is None: raise MissingCertException("Missing user private key for '%s'" % user_cert.email) log.debug("Sign reload request with private key of user '%s' for gateway '%s' in volume %s" % (user_cert.email, gateway_cert.name, volume_id)) signing_key = user_pkey else: # send to volume volume_cert = object_stub.load_volume_cert( config, str(volume_id) ) if volume_cert is None: raise MissingCertException("Missing cert for volume %s" % (volume_id)) owner_cert = object_stub.load_user_cert( config, str(volume_cert.owner_id) ) if owner_cert is None: raise MissingCertException("Missing cert for user %s" % volume_cert.owner_id) volume_pkey = storage.load_private_key( config, "user", owner_cert.email ) if volume_pkey is None: raise MissingKeyException("Missing both gateway and volume private keys") log.debug("Sign reload request with private key of volume owner '%s' in volume %s" % (owner_cert.email, volume_cert.name)) signing_key = volume_pkey if volume_version is None: # look up volume cert version volume_cert = object_stub.load_volume_cert( config, str(volume_id) ) if volume_cert is None: raise MissingCertException("Missing volume cert, and volume cert version is not given") volume_version = volume_cert.volume_version if cert_bundle_version is None: # look up version vector; cross-check with volume version version_vector_txt = object_stub.load_object_file( config, "volume", str(volume_id) + ".bundle.version" ) if version_vector_txt is None: raise MissingCertException("No cert bundle version information for volume '%s'" % volume_name) try: version_vector = json.loads(version_vector_txt) except: raise MissingCertException("Invalid version vector JSON") cert_bundle_version = version_vector.get('bundle_version', None) onfile_volume_version = version_vector.get('volume_version', None) assert cert_bundle_version is not None, "Missing bundle version in cert bundle version vector" assert onfile_volume_version is not None, "Missing volume version in cert bundle version vector" try: cert_bundle_version = int(cert_bundle_version) onfile_volume_version = int(onfile_volume_version) except: raise MissingCertException("Missing valid version information for cert bundle") assert onfile_volume_version == volume_version, "BUG: On-file cert bundle volume version (%s) does not match given volume version (%s)" % (onfile_volume_version, volume_version) req = sg_pb2.Request() req.request_type = sg_pb2.Request.RELOAD req.user_id = user_id req.volume_id = volume_id if gateway_id is not None: req.coordinator_id = gateway_id else: req.coordinator_id = 0 req.src_gateway_id = libsyndicate.Syndicate.GATEWAY_TOOL req.message_nonce = random.randint(0, 2**64-1) req.volume_version = volume_version req.cert_version = cert_bundle_version req.file_id = 0 # ignored req.file_version = 0 # ignored req.fs_path = "" # ignored if gateway_cert_version is not None: req.gateway_cert_version = gateway_cert_version # sign req.signature = "" reqstr = req.SerializeToString() sig = crypto.sign_data( signing_key, reqstr ) req.signature = base64.b64encode( sig ) return req
def broadcast_reload(config, user_id, volume_id, cert_bundle_version=None, volume_version=None, gateway_names=None): """ Generate and broadcast a set of requests to all gateways that: * are write-capable * can receive writes * can coordinate writes. The message will have them synchronously reload their configurations. If gateway_names is given, then send to those gateways instead. Send it off and wait for their acknowledgements (or timeouts). This method is used when adding/removing gateways, and updating volume capability information. We'll need the volume private key. Return {"gateway_name": True|False|None} on success None indicates "unknown" """ import grequests logging.getLogger("requests").setLevel(logging.CRITICAL) logging.getLogger("grequests").setLevel(logging.CRITICAL) gateway_certs = None gateway_status = {} # sanity check--volume key is on file volume_cert = object_stub.load_volume_cert(config, str(volume_id)) if volume_cert is None: raise MissingCertException("No volume cert for '%s'" % str(volume_id)) owner_cert = object_stub.load_user_cert(config, str(volume_cert.owner_id)) if owner_cert is None: raise MissingCertException("Missing user cert for %s, owner of volume '%s'" % (volume_cert.owner_id, volume_cert.name)) volume_pkey = storage.load_private_key(config, "user", owner_cert.email) if volume_pkey is None: raise MissingKeyException("No volume key for owner '%s' of '%s'" % (owner_cert.email, volume_cert.name)) if gateway_names is None: writer_certs = list_volume_writers(config, volume_id) coord_certs = list_volume_coordinators(config, volume_id) recver_certs = list_gateways_by_type(config, volume_id, "RG") gateway_certs = writer_certs + coord_certs + recver_certs else: gateway_certs = [] for gateway_name in gateway_names: gateway_cert = object_stub.load_gateway_cert(config, gateway_name) if gateway_cert is None: raise MissingCertException("No gateway cert for '%s'" % gateway_name) gateway_certs.append(gateway_cert) for gateway_cert in gateway_certs: gateway_status[gateway_cert.name] = None gateway_url_names = dict([('http://%s:%s' % (cert.host, cert.port), cert.name) for cert in gateway_certs]) urls = gateway_url_names.keys() msg = make_reload_request(config, user_id, volume_id, cert_bundle_version=cert_bundle_version, volume_version=volume_version) if msg is None: raise Exception("BUG: failed to generate config-reload request") def req_exception(request, exception): log.info("Caught exception on broadcast to '%s'" % request.url) log.info(traceback.format_exception(type(exception), exception, None)) gateway_name = gateway_url_names[request.url] gateway_status[gateway_name] = False msg_txt = msg.SerializeToString() reqs = [grequests.post(url, data={"control-plane": msg_txt}) for url in urls] # send all! iresps = grequests.imap(reqs, exception_handler=req_exception) for resp in iresps: url = resp.url purl = urlparse.urlparse(url) hostname = purl.hostname port = purl.port gateway_name = gateway_url_names.get('http://%s:%s' % (hostname,port), None) if gateway_name is None: log.warn("Unknown URL '%s'" % url) if resp.status_code == 200: gateway_status[gateway_name] = True else: gateway_status[gateway_name] = False log.warn("HTTP %s on broadcast to '%s'" % (resp.status_code, gateway_name)) return gateway_status
def make_reload_request(config, user_id, volume_id, gateway_id=None, gateway_name=None, cert_bundle_version=None, volume_version=None): """ Make a signed, serialized gateway-reload request. If gateway_id or gateway_name is not None, then the request will be destined to a particular gateway, and will be signed with the owner's private key. Otherwise, the request will be destined to all write/coordinate gateways in the volume, and will be signed with the volume owner's private key. Return the signed request. Raise on error. """ signing_key = None gateway_cert_version = None # need either volume key or gateway key if gateway_id is None and gateway_name is not None: gateway_id = object_stub.load_gateway_id(config, gateway_name) if gateway_name is None and gateway_id is not None: gateway_name = object_stub.load_gateway_name(config, gateway_id) if gateway_name is not None: # look up the gateway's cert--its version it must match gateway_cert_version gateway_cert = object_stub.load_gateway_cert(config, gateway_name) if gateway_cert is None: raise MissingCertException("Missing gateway certificate for %s" % gateway_name) assert volume_id == gateway_cert.volume_id, "Gateway '%s' is not in volume %s (but %s)" % (gateway_cert.name, volume_id, gateway_cert.volume_id) gateway_cert_version = gateway_cert.version # look up the owner's user user_cert = object_stub.load_user_cert(config, str(gateway_cert.owner_id)) if user_cert is None: raise MissingCertException("Missing user certificate for %s, owner of '%s'" % (gateway_cert.owner_id, gateway_cert.name)) # look up the user's private key, to sign with that user_pkey = storage.load_private_key(config, "user", user_cert.email) if user_pkey is None: raise MissingCertException("Missing user private key for '%s'" % user_cert.email) log.debug("Sign reload request with private key of user '%s' for gateway '%s' in volume %s" % (user_cert.email, gateway_cert.name, volume_id)) signing_key = user_pkey else: # send to volume volume_cert = object_stub.load_volume_cert(config, str(volume_id)) if volume_cert is None: raise MissingCertException("Missing cert for volume %s" % (volume_id)) owner_cert = object_stub.load_user_cert(config, str(volume_cert.owner_id)) if owner_cert is None: raise MissingCertException("Missing cert for user %s" % volume_cert.owner_id) volume_pkey = storage.load_private_key(config, "user", owner_cert.email) if volume_pkey is None: raise MissingKeyException("Missing both gateway and volume private keys") log.debug("Sign reload request with private key of volume owner '%s' in volume %s" % (owner_cert.email, volume_cert.name)) signing_key = volume_pkey if volume_version is None: # look up volume cert version volume_cert = object_stub.load_volume_cert(config, str(volume_id)) if volume_cert is None: raise MissingCertException("Missing volume cert, and volume cert version is not given") volume_version = volume_cert.volume_version if cert_bundle_version is None: # look up version vector; cross-check with volume version version_vector_txt = object_stub.load_object_file(config, "volume", str(volume_id) + ".bundle.version") if version_vector_txt is None: raise MissingCertException("No cert bundle version information for volume '%s'" % volume_cert.name) try: version_vector = json.loads(version_vector_txt) except: raise MissingCertException("Invalid version vector JSON") cert_bundle_version = version_vector.get('bundle_version', None) onfile_volume_version = version_vector.get('volume_version', None) assert cert_bundle_version is not None, "Missing bundle version in cert bundle version vector" assert onfile_volume_version is not None, "Missing volume version in cert bundle version vector" try: cert_bundle_version = int(cert_bundle_version) onfile_volume_version = int(onfile_volume_version) except: raise MissingCertException("Missing valid version information for cert bundle") assert onfile_volume_version == volume_version, "BUG: On-file cert bundle volume version (%s) does not match given volume version (%s)" % (onfile_volume_version, volume_version) req = sg_pb2.Request() req.request_type = sg_pb2.Request.RELOAD req.user_id = user_id req.volume_id = volume_id if gateway_id is not None: req.coordinator_id = gateway_id else: req.coordinator_id = 0 req.src_gateway_id = libsyndicate.Syndicate.GATEWAY_TOOL req.message_nonce = random.randint(0, 2**64-1) req.volume_version = volume_version req.cert_version = cert_bundle_version req.file_id = 0 # ignored req.file_version = 0 # ignored req.fs_path = "" # ignored if gateway_cert_version is not None: req.gateway_cert_version = gateway_cert_version # sign req.signature = "" reqstr = req.SerializeToString() sig = crypto.sign_data(signing_key, reqstr) req.signature = base64.b64encode(sig) return req
def upload_keys(new_emails, user_infos): """ Save all private keys for the users we just provisioned (user and volume keys) Return True if they all succeed Return False if at least one fails """ syndicate_config = conf.get_config_from_argv(sys.argv) user_bundles = {} # make initial user bundles for user_name in new_emails: user_pkey = storage.load_private_key( syndicate_config, "user", user_name ) if user_pkey is None: log.error("Automount daemon failed to produce key for {}".format(user_name)) return False user_cert = object_stub.load_user_cert(syndicate_config, user_name) if user_cert is None: log.error("Automount daemon failed to produce cert for {}".format(user_name)) return False ug_name = provisioning.make_gateway_name('demo', 'UG', sanitize_name('volume-{}'.format(user_name)), 'localhost') rg_name = provisioning.make_gateway_name('demo', 'RG', sanitize_name('volume-{}'.format(user_name)), 'localhost') ug_pkey = storage.load_private_key( syndicate_config, "gateway", ug_name) if ug_pkey is None: log.error("Automount daemon failed to produce key for {}".format(ug_name)) return False rg_pkey = storage.load_private_key( syndicate_config, "gateway", rg_name) if rg_pkey is None: log.error("Automount daemon failed to produce key for {}".format(rg_name)) return False ug_cert = object_stub.load_gateway_cert(syndicate_config, ug_name) if ug_cert is None: log.error("Automount daemon failed to produce cert for {}".format(ug_name)) return False rg_cert = object_stub.load_gateway_cert(syndicate_config, rg_name) if rg_cert is None: log.error("Automount daemon failed to produce cert for {}".format(rg_name)) return False # gateway keys for the same volume must be the same, initially if ug_pkey.exportKey() != rg_pkey.exportKey(): log.error("Automount daemon did not produce the same initial key for {} and {}".format(ug_name, rg_name)) return False user_bundles[user_name] = { 'user_pkey': user_pkey.exportKey(), 'user_cert': base64.b64encode(user_cert.SerializeToString()), 'ug_cert': base64.b64encode(ug_cert.SerializeToString()), 'rg_cert': base64.b64encode(rg_cert.SerializeToString()), 'gateway_pkey': ug_pkey.exportKey() } # encrypt private keys for user_info in user_infos: user_name = user_info['email'] user_password = user_info['password'] if user_name not in new_emails: continue if len(user_password) == 0: # skip this user log.debug("Skipping already-processed user {}".format(user_name)) del user_bundles[user_name] continue user_password = base64.urlsafe_b64encode(base64.b64decode(user_password)) for keyname in ['user_pkey', 'gateway_pkey']: f = Fernet(user_password) user_bundles[user_name][keyname] = f.encrypt(user_bundles[user_name][keyname]) user_bundles[user_name][keyname] = base64.b64encode( base64.urlsafe_b64decode(user_bundles[user_name][keyname]) ) log.debug("Upload key bundles for {} users".format(len(user_bundles.keys()))) for user_name in user_bundles.keys(): # send encrypted keys try: log.debug("Upload keys for {}".format(user_name)) data = { 'demo_payload': json.dumps(user_bundles[user_name]) } req = requests.post(SIGNUP_URL + '/provision/{}'.format(urllib.quote(user_name)), headers=make_auth_headers(), data=data) if req.status_code != 200: if req.status_code != 202: log.error("Failed to provision {}: HTTP {} ({})".format(user_name, req.status_code, req.text)) except Exception as e: if DEBUG: log.exception(e) return False return True