def serialize(models, sign=True, increment_counters=True, *args, **kwargs): """ This function encapsulates serialization, and ensures that any final steps needed before syncing (e.g. signing, incrementing counters, etc) are done. """ from securesync.devices.models import Device from .models import SyncedModel own_device = Device.get_own_device() for model in models: resave = False if increment_counters or sign: assert isinstance(model, SyncedModel), "Can only serialize SyncedModel instances" if increment_counters and not model.counter: model.counter = own_device.increment_counter_position() resave = True if sign and not model.signature: model.sign() resave = True if resave: super(SyncedModel, model).save() return serializers.serialize("versioned-json", models, *args, **kwargs)
def serialize(models, sign=True, increment_counters=True, *args, **kwargs): """ This function encapsulates serialization, and ensures that any final steps needed before syncing (e.g. signing, incrementing counters, etc) are done. """ from securesync.devices.models import Device from .models import SyncedModel own_device = Device.get_own_device() for model in models: resave = False if increment_counters or sign: assert isinstance( model, SyncedModel), "Can only serialize SyncedModel instances" if increment_counters and not model.counter: model.counter = own_device.increment_counter_position() resave = True if sign and not model.signature: model.sign() resave = True if resave: super(SyncedModel, model).save() return serializers.serialize("versioned-json", models, *args, **kwargs)
def register(self): """Register a device with the central server. Happens outside of a session.""" own_device = Device.get_own_device() # Since we can't know the version of the remote device (yet), # we give it everything we possibly can (don't specify a dest_version) # # Note that (currently) this should never fail--the central server (which we're sending # these objects to) should always have a higher version. r = self.post("register", { "client_device": serializers.serialize("versioned-json", [own_device], ensure_ascii=False) }) # If they don't understand, our assumption is broken. if r.status_code == 500: if "Device has no field named 'version'" in r.content: raise Exception("Central server is of an older version than us?") elif r.headers.get("content-type", "") == "text/html": raise Exception("Unhandled server-side exception: %s" % r.content) elif r.status_code == 200: # Save to our local store. By NOT passing a src_version, # we're saying it's OK to just store what we can. models = serializers.deserialize("versioned-json", r.content, src_version=None, dest_version=own_device.get_version()) for model in models: if not model.object.verify(): continue # save the imported model, and mark the returned Device as trusted if isinstance(model.object, Device): model.object.save(is_trusted=True, imported=True) else: model.object.save(imported=True) return {"code": "registered"} return json.loads(r.content)
def device_download(data, session): """This device is having its own devices downloaded""" zone = session.client_device.get_zone() devicezones = list(DeviceZone.objects.filter(zone=zone, device__in=data["devices"])) devices = [devicezone.device for devicezone in devicezones] session.models_downloaded += len(devices) + len(devicezones) # Return the objects serialized to the version of the other device. return JsonResponse({"devices": serializers.serialize("versioned-json", devices + devicezones, dest_version=session.client_version, ensure_ascii=False)})
def jsonify(object): try: if isinstance(object, QuerySet): return serialize('json', object) except: pass str = mark_safe(simplejson.dumps(object, default=_dthandler)) if str == "null": str = mark_safe("[%s]" % (",".join([simplejson.dumps(o, default=_dthandler) for o in object]))) return str
def jsonify(object): try: if isinstance(object, QuerySet): return serialize('json', object) except: pass str = mark_safe(simplejson.dumps(object, default=_dthandler)) if str == "null": str = mark_safe("[%s]" % (",".join( [simplejson.dumps(o, default=_dthandler) for o in object]))) return str
def register(self): """Register a device with the central server. Happens outside of a session.""" own_device = Device.get_own_device() # Todo: registration process should always use one of these--and it needs to use # Device.public_key. So, should migrate over the rest of the registration code # to do the same. assert own_device.public_key == own_device.get_key( ).get_public_key_string( ), "Make sure these somehow didn't get out of sync (can happen when people muck around with the data manually." # Since we can't know the version of the remote device (yet), # we give it everything we possibly can (don't specify a dest_version) # # Note that (currently) this should never fail--the central server (which we're sending # these objects to) should always have a higher version. r = self.post( "register", { "client_device": serializers.serialize("versioned-json", [own_device], ensure_ascii=False) }) # If they don't understand, our assumption is broken. if r.status_code == 500: if "Device has no field named 'version'" in r.content: raise Exception( "Central server is of an older version than us?") elif r.headers.get("content-type", "") == "text/html": raise Exception("Unhandled server-side exception: %s" % r.content) elif r.status_code == 200: # Save to our local store. By NOT passing a src_version, # we're saying it's OK to just store what we can. models = serializers.deserialize("versioned-json", r.content, src_version=None, dest_version=own_device.version) for model in models: if not model.object.verify(): continue # save the imported model, and mark the returned Device as trusted if isinstance(model.object, Device): model.object.save(is_trusted=True, imported=True) else: model.object.save(imported=True) return {"code": "registered"} return json.loads(r.content)
def create_session(request): data = simplejson.loads(request.raw_post_data or "{}") if "client_nonce" not in data: return JsonResponse({"error": "Client nonce must be specified."}, status=500) if len(data["client_nonce"]) != 32 or re.match("[^0-9a-fA-F]", data["client_nonce"]): return JsonResponse({"error": "Client nonce is malformed (must be 32-digit hex)."}, status=500) if "client_device" not in data: return JsonResponse({"error": "Client device must be specified."}, status=500) if "server_nonce" not in data: if SyncSession.objects.filter(client_nonce=data["client_nonce"]).count(): return JsonResponse({"error": "Session already exists; include server nonce and signature."}, status=500) session = SyncSession() session.client_nonce = data["client_nonce"] session.client_os = data.get("client_os", "") session.client_version = data.get("client_version", "") try: client_device = Device.objects.get(pk=data["client_device"]) session.client_device = client_device except Device.DoesNotExist: return JsonResponse({"error": "Client device matching id could not be found. (id=%s)" % data["client_device"]}, status=500) session.server_nonce = uuid.uuid4().hex session.server_device = Device.get_own_device() session.ip = request.META.get("HTTP_X_FORWARDED_FOR", request.META.get('REMOTE_ADDR', "")) if session.client_device.pk == session.server_device.pk: return JsonResponse({"error": "I know myself when I see myself, and you're not me."}, status=500) session.save() else: try: session = SyncSession.objects.get(client_nonce=data["client_nonce"]) except SyncSession.DoesNotExist: return JsonResponse({"error": "Session with specified client nonce could not be found."}, status=500) if session.server_nonce != data["server_nonce"]: return JsonResponse({"error": "Server nonce did not match saved value."}, status=500) if not data.get("signature", ""): return JsonResponse({"error": "Must include signature."}, status=500) if not session.verify_client_signature(data["signature"]): return JsonResponse({"error": "Signature did not match."}, status=500) session.verified = True session.save() # Return the serializd session, in the version intended for the other device return JsonResponse({ "session": serializers.serialize("versioned-json", [session], dest_version=session.client_version, ensure_ascii=False ), "signature": session.sign(), })
def register_device(request): data = simplejson.loads(request.raw_post_data or "{}") # attempt to load the client device data from the request data if "client_device" not in data: return JsonResponse({"error": "Serialized client device must be provided."}, status=500) try: # When hand-shaking on the device models, since we don't yet know the version, # we have to just TRY with our own version. # # This is currently "central server" code, so # this will only fail (currently) if the central server version # is less than the version of a client--something that should never happen try: models = serializers.deserialize("versioned-json", data["client_device"], src_version=version.VERSION, dest_version=version.VERSION) except db_models.FieldDoesNotExist as fdne: raise Exception("Central server version is lower than client version. This is ... impossible!") client_device = models.next().object except Exception as e: return JsonResponse({ "error": "Could not decode the client device model: %r" % e, "code": "client_device_corrupted", }, status=500) if not isinstance(client_device, Device): return JsonResponse({ "error": "Client device must be an instance of the 'Device' model.", "code": "client_device_not_device", }, status=500) if not client_device.verify(): return JsonResponse({ "error": "Client device must be self-signed with a signature matching its own public key.", "code": "client_device_invalid_signature", }, status=500) # we have a valid self-signed Device, so now check if its public key has been registered try: registration = RegisteredDevicePublicKey.objects.get(public_key=client_device.public_key) except RegisteredDevicePublicKey.DoesNotExist: try: device = Device.objects.get(public_key=client_device.public_key) return JsonResponse({ "error": "This device has already been registered", "code": "device_already_registered", }, status=500) except Device.DoesNotExist: return JsonResponse({ "error": "Device registration with public key not found; login and register first?", "code": "public_key_unregistered", }, status=500) client_device.signed_by = client_device # the device checks out; let's save it! client_device.save(imported=True) # create the DeviceZone for the new device device_zone = DeviceZone(device=client_device, zone=registration.zone) device_zone.save() # delete the RegisteredDevicePublicKey, now that we've initialized the device and put it in its zone registration.delete() # return our local (server) Device, its Zone, and the newly created DeviceZone, to the client return JsonResponse( serializers.serialize("versioned-json", [Device.get_own_device(), registration.zone, device_zone], dest_version=client_device.version, ensure_ascii=False) )
def get_serialized_models(device_counters=None, limit=100, zone=None, include_count=False, dest_version=None): """Serialize models for some intended version (dest_version) Default is our own version--i.e. include all known fields. If serializing for a device of a lower version, pass in that device's version! """ from securesync.devices.models import Device # cannot be top-level, otherwise inter-dependency of this and models fouls things up own_device = Device.get_own_device() # Get the current version if none was specified if not dest_version: dest_version = own_device.get_version() # use the current device's zone if one was not specified if not zone: zone = own_device.get_zone() # if no devices specified, assume we're starting from zero, and include all devices in the zone if device_counters is None: device_counters = dict((device.id, 0) for device in Device.objects.by_zone(zone)) # remove all requested devices that either don't exist or aren't in the correct zone for device_id in device_counters.keys(): device = get_object_or_None(Device, pk=device_id) if not device or not (device.in_zone(zone) or device.get_metadata().is_trusted): del device_counters[device_id] models = [] boost = 0 # loop until we've found some models, or determined that there are none to get while True: # assume no instances remaining until proven otherwise instances_remaining = False # loop through all the model classes marked as syncable for Model in _syncing_models: # loop through each of the devices of interest for device_id, counter in device_counters.items(): device = Device.objects.get(pk=device_id) queryset = Model.objects.filter(signed_by=device) # for trusted (central) device, only include models with the correct fallback zone if not device.in_zone(zone): if device.get_metadata().is_trusted: queryset = queryset.filter(zone_fallback=zone) else: continue # check whether there are any models that will be excluded by our limit, so we know to ask again if not instances_remaining and queryset.filter(counter__gt=counter+limit+boost).count() > 0: instances_remaining = True # pull out the model instances within the given counter range models += queryset.filter(counter__gt=counter, counter__lte=counter+limit+boost) # if we got some models, or there were none to get, then call it quits if len(models) > 0 or not instances_remaining: break # boost the effective limit, so we have a chance of catching something when we do another round boost += limit # serialize the models we found serialized_models = serializers.serialize("versioned-json", models, ensure_ascii=False, dest_version=dest_version) if include_count: return {"models": serialized_models, "count": len(models)} else: return serialized_models
def save_serialized_models(data, increment_counters=True, src_version=None): """Unserializes models (from a device of version=src_version) in data and saves them to the django database. If src_version is None, all unrecognized fields are (silently) stripped off. If it is set to some value, then only fields of versions higher than ours are stripped off. By defaulting to src_version=None, we're expecting a perfect match when we come in (i.e. that wherever we got this data from, they were smart enough to "dumb it down" for us, or they were old enough to have nothing unexpecting) So, care must be taken in calling this function Returns a dictionary of the # of saved models, # unsaved, and any exceptions during saving""" from .models import ImportPurgatory # cannot be top-level, otherwise inter-dependency of this and models fouls things up from securesync.devices.models import Device own_device = Device.get_own_device() if not src_version: # default version: our own src_version = own_device.get_version() # if data is from a purgatory object, load it up if isinstance(data, ImportPurgatory): purgatory = data data = purgatory.serialized_models else: purgatory = None # deserialize the models, either from text or a list of dictionaries if isinstance(data, str) or isinstance(data, unicode): models = serializers.deserialize("versioned-json", data, src_version=src_version, dest_version=own_device.get_version()) else: models = serializers.deserialize("versioned-python", data, src_version=src_version, dest_version=own_device.get_version()) # try importing each of the models in turn unsaved_models = [] exceptions = "" saved_model_count = 0 try: for modelwrapper in models: try: # extract the model from the deserialization wrapper model = modelwrapper.object # only allow the importing of models that are subclasses of SyncedModel if not hasattr(model, "verify"): raise ValidationError("Cannot save model: %s does not have a verify method (not a subclass of SyncedModel?)" % model.__class__) # TODO(jamalex): more robust way to do this? (otherwise, it might barf about the id already existing) model._state.adding = False # verify that all fields are valid, and that foreign keys can be resolved model.full_clean() # save the imported model (checking that the signature is valid in the process) model.save(imported=True, increment_counters=increment_counters) # keep track of how many models have been successfully saved saved_model_count += 1 except ValidationError as e: # the model could not be saved # keep a running list of models and exceptions, to be stored in purgatory exceptions += "%s: %s\n" % (model.pk, e) unsaved_models.append(model) # if the model is at least properly signed, try incrementing the counter for the signing device # (because otherwise we may never ask for additional models) try: if increment_counters and model.verify(): model.signed_by.set_counter_position(model.counter) except: pass except Exception as e: exceptions += str(e) # deal with any models that didn't validate properly; throw them into purgatory so we can try again later if unsaved_models: if not purgatory: purgatory = ImportPurgatory() # These models were successfully unserialized, so re-save in our own version. purgatory.serialized_models = serializers.serialize("versioned-json", unsaved_models, ensure_ascii=False, dest_version=own_device.get_version()) purgatory.exceptions = exceptions purgatory.model_count = len(unsaved_models) purgatory.retry_attempts += 1 purgatory.save() elif purgatory: # everything saved properly this time, so we can eliminate the purgatory instance purgatory.delete() out_dict = { "unsaved_model_count": len(unsaved_models), "saved_model_count": saved_model_count, } if exceptions: out_dict["exceptions"] = exceptions return out_dict
def get_serialized_models(device_counters=None, limit=100, zone=None, include_count=False, dest_version=version.VERSION): """Serialize models for some intended version (dest_version) Default is our own version--i.e. include all known fields. If serializing for a device of a lower version, pass in that device's version! """ from models import Device # cannot be top-level, otherwise inter-dependency of this and models fouls things up # use the current device's zone if one was not specified if not zone: zone = Device.get_own_device().get_zone() # if no devices specified, assume we're starting from zero, and include all devices in the zone if device_counters is None: device_counters = dict( (device.id, 0) for device in Device.objects.by_zone(zone)) # remove all requested devices that either don't exist or aren't in the correct zone for device_id in device_counters.keys(): device = get_object_or_None(Device, pk=device_id) if not device or not (device.in_zone(zone) or device.get_metadata().is_trusted): del device_counters[device_id] models = [] boost = 0 # loop until we've found some models, or determined that there are none to get while True: # assume no instances remaining until proven otherwise instances_remaining = False # loop through all the model classes marked as syncable for Model in _syncing_models: # loop through each of the devices of interest for device_id, counter in device_counters.items(): device = Device.objects.get(pk=device_id) queryset = Model.objects.filter(signed_by=device) # for trusted (central) device, only include models with the correct fallback zone if not device.in_zone(zone): if device.get_metadata().is_trusted: queryset = queryset.filter(zone_fallback=zone) else: continue # check whether there are any models that will be excluded by our limit, so we know to ask again if not instances_remaining and queryset.filter( counter__gt=counter + limit + boost).count() > 0: instances_remaining = True # pull out the model instances within the given counter range models += queryset.filter(counter__gt=counter, counter__lte=counter + limit + boost) # if we got some models, or there were none to get, then call it quits if len(models) > 0 or not instances_remaining: break # boost the effective limit, so we have a chance of catching something when we do another round boost += limit # serialize the models we found serialized_models = serializers.serialize("versioned-json", models, ensure_ascii=False, dest_version=dest_version) if include_count: return {"models": serialized_models, "count": len(models)} else: return serialized_models
def save_serialized_models(data, increment_counters=True, src_version=version.VERSION): """Unserializes models (from a device of version=src_version) in data and saves them to the django database. If src_version is None, all unrecognized fields are (silently) stripped off. If it is set to some value, then only fields of versions higher than ours are stripped off. By defaulting to src_version=None, we're expecting a perfect match when we come in (i.e. that wherever we got this data from, they were smart enough to "dumb it down" for us, or they were old enough to have nothing unexpecting) So, care must be taken in calling this function Returns a dictionary of the # of saved models, # unsaved, and any exceptions during saving""" from models import ImportPurgatory # cannot be top-level, otherwise inter-dependency of this and models fouls things up # if data is from a purgatory object, load it up if isinstance(data, ImportPurgatory): purgatory = data data = purgatory.serialized_models else: purgatory = None # deserialize the models, either from text or a list of dictionaries if isinstance(data, str) or isinstance(data, unicode): models = serializers.deserialize("versioned-json", data, src_version=src_version, dest_version=version.VERSION) else: models = serializers.deserialize("versioned-python", data, src_version=src_version, dest_version=version.VERSION) # try importing each of the models in turn unsaved_models = [] exceptions = "" saved_model_count = 0 try: for modelwrapper in models: try: # extract the model from the deserialization wrapper model = modelwrapper.object # only allow the importing of models that are subclasses of SyncedModel if not hasattr(model, "verify"): raise ValidationError( "Cannot save model: %s does not have a verify method (not a subclass of SyncedModel?)" % model.__class__) # TODO(jamalex): more robust way to do this? (otherwise, it might barf about the id already existing) model._state.adding = False # verify that all fields are valid, and that foreign keys can be resolved model.full_clean() # save the imported model (checking that the signature is valid in the process) model.save(imported=True, increment_counters=increment_counters) # keep track of how many models have been successfully saved saved_model_count += 1 except ValidationError as e: # the model could not be saved # keep a running list of models and exceptions, to be stored in purgatory exceptions += "%s: %s\n" % (model.pk, e) unsaved_models.append(model) # if the model is at least properly signed, try incrementing the counter for the signing device # (because otherwise we may never ask for additional models) try: if increment_counters and model.verify(): model.signed_by.set_counter_position(model.counter) except: pass except Exception as e: exceptions += str(e) # deal with any models that didn't validate properly; throw them into purgatory so we can try again later if unsaved_models: if not purgatory: purgatory = ImportPurgatory() # These models were successfully unserialized, so re-save in our own version. purgatory.serialized_models = serializers.serialize( "versioned-json", unsaved_models, ensure_ascii=False, dest_version=version.VERSION) purgatory.exceptions = exceptions purgatory.model_count = len(unsaved_models) purgatory.retry_attempts += 1 purgatory.save() elif purgatory: # everything saved properly this time, so we can eliminate the purgatory instance purgatory.delete() out_dict = { "unsaved_model_count": len(unsaved_models), "saved_model_count": saved_model_count, } if exceptions: out_dict["exceptions"] = exceptions return out_dict
def register_device(request): data = simplejson.loads(request.raw_post_data or "{}") # attempt to load the client device data from the request data if "client_device" not in data: return JsonResponse({"error": "Serialized client device must be provided."}, status=500) try: # When hand-shaking on the device models, since we don't yet know the version, # we have to just TRY with our own version. # # This is currently "central server" code, so # this will only fail (currently) if the central server version # is less than the version of a client--something that should never happen try: own_device = Device.get_own_device() models = serializers.deserialize("versioned-json", data["client_device"], src_version=own_device.get_version(), dest_version=own_device.get_version()) except db_models.FieldDoesNotExist as fdne: raise Exception("Central server version is lower than client version. This is ... impossible!") client_device = models.next().object except Exception as e: return JsonResponse({ "error": "Could not decode the client device model: %r" % e, "code": "client_device_corrupted", }, status=500) if not isinstance(client_device, Device): return JsonResponse({ "error": "Client device must be an instance of the 'Device' model.", "code": "client_device_not_device", }, status=500) if not client_device.verify(): return JsonResponse({ "error": "Client device must be self-signed with a signature matching its own public key.", "code": "client_device_invalid_signature", }, status=500) # we have a valid self-signed Device, so now check if its public key has been registered try: registration = RegisteredDevicePublicKey.objects.get(public_key=client_device.public_key) if registration.is_used(): if Device.objects.get(public_key=client_device.public_key): return JsonResponse({ "error": "This device has already been registered", "code": "device_already_registered", }, status=500) else: # If not... we're in a very weird state--we have a record of their # registration, but no device record. # Let's just let the registration happens, so we can refresh things here. # No harm, and some failsafe benefit. # So, pass through... no code :) pass except RegisteredDevicePublicKey.DoesNotExist: # This is the codepath for unregistered devices trying to start a session. # This would only get hit, however, if they visit the registration page. # But still, good to keep track of! UnregisteredDevicePing.record_ping(id=client_device.id, ip=get_request_ip(request)) return JsonResponse({ "error": "Device registration with public key not found; login and register first?", "code": "public_key_unregistered", }, status=500) client_device.signed_by = client_device # the device checks out; let's save it! client_device.save(imported=True) # create the DeviceZone for the new device device_zone = DeviceZone(device=client_device, zone=registration.zone) device_zone.save() # Use the RegisteredDevicePublicKey, now that we've initialized the device and put it in its zone registration.use() # return our local (server) Device, its Zone, and the newly created DeviceZone, to the client return JsonResponse( serializers.serialize("versioned-json", [Device.get_own_device(), registration.zone, device_zone], dest_version=client_device.version, ensure_ascii=False) )
def register_device(request): data = simplejson.loads(request.raw_post_data or "{}") # attempt to load the client device data from the request data if "client_device" not in data: return JsonResponse( {"error": "Serialized client device must be provided."}, status=500) try: # When hand-shaking on the device models, since we don't yet know the version, # we have to just TRY with our own version. # # This is currently "central server" code, so # this will only fail (currently) if the central server version # is less than the version of a client--something that should never happen try: own_device = Device.get_own_device() models = serializers.deserialize( "versioned-json", data["client_device"], src_version=own_device.get_version(), dest_version=own_device.get_version()) except db_models.FieldDoesNotExist as fdne: raise Exception( "Central server version is lower than client version. This is ... impossible!" ) client_device = models.next().object except Exception as e: return JsonResponse( { "error": "Could not decode the client device model: %r" % e, "code": "client_device_corrupted", }, status=500) if not isinstance(client_device, Device): return JsonResponse( { "error": "Client device must be an instance of the 'Device' model.", "code": "client_device_not_device", }, status=500) if not client_device.verify(): return JsonResponse( { "error": "Client device must be self-signed with a signature matching its own public key.", "code": "client_device_invalid_signature", }, status=500) # we have a valid self-signed Device, so now check if its public key has been registered try: registration = RegisteredDevicePublicKey.objects.get( public_key=client_device.public_key) if registration.is_used(): if Device.objects.get(public_key=client_device.public_key): return JsonResponse( { "error": "This device has already been registered", "code": "device_already_registered", }, status=500) else: # If not... we're in a very weird state--we have a record of their # registration, but no device record. # Let's just let the registration happens, so we can refresh things here. # No harm, and some failsafe benefit. # So, pass through... no code :) pass except RegisteredDevicePublicKey.DoesNotExist: # This is the codepath for unregistered devices trying to start a session. # This would only get hit, however, if they visit the registration page. # But still, good to keep track of! UnregisteredDevicePing.record_ping(id=client_device.id, ip=get_request_ip(request)) return JsonResponse( { "error": "Device registration with public key not found; login and register first?", "code": "public_key_unregistered", }, status=500) client_device.signed_by = client_device # the device checks out; let's save it! client_device.save(imported=True) # create the DeviceZone for the new device device_zone = DeviceZone(device=client_device, zone=registration.zone) device_zone.save() # Use the RegisteredDevicePublicKey, now that we've initialized the device and put it in its zone registration.use() # return our local (server) Device, its Zone, and the newly created DeviceZone, to the client return JsonResponse( serializers.serialize( "versioned-json", [Device.get_own_device(), registration.zone, device_zone], dest_version=client_device.version, ensure_ascii=False))