def _delete(self, model, id): if not id: raise AppError("Must provide an id.") model = model.lower() if model != "users": if _config.DEFINED_MODELS: cls = _config.DEFINED_MODELS.get(model) if _config.RESTRICT_TO_DEFINED_MODELS and not cls: raise RestrictedModelError if cls: model = cls.__name__ validate_modelname(model) u = current_user(required=True) if model == "users": if id != "me" and id != u: raise AppError( "Id must be the current " + "user_id or me. User {} tried to modify user {}.".format( u, id)) id = u key = parse_id(id, model) key.delete() search.delete(key) if _config.METADATA: counter.decrement(model) return {}
def set_or_create(self, model, id, parent_key=None): model = model.lower() u = current_user(required=True) if model == "users": if not (id == "me" or id == "" or id == u): raise AppError( "Id must be the current " + "user_id or me. User {} tried to modify user {}.".format( u, id)) id = u cls = users else: cls = None if _config.DEFINED_MODELS: cls = _config.DEFINED_MODELS.get(model) if not cls and _config.RESTRICT_TO_DEFINED_MODELS: raise RestrictedModelError if cls: model = cls.__name__ if not cls: validate_modelname(model) cls = type(model, (ScopedExpando, ), {}) data = parse_body(self) key = parse_id(id, model, data.get("Id")) clean_data(data) validate(cls.__name__, data) already_exists = False if key: old_model = key.get() if old_model: if model != "users" and not old_model.can_write(u): raise AppError("You do not have sufficient privileges.") already_exists = True # TODO: might want to add this post creation since you already have the key if parent_key: data[parent_key.kind()] = parent_key.urlsafe() m = reflective_create(cls, data) if key: m.key = key if model != "users": if len(m.owners) == 0: m.owners.append(u) m.put() # increment count if not already_exists and _config.METADATA: counter.increment(model) # update indexes search.put(m) redirect = self.request.get("redirect") if redirect: self.redirect(redirect) # Raising break error to avoid header and body writes from @as_json decorator since we override as a redirect raise BreakError() return m.to_dict()
def post(self, mesh_id, client_id): if mesh_id == "": raise AppError("Must specify mesh id.") if client_id == "": raise AppError("Must specify client id.") clients, payload = json.loads(self.request.body) msg = json.dumps([client_id, time.time(), payload]) logging.info(msg) for cid in clients: # TODO: assert client_id is in mesh_id channel.send_message(cid, msg)
def get(self): username = self.request.get("username") if not username: raise AppError("Must provide username.") instance = LoadBalancer.find(TailboneTurnInstance, self.request) if not instance: raise AppError('Instance not found, try again later.') username, password = credentials(username, instance.secret) return { "username": username, "password": password, "turn": instance.address }
def _validate(validator, data, ignored=None): if isinstance(validator, re_type): if validator.pattern == "": return if type(data) not in [str, unicode]: data = json.dumps(data) if not validator.match(data): raise AppError("Validator '{}' does not match '{}'".format(validator.pattern, data)) elif isinstance(validator, dict) and isinstance(data, dict): for name, val in data.iteritems(): if name not in ignored: _validate(validator.get(name), val) else: raise AppError("Unsupported validator type {} : {}".format(validator, type(validator)))
def parse_id(id, model, data_id=None): if data_id: if id: if data_id != id: raise AppError("Url id {%s} must match object id {%s}" % (id, data_id)) else: id = data_id if id: key = ndb.Key(urlsafe=id) if model != key.kind(): raise AppError("Key kind must match id kind: {} != {}.".format( model, key.kind())) return key return None
def _get(self, model, id, *extra_filters): model = model.lower() cls = None if _config.DEFINED_MODELS: cls = users if model == "users" else _config.DEFINED_MODELS.get( model) if not cls and _config.RESTRICT_TO_DEFINED_MODELS: raise RestrictedModelError if cls: model = cls.__name__ if not cls: validate_modelname(model) cls = users if model == "users" else type(model, (ScopedExpando, ), {}) if id: me = False if model == "users": if id == "me": me = True id = current_user(required=True) key = parse_id(id, model) m = key.get() if not m: if model == "users" and me: m = users() m.key = key setattr(m, "$unsaved", True) u = config.get_current_user() if hasattr(u, "email"): m.email = u.email() else: raise AppError("No {} with id {}.".format(model, id)) return m.to_dict() else: return query(self, cls, *extra_filters)
def _get(self, model, id, *extra_filters): # TODO(doug) does the model name need to be ascii encoded since types don't support utf-8? cls = users if model == "users" else type(model.lower(), (ScopedExpando, ), {}) if id: me = False if model == "users": if id == "me": me = True id = current_user(required=True) key = parse_id(id, model) m = key.get() if not m: if model == "users" and me: m = users() m.key = key setattr(m, "$unsaved", True) environ = webapp2.get_request().environ for k, v in environ.iteritems(): if k[: 14] == "TAILBONE_USER_" and k != "TAILBONE_USER_ID" and v: setattr(m, k[14:].lower(), v) else: raise AppError("No {} with id {}.".format(model, id)) return m.to_dict() else: return query(self, cls, *extra_filters)
def _pre_delete_hook(cls, key): m = key.get() u = current_user(required=True) if not m.can_write(u): raise AppError( "You ({}) do not have permission to delete this model ({}).". format(u, key.id()))
def start_instance(pool): """Start a new instance with a given configuration.""" # start instance # defer an update load call instance_class = string_to_class(pool.instance_type) name = rfc1035(instance_class.__name__) # max length of a name is 63 name = "{}-{}".format(name, uuid.uuid4())[:63] instance = instance_class(id=name) instance.pool = pool.key instance.zone = random.choice(LOCATIONS[pool.region]["zones"]) instance.put() compute = compute_api() if compute: def update_zone(s): return re.sub(r"\/zones\/([^/]*)", "/zones/{}".format(instance.zone), s) instance.PARAMS.update({ "name": name, "zone": update_zone(instance.PARAMS.get("zone")), "machineType": update_zone(instance.PARAMS.get("machineType")), }) operation = compute.instances().insert( project=PROJECT_ID, zone=instance.zone, body=instance.PARAMS).execute() logging.info("Create instance operation {}".format(operation)) instance.status = operation.get("status") name = "update_instance_status_{}_{}".format(instance.key.urlsafe(), int(time.time())) deferred.defer(update_instance_status, instance.key.urlsafe(), _countdown=STARTING_STATUS_DELAY, _name=name) else: logging.warn("No compute api defined.") raise AppError("No compute api defined.")
def get_or_create_room(request, name=None): if not name: name, room, address = unique_name() else: room = room_hash(name) address = memcache.get(room) if not address: if _config.ENABLE_WEBSOCKET: if DEBUG: class DebugInstance(object): if _config.DEBUG_WEBSOCKET_ADDRESS: address = _config.DEBUG_WEBSOCKET_ADDRESS elif str(request.remote_addr) == "::1": address = "localhost" else: address = request.remote_addr instance = DebugInstance() else: instance = LoadBalancer.find(TailboneWebsocketInstance) if not instance: raise AppError('Instance not yet ready, try again later.') address = "ws://{}:{}/{}".format(instance.address, _config.PORT, name) else: address = "/api/channel/{}".format(name) if not memcache.add(room, address, time=_config.ROOM_EXPIRATION): return get_or_create_room(request, name) return name, address
def get(self): instance = LoadBalancer.find(TailboneCustomInstance) if not instance: raise AppError('Instance not found, try again later.') return { "ip": instance.address }
def query(self, cls, *extra_filters): params = self.request.get("params") if params: params = json.loads(params) page_size = params.get("page_size", 100) cursor = params.get("cursor") filters = params.get("filter") orders = params.get("order") projection = params.get("projection") or None q = construct_query_from_json(cls, filters, orders) else: page_size = int(self.request.get("page_size", default_value=100)) cursor = self.request.get("cursor") projection = self.request.get_all("projection") projection = [i for sublist in projection for i in sublist.split(",")] if projection else None filters = self.request.get_all("filter") orders = self.request.get_all("order") q = construct_query_from_url_args(cls, filters, orders) for f in extra_filters: q = f(q) cursor = ndb.Cursor.from_websafe_string(cursor) if cursor else None if projection: # if asking for private variables and not specifing owners and viewers append them private = [p for p in projection if not re_public.match(p)] if len(private) > 0: acl = [p for p in private if p in acl_attributes] if len(acl) == 0: raise AppError("Requesting projection of private properties, but did not specify 'owners' or 'viewers' to verify access.") results, cursor, more = q.fetch_page(page_size, start_cursor=cursor, projection=projection) self.response.headers["More"] = "true" if more else "false" if cursor: self.response.headers["Cursor"] = cursor.urlsafe() # The Reverse-Cursor is used if you construct a query in the opposite direction self.response.headers["Reverse-Cursor"] = cursor.reversed().urlsafe() return [m.to_dict() for m in results]
def get_model(urlsafekey): key = ndb.Key(urlsafe=urlsafekey) # dynamic class defined if doesn't exists for reflective creation later type(key.kind(), (ScopedExpando, ), {}) m = key.get() if not key: raise AppError("Model {} does not exists.".format(urlsafekey)) return m
def get(self): username = self.request.get("username") if not username: raise AppError("Must provide username.") instance = LoadBalancer.find(TailboneTurnInstance) if not instance: raise AppError('Instance not found, try again later.') username, password = credentials(username, instance.secret) return { "username": username, "password": password, "uris": [ "turn:{}:3478?transport=udp".format(instance.address), "turn:{}:3478?transport=tcp".format(instance.address), ], }
def _delete(self, model, id): if not id: raise AppError("Must provide an id.") model = model.lower() u = current_user(required=True) if model == "users": if id != "me" and id != u: raise AppError( "Id must be the current " + "user_id or me. User {} tried to modify user {}.".format( u, id)) id = u key = parse_id(id, model) key.delete() search.delete(key) if store_metadata: decrement(model) return {}
def delete(self, key): if not config.is_current_user_admin(): raise AppError("User must be administrator.") key = blobstore.BlobKey(str(urllib.unquote(key))) blob_info = BlobInfo.get(key) if blob_info: blob_info.delete() if HAS_PIL and re_image.match(blob_info.content_type): delete_serving_url(key) return {} else: self.error(404) return {"error": "File not found with key " + key}
def stop_instance(instance): """Stop an instance.""" # cancel update load defered call # stop instance # TODO: need some way of clearing externally assciated instances compute = compute_api() if compute: compute.instances().delete( project=PROJECT_ID, zone=instance.zone, instance=instance.key.id()).execute() else: logging.warn("No compute api defined.") raise AppError("No compute api defined.") instance.key.delete()
def find(instance_class, request): """Return an instance of this instance type from the nearest pool or create it.""" if DEBUG: # check to make sure we can access the api before we do anything compute_api() region, zone = LoadBalancer.nearest_zone(request) instance_str = class_to_string(instance_class) pool = LoadBalancer.get_or_create_pool(instance_str, region) instance = pool.instance() if instance and instance.address: return instance raise AppError("Instance not yet ready, please try again.")
def validate(cls_name, data): # properties = data.keys() # confirm the format of any tailbone specific types for name in acl_attributes: val = data.get(name) if val: # TODO(doug): validate list, can't be empty list, must contain id like objects pass # run validation over remaining properties if _validation: validations = _validation.get(cls_name) if not validations: raise AppError("Validation requires all valid models to be listed, use empty quote to skip.") _validate(validations, data, acl_attributes)
def construct_filter(filter_str): m = re_composite_filter.match(filter_str) if m: filters = [construct_filter(f) for f in re_split.split(m.group(2))] if m.group(1) == "AND": return ndb.query.AND(*filters) else: return ndb.query.OR(*filters) m = re_filter.match(filter_str) if m: name, opsymbol, value = m.groups() return ndb.query.FilterNode(name, convert_opsymbol(opsymbol), convert_value(value)) if re_split.match(filter_str): return construct_filter("AND({})".format(filter_str)) raise AppError("Filter format is unsupported: {}".format(filter_str))
def get(self, key): if key == "": # query if not config.is_current_user_admin(): raise AppError("User must be administrator.") return restful.query(self, BlobInfo) elif key == "create": return { "upload_url": blobstore.create_upload_url("/api/files/upload") } key = str(urllib.unquote(key)) blob_info = bs.BlobInfo.get(key) if blob_info: self.send_blob(blob_info) raise BreakError else: self.error(404) return {"error": "File not found with key " + key}
def _get(self, model, id, *extra_filters): model = model.lower() cls = None if _config.DEFINED_MODELS: cls = users if model == "users" else _config.DEFINED_MODELS.get( model) if not cls and _config.RESTRICT_TO_DEFINED_MODELS: raise RestrictedModelError if cls: model = cls.__name__ if not cls: validate_modelname(model) cls = users if model == "users" else type(model, (ScopedExpando, ), {}) logging.info("ID %s" % id) if id: me = False if model == "users": if id == "me": me = True id = current_user(required=True).urlsafe() logging.info("users me %s" % current_user(required=True)) if "," in id: ids = id.split(",") keys = [parse_id(i, model) for i in ids] results = ndb.get_multi(keys) return [m.to_dict() if m else m for m in results] key = parse_id(id, model) m = key.get() if not m: if model == "users" and me: m = users() m.key = key else: raise AppError("No {} with id {}.".format(model, id)) if model == "users" and me: u = config.get_current_user() if u: for k, v in getAttributes(u).items(): if k.startswith("_"): k = "$" + k[1:] setattr(m, k, v) return m.to_dict() else: return query(self, cls, *extra_filters)
def set_or_create(self, model, id): model = model.lower() u = current_user(required=True) if model == "users": if not (id == "me" or id == "" or id == u): raise AppError( "Id must be the current " + "user_id or me. User {} tried to modify user {}.".format( u, id)) id = u cls = users else: cls = None if _config.DEFINED_MODELS: cls = _config.DEFINED_MODELS.get(model) if not cls and _config.RESTRICT_TO_DEFINED_MODELS: raise RestrictedModelError if cls: model = cls.__name__ if not cls: validate_modelname(model) cls = type(model, (ScopedExpando, ), {}) data = parse_body(self) key = parse_id(id, model, data.get("Id")) clean_data(data) validate(cls.__name__, data) m = reflective_create(cls, data) if key: m.key = key if model != "users": if len(m.owners) == 0: m.owners.append(u) m.put() redirect = self.request.get("redirect") if redirect: self.redirect(redirect) # Raising break error to avoid header and body writes from @as_json decorator since we override as a redirect raise BreakError() return m.to_dict()
def _pre_put_hook(self): super(ScopedModel, self)._pre_put_hook() if config.is_current_user_admin(): return # check for writable and for any admin properties if self._previous is not None: u = current_user(required=True) if not self._previous.can_write(u): raise AppError("You do not have sufficient privileges.") keys = [p._code_name for p in self._properties.itervalues()] for k in keys: if re_admin.match(k): attr = getattr(self._previous, k, None) if attr: setattr(self, k, attr) else: delattr(self, k) else: keys = [p._code_name for p in self._properties.itervalues()] for k in keys: if re_admin.match(k): delattr(self, k)
def stop_instance(instance): """Stop an instance.""" # cancel update load defered call # stop instance # TODO: need some way of clearing externally assciated instances compute = compute_api() if compute: name = instance.key.id() result = compute.instances().delete( project=PROJECT_ID, zone=instance.zone, instance=name).execute() # As of v1 this call needs to be blocking to ensure the instance is stopped # before we can delete the boot disk _blocking_call(compute, result) # As scratch is is depricated in v1 we are treating the persistent disk as # a scratch disk by deleting it when stopping the instance. result = compute.disks().delete( project=PROJECT_ID, zone=instance.zone, disk=name).execute() _blocking_call(compute, result) else: logging.warn("No compute api defined.") raise AppError("No compute api defined.") instance.key.delete()
DEFINED_MODELS = None RESTRICT_TO_DEFINED_MODELS = True PROTECTED_MODEL_NAMES = [ "(?i)(mesh|messages|files|events|admin|proxy)", "(?i)tailbone.*" ] _config = api.lib_config.register('tailboneRestful', _ConfigDefaults.__dict__) re_public = re.compile(r"^[A-Z].*") re_type = type(re_public) re_admin = re.compile(r"^(A|a)dmin.*") acl_attributes = [u"owners", u"viewers"] ProtectedModelError = AppError("This is a protected Model.") RestrictedModelError = AppError("Models are restricted.") def validate_modelname(model): if [r for r in _config.PROTECTED_MODEL_NAMES if re.match(r, model)]: raise ProtectedModelError def current_user(required=False): u = config.get_current_user() if u: return ndb.Key("users", u.user_id()).urlsafe() if required: raise LoginError("User must be logged in.") return None
def start_instance(pool): """Start a new instance with a given configuration.""" # start instance # defer an update load call instance_class = string_to_class(pool.instance_type) name = rfc1035(instance_class.__name__) # max length of a name is 63 name = "{}-{}".format(name, uuid.uuid4())[:63] instance = instance_class(id=name) instance.pool = pool.key locations = get_locations() instance.zone = random.choice(locations[pool.region]["zones"]) compute = compute_api() if compute: def update_zone(s): return re.sub(r"\/zones\/([^/]*)", "/zones/{}".format(instance.zone), s) # create disk from source snapshot if instance.SOURCE_SNAPSHOT: operation = compute.disks().insert( project=PROJECT_ID, zone=instance.zone, body={ "name": name, "sizeGb": 10, "sourceSnapshot": instance.SOURCE_SNAPSHOT, }).execute() _blocking_call(compute, operation) instance.PARAMS.update({ "disks": [{ "kind": "compute#attachedDisk", "boot": True, "type": "PERSISTENT", "mode": "READ_WRITE", "deviceName": name, "zone": update_zone(instance.PARAMS.get("zone")), "source": api_url(PROJECT_ID, "zones", instance.zone, "disks", name), }], }) instance.PARAMS.update({ "name": name, "zone": update_zone(instance.PARAMS.get("zone")), "machineType": update_zone(instance.PARAMS.get("machineType")), }) operation = compute.instances().insert( project=PROJECT_ID, zone=instance.zone, body=instance.PARAMS).execute() logging.info("Create instance operation {}".format(operation)) instance.status = operation.get("status") name = "update_instance_status_{}_{}".format( instance.key.urlsafe(), int(time.time())) deferred.defer(update_instance_status, instance.key.urlsafe(), _countdown=STARTING_STATUS_DELAY, _name=name) else: logging.warn("No compute api defined.") raise AppError("No compute api defined.") # finally put the instance after the resource has been created instance.put()
def put(self, _): raise AppError("PUT is not supported for the files api.")
def post(self, _): raise AppError("You must make a GET call to /api/files/create to get a POST url.")