def _delete(self, model, id):
if not id:
raise AppError("Must provide an id.")
model = model.lower()
if model != "users":
if _config.DEFINED_MODELS:
cls = _config.DEFINED_MODELS.get(model)
if _config.RESTRICT_TO_DEFINED_MODELS and not cls:
raise RestrictedModelError
if cls:
model = cls.__name__
validate_modelname(model)
u = current_user(required=True)
if model == "users":
if id != "me" and id != u:
raise AppError(
"Id must be the current " +
"user_id or me. User {} tried to modify user {}.".format(
u, id))
id = u
key = parse_id(id, model)
key.delete()
search.delete(key)
if _config.METADATA:
counter.decrement(model)
return {}
def set_or_create(self, model, id, parent_key=None):
model = model.lower()
u = current_user(required=True)
if model == "users":
if not (id == "me" or id == "" or id == u):
raise AppError(
"Id must be the current " +
"user_id or me. User {} tried to modify user {}.".format(
u, id))
id = u
cls = users
else:
cls = None
if _config.DEFINED_MODELS:
cls = _config.DEFINED_MODELS.get(model)
if not cls and _config.RESTRICT_TO_DEFINED_MODELS:
raise RestrictedModelError
if cls:
model = cls.__name__
if not cls:
validate_modelname(model)
cls = type(model, (ScopedExpando, ), {})
data = parse_body(self)
key = parse_id(id, model, data.get("Id"))
clean_data(data)
validate(cls.__name__, data)
already_exists = False
if key:
old_model = key.get()
if old_model:
if model != "users" and not old_model.can_write(u):
raise AppError("You do not have sufficient privileges.")
already_exists = True
# TODO: might want to add this post creation since you already have the key
if parent_key:
data[parent_key.kind()] = parent_key.urlsafe()
m = reflective_create(cls, data)
if key:
m.key = key
if model != "users":
if len(m.owners) == 0:
m.owners.append(u)
m.put()
# increment count
if not already_exists and _config.METADATA:
counter.increment(model)
# update indexes
search.put(m)
redirect = self.request.get("redirect")
if redirect:
self.redirect(redirect)
# Raising break error to avoid header and body writes from @as_json decorator since we override as a redirect
raise BreakError()
return m.to_dict()
def post(self, mesh_id, client_id):
if mesh_id == "":
raise AppError("Must specify mesh id.")
if client_id == "":
raise AppError("Must specify client id.")
clients, payload = json.loads(self.request.body)
msg = json.dumps([client_id, time.time(), payload])
logging.info(msg)
for cid in clients:
# TODO: assert client_id is in mesh_id
channel.send_message(cid, msg)
def get(self):
username = self.request.get("username")
if not username:
raise AppError("Must provide username.")
instance = LoadBalancer.find(TailboneTurnInstance, self.request)
if not instance:
raise AppError('Instance not found, try again later.')
username, password = credentials(username, instance.secret)
return {
"username": username,
"password": password,
"turn": instance.address
}
def _validate(validator, data, ignored=None):
if isinstance(validator, re_type):
if validator.pattern == "":
return
if type(data) not in [str, unicode]:
data = json.dumps(data)
if not validator.match(data):
raise AppError("Validator '{}' does not match '{}'".format(validator.pattern, data))
elif isinstance(validator, dict) and isinstance(data, dict):
for name, val in data.iteritems():
if name not in ignored:
_validate(validator.get(name), val)
else:
raise AppError("Unsupported validator type {} : {}".format(validator, type(validator)))
def parse_id(id, model, data_id=None):
if data_id:
if id:
if data_id != id:
raise AppError("Url id {%s} must match object id {%s}" %
(id, data_id))
else:
id = data_id
if id:
key = ndb.Key(urlsafe=id)
if model != key.kind():
raise AppError("Key kind must match id kind: {} != {}.".format(
model, key.kind()))
return key
return None
def _get(self, model, id, *extra_filters):
model = model.lower()
cls = None
if _config.DEFINED_MODELS:
cls = users if model == "users" else _config.DEFINED_MODELS.get(
model)
if not cls and _config.RESTRICT_TO_DEFINED_MODELS:
raise RestrictedModelError
if cls:
model = cls.__name__
if not cls:
validate_modelname(model)
cls = users if model == "users" else type(model,
(ScopedExpando, ), {})
if id:
me = False
if model == "users":
if id == "me":
me = True
id = current_user(required=True)
key = parse_id(id, model)
m = key.get()
if not m:
if model == "users" and me:
m = users()
m.key = key
setattr(m, "$unsaved", True)
u = config.get_current_user()
if hasattr(u, "email"):
m.email = u.email()
else:
raise AppError("No {} with id {}.".format(model, id))
return m.to_dict()
else:
return query(self, cls, *extra_filters)
def _get(self, model, id, *extra_filters):
# TODO(doug) does the model name need to be ascii encoded since types don't support utf-8?
cls = users if model == "users" else type(model.lower(),
(ScopedExpando, ), {})
if id:
me = False
if model == "users":
if id == "me":
me = True
id = current_user(required=True)
key = parse_id(id, model)
m = key.get()
if not m:
if model == "users" and me:
m = users()
m.key = key
setattr(m, "$unsaved", True)
environ = webapp2.get_request().environ
for k, v in environ.iteritems():
if k[:
14] == "TAILBONE_USER_" and k != "TAILBONE_USER_ID" and v:
setattr(m, k[14:].lower(), v)
else:
raise AppError("No {} with id {}.".format(model, id))
return m.to_dict()
else:
return query(self, cls, *extra_filters)
def _pre_delete_hook(cls, key):
m = key.get()
u = current_user(required=True)
if not m.can_write(u):
raise AppError(
"You ({}) do not have permission to delete this model ({}).".
format(u, key.id()))
def start_instance(pool):
"""Start a new instance with a given configuration."""
# start instance
# defer an update load call
instance_class = string_to_class(pool.instance_type)
name = rfc1035(instance_class.__name__)
# max length of a name is 63
name = "{}-{}".format(name, uuid.uuid4())[:63]
instance = instance_class(id=name)
instance.pool = pool.key
instance.zone = random.choice(LOCATIONS[pool.region]["zones"])
instance.put()
compute = compute_api()
if compute:
def update_zone(s):
return re.sub(r"\/zones\/([^/]*)",
"/zones/{}".format(instance.zone),
s)
instance.PARAMS.update({
"name": name,
"zone": update_zone(instance.PARAMS.get("zone")),
"machineType": update_zone(instance.PARAMS.get("machineType")),
})
operation = compute.instances().insert(
project=PROJECT_ID, zone=instance.zone, body=instance.PARAMS).execute()
logging.info("Create instance operation {}".format(operation))
instance.status = operation.get("status")
name = "update_instance_status_{}_{}".format(instance.key.urlsafe(), int(time.time()))
deferred.defer(update_instance_status, instance.key.urlsafe(), _countdown=STARTING_STATUS_DELAY, _name=name)
else:
logging.warn("No compute api defined.")
raise AppError("No compute api defined.")
def get_or_create_room(request, name=None):
if not name:
name, room, address = unique_name()
else:
room = room_hash(name)
address = memcache.get(room)
if not address:
if _config.ENABLE_WEBSOCKET:
if DEBUG:
class DebugInstance(object):
if _config.DEBUG_WEBSOCKET_ADDRESS:
address = _config.DEBUG_WEBSOCKET_ADDRESS
elif str(request.remote_addr) == "::1":
address = "localhost"
else:
address = request.remote_addr
instance = DebugInstance()
else:
instance = LoadBalancer.find(TailboneWebsocketInstance)
if not instance:
raise AppError('Instance not yet ready, try again later.')
address = "ws://{}:{}/{}".format(instance.address, _config.PORT,
name)
else:
address = "/api/channel/{}".format(name)
if not memcache.add(room, address, time=_config.ROOM_EXPIRATION):
return get_or_create_room(request, name)
return name, address
def get(self):
instance = LoadBalancer.find(TailboneCustomInstance)
if not instance:
raise AppError('Instance not found, try again later.')
return {
"ip": instance.address
}
def query(self, cls, *extra_filters):
params = self.request.get("params")
if params:
params = json.loads(params)
page_size = params.get("page_size", 100)
cursor = params.get("cursor")
filters = params.get("filter")
orders = params.get("order")
projection = params.get("projection") or None
q = construct_query_from_json(cls, filters, orders)
else:
page_size = int(self.request.get("page_size", default_value=100))
cursor = self.request.get("cursor")
projection = self.request.get_all("projection")
projection = [i for sublist in projection for i in sublist.split(",")] if projection else None
filters = self.request.get_all("filter")
orders = self.request.get_all("order")
q = construct_query_from_url_args(cls, filters, orders)
for f in extra_filters:
q = f(q)
cursor = ndb.Cursor.from_websafe_string(cursor) if cursor else None
if projection:
# if asking for private variables and not specifing owners and viewers append them
private = [p for p in projection if not re_public.match(p)]
if len(private) > 0:
acl = [p for p in private if p in acl_attributes]
if len(acl) == 0:
raise AppError("Requesting projection of private properties, but did not specify 'owners' or 'viewers' to verify access.")
results, cursor, more = q.fetch_page(page_size, start_cursor=cursor, projection=projection)
self.response.headers["More"] = "true" if more else "false"
if cursor:
self.response.headers["Cursor"] = cursor.urlsafe()
# The Reverse-Cursor is used if you construct a query in the opposite direction
self.response.headers["Reverse-Cursor"] = cursor.reversed().urlsafe()
return [m.to_dict() for m in results]
def get_model(urlsafekey):
key = ndb.Key(urlsafe=urlsafekey)
# dynamic class defined if doesn't exists for reflective creation later
type(key.kind(), (ScopedExpando, ), {})
m = key.get()
if not key:
raise AppError("Model {} does not exists.".format(urlsafekey))
return m
def get(self):
username = self.request.get("username")
if not username:
raise AppError("Must provide username.")
instance = LoadBalancer.find(TailboneTurnInstance)
if not instance:
raise AppError('Instance not found, try again later.')
username, password = credentials(username, instance.secret)
return {
"username":
username,
"password":
password,
"uris": [
"turn:{}:3478?transport=udp".format(instance.address),
"turn:{}:3478?transport=tcp".format(instance.address),
],
}
def _delete(self, model, id):
if not id:
raise AppError("Must provide an id.")
model = model.lower()
u = current_user(required=True)
if model == "users":
if id != "me" and id != u:
raise AppError(
"Id must be the current " +
"user_id or me. User {} tried to modify user {}.".format(
u, id))
id = u
key = parse_id(id, model)
key.delete()
search.delete(key)
if store_metadata:
decrement(model)
return {}
def delete(self, key):
if not config.is_current_user_admin():
raise AppError("User must be administrator.")
key = blobstore.BlobKey(str(urllib.unquote(key)))
blob_info = BlobInfo.get(key)
if blob_info:
blob_info.delete()
if HAS_PIL and re_image.match(blob_info.content_type):
delete_serving_url(key)
return {}
else:
self.error(404)
return {"error": "File not found with key " + key}
def stop_instance(instance):
"""Stop an instance."""
# cancel update load defered call
# stop instance
# TODO: need some way of clearing externally assciated instances
compute = compute_api()
if compute:
compute.instances().delete(
project=PROJECT_ID, zone=instance.zone, instance=instance.key.id()).execute()
else:
logging.warn("No compute api defined.")
raise AppError("No compute api defined.")
instance.key.delete()
def find(instance_class, request):
"""Return an instance of this instance type from the nearest pool or create it."""
if DEBUG:
# check to make sure we can access the api before we do anything
compute_api()
region, zone = LoadBalancer.nearest_zone(request)
instance_str = class_to_string(instance_class)
pool = LoadBalancer.get_or_create_pool(instance_str, region)
instance = pool.instance()
if instance and instance.address:
return instance
raise AppError("Instance not yet ready, please try again.")
def validate(cls_name, data):
# properties = data.keys()
# confirm the format of any tailbone specific types
for name in acl_attributes:
val = data.get(name)
if val:
# TODO(doug): validate list, can't be empty list, must contain id like objects
pass
# run validation over remaining properties
if _validation:
validations = _validation.get(cls_name)
if not validations:
raise AppError("Validation requires all valid models to be listed, use empty quote to skip.")
_validate(validations, data, acl_attributes)
def construct_filter(filter_str):
m = re_composite_filter.match(filter_str)
if m:
filters = [construct_filter(f) for f in re_split.split(m.group(2))]
if m.group(1) == "AND":
return ndb.query.AND(*filters)
else:
return ndb.query.OR(*filters)
m = re_filter.match(filter_str)
if m:
name, opsymbol, value = m.groups()
return ndb.query.FilterNode(name, convert_opsymbol(opsymbol), convert_value(value))
if re_split.match(filter_str):
return construct_filter("AND({})".format(filter_str))
raise AppError("Filter format is unsupported: {}".format(filter_str))
def get(self, key):
if key == "": # query
if not config.is_current_user_admin():
raise AppError("User must be administrator.")
return restful.query(self, BlobInfo)
elif key == "create":
return {
"upload_url": blobstore.create_upload_url("/api/files/upload")
}
key = str(urllib.unquote(key))
blob_info = bs.BlobInfo.get(key)
if blob_info:
self.send_blob(blob_info)
raise BreakError
else:
self.error(404)
return {"error": "File not found with key " + key}
def _get(self, model, id, *extra_filters):
model = model.lower()
cls = None
if _config.DEFINED_MODELS:
cls = users if model == "users" else _config.DEFINED_MODELS.get(
model)
if not cls and _config.RESTRICT_TO_DEFINED_MODELS:
raise RestrictedModelError
if cls:
model = cls.__name__
if not cls:
validate_modelname(model)
cls = users if model == "users" else type(model,
(ScopedExpando, ), {})
logging.info("ID %s" % id)
if id:
me = False
if model == "users":
if id == "me":
me = True
id = current_user(required=True).urlsafe()
logging.info("users me %s" % current_user(required=True))
if "," in id:
ids = id.split(",")
keys = [parse_id(i, model) for i in ids]
results = ndb.get_multi(keys)
return [m.to_dict() if m else m for m in results]
key = parse_id(id, model)
m = key.get()
if not m:
if model == "users" and me:
m = users()
m.key = key
else:
raise AppError("No {} with id {}.".format(model, id))
if model == "users" and me:
u = config.get_current_user()
if u:
for k, v in getAttributes(u).items():
if k.startswith("_"):
k = "$" + k[1:]
setattr(m, k, v)
return m.to_dict()
else:
return query(self, cls, *extra_filters)
def set_or_create(self, model, id):
model = model.lower()
u = current_user(required=True)
if model == "users":
if not (id == "me" or id == "" or id == u):
raise AppError(
"Id must be the current " +
"user_id or me. User {} tried to modify user {}.".format(
u, id))
id = u
cls = users
else:
cls = None
if _config.DEFINED_MODELS:
cls = _config.DEFINED_MODELS.get(model)
if not cls and _config.RESTRICT_TO_DEFINED_MODELS:
raise RestrictedModelError
if cls:
model = cls.__name__
if not cls:
validate_modelname(model)
cls = type(model, (ScopedExpando, ), {})
data = parse_body(self)
key = parse_id(id, model, data.get("Id"))
clean_data(data)
validate(cls.__name__, data)
m = reflective_create(cls, data)
if key:
m.key = key
if model != "users":
if len(m.owners) == 0:
m.owners.append(u)
m.put()
redirect = self.request.get("redirect")
if redirect:
self.redirect(redirect)
# Raising break error to avoid header and body writes from @as_json decorator since we override as a redirect
raise BreakError()
return m.to_dict()
def _pre_put_hook(self):
super(ScopedModel, self)._pre_put_hook()
if config.is_current_user_admin():
return
# check for writable and for any admin properties
if self._previous is not None:
u = current_user(required=True)
if not self._previous.can_write(u):
raise AppError("You do not have sufficient privileges.")
keys = [p._code_name for p in self._properties.itervalues()]
for k in keys:
if re_admin.match(k):
attr = getattr(self._previous, k, None)
if attr:
setattr(self, k, attr)
else:
delattr(self, k)
else:
keys = [p._code_name for p in self._properties.itervalues()]
for k in keys:
if re_admin.match(k):
delattr(self, k)
def stop_instance(instance):
"""Stop an instance."""
# cancel update load defered call
# stop instance
# TODO: need some way of clearing externally assciated instances
compute = compute_api()
if compute:
name = instance.key.id()
result = compute.instances().delete(
project=PROJECT_ID, zone=instance.zone, instance=name).execute()
# As of v1 this call needs to be blocking to ensure the instance is stopped
# before we can delete the boot disk
_blocking_call(compute, result)
# As scratch is is depricated in v1 we are treating the persistent disk as
# a scratch disk by deleting it when stopping the instance.
result = compute.disks().delete(
project=PROJECT_ID, zone=instance.zone, disk=name).execute()
_blocking_call(compute, result)
else:
logging.warn("No compute api defined.")
raise AppError("No compute api defined.")
instance.key.delete()
DEFINED_MODELS = None
RESTRICT_TO_DEFINED_MODELS = True
PROTECTED_MODEL_NAMES = [
"(?i)(mesh|messages|files|events|admin|proxy)", "(?i)tailbone.*"
]
_config = api.lib_config.register('tailboneRestful', _ConfigDefaults.__dict__)
re_public = re.compile(r"^[A-Z].*")
re_type = type(re_public)
re_admin = re.compile(r"^(A|a)dmin.*")
acl_attributes = [u"owners", u"viewers"]
ProtectedModelError = AppError("This is a protected Model.")
RestrictedModelError = AppError("Models are restricted.")
def validate_modelname(model):
if [r for r in _config.PROTECTED_MODEL_NAMES if re.match(r, model)]:
raise ProtectedModelError
def current_user(required=False):
u = config.get_current_user()
if u:
return ndb.Key("users", u.user_id()).urlsafe()
if required:
raise LoginError("User must be logged in.")
return None
def start_instance(pool):
"""Start a new instance with a given configuration."""
# start instance
# defer an update load call
instance_class = string_to_class(pool.instance_type)
name = rfc1035(instance_class.__name__)
# max length of a name is 63
name = "{}-{}".format(name, uuid.uuid4())[:63]
instance = instance_class(id=name)
instance.pool = pool.key
locations = get_locations()
instance.zone = random.choice(locations[pool.region]["zones"])
compute = compute_api()
if compute:
def update_zone(s):
return re.sub(r"\/zones\/([^/]*)",
"/zones/{}".format(instance.zone), s)
# create disk from source snapshot
if instance.SOURCE_SNAPSHOT:
operation = compute.disks().insert(
project=PROJECT_ID,
zone=instance.zone,
body={
"name": name,
"sizeGb": 10,
"sourceSnapshot": instance.SOURCE_SNAPSHOT,
}).execute()
_blocking_call(compute, operation)
instance.PARAMS.update({
"disks": [{
"kind":
"compute#attachedDisk",
"boot":
True,
"type":
"PERSISTENT",
"mode":
"READ_WRITE",
"deviceName":
name,
"zone":
update_zone(instance.PARAMS.get("zone")),
"source":
api_url(PROJECT_ID, "zones", instance.zone, "disks",
name),
}],
})
instance.PARAMS.update({
"name":
name,
"zone":
update_zone(instance.PARAMS.get("zone")),
"machineType":
update_zone(instance.PARAMS.get("machineType")),
})
operation = compute.instances().insert(
project=PROJECT_ID, zone=instance.zone,
body=instance.PARAMS).execute()
logging.info("Create instance operation {}".format(operation))
instance.status = operation.get("status")
name = "update_instance_status_{}_{}".format(
instance.key.urlsafe(), int(time.time()))
deferred.defer(update_instance_status,
instance.key.urlsafe(),
_countdown=STARTING_STATUS_DELAY,
_name=name)
else:
logging.warn("No compute api defined.")
raise AppError("No compute api defined.")
# finally put the instance after the resource has been created
instance.put()
def put(self, _):
raise AppError("PUT is not supported for the files api.")
def post(self, _):
raise AppError("You must make a GET call to /api/files/create to get a POST url.")