def delete(self, obj, datastore_name="", del_associations=False):
if not isinstance(obj, IonObjectBase) and not isinstance(obj, str):
raise BadRequest(
"Obj param is not instance of IonObjectBase or string id")
if type(obj) is str:
self.delete_doc(obj, datastore_name=datastore_name)
else:
if '_id' not in obj:
raise BadRequest("Doc must have '_id'")
if '_rev' not in obj:
raise BadRequest("Doc must have '_rev'")
self.delete_doc(self._ion_object_to_persistence_dict(obj),
datastore_name=datastore_name,
del_associations=del_associations)
def create_object_type(self, object_type=None):
""" Should receive an ObjectType object
"""
# Return Value
# ------------
# {object_type_id: ''}
#
if not is_basic_identifier(object_type.name):
raise BadRequest("Invalid object_type name: %s" % object_type.name)
if not is_yaml_string_valid(object_type.definition):
raise BadRequest("Invalid YAML definition")
object_type_id, version = self.clients.resource_registry.create(
object_type)
return object_type_id
def create_datastore(self, datastore_name=None, **kwargs):
"""
Create a data store with the given name. This is
equivalent to creating a database on a database server.
@param datastore_name Datastore to work on. Will be scoped if scope was provided.
"""
datastore_name = self._get_datastore_name(datastore_name)
try:
self.server.create(datastore_name)
except PreconditionFailed:
raise BadRequest("Data store with name %s already exists" %
datastore_name)
except ValueError:
raise BadRequest("Data store name %s invalid" % datastore_name)
def __init__(self,
process=None,
stream_id='',
stream_route=None,
exchange_point='',
routing_key=''):
"""
Creates a StreamPublisher which publishes to the specified stream by default and is attached to the
specified process.
@param process The process which the subscriber is to be attached.
@param stream_id Stream identifier for the publishing stream.
@param stream_route A StreamRoute corresponding to the stream_id
@param exchange_point The name of the exchange point, to be used in lieu of stream_route or stream_id
@param routing_key The routing key to be used in lieu of stream_route or stream_id
"""
super(StreamPublisher, self).__init__()
if not isinstance(process, BaseService):
raise BadRequest('No valid process provided.')
#--------------------------------------------------------------------------------
# The important part of publishing is the stream_route and there are three ways
# to the stream route
# - The Route is obtained from Pubsub Management with a stream id.
# - The Route is obtained by combining exchange_point and the routing_key
# but all other information is lost (credentials, etc.)
# - The Route is obtained by being provided directly to __init__
#--------------------------------------------------------------------------------
self.stream_id = stream_id
if stream_id:
# Regardless of what's passed in for stream_route look it up, prevents mismatching
pass
elif not stream_route:
self.stream_route = None
if exchange_point and routing_key:
self.stream_route = StreamRoute(exchange_point=exchange_point,
routing_key=routing_key)
else:
# Create stream
self.stream_id = stream_id
self.stream_route = stream_route
else:
self.stream_route = stream_route
if not isinstance(self.stream_route, StreamRoute):
raise BadRequest('No valid stream route provided to publisher.')
self.container = process.container
self.xp = self.container.ex_manager.create_xp(
self.stream_route.exchange_point)
self.xp_route = self.xp.create_route(self.stream_route.routing_key)
def find_by_value(self, subtree='/', attribute=None, value=None, **kwargs):
"""
Returns a list of DirEntry with entries that have an attribute with the given value.
"""
if attribute is None:
raise BadRequest("Illegal arguments")
if subtree is None:
raise BadRequest("Illegal arguments")
start_key = [self.orgname, attribute, value, subtree]
end_key = [self.orgname, attribute, value, subtree+"ZZZZZZ"]
res = self.dir_store.find_by_view('directory', 'by_attribute',
start_key=start_key, end_key=end_key, id_only=True, convert_doc=True, **kwargs)
match = [doc for docid, indexkey, doc in res]
return match
def update_service_definition(self, service_definition=None):
""" Should receive a ServiceDefinition object
"""
# Return Value
# ------------
# {success: true}
#
if not is_basic_identifier(service_definition.name):
raise BadRequest("Invalid service_definition name: %s" %
service_definition.name)
if not is_yaml_string_valid(service_definition.definition):
raise BadRequest("Invalid YAML definition")
service_id, version = self.clients.resource_registry.update(
service_definition)
return service_id
def await_launch(self, timeout, process_id=None):
if None is process_id:
if None is self.process_id:
raise BadRequest("No process_id was supplied to await_launch, and " +
"no process_id was available from launch")
else:
process_id = self.process_id
log.debug("waiting %s seconds for agent launch", timeout)
psg = ProcessStateGate(self.process_dispatcher_client.read_process, process_id, ProcessStateEnum.RUNNING)
if not psg.await(20):
# todo: different error
raise BadRequest("The agent process '%s' failed to launch in %s seconds" %
(process_id, timeout))
def update_doc(self, doc, datastore_name=""):
ds, datastore_name = self._get_datastore(datastore_name)
if '_id' not in doc:
raise BadRequest("Doc must have '_id'")
if '_rev' not in doc:
raise BadRequest("Doc must have '_rev'")
log.debug('update doc contents: %s', doc)
try:
res = ds.save(doc)
except ResourceConflict:
raise Conflict('Object not based on most current version')
log.debug('Update result: %s', str(res))
id, version = res
return (id, version)
def create_doc_mult(self, docs, object_ids=None, datastore_name=None):
"""
Create multiple raw docs.
Returns list of (Success, Oid, rev)
"""
if object_ids and any(["_id" in doc for doc in docs]):
raise BadRequest("Docs must not have '_id'")
if any(["_rev" in doc for doc in docs]):
raise BadRequest("Docs must not have '_rev'")
if object_ids and len(object_ids) != len(docs):
raise BadRequest("Invalid object_ids length")
return self.save_doc_mult(docs,
object_ids,
datastore_name=datastore_name)
def _do_service_call(self, service, method, params):
post_data = self._service_request_template()
post_data['serviceRequest']['serviceName'] = service
post_data['serviceRequest']['serviceOp'] = method
for k, v in params.iteritems():
post_data['serviceRequest']['params'][k] = v
url = "http://%s:%s/ion-service/%s/%s" % (
CFG.container.service_gateway.web_server.hostname,
CFG.container.service_gateway.web_server.port, service, method)
try:
json.dumps(post_data)
except:
raise NotImplementedError("%s.%s: %s" %
(service, method, str(params)))
service_gateway_call = requests.post(
url, data={'payload': json.dumps(post_data)})
if service_gateway_call.status_code != 200:
raise BadRequest(
"The service gateway returned the following error: %d" %
service_gateway_call.status_code)
# debug lines
#sys.stderr.write(str(url) + "\n")
#sys.stderr.write(str(post_data['serviceRequest']) + "\n")
resp = json.loads(service_gateway_call.content)
#sys.stderr.write(str(resp) + "\n")
if "GatewayResponse" not in resp["data"]:
if "GatewayError" in resp["data"]:
#raise BadRequest("%s: %s" % (resp["data"]["GatewayError"]["Exception"],
# resp["data"]["GatewayError"]["Message"]))
raise BadRequest("%s: %s\nFROM\n%s.%s(%s)" %
(resp["data"]["GatewayError"]["Exception"],
resp["data"]["GatewayError"]["Message"],
service, method, str(params)))
else:
raise BadRequest("Unknown error object: %s" % str(resp))
ret = resp["data"]["GatewayResponse"]
return ret
def _validate_resource_obj(self,
arg_name,
resource_obj,
res_type=None,
optional=False,
checks=""):
"""
Check that the given argument (object) exists and is a resource object of given type.
Can be None if optional==True.
Optional checks in comma separated string:
- id: resource referenced by ID is compatible and returns it.
- noid: object contains no id
- name: object has non empty name
- unique: name is not used yet in system for given res_type (must be set)
"""
checks = checks.split(",") if isinstance(checks,
basestring) else checks
if optional and resource_obj is None:
return
if not resource_obj:
raise BadRequest("Argument '%s': missing" % arg_name)
from interface.objects import Resource
if not isinstance(resource_obj, Resource):
raise BadRequest("Argument '%s': not a resource object" % arg_name)
if "noid" in checks and "_id" in resource_obj:
raise BadRequest("Argument '%s': resource object has an id" %
arg_name)
if ("name" in checks or "unique" in checks) and not resource_obj.name:
raise BadRequest("Argument '%s': resource has invalid name" %
arg_name)
if "unique" in checks:
if not res_type:
raise BadRequest("Must provide resource type")
res_list, _ = self.clients.resource_registry.find_resources(
restype=res_type, name=resource_obj.name)
if res_list:
raise BadRequest(
"Argument '%s': resource name already exists" % arg_name)
if res_type and resource_obj.type_ != res_type:
raise BadRequest(
"Argument '%s': resource object type is not a '%s' -- SPOOFING ALERT"
% (arg_name, res_type))
if "id" in checks:
if "_id" not in resource_obj:
raise BadRequest("Argument '%s': resource object has no id" %
arg_name)
old_resource_obj = self.clients.resource_registry.read(
resource_obj._id)
if res_type and old_resource_obj.type_ != res_type:
raise BadRequest(
"Argument '%s': existing resource is not a '%s' -- SPOOFING ALERT"
% (arg_name, res_type))
return old_resource_obj
def read_mult(self, object_ids, datastore_name=""):
if any([not isinstance(object_id, str) for object_id in object_ids]):
raise BadRequest("Object id param is not string")
docs = self.read_doc_mult(object_ids, datastore_name)
# Convert docs into Ion objects
obj_list = [self._persistence_dict_to_ion_object(doc) for doc in docs]
return obj_list
def create_doc_mult(self, docs, object_ids=None):
if any(["_id" in doc for doc in docs]):
raise BadRequest("Docs must not have '_id'")
if any(["_rev" in doc for doc in docs]):
raise BadRequest("Docs must not have '_rev'")
if object_ids and len(object_ids) != len(docs):
raise BadRequest("Invalid object_ids")
# Assign an id to doc (recommended in CouchDB documentation)
object_ids = object_ids or [uuid4().hex for i in xrange(len(docs))]
res = []
for doc, oid in zip(docs, object_ids):
oid, rev = self.create_doc(doc, oid)
res.append((True, oid, rev))
return res
def terminate_process(self, process_id, do_notifications=True):
"""
Terminates a process and all its resources. Termination is graceful with timeout.
@param process_id The id of the process to terminate. Should exist in the container's
list of processes or this will raise.
@param do_notifications If True, emits process state changes for TERMINATING and TERMINATED.
If False, supresses any state changes. Used near EXITED and FAILED.
"""
process_instance = self.procs.get(process_id, None)
if not process_instance:
raise BadRequest(
"Cannot terminate. Process id='%s' unknown on container id='%s'"
% (process_id, self.container.id))
log.info("ProcManager.terminate_process: %s -> pid=%s",
process_instance._proc_name, process_id)
if do_notifications:
self._call_proc_state_changed(process_instance,
ProcessStateEnum.TERMINATING)
self._process_quit(process_instance)
self._unregister_process(process_id, process_instance)
if do_notifications:
self._call_proc_state_changed(process_instance,
ProcessStateEnum.TERMINATED)
def update_index(self, index=None):
if index is None:
raise BadRequest("No index specified")
validate_is_instance(
index, Index,
'The specified index is not of type interface.objects.Index')
return self.clients.resource_registry.update(index)
def update_dataset(self, dataset=None):
if not (dataset and dataset._id):
raise BadRequest('%s: Dataset either not provided or malformed.' %
self.logging_name)
self.clients.resource_registry.update(dataset)
#@todo: Check to make sure retval is boolean
return True
def cancel_member_enrollment(self, org_id='', user_id=''):
"""Cancels the membership of a specified user within the specified Org. Once canceled, the user will no longer
have access to the resource of that Org. Throws a NotFound exception if neither id is found.
@param org_id str
@param user_id str
@retval success bool
@throws NotFound object with specified id does not exist
"""
param_objects = self._validate_parameters(org_id=org_id,
user_id=user_id)
org = param_objects['org']
user = param_objects['user']
if org.name == ROOT_ION_ORG_NAME:
raise BadRequest(
"A request to cancel enrollment in the root ION Org is not allowed"
)
#First remove all associations to any roles
role_list = self.find_roles_by_user(org_id, user_id)
for user_role in role_list:
self._delete_role_association(org, user, user_role)
#Finally remove the association to the Org
aid = self.clients.resource_registry.get_association(
org, PRED.hasMembership, user)
if not aid:
raise NotFound(
"The membership association between the specified user and Org is not found"
)
self.clients.resource_registry.delete_association(aid)
return True
def find_res_by_lcstate(self, lcstate, restype=None, id_only=False):
log.debug("find_res_by_lcstate(lcstate=%s, restype=%s)", lcstate, restype)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
ds, datastore_name = self._get_datastore()
view = ds.view(self._get_viewname("resource","by_lcstate"), include_docs=(not id_only))
is_hierarchical = (lcstate in CommonResourceLifeCycleSM.STATE_ALIASES)
# lcstate is a hiearachical state and we need to treat the view differently
if is_hierarchical:
key = [1, lcstate]
else:
key = [0, lcstate]
if restype:
key.append(restype)
endkey = list(key)
endkey.append(END_MARKER)
rows = view[key:endkey]
if is_hierarchical:
res_assocs = [dict(lcstate=row['key'][3], type=row['key'][2], name=row['key'][4], id=row.id) for row in rows]
else:
res_assocs = [dict(lcstate=row['key'][1], type=row['key'][2], name=row['key'][3], id=row.id) for row in rows]
log.debug("find_res_by_lcstate() found %s objects", len(res_assocs))
self._count(find_res_by_lcstate_call=1, find_res_by_lcstate_obj=len(res_assocs))
if id_only:
res_ids = [row.id for row in rows]
return (res_ids, res_assocs)
else:
res_docs = [self._persistence_dict_to_ion_object(row.doc) for row in rows]
return (res_docs, res_assocs)
def incoming(self, invocation):
if 'signature' in invocation.headers and 'signer' in invocation.headers:
hash = hashlib.sha1(invocation.message).hexdigest()
if not self.auth.verify_message(hash, Container.instance.cert,
invocation.headers['signature']):
raise BadRequest("Digital signature invalid")
return invocation
def create_mult(self, objects, object_ids=None, allow_ids=False):
if any([not isinstance(obj, IonObjectBase) for obj in objects]):
raise BadRequest("Obj param is not instance of IonObjectBase")
return self.create_doc_mult(
[self._ion_object_to_persistence_dict(obj) for obj in objects],
object_ids,
allow_ids=allow_ids)
def get_datastore_class(cls, server_cfg, variant=None):
server_type = server_cfg.get('type', 'postgresql')
if server_type == 'postgresql':
store_cls = "pyon.datastore.postgresql.base_store.PostgresDataStore"
else:
raise BadRequest("Unknown datastore server type: %s" % server_type)
return store_cls
def buy_bonds(self, account_id='', cash_amount=0.0):
"""
Purchase the specified amount of bonds. Check is first made
that the cash account has sufficient funds.
"""
account_obj = self.clients.resource_registry.read(account_id)
if not account_obj:
raise NotFound("Account %s does not exist" % account_id)
if account_obj.cash_balance < cash_amount:
raise BadRequest("Insufficient funds")
owner_obj = self.clients.resource_registry.find_subjects(
"BankCustomer", PRED.hasAccount, account_obj, False)[0][0]
# Create order object and call trade service
order_obj = IonObject("Order",
type="buy",
on_behalf=owner_obj.name,
cash_amount=cash_amount)
# Make the call to trade service
confirmation_obj = self.clients.trade.exercise(order_obj)
if confirmation_obj.status == "complete":
account_obj.cash_balance -= cash_amount
account_obj.bond_balance += confirmation_obj.proceeds
self.clients.resource_registry.update(account_obj)
return "Balances after bond purchase: cash %f, bonds: %s" % (
account_obj.cash_balance, account_obj.bond_balance)
return "Bond purchase status is: %s" % confirmation_obj.status
def delete_doc(self, doc, datastore_name=""):
if not datastore_name:
datastore_name = self.datastore_name
try:
datastore_dict = self.root[datastore_name]
except KeyError:
raise BadRequest('Data store ' + datastore_name +
' does not exist.')
if type(doc) is str:
object_id = doc
else:
object_id = doc["_id"]
log.info('Deleting object %s/%s' % (datastore_name, object_id))
if object_id in datastore_dict.keys():
if self._is_in_association(object_id, datastore_name):
obj = self.read(object_id, "", datastore_name)
log.warn(
"XXXXXXX Attempt to delete object %s that still has associations"
% str(obj))
# raise BadRequest("Object cannot be deleted until associations are broken")
# Find all version dicts and delete them
for key in datastore_dict.keys():
if key.find(object_id + '_version_') == 0:
del datastore_dict[key]
# Delete the HEAD dict
del datastore_dict[object_id]
# Delete the version counter dict
del datastore_dict['__' + object_id + '_version_counter']
else:
raise NotFound('Object with id ' + object_id + ' does not exist.')
log.info('Delete result: True')
def stop(self):
if not self.started:
raise BadRequest('Can\'t stop a subscriber that hasn\'t started.')
self.close()
self.greenlet.join(timeout=10)
self.started = False
def _validate_object(self, arg):
''' check that arg is IonObject or list of IonObjects '''
if isinstance(arg, IonObjectBase):
return
if isinstance(arg, list) and len(arg) and all([isinstance(o, IonObjectBase) for o in arg]):
return
raise BadRequest('invalid repository entry type ' + arg.__class__.__name__ + ': ' + repr(arg))
def my_get_agent_client(self, device_id, **kwargs):
try:
return self.device_agents[device_id]
except KeyError:
raise BadRequest(
"Tried to retrieve status for undefined device '%s'" %
device_id)
def persist_data_stream(self,
stream_id='',
ingestion_configuration_id='',
dataset_id='',
config=None):
#--------------------------------------------------------------------------------
# Validate that the method call was indeed valid
#--------------------------------------------------------------------------------
config = config or {}
validate_is_instance(stream_id, basestring,
'stream_id %s is not a valid string' % stream_id)
validate_true(dataset_id,
'Clients must specify the dataset to persist')
log.info('Persisting stream %s to dataset %s.', stream_id, dataset_id)
ingestion_config = self.read_ingestion_configuration(
ingestion_configuration_id)
if self.is_persisted(stream_id):
raise BadRequest('This stream is already being persisted')
#--------------------------------------------------------------------------------
# Set up the stream subscriptions and associations for this stream and its ingestion_type
#--------------------------------------------------------------------------------
if self.setup_queues(ingestion_config, stream_id, dataset_id, config):
self.clients.pubsub_management.persist_stream(stream_id)
return dataset_id
def enroll_member(self, org_id='', user_id=''):
"""Enrolls a specified user into the specified Org so that they may find and negotiate to use resources
of the Org. Membership in the ION Org is implied by registration with the system, so a membership
association to the ION Org is not maintained. Throws a NotFound exception if neither id is found.
@param org_id str
@param user_id str
@retval success bool
@throws NotFound object with specified id does not exist
"""
param_objects = self._validate_parameters(org_id=org_id,
user_id=user_id)
org = param_objects['org']
user = param_objects['user']
if org.name == ROOT_ION_ORG_NAME:
raise BadRequest(
"A request to enroll in the root ION Org is not allowed")
aid = self.clients.resource_registry.create_association(
org, PRED.hasMembership, user)
if not aid:
return False
return True
def find_child_entries(self, parent='/', direct_only=True, **kwargs):
"""
Return all child entries (ordered by path) for the given parent path.
Does not return the parent itself. Optionally returns child of child entries.
Additional kwargs are applied to constrain the search results (limit, descending, skip).
@param parent Path to parent (must start with "/")
@param direct_only If False, includes child of child entries
@retval A list of DirEntry objects for the matches
"""
if not type(parent) is str or not parent.startswith("/"):
raise BadRequest("Illegal argument parent: %s" % parent)
if direct_only:
start_key = [self.orgname, parent, 0]
end_key = [self.orgname, parent]
res = self.datastore.find_docs_by_view('directory', 'by_parent',
start_key=start_key, end_key=end_key, id_only=True, **kwargs)
else:
path = parent[1:].split("/")
start_key = [self.orgname, path, 0]
end_key = [self.orgname, list(path) + ["ZZZZZZ"]]
res = self.datastore.find_docs_by_view('directory', 'by_path',
start_key=start_key, end_key=end_key, id_only=True, **kwargs)
match = [value for docid, indexkey, value in res]
return match
def find_res_by_type(self, restype, lcstate=None, id_only=False):
log.debug("find_res_by_type(restype=%s, lcstate=%s)", restype, lcstate)
if type(id_only) is not bool:
raise BadRequest('id_only must be type bool, not %s' % type(id_only))
ds, datastore_name = self._get_datastore()
view = ds.view(self._get_viewname("resource","by_type"), include_docs=(not id_only))
if restype:
key = [restype]
if lcstate:
key.append(lcstate)
endkey = list(key)
endkey.append(END_MARKER)
rows = view[key:endkey]
else:
rows = view
res_assocs = [dict(type=row['key'][0], lcstate=row['key'][1], name=row['key'][2], id=row.id) for row in rows]
log.debug("find_res_by_type() found %s objects", len(res_assocs))
self._count(find_res_by_type_call=1, find_res_by_type_obj=len(res_assocs))
if id_only:
res_ids = [row.id for row in rows]
return (res_ids, res_assocs)
else:
res_docs = [self._persistence_dict_to_ion_object(row.doc) for row in rows]
return (res_docs, res_assocs)
