def test_init_fail(self):
self.start_test()
config = {'shock-url': self.shock_url + '/' + 'fake_endpoint_100'}
with self.assertRaises(ValueError) as context:
ShockUtil(config)
self.assertIn('Connot connect to shock server', str(context.exception.args))
config = {'shock-url': self.shock_url}
with self.assertRaises(ValueError) as context:
ShockUtil(config)
self.assertIn('Unexpected response from shock server', str(context.exception.args))
def __init__(self, config):
self.mongo_util = MongoUtil(config)
self.shock_util = ShockUtil(config)
self.token_cache = TokenCache(1000, self.CACHE_EXPIRE_TIME)
self.auth_url = config.get('auth-url')
self.admin_roles = [
role.strip() for role in config.get('admin-roles').split(',')
]
logging.basicConfig(format='%(created)s %(levelname)s: %(message)s',
level=logging.INFO)
def setUpClass(cls):
cls.token = os.environ.get('KB_AUTH_TOKEN', None)
config_file = os.environ.get('KB_DEPLOYMENT_CONFIG', None)
cls.cfg = {}
config = ConfigParser()
config.read(config_file)
for nameval in config.items('AbstractHandle'):
cls.cfg[nameval[0]] = nameval[1]
cls.cfg['admin-token'] = cls.token
# Getting username from Auth profile for token
authServiceUrl = cls.cfg['auth-service-url']
auth_client = _KBaseAuth(authServiceUrl)
cls.user_id = auth_client.get_user(cls.token)
cls.shock_util = ShockUtil(cls.cfg)
cls.shock_url = cls.cfg['shock-url']
cls.shock_ids_to_delete = list()
class Handler:
FIELD_NAMES = [
'hid', 'id', 'file_name', 'type', 'url', 'remote_md5', 'remote_sha1',
'created_by', 'creation_date'
]
AUTH_API_PATH = 'api/V2'
CACHE_EXPIRE_TIME = 300 # seconds
@staticmethod
def validate_params(params, expected, opt_param=set()):
"""Validates that required parameters are present. Warns if unexpected parameters appear"""
expected = set(expected)
opt_param = set(opt_param)
pkeys = set(params)
if expected - pkeys:
raise ValueError(
"Required keys {} not in supplied parameters".format(
", ".join(expected - pkeys)))
defined_param = expected | opt_param
for param in params:
if param not in defined_param:
logging.warning(
"Unexpected parameter {} supplied".format(param))
def _process_handle(self, handle, user_id):
"""
pre-process handle: check/remove/add fields
"""
logging.info('start processing handle')
handle = {k: v
for k, v in handle.items()
if k in self.FIELD_NAMES} # remove unnecessary fields
if handle.get('hid'):
raise ValueError(
'Please do not specify hid. HandleService will auto-create a new hid.'
)
else:
hid_counter = self.mongo_util.increase_counter()
handle['hid'] = int(hid_counter)
handle['_id'] = int(handle['hid']) # assign _id to 'hid'
required_fields = ['id', 'type', 'url']
fields_values = [v for k, v in handle.items() if k in required_fields]
if (len(fields_values) !=
len(required_fields)) or (not all(fields_values)):
error_msg = 'Missing one or more required positional field\n'
error_msg += 'Requried fields: {}'.format(required_fields)
raise ValueError(error_msg)
if not handle.get(
'file_name'
): # assign None to remote_md5/remote_sha1 if missing/empty
handle['file_name'] = None
if not handle.get(
'remote_md5'
): # assign None to remote_md5/remote_sha1 if missing/empty
handle['remote_md5'] = None
if not handle.get('remote_sha1'):
handle['remote_sha1'] = None
if not handle.get(
'created_by'
): # assign created_by to current token user if missing
handle['created_by'] = user_id
if not handle.get('creation_date'): # assign creation_date if missing
handle['creation_date'] = datetime.datetime.utcnow()
if not isinstance(handle['creation_date'], datetime.datetime):
# cast due to fetch_handles_by return epoch creation_date
try:
handle['creation_date'] = datetime.datetime.fromtimestamp(
handle['creation_date'])
except Exception:
raise ValueError(
'Cannot convert creation_date field to datetime')
return handle
def _get_token_roles(self, token):
headers = {'Authorization': token}
end_point = os.path.join(self.auth_url, self.AUTH_API_PATH, 'me')
resp = _requests.get(end_point, headers=headers)
if resp.status_code != 200:
raise ValueError(
'Request auth roles.\nError Code: {}\n{}\n'.format(
resp.status_code, resp.text))
else:
data = resp.json()
customroles = data.get('customroles')
return customroles
def _is_admin_user(self, token):
fetched_token_info = self.token_cache.get(token)
if fetched_token_info:
customroles = fetched_token_info.get('customroles')
else:
customroles = self._get_token_roles(token)
self.token_cache[token] = {'customroles': customroles}
return not set(self.admin_roles).isdisjoint(customroles)
def __init__(self, config):
self.mongo_util = MongoUtil(config)
self.shock_util = ShockUtil(config)
self.token_cache = TokenCache(1000, self.CACHE_EXPIRE_TIME)
self.auth_url = config.get('auth-url')
self.admin_roles = [
role.strip() for role in config.get('admin-roles').split(',')
]
self.namespace = config.get('namespace')
logging.basicConfig(format='%(created)s %(levelname)s: %(message)s',
level=logging.INFO)
def fetch_handles_by(self, params):
"""
query DB and return if element match one of entry in field column
"""
logging.info('start fetching handles')
self.validate_params(params, ['elements', 'field_name'])
elements = params.get('elements')
field_name = params.get('field_name')
if field_name == 'hid':
# remove prefix for hids
elements = [
int(hid.split(self.namespace + '_')[-1]) for hid in elements
]
docs = self.mongo_util.find_in(elements, field_name)
handles = list()
for doc in docs:
# append prefix for returned hids
doc['hid'] = self.namespace + '_' + str(doc['hid'])
doc['creation_date'] = doc['creation_date'].timestamp()
handles.append(doc)
return handles
def persist_handle(self, handle, user_id):
"""
writes the handle to a persistent store
insert handle if handle does not exist
otherwise update handle if it's created by token user
"""
logging.info('start persisting handle')
handle = self._process_handle(handle, user_id)
hid = handle.get('hid')
# handle doesn't exist, insert handle
self.mongo_util.insert_one(handle)
return str(self.namespace + '_' + str(hid))
def delete_handles(self, handles, user_id):
"""
delete handles
raise error if any of handles are not created by token user
"""
handle_user = set([h.get('created_by') for h in handles])
if not (handle_user == set([user_id])):
raise ValueError('Cannot delete handles not created by owner')
# remove 'KBH_' prefix for hids
for handle in handles:
handle['hid'] = int(handle['hid'].split(self.namespace + '_')[-1])
deleted_count = self.mongo_util.delete_many(handles)
return deleted_count
def is_owner(self, hids, token, user_id):
"""
check and see if token user is owner.username from shock node
"""
try:
handles = self.fetch_handles_by({
'elements': hids,
'field_name': 'hid'
})
except Exception:
return 0
for handle in handles:
node_type = handle.get('type')
if node_type.lower() != 'shock':
raise ValueError('Do not support node type other than Shock')
node_id = handle.get('id')
owner = self.shock_util.get_owner(node_id, token)
if owner != user_id:
return 0
return 1
def are_readable(self, hids, token):
"""
check if nodes associated with handles is reachable/readable
"""
try:
handles = self.fetch_handles_by({
'elements': hids,
'field_name': 'hid'
})
except Exception:
return 0
for handle in handles:
node_type = handle.get('type')
if node_type.lower() != 'shock':
raise ValueError('Do not support node type other than Shock')
node_id = handle.get('id')
is_readable = self.shock_util.is_readable(node_id, token)
if not is_readable:
return 0
return 1
def add_read_acl(self, hids, token, username=None):
"""
grand readable acl for username or global if username is empty
"""
if not self._is_admin_user(token):
raise ValueError(
'User may not run add_read_acl/set_public_read method')
handles = self.fetch_handles_by({
'elements': hids,
'field_name': 'hid'
})
for handle in handles:
node_type = handle.get('type')
if node_type.lower() != 'shock':
raise ValueError('Do not support node type other than Shock')
node_id = handle.get('id')
try:
self.shock_util.add_read_acl(node_id, token, username=username)
except Exception:
raise ValueError("Unable to set acl(s) on handles {}".format(
handle.get('hid')))
return 1