def validate(self, value):
if utils.is_string_type(value):
try:
value = json.loads(value)
except JSONDecodeError as e:
return str(e)
if isinstance(value, collections.Mapping):
value = [value]
if utils.is_list_type(value):
if len(value) < self.length_min:
return _('length[%(length)s] < minimum length[%(minimum)s]') % {
'length': len(value),
'minimum': self.length_min
}
if self.length_max is not None and len(value) > self.length_max:
return _('length[%(length)s] > maximum length[%(maximum)s]') % {
'length': len(value),
'maximum': self.length_max
}
for idx, item in enumerate(value):
try:
self.validate_func(self.rules, item, self.situation)
except exceptions.Error as e:
return _('validate failed for index[%(index)s], because: %(msg)s' % {
'index': idx + 1,
'msg': str(e)
})
else:
return _('expected type of list/tuple/set, not %(type)s ') % {'type': type(value).__name__}
return True
def validate(self, value):
if not utils.is_string_type(value):
return _('expected string, not %(type)s ') % {'type': type(value).__name__}
if self._minimum <= len(value) and len(value) <= self._maximum:
return True
return _('length required: %(min)d <= %(value)d <= %(max)d') % {
'min': self._minimum,
'value': len(value),
'max': self._maximum
}
def validate(self, value):
if utils.is_string_type(value):
try:
value = json.loads(value)
except JSONDecodeError as e:
return str(e)
if not isinstance(value, collections.Mapping):
return _('type invalid: %(type)s, expected: %(expected)s') % {'type': type(value), 'expected': 'dict'}
try:
self.validate_func(self.rules, value, self.situation)
except exceptions.Error as e:
return _('validate failed , because: %(msg)s' % {'msg': str(e)})
return True
def download_file(self, filepath, access_key, secret_key):
secure = True if self.schema == 'https' else False
ca_certs = os.environ.get('SSL_CERT_FILE') or certifi.where()
http_client = urllib3.PoolManager(
timeout=3,
maxsize=10,
cert_reqs='CERT_REQUIRED',
ca_certs=ca_certs,
retries=urllib3.Retry(total=1,
backoff_factor=0.2,
status_forcelist=[500, 502, 503, 504]))
client = minio.Minio(self.host,
access_key,
secret_key,
secure=secure,
http_client=http_client)
try:
return client.fget_object(self.bucket, self.object_key, filepath)
except Exception as e:
raise exceptions.PluginError(message=_(
'failed to download file[%(filepath)s] from s3: %(reason)s') %
{
'filepath': self.object_key,
'reason': str(e)
})
def on_patch(self, req, resp, **kwargs):
self._validate_method(req)
self._validate_data(req)
datas = req.json
if not utils.is_list_type(datas):
raise exceptions.BodyParseError(_('must be list type'))
rets = []
ex_rets = []
for idx, data in enumerate(datas):
try:
res_instance = self.make_resource(req)
if res_instance.primary_keys not in data:
raise exceptions.FieldRequired(
attribute=res_instance.primary_keys)
rid = data.pop(res_instance.primary_keys)
before_update, after_update = self.update(req, data, rid=rid)
if after_update is None:
raise exceptions.NotFoundError(
resource='%s[%s]' % (self.resource.__name__, rid))
rets.append(after_update)
except exceptions.Error as e:
ex_rets.append({'index': idx + 1, 'message': str(e)})
if len(ex_rets):
raise my_exceptions.BatchPartialError(
num=len(ex_rets),
action='update',
exception_data={'data': ex_rets})
resp.json = {
'code': 200,
'status': 'OK',
'data': rets,
'message': 'success'
}
resp.status = falcon.HTTP_200
def on_delete(self, req, resp, **kwargs):
self._validate_method(req)
self._validate_data(req)
datas = req.json
if not utils.is_list_type(datas):
raise exceptions.BodyParseError(_('must be list type'))
rets = []
ex_rets = []
for idx, data in enumerate(datas):
try:
res_instance = self.make_resource(req)
ref_count, ref_details = self.delete(req, rid=data)
rets.append(ref_details[0])
except exceptions.Error as e:
ex_rets.append({'index': idx + 1, 'message': str(e)})
if len(ex_rets):
raise my_exceptions.BatchPartialError(
num=len(ex_rets),
action='delete',
exception_data={'data': ex_rets})
resp.json = {
'code': 200,
'status': 'OK',
'data': rets,
'message': 'success'
}
resp.status = falcon.HTTP_200
def validate(self, value):
if self.cls_res().count(filters={'id': value}) == 0:
return _('reference of %(resource)s(%(id)s) not found') % {
'resource': self.cls_res.__name__,
'id': value
}
return True
def on_post(self, req, resp, **kwargs):
self._validate_method(req)
self._validate_data(req)
datas = req.json
if not utils.is_list_type(datas):
raise exceptions.PluginError(_('data must be list type'))
rets = []
ex_rets = []
for idx, data in enumerate(datas):
try:
rets.append(self.create(req, data, **kwargs))
except base_ex.Error as e:
ex_rets.append({'index': idx + 1, 'message': str(e)})
if len(ex_rets):
raise exceptions.BatchPartialError(
num=len(ex_rets),
action='create',
exception_data={'data': ex_rets})
resp.json = {
'code': 200,
'status': 'OK',
'data': rets,
'message': 'success'
}
resp.status = falcon.HTTP_200
def ensure_url_cached(url):
'''download file form url, use cached file if available
:param url: file url
:type url: str
:raises OSError: if timeout waiting for lock
:return: local file path, filename
:rtype: tuple(str, str)
'''
cache_dir = CONF.pakcage_cache_dir
# make dir if not exist
if not os.path.exists(cache_dir):
os.makedirs(cache_dir, exist_ok=True)
filename = url.rsplit('/', 1)[-1]
new_filename = hashlib.sha1(url.encode()).hexdigest() + '-' + filename
cached_file_path = os.path.join(cache_dir, new_filename)
with plugin_utils.lock(new_filename, timeout=300) as locked:
if locked:
if os.path.exists(cached_file_path):
LOG.info('using cache: %s for package: %s', cached_file_path, url)
else:
with tempfile.TemporaryDirectory() as download_path:
LOG.info('download from: %s for pakcage: %s', url, url)
filepath = download_from_url(download_path, url)
LOG.info('download complete')
os.rename(filepath, cached_file_path)
else:
raise OSError(_('failed to acquire lock, package cache may not be available'))
return cached_file_path, filename
def list(self, filters=None, orders=None, offset=None, limit=None, hooks=None):
results = []
service_name = filters.get('serviceName', '') or ''
if not service_name:
raise exceptions.FieldRequired(
message=_('missing query param: %(attribute)s, eg. /v1/api?%(attribute)s=value') %
{'attribute': 'serviceName'})
client = wecube.WeCubeClient(CONF.wecube.base_url)
# subsys_token = cache.get_or_create(TOKEN_KEY, client.login_subsystem, expires=600)
# client.token = subsys_token
key = '/platform/v1/plugins/interfaces/enabled'
cached = cache.get(key, 15)
if cache.validate(cached):
resp = cached
else:
resp = client.retrieve('/platform/v1/plugins/interfaces/enabled')
cache.set(key, resp)
for interface in resp['data']:
if interface['serviceName'] == service_name:
for param in interface['inputParameters']:
results.append({
'type': param['dataType'],
'name': param['name'],
'value': 'inputParams.' + param['name']
})
return results
def _before_update(self, rid, resource, validate):
super()._before_update(rid, resource, validate)
if 'content_field' in resource and not resource[
'content_field'] and 'endpoint_field' in resource and not resource[
'endpoint_field']:
raise exceptions.ValidationError(
attribute='content_field and endpoint_field',
msg=_('specify at least one field content'))
def __init__(self, url):
self.url = url
m = R_S3_ENDPOINT.match(url)
if m:
result = m.groupdict()
self.schema = result['schema']
self.host = result['host']
self.bucket = result['bucket']
self.object_key = result['object_key']
else:
raise ValueError(_('invalid s3 endpoint url, eg: schema://host[:port]/bucket/object'))
def validate(self, value):
choices = ['?', '+', '*']
if utils.is_string_type(value):
if value not in choices:
return _('expected %(choices)s, not %(value)s') % {
'choices': choices,
'value': value
}
elif isinstance(value, int):
if value < 1:
return _('value should be >= 1, not %(value)s') % {
'value': value
}
else:
return _(
'expected string in %(choices)s or int(>=1), not %(type)s '
) % {
'choices': choices,
'type': type(value).__name__
}
return True
def on_post(self, req, resp, **kwargs):
download_url = req.params.get('downloadUrl', None)
if not download_url:
raise exceptions.ValidationError(
message=_('missing query: downloadUrl'))
form = cgi.FieldStorage(fp=req.stream, environ=req.env)
resp.json = {
'code': 200,
'status': 'OK',
'data': self.upload(req, download_url, **kwargs),
'message': 'success'
}
def process_post(self, req, data, **kwargs):
result = {
'resultCode': '0',
'resultMessage': 'success',
'results': {
'outputs': []
}
}
is_item_error = False
error_indexes = []
try:
clean_data = crud.ColumnValidator.get_clean_data(
self._param_rules, data, 'check')
reqid = clean_data.get('requestId', None) or 'N/A'
operator = clean_data.get('operator', None) or 'N/A'
for idx, item in enumerate(clean_data['inputs']):
single_result = {
'callbackParameter': item.get('callbackParameter', None),
'errorCode': '0',
'errorMessage': 'success'
}
try:
validate_item_func = getattr(
self, 'validate_item_' + self._default_action,
self.validate_item)
clean_item = validate_item_func(idx, item)
process_func = getattr(self, self._default_action,
self.process)
process_result = process_func(reqid, operator, idx,
clean_item, **kwargs)
if process_result:
single_result.update(process_result)
result['results']['outputs'].append(single_result)
except Exception as e:
LOG.exception(e)
single_result['errorCode'] = '1'
single_result['errorMessage'] = str(e)
result['results']['outputs'].append(single_result)
is_item_error = True
error_indexes.append(str(idx + 1))
except Exception as e:
LOG.exception(e)
result['resultCode'] = '1'
result['resultMessage'] = str(e)
if is_item_error:
result['resultCode'] = '1'
result['resultMessage'] = _(
'Fail to process [%(num)s] record, detail error in the data block'
) % dict(num=','.join(error_indexes))
return result
def remove(self, data):
cluster_info = db_resource.Cluster().list({'name': data['cluster']})
if not cluster_info:
raise exceptions.ValidationError(attribute='cluster',
msg=_('name of cluster(%(name)s) not found' % {'name': data['cluster']}))
cluster_info = cluster_info[0]
k8s_auth = k8s.AuthToken(cluster_info['api_server'], cluster_info['token'])
k8s_client = k8s.Client(k8s_auth)
resource_name = api_utils.escape_name(data['name'])
exists_resource = k8s_client.get_service(resource_name, data['namespace'])
if exists_resource is not None:
k8s_client.delete_service(resource_name, data['namespace'])
# TODO: k8s为异步接口，是否需要等待真正执行完毕
return {'id': '', 'name': '', 'correlation_id': ''}
def _json_or_error(url, **kwargs):
try:
try:
return func(url, **kwargs)
except requests.ConnectionError as e:
LOG.error('http error: %s %s, reason: %s', func.__name__.upper(), url, str(e))
raise base_ex.CallBackError(message={
'code': 50002,
'title': _('Connection Error'),
'description': _('Failed to establish a new connection')
})
except requests.Timeout as e:
LOG.error('http error: %s %s, reason: %s', func.__name__.upper(), url, str(e))
raise base_ex.CallBackError(message={
'code': 50004,
'title': _('Timeout Error'),
'description': _('Server do not respond')
})
except requests.HTTPError as e:
LOG.error('http error: %s %s, reason: %s', func.__name__.upper(), url, str(e))
code = int(e.response.status_code)
message = RestfulJson.get_response_json(e.response, default={'code': code})
if code == 401:
raise base_ex.AuthError()
if code == 404:
message['title'] = _('Not Found')
message['description'] = _('The resource you request not exist')
# 如果后台返回的数据不符合要求，强行修正
if 'code' not in message:
message['code'] = code
if 'title' not in message:
message['title'] = message.get('title', e.response.reason)
if 'description' not in message:
message['description'] = message.get('message', str(e))
raise base_ex.CallBackError(message=message)
except Exception as e:
LOG.error('http error: %s %s, reason: %s', func.__name__.upper(), url, str(e))
message = RestfulJson.get_response_json(e.response,
default={
'code': 500,
'title': _('Server Error'),
'description': str(e)
})
if 'code' not in message:
message['code'] = 500
if 'title' not in message:
message['title'] = message.get('title', str(e))
if 'description' not in message:
message['description'] = message.get('message', str(e))
raise base_ex.CallBackError(message=message)
except base_ex.CallBackError as e:
raise exceptions.PluginError(message=e.message, error_code=e.code)
def _addtional_validation(self, resource):
if resource['effect_on'] == 'param' and resource['match_type'] not in (
'filter', ):
raise exceptions.ValidationError(
attribute='match_type',
msg=_(
'%(value)s is not acceptable, use %(fixed_value)s instead')
% {
'value': resource['match_type'],
'fixed_value': 'filter'
})
if resource['effect_on'] == 'script' and resource[
'match_type'] not in ('cli', 'sql', 'text', 'fulltext'):
raise exceptions.ValidationError(
attribute='match_type',
msg=_(
'%(value)s is not acceptable, use %(fixed_value)s instead')
% {
'value': resource['match_type'],
'fixed_value': ('cli', 'sql', 'text', 'fulltext')
})
if resource['effect_on'] == 'script' and resource[
'match_type'] == 'cli' and not resource['match_param_id']:
raise exceptions.FieldRequired(attribute='match_param_id')
def script_check(self, data, boxes=None):
'''run boxes rule check
:param data: see function check
:type data: dict
:param boxes: box ids
:type boxes: list
:return: see function check
:rtype: see function check
'''
# check box is enabled
filters = {'policy.enabled': 1, 'subject.enabled': 1, 'enabled': 1}
if boxes:
filters['id'] = boxes
refs = self.list(filters)
results = self.check(data, boxes=refs, without_subject_test=False)
associate_instances = data['entityInstances']
text_output = ''
if results:
table = texttable.Texttable(max_width=120)
# {
# 'lineno': [start, end], 'level': level of rule,
# 'content': content, 'message': rule name, 'script_name': script name
# }
table.set_cols_align(["l", "c", "l", "l", "l"])
table.set_cols_valign(["m", "m", "m", "m", "m"])
table.header([_("Instance Ids"), _("Line"), _("Content"), _("Message"), _('Source Script')])
for ret in results:
associate_ids = ','.join([(inst.get('displayName', '') or '#' + str(inst.get('id', '')))
for inst in associate_instances])
table.add_row([
associate_ids,
'%s-%s' % (ret['lineno'][0], ret['lineno'][1]), ret['content'], ret['message'], ret['script_name']
])
text_output = table.draw()
return {'text': text_output, 'data': results}
def convert_env(items):
# convert envs
rets = []
for idx, item in enumerate(items):
if 'valueRef' not in item:
item['valueRef'] = None
if 'valueFrom' not in item or not item['valueFrom']:
item['valueFrom'] = 'value'
if item['valueRef'] is None and item['valueFrom'] != 'value':
raise exceptions.ValidationError(
attribute='envs[%s]' % (idx + 1),
msg=_('valueRef is NULL, while valueFrom is %(value)s') %
{'value': item['valueFrom']})
if item['valueFrom'] == 'value':
rets.append({'name': item['name'], 'value': item['value']})
elif item['valueFrom'] == 'configMap':
rets.append({
'name': item['name'],
'valueFrom': {
'configMapKeyRef': {
'name': item['valueRef']['name'],
'key': item['valueRef']['value']
}
}
})
elif item['valueFrom'] == 'secretKey':
rets.append({
'name': item['name'],
'valueFrom': {
'secretKeyRef': {
'name': item['valueRef']['name'],
'key': item['valueRef']['value']
}
}
})
elif item['valueFrom'] == 'fieldRef':
rets.append({
'name': item['name'],
'valueFrom': {
'fieldRef': {
'fieldPath': item['valueRef']['name']
}
}
})
return rets
def on_patch(self, req, resp, **kwargs):
self._validate_method(req)
self._validate_data(req)
data = req.json
if data is not None and not isinstance(data, dict):
raise exceptions.PluginError(_('data must be dict type'))
ref_before, ref_after = self.update(req, data, **kwargs)
if ref_after is not None:
resp.json = {
'code': 200,
'status': 'OK',
'data': ref_after,
'message': 'success'
}
else:
raise exceptions.NotFoundError(
resource='%s[%s]' %
(self.resource.__name__, kwargs.get('rid', '-')))
def create(self, req, data):
result = {'resultCode': '0', 'resultMessage': 'success', 'results': {'outputs': []}}
is_error = False
error_indexes = []
try:
clean_data = crud.ColumnValidator.get_clean_data(self.param_rules, data, 'check')
operator = clean_data.get('operator', None) or 'N/A'
for idx, item in enumerate(clean_data['inputs']):
single_result = {
'callbackParameter': item.get('callbackParameter', None),
'errorCode': '0',
'errorMessage': 'success',
'guid': None,
'deploy_package_url': None
}
try:
clean_item = crud.ColumnValidator.get_clean_data(self.input_rules, item, 'check')
package = plugin_api.Package().create_from_image_name(clean_item['image_name'], clean_item['tag'],
clean_item.get('namespace', None),
clean_item.get('md5', None),
clean_item.get('nexus_url', None),
clean_item.get('connector_port', None),
clean_item['unit_design'],
clean_item.get('baseline_package',
None), operator)
single_result.update(package)
result['results']['outputs'].append(single_result)
except Exception as e:
single_result['errorCode'] = '1'
single_result['errorMessage'] = str(e)
result['results']['outputs'].append(single_result)
is_error = True
error_indexes.append(str(idx + 1))
except Exception as e:
result['resultCode'] = '1'
result['resultMessage'] = str(e)
if is_error:
result['resultCode'] = '1'
result['resultMessage'] = _('Fail to %(action)s [%(num)s] record, detail error in the data block') % dict(
action='process', num=','.join(error_indexes))
return result
def apply(self, data):
resource_id = data['correlation_id']
cluster_info = db_resource.Cluster().list({'name': data['cluster']})
if not cluster_info:
raise exceptions.ValidationError(attribute='cluster',
msg=_('name of cluster(%(name)s) not found' % {'name': data['cluster']}))
cluster_info = cluster_info[0]
k8s_auth = k8s.AuthToken(cluster_info['api_server'], cluster_info['token'])
k8s_client = k8s.Client(k8s_auth)
k8s_client.ensure_namespace(data['namespace'])
resource_name = api_utils.escape_name(data['name'])
exists_resource = k8s_client.get_deployment(resource_name, data['namespace'])
if exists_resource is None:
exists_resource = k8s_client.create_deployment(data['namespace'], self.to_resource(k8s_client, data))
else:
exists_resource = k8s_client.update_deployment(resource_name, data['namespace'],
self.to_resource(k8s_client, data))
# TODO: k8s为异步接口，是否需要等待真正执行完毕
return {
'id': exists_resource.metadata.uid,
'name': exists_resource.metadata.name,
'correlation_id': resource_id
}
def __call__(self, req, resp, package_name, entity_name, action_name):
server = CONF.wecube.server
token = req.auth_token
client = wecube.WeCubeClient(server, token)
url = '/platform/v1/packages/%(package_name)s/entities/%(entity_name)s/%(action_name)s' % {
'package_name': package_name,
'entity_name': entity_name,
'action_name': action_name,
}
if action_name == 'retrieve':
url = '/%(package_name)s/entities/%(entity_name)s/query' % {
'package_name': package_name,
'entity_name': entity_name
}
data = client.retrieve(url)
resp.json = {
'code': 200,
'status': 'OK',
'data': data['data'],
'message': 'success'
}
elif action_name == 'update':
data = client.update(url, req.json)
resp.json = {
'code': 200,
'status': 'OK',
'data': data['data'],
'message': 'success'
}
else:
raise exceptions.NotFoundError(
_('%(action_name)s for %(package_name)s:%(entity_name)s not supported'
) % {
'package_name': package_name,
'entity_name': entity_name,
'action_name': action_name,
})
def update(self, req, data, **kwargs):
operation = req.params.get('operation', None)
if not operation:
raise exceptions.ValidationError(message=_(
'missing query: operation, eg. ?operation=confirm/discard'))
return self.make_resource(req).update_state(data, operation)
def message_format(self):
return _(
'fail to %(action)s [%(num)s] record, detail error in the data block'
)
def title(self):
return _('Batch Operation Partial Error')
def message_format(self):
return _('the resource(%(resource)s) you request not found')
def message_format(self):
return _('column %(attribute)s validate failed, because: %(msg)s')
def title(self):
return _('Validation Error')