def __init__(cls, name, bases, attrs):
# Initialize the data store of the class, if necessary
cls._data = getattr(cls, "_data", dict())
# Retrieve and process attribute names
fields = getattr(cls, "_FIELDS", dict())
_fields_read_only = getattr(cls, "_FIELDS_READ_ONLY", list())
_fields_required = getattr(cls, "_FIELDS_REQUIRED", list())
# Post-process _FIELDS list/dictionary
if isinstance(fields, list):
fields = {key: (str, None) for key in fields}
setattr(
cls, "_pre_save_hook", lambda self: self._cache and map(
lambda obj: obj.save(), self._cache.values()))
setattr(cls, "_pre_save_hook",
lambda self: APIResourceMetaclass.__pre_save_hook(self))
# Define special getter for the ID field
setattr(
cls, "id",
property(fget=APIResourceMetaclass.__getid,
doc="The API-provided ID of the instantiated resource."))
for field_name in fields:
field_type = fields[field_name][0]
field_doc = fields[field_name][1]
setattr(
cls, field_name,
APIResourceMetaclass.__mk_property(cls,
field_name=field_name,
field_type=field_type,
field_doc=field_doc))
if _forge:
# Only works in Python 3.5+ which has
# the python-forge package
if _crud.CreatableAPIResource in bases:
cls.create = _forge.sign(
*APIResourceMetaclass._build_signature(
obj=cls, all_optional=False, with_id=False))(
cls.create)
cls.saveInstanceAsNew = _forge.sign(
*APIResourceMetaclass._build_signature(
obj=cls, all_optional=True, with_id=False))(
cls.duplicate)
if _crud.UpdatableAPIResource in bases:
cls.update = _forge.sign(
*APIResourceMetaclass._build_signature(
obj=cls, all_optional=True, with_id=True))(cls.update)
cls.saveInstance = _forge.sign(
*APIResourceMetaclass._build_signature(
obj=cls, all_optional=True, with_id=False))(cls.save)
def _eval_lambda(self, node):
sig_list, sig_dict = self._eval(node.args)
_class = self.__class__
def _func(*args, **kwargs):
local_scope = {
inspect.getfullargspec(_func).varargs: args,
**{
kwo: kwargs.pop(kwo)
for kwo in inspect.getfullargspec(_func).kwonlyargs + inspect.getfullargspec(_func).args
},
inspect.getfullargspec(_func).varkw: kwargs,
}
s = _class(modules=self.modules,
scope=copy(self.scope),
call_stack=self.call_stack)
s.scope.push(local_scope)
s.expr = self.expr
try:
return s._eval(node.body)
finally:
self.track(s)
# prevent unwrap from detecting this nested function
# del _func.__wrapped__
_func.__name__ = "<lambda>"
_func.__qualname__ = "<lambda>"
_func = forge.sign(*sig_list, **sig_dict)(_func)
return _func
def sign_defaults(symbol, expr, composition):
'''gets defaults from an expression using composition'''
defaults = {}
for f_ in get_undefined_funcs(expr):
# includes {h(f(g)), f(g)}
if str(f_) in composition:
# ignores h(f(g))
f_defaults = get_defaults(composition[str(f_)])
# flatten defaults
for arg, arg_default in f_defaults.items():
defaults[arg] = arg_default
arg_signatures = []
# defaults have to go last, which may conflict with user's ordering
for arg in symbol.args:
str_arg = str(arg)
if str_arg in defaults:
arg_default = defaults[str(arg)]
arg_signatures.append(forge.arg(str_arg, default=arg_default))
else:
arg_signatures.append(forge.arg(str_arg))
# will raise an error if defaults are not last
signature = forge.sign(*arg_signatures)
return signature
def copy_signatures(
target_function: Callable,
template_functions: List[TemplateFunction],
exclude_args: Iterable[str] = None,
) -> Callable:
"""A decorator that copies function signatures from one or more template functions to a
target function.
Args:
target_function: Function to modify
template_functions: Functions containing params to apply to ``target_function``
"""
# Start with 'self' parameter if this is an instance method
fparams = {}
if 'self' in signature(target_function).parameters or ismethod(
target_function):
fparams['self'] = forge.self
# Add and combine parameters from all template functions, excluding duplicates, self, and *args
for func in template_functions:
new_fparams = {
k: v
for k, v in forge.copy(func).signature.parameters.items()
if k != 'self' and v.kind != Parameter.VAR_POSITIONAL
}
fparams.update(new_fparams)
# Manually remove any excluded parameters
for key in ensure_list(exclude_args):
fparams.pop(key, None)
fparams = deduplicate_var_kwargs(fparams)
revision = forge.sign(*fparams.values())
return revision(target_function)
def get_combined_revision(*functions: Callable):
"""Combine the parameters of all revisions into a single revision"""
import forge
params = {}
for func in functions:
params.update(forge.copy(func).signature.parameters)
params = deduplicate_kwargs(params)
return forge.sign(*params.values())
def _get_combined_revision(template_functions: List[TemplateFunction]):
""" Create a :py:class:`forge.Revision` from the combined parameters of multiple functions """
import forge
# Use forge.copy to create a revision for each template function
revisions = [forge.copy(func) for func in template_functions]
# Combine the parameters of all revisions into a single revision
fparams = [list(rev.signature.parameters.values()) for rev in revisions]
return forge.sign(*list(chain.from_iterable(fparams)))
def _eval_functiondef(self, node):
sig_list, sig_dict = self._eval(node.args)
_annotations = {
a.arg: self._eval(a.annotation)
for a in node.args.args +
getattr(node.args, "posonlyargs", [None]) # for backwards compat
+ getattr(node.args, "kwonlyargs", [None]) # for backwards compat
+ [node.args.kwarg] # is a single element
if a and a.annotation
}
_class = self.__class__
def _func(*args, **kwargs):
# reconostruct what the orignial function arguments would have been
local_scope = {
inspect.getfullargspec(_func).varargs: args,
**{
kwo: kwargs.pop(kwo)
for kwo in inspect.getfullargspec(_func).kwonlyargs + inspect.getfullargspec(_func).args
},
inspect.getfullargspec(_func).varkw: kwargs,
}
s = _class(modules=self.modules,
scope=copy(self.scope),
call_stack=self.call_stack)
s.scope.push(local_scope)
s.expr = self.expr
for b in node.body:
try:
s._eval(b)
except Return as r:
return r.value
finally:
self.track(s)
_func.__name__ = node.name
_func.__annotations__ = _annotations
_func.__qualname__ = node.name
_func = forge.sign(*sig_list, **sig_dict)(_func)
# prevent unwrap from detecting this nested function
del _func.__wrapped__
_func.__doc__ = ast.get_docstring(node)
decorated_func = _func
decorators = [self._eval(d) for d in node.decorator_list]
for decorator in decorators[::-1]:
decorated_func = decorator(decorated_func)
self.scope[node.name] = decorated_func
def combine(functions: List[Callable]):
args_list = []
funcs = []
combined_args = {}
for func in functions:
if not callable(func):
raise TypeError("Arguments must be callable")
signature = get_typed_signature(func)
signature_params = signature.parameters
func_arg_names = set({})
for param_name, param in signature_params.items():
param_field = get_param_field(param=param,
param_name=param_name,
default_field_info=param.default)
arg = forge.arg(name=param_field.name,
type=param_field.type_,
default=param.default)
if param_name in combined_args:
current_arg = combined_args[param_name]
if str(arg) != str(current_arg):
raise TypeError(
f"{arg} and {current_arg} are incompatible.")
else:
combined_args[param_name] = arg
func_arg_names.add(param_name)
args_list.append(arg)
funcs.append((func, func_arg_names))
def combined_functions(*args, **kwargs):
result = []
for (func, arg_names) in funcs:
func_kwargs = dict((k, kwargs[k]) for k in arg_names)
result.append(func(*args, **func_kwargs))
return result
new_args = tuple(combined_args.values())
return forge.sign(*new_args)(combined_functions)
def _initialize(self, instance):
self.request = instance.request
available_parameters = {
# Json body is only available for POST/PUT
('create', 'update'):
self._params['body'],
# Query parameter is only available for GET/DELETE
('get', 'list', 'delete'):
self._params['query'],
}
for method, path in self._paths.items():
# Path parameter would be postional argument
sigs = [forge.pos(f) for f in path.fields]
for methods, params in available_parameters.items():
if method in methods:
# Query/Body parameter would be keyword only argument
sigs.extend(
forge.kwo(
param.name, default=param.default, type=param.type)
for param in params)
# Any other keyword argument would be matched to 'kwargs'
sigs.append(forge.vkw('kwargs'))
setattr(self, method, forge.sign(*sigs)(getattr(self, method)))
for method in list(self.methods):
def unsupported(*args, **kwargs):
raise NotImplementedError(f"'{self._name}' has no method "
f"what you called")
if method not in self._paths:
setattr(self, method, unsupported)
self.methods.remove(method)
def get_sdk_node(
self,
pipeline_context,
instance,
pipeline_run,
step_key,
task_type=constants.SdkTaskType.PYTHON_TASK,
cache_version="",
retries=0,
interruptible=False,
deprecated="",
storage_request=None,
cpu_request=None,
gpu_request=None,
memory_request=None,
storage_limit=None,
cpu_limit=None,
gpu_limit=None,
memory_limit=None,
cache=False,
timeout=datetime.timedelta(seconds=0),
environment=None,
):
execution_step = self.execution_plan.get_step_by_key(step_key)
flyte_inputs = self.flyte_inputs(execution_step.step_input_dict,
execution_step.solid_name)
flyte_outputs = self.flyte_outputs(execution_step.step_output_dict,
execution_step.solid_name)
def wrapper(wf_params, *args, **kwargs): # pylint: disable=unused-argument
# TODO: We can't update config values via inputs from Flyte, because they are immutable
plan = self.execution_plan.build_subset_plan([step_key])
for param, arg in kwargs.items():
self.inject_intermediates(pipeline_context, execution_step,
param, arg)
results = list(
execute_plan(
plan,
instance,
run_config=self.run_config,
pipeline_run=pipeline_run,
))
for result in results:
step_context = pipeline_context.for_step(execution_step)
self.output_value(step_context, step_key, result,
execution_step, kwargs)
# This will take the wrapper definition and re-create it with explicit parameters as keyword argumentss
wrapper = forge.sign(forge.arg("wf_params"),
*map(forge.arg, flyte_inputs.keys()),
*map(forge.arg, flyte_outputs.keys()))(wrapper)
# flytekit uses this name for an internal representation, make it unique to the step key
wrapper.__name__ = execution_step.solid_name
task = sdk_runnable.SdkRunnableTask(
task_function=wrapper,
task_type=task_type,
discovery_version=cache_version,
retries=retries,
interruptible=interruptible,
deprecated=deprecated,
storage_request=storage_request,
cpu_request=cpu_request,
gpu_request=gpu_request,
memory_request=memory_request,
storage_limit=storage_limit,
cpu_limit=cpu_limit,
gpu_limit=gpu_limit,
memory_limit=memory_limit,
discoverable=cache,
timeout=timeout,
environment=environment,
custom={},
)
if flyte_inputs:
task = inputs(task, **flyte_inputs)
if flyte_outputs:
task = outputs(task, **flyte_outputs)
return task
def get_combined_revision(functions: Iterable[Callable]) -> forge.Revision:
"""Combine the parameters of all revisions into a single revision"""
params = {}
for func in functions:
params.update(forge.copy(func).signature.parameters)
return forge.sign(*params.values())
def create_method(client, params, body, method_info):
_sig_url_params = []
_sig_params_req = []
_sig_params_opt = []
_sig_body_req = []
_sig_body_opt = []
for _key, _url_param in method_info['url_parameters'].items():
_sig_url_params.append(forge.arg(name=_key, type=_url_param['type']))
if params['data']:
for _key, _param_item in params['data'].items():
if _param_item['required']:
_sig_params_req.append(
forge.arg(name=_key, type=_param_item['type']))
else:
_sig_params_opt.append(
forge.kwo(name=_key,
type=_param_item['type'],
default=None))
if body['data']:
for _key, _body_item in body['data'].items():
if _body_item['required']:
_sig_body_req.append(
forge.arg(name=_key, type=_body_item['type']))
else:
_sig_body_opt.append(
forge.kwo(name=_key, type=_body_item['type'],
default=None))
async def inner_method(self, **kwargs):
# Verify signature
endpoint_signature = {}
for url_param, url_param_info in method_info['url_parameters'].items():
if url_param not in kwargs:
raise SyntaxError(
f'Argument {url_param!r} is a required keyword argument.')
if 'validator' in url_param_info:
tested = url_param_info['validator'](kwargs[url_param])
if not tested:
# TODO: less generic error messages by abstracting this to the endpoint data structure
raise ValueError(
f'Supplied argument {url_param!r} failed to validate')
endpoint_signature[url_param] = kwargs[url_param]
endpoint = method_info['endpoint'].format(**endpoint_signature)
# Verify params
# TODO: DRY
params_signature = []
for param in params['required']:
item = None
if param not in kwargs:
if 'default' not in params['data'][param]:
raise SyntaxError(
f'Parameter {param!r} is a required keyword argument')
else:
item = params['data'][param]
if not item:
item = kwargs[param]
if 'validator' in params['data'][param]:
tested = params['data'][param]['validator'](item)
if not tested:
# TODO: less generic error messages by abstracting this to the endpoint data structure
raise ValueError(
f'Supplied argument {param!r} failed to validate')
if '_internal_name' in params['data'][param]:
params_signature.append(
(params['data'][param]['_internal_name'],
format_parameter_value(item)))
else:
params_signature.append((param, format_parameter_value(item)))
for param in params['optional']:
if param in kwargs and kwargs[param] is not None:
if 'validator' in params['data'][param]:
tested = params['data'][param]['validator'](kwargs[param])
if not tested:
# TODO: less generic error messages by abstracting this to the endpoint data structure
raise ValueError(
f'Supplied argument {param!r} failed to validate')
if '_internal_name' in params['data'][param]:
params_signature.append(
(params['data'][param]['_internal_name'],
format_parameter_value(kwargs[param])))
else:
params_signature.append(
(param, format_parameter_value(kwargs[param])))
# Verify body
# TODO: DRY
body_signature = {}
for key in body['required']:
item = None
if key not in kwargs:
if 'default' not in body['data'][key]:
raise SyntaxError(
f'Body key {key!r} is a required keyword argument')
else:
item = body['data'][key]['default']
if not item:
item = kwargs[key]
if 'validator' in body['data'][key]:
tested = body['data'][key]['validator'](item)
if not tested:
# TODO: less generic error messages by abstracting this to the endpoint data structure
raise ValueError(
f'Supplied argument {key!r} failed to validate')
if '_internal_name' in body['data'][key]:
body_signature[body['data'][key]['_internal_name']] = item if method_info['body_type'] == 'json' else \
format_parameter_value(item)
else:
body_signature[key] = item if method_info['body_type'] == 'json' else \
format_parameter_value(item)
for key in body['optional']:
if key in kwargs and kwargs[key] is not None:
if 'validator' in body['data'][key]:
tested = body['data'][key]['validator'](kwargs[key])
if not tested:
# TODO: less generic error messages by abstracting this to the endpoint data structure
raise ValueError(
f'Supplied argument {key!r} failed to validate')
if '_internal_name' in body['data'][key]:
body_signature[body['data'][key]['_internal_name']] = kwargs[key] if \
method_info['body_type'] == 'json' else \
format_parameter_value(kwargs[key])
else:
body_signature[key] = kwargs[key] if method_info['body_type'] == 'json' else \
format_parameter_value(kwargs[key])
if method_info['http_method'] in ['POST', 'PUT', 'PATCH']:
# Have a body
if method_info['body_type'] == 'kv':
resp = await self.signed_request(
method_info['http_method'],
endpoint,
params=params_signature,
data=body_signature,
headers={'content-type': method_info['content_type']})
elif method_info['body_type'] == 'json':
resp = await self.signed_request(
method_info['http_method'],
endpoint,
params=params_signature,
json=body_signature,
headers={'content-type': method_info['content_type']})
else:
raise RuntimeError(
f'Unknown body type {method_info["body_type"]!r} in method {method_info["method_name"]!r}'
)
elif method_info['http_method'] in ['GET', 'DELETE']:
# No body for these methods, nor a specific content-type
resp = await self.signed_request(method_info['http_method'],
endpoint,
params=params_signature)
else:
raise NotImplementedError(
f'Unsupported HTTP verb {method_info["http_method"]!r}.')
return resp
revised = forge.sign(
forge.pos(name='self'),
*_sig_url_params,
*_sig_params_req,
*_sig_body_req,
*_sig_params_opt,
*_sig_body_opt,
)(inner_method)
revised.__name__ = method_info['method_name']
revised.__doc__ = generate_docstring(params, body, method_info)
setattr(client, method_info['method_name'], revised)