def request_with_bearer(self, *args, **kwargs):
sh.debug(" ".join(args))
response = self.bearer_session.request(*args, **kwargs)
if response.status_code == 401:
self.exchange_tokens()
return self.bearer_session.request(*args, **kwargs)
return response
def set_creds(self, rd, creds):
config = rd.get('credentialsConfiguration')
if config is not None:
cred_id = config['id']['value']
apply_creds(config, 'function', cred_id, creds)
else:
sh.debug(f'no credential config for {rd}')
def set_creds(self, rd, creds):
id = rd.get('credentialId')
if id is not None:
cred_id = id['value']
apply_creds(rd, 'snowflake', cred_id, creds)
staging_type = (set(self.staging_reg.keys())
& set(rd.keys())).pop()
staging_creds = self.staging_reg[staging_type](rd, staging_type)
staging_creds.set_creds(rd[staging_type], creds)
else:
sh.debug(f'no credentialId for {rd}')
def apply_creds(cont, typ, cred_id, creds):
sh.debug(f'apply cred: {cred_id}')
if cred_id in creds:
cred = creds[cred_id]
if cred.credential_type != typ:
raise ValueError(
f'Wrong type for {cred_id} '
f'(expected {typ}, but found {cred.credential_type})')
cont['credentials'] = cred.credential_value
else:
sh.debug(f'{cred_id} not found in creds')
def _set_creds(self, rd, typ, creds, staging=False):
cred_id = rd.get('credentialId', None)
if cred_id is not None:
v = cred_id['value']
apply_creds(rd, typ, v, creds)
if not staging and self.staging_type() is not None:
self._set_creds(rd['stagingContainer'],
self.staging_type(),
creds,
staging=True)
else:
sh.debug(f'no credentialId for {rd}')
def get(self, path_str, options):
res_path = ResourcePath.from_arg(path_str, self)
if not res_path:
raise ValueError(f'{path_str} is not exportable!')
if res_path.rd_path[0] is None:
raise ValueError('Cannot get root path!')
sh.debug(f'origin: {res_path}')
creds_snippet = {}
self._get(res_path, res_path, options, creds_snippet)
if options.output is not None:
sys.stdout.write(
ascend.credentials.dump_credentials(creds_snippet))
def get_dump_target(self, origin_path: 'ResourcePath',
options) -> Optional[str]:
if options.directory is None:
return options.output
base_path = options.output
dirs, t = os.path.split(base_path)
if t == "" and origin_path.rd_path[0] is not None:
t = origin_path.resource.resource_id
suff = self.dir_path(origin_path.rd_path, t) + '.yaml'
target = os.path.join(dirs, suff)
sh.debug(f'target for {self.path}: {target}')
os.makedirs(os.path.split(target)[0], exist_ok=True)
return target
def resolve_path(arg_path, base_path, res_path) -> str:
'''
e.g.
arg = new_resource
base_path = ('ds_id', 'df_id', None)
res_path = ('ds_id', 'df_id', 'comp_id')
returns: new_resource/comp_id
:param path string
:param base resource path
:param target resource path
:return:
'''
assert len(filter_none(base_path)) <= len(filter_none(res_path))
if base_path == res_path:
return arg_path
strip_length = len(filter_none(base_path))
sh.debug(res_path)
overrides = '/'.join(filter_none(res_path[strip_length:]))
return os.path.join(arg_path, overrides)
def make_resource(self, json: dict, rs: 'ResourceSession') -> Resource:
input = rs.get_resource(self.data_service_id, self.dataflow_id,
self.input_id)
json['inputType'] = input.api_type
json['inputUUID'] = input.uuid
proto = resource_pb2.DataFeed()
ParseDict(self.rd, proto)
if proto.group_id != '':
group_path = (self.data_service_id, self.dataflow_id,
proto.group_id)
else:
group_path = None
rs.update_group(group_path, self.path)
if proto.sharing.all:
json['open'] = True
json['pubToRoles'] = ""
else:
json['open'] = False
ds_ids = proto.sharing.data_services
shared_roles = []
for ds_id in ds_ids:
try:
orgUUID = rs.get_ds(ds_id).uuid
except KeyError:
# do not have this org
sh.debug(f'cannot get uuid for {ds_id}')
continue
everyone_role = next(
r for r in rs.roles
if r['orgId'] == orgUUID and r['id'] == 'Everyone')
shared_roles.append(everyone_role['uuid'])
if not proto.sharing.hidden:
host_ds = rs.get_resource(self.data_service_id, None, None)
everyone_role = next(
r for r in rs.roles
if r['orgId'] == host_ds.uuid and r['id'] == 'Everyone')
shared_roles.append(everyone_role['uuid'])
json['pubToRoles'] = ','.join(shared_roles)
return model.DataFeed(self.data_service_id, self.dataflow_id,
self.resource_id, json, rs.session)
def from_resource_proto(
m: resource_pb2.Resource, override_path=(None, None, None)
) -> 'ResourceDefinition':
if m.version > global_values.API_VERSION:
raise ValueError(f'Version out of date (found {m.version}, '
f'running {global_values.API_VERSION})')
sh.debug(m)
t = m.WhichOneof('type')
typed_resource = getattr(m, t)
if isinstance(typed_resource, resource_pb2.DataService):
resource = DataServiceDef(m, override_path)
elif isinstance(typed_resource, resource_pb2.Dataflow):
resource = DataflowDef(m, override_path)
elif isinstance(typed_resource, resource_pb2.Component):
resource = ComponentDef(m, override_path)
elif isinstance(typed_resource, resource_pb2.Group):
resource = GroupDef(m, override_path)
elif isinstance(typed_resource, resource_pb2.DataFeed):
resource = DataFeedDef(m, override_path)
else:
raise KeyError(f"Unrecognizable resource {m}")
return resource
def apply(self, rs: 'ResourceSession') -> Resource:
try:
old = rs.get_resource(*self.path)
prev_uuids = [i['uuid'] for i in old.json_definition['content']]
except KeyError:
prev_uuids = []
sh.debug(f'prev: {prev_uuids}')
cur_rds = rs.group_path_to_content_path.get(self.path, [])
cur_uuids = [rs.get_resource(*p).uuid for p in cur_rds]
sh.debug(f'cur: {cur_uuids}')
ress = [rs.uuid_to_resource[uuid] for uuid in cur_uuids]
json = {}
json['id'] = self.resource_id
json['name'] = self.rd['name']
json['description'] = self.rd.get('description', '')
json['content'] = [{
'uuid': res.uuid,
'type': res.api_type
} for res in ress]
sh.debug(json)
res = model.Group(self.data_service_id, self.dataflow_id,
self.resource_id, json, rs.session)
res.apply()
return res
def context_gen(self, args):
global_values.DEBUG = args.debug
sh.debug(f'DEBUG = {args.debug}')
yield
def apply(self, res_path_str, creds, options):
# load resource defs from directory: path_to_def
self.path_to_def = {}
# need to handle case where there is no resource in api for path
rd_path = ResourcePath.rd_from_arg(res_path_str)
self.load_defs(rd_path, options)
marked_paths = {rd_path}
g = nx.DiGraph()
for path, res_def in self.path_to_def.items():
marked_paths.add(path)
g.add_node(path)
for dep in res_def.dependencies():
g.add_edge(path, dep)
for dep in res_def.dependees():
g.add_edge(dep, path)
groups = set()
for path in g.nodes:
if path not in self.path_to_def:
# just load info from api to supply a dependency, do not need to apply
try:
self.get_resource(*path)
except Exception as e:
raise KeyError(f'Unable to load dependency {path}') from e
for path in reversed(list(nx.topological_sort(g))):
if path not in self.path_to_def:
continue
else:
res_def = self.path_to_def[path]
if isinstance(res_def, GroupDef):
# groups need to go last
groups.add(res_def)
continue
for transform in res_def.transforms:
transform.to_api(creds=creds)
sh.info(f'APPLY: {res_def.path}')
sh.debug(res_def.resource)
if not options.dry_run:
res = res_def.apply(self)
self.path_to_uuid[path] = res.uuid
self.uuid_to_resource[res.uuid] = res
else:
try:
res = self.get_resource(*res_def.path)
self.path_to_uuid[path] = res.uuid
self.uuid_to_resource[res.uuid] = res
except KeyError:
sh.info(f'Unable to get {res_def.path} during dry run')
group_paths = self.dirty_groups | {g.path for g in groups}
for group_path in group_paths:
group_def = self.path_to_def.get(group_path)
if group_def is None:
# Not creating, must already exist
group_uuid = self.get_resource(*group_path).uuid
if group_uuid is None:
raise KeyError(f'Unable to find group {group_path}')
group_res = self.uuid_to_resource[group_uuid]
group_def = ResourceDefinition.from_resource_proto(
group_res.to_resource_proto(self, options),
(group_res.data_service_id, group_res.dataflow_id,
group_res.resource_id))
sh.info(f'APPLY: {group_def.path}')
if not options.dry_run:
group_def.apply(self)
if options.delete and options.recursive:
for child in ResourcePath.from_arg(res_path_str,
self).children(self):
if child.exportable and child.rd_path not in marked_paths:
self._delete(child, options)