def contained(self):
res_defs = []
for dataflow in self.rd.get('dataflows', []):
m = resource_pb2.Resource()
d = {'dataflow': dataflow}
ParseDict(d, m)
res_defs.append(
ResourceDefinition.from_resource_proto(m, self.path))
for data_feed in self.rd.get('dataFeeds', []):
m = resource_pb2.Resource()
d = {'dataFeed': data_feed}
ParseDict(d, m)
res_defs.append(
ResourceDefinition.from_resource_proto(m, self.path))
return res_defs
def contained(self):
res_defs = []
for component in self.rd.get('components', []):
m = resource_pb2.Resource()
d = {'component': component}
ParseDict(d, m)
res_defs.append(
ResourceDefinition.from_resource_proto(m, self.path))
for group in self.rd.get('groups', []):
m = resource_pb2.Resource()
d = {'group': group}
ParseDict(d, m)
res_defs.append(
ResourceDefinition.from_resource_proto(m, self.path))
return res_defs
def setUp(self):
# import ascend.cli.global_values as globals
# globals.DEBUG = True
self.maxDiff = None
self.creds = credentials.load_credentials(self.creds_file)
with open(self.comps_file) as f:
raw = yaml.load(f.read(), Loader=yaml.SafeLoader)
proto = resource_pb2.Resource()
ParseDict(raw, proto)
res_def = resource_definitions.ResourceDefinition.from_resource_proto(
proto, ('df', 'ds', None))
self.components = {
comp.resource_id: comp
for comp in res_def.contained()
}
for k in self.components.keys():
name, suff = k.split('.')
if suff == 'before':
assert f'{name}.after' in self.components
elif suff == 'after':
assert f'{name}.before' in self.components
elif suff == 'none':
continue
else:
raise ValueError(f'Unexpected suffix {suff}')
def delete_deps(self):
result = set()
for df_rd in self.rd.get('dataFeeds', []):
m = resource_pb2.Resource()
d = {'dataFeed': df_rd}
ParseDict(d, m)
data_feed = ResourceDefinition.from_resource_proto(m, self.path)
result.add((data_feed.data_service_id, None, None))
return list(result - set([(self.resource_id, None, None)]))
def delete_deps(self):
result = set()
for comp_rd in self.rd.get('components', []):
m = resource_pb2.Resource()
d = {'component': comp_rd}
ParseDict(d, m)
component = ResourceDefinition.from_resource_proto(m, self.path)
result |= set(component.dependencies())
return list(result -
set([(self.data_service_id, self.resource_id, None)]))
def to_resource_proto(self, rs: 'ResourceSession',
options) -> 'resource_pb2.Resource':
d = {
'version': global_values.API_VERSION,
}
self.set_boilerplate(d)
type_message = self.to_type_proto(rs, options)
message = resource_pb2.Resource(**{
**d,
**{
self.resource_type: type_message
}
})
return message
def load_defs_recur(self, origin, path, rd_path, options):
p = path
if os.path.isdir(path):
p = os.path.join(path, '__metadata__.yaml')
if not os.path.exists(p):
if origin == path:
raise ValueError(f"Path {p} does not exist!")
else:
# if we are not at the top level, we will skip
# directories missing a metadata file
return
_, ext = os.path.splitext(p)
if ext != '.yaml':
# only load yaml files
if origin == path:
raise ValueError(
f"Resource definitions must be yaml files (found {p})")
return
ascend_dir, _ = os.path.split(p)
try:
with open(p, 'r') as f:
raw_template = f.read()
env = jinja.setup_jinja_env(
loader=jinja2.FileSystemLoader(ascend_dir))
data = env.from_string(raw_template).render(
config=options.config, resourceParentDirectory=ascend_dir)
res_proto = resource_pb2.Resource()
d = yaml.load(data, Loader=yaml.SafeLoader)
ParseDict(d, res_proto)
except Exception as e:
raise Exception(
f"Unable to load resource definition from {p}") from e
res_def = ResourceDefinition.from_resource_proto(res_proto, rd_path)
self.path_to_def[res_def.path] = res_def
if options.recursive:
self.load_contained_defs(res_def)
if os.path.isdir(path):
child_paths = os.listdir(path)
for child_path in child_paths:
if child_path == '__metadata__.yaml':
continue
name, _ = os.path.splitext(child_path)
self.load_defs_recur(origin,
os.path.join(path, child_path),
extend_rd(rd_path, name), options)