def get_local(self, local_item: Item, overwrite=False):
if overwrite or local_item.content is None:
local_item.content = Local.load_json(local_item.path)
for attribute in self._c.conf.jobs.strip_attributes:
local_item.content.pop(attribute, None)
c = local_item.content
# apply default values
c['timeout_seconds'] = c.get('timeout_seconds', 0)
# find the cluster
if c.get('existing_cluster_name') and self._clusters:
ec = self._clusters.get_single_item(c['existing_cluster_name'])
assert ec is not None, f'Cluster "{c["existing_cluster_name"]}", ' \
f'referenced in job "{c["name"]}" not found'
c['existing_cluster_id'] = ec.path
c.pop('existing_cluster_name', None)
# find the right notebook
notebook_path = c.get('notebook_task', {}).get('notebook_path')
if notebook_path:
remote_notebook_path = self._workspace.find_notebook(
notebook_path)
assert remote_notebook_path is not None, \
f'Notebook "{notebook_path}" referenced in job "{c["name"]}" not found'
c['notebook_task']['notebook_path'] = remote_notebook_path
c['name'] = self.remote_path(c['name'])
def validate_cli(**kwargs):
display_log(logging.DEBUG if kwargs['verbose'] else logging.INFO)
conf = Conf({
'global': {'local_path': kwargs['local_path']},
},
kwargs['config_file'])
_log.info('''databricks-cicd validate initialized. Current configuration:
---------------------------------------------------------------------------------------------------------
%s
---------------------------------------------------------------------------------------------------------''', conf)
context = Context(conf)
workspace = helpers.WorkspaceHelper(context)
if conf.workspace.deploy:
_log.info('Validating workspace...')
for i in workspace.local_items:
local_item = workspace.local_items[i] # type: Item
# TODO: check if references to other notebooks are valid
instance_pools = helpers.InstancePoolsHelper(context)
if conf.instance_pools.deploy:
_log.info('Validating instance pools...')
for i in instance_pools.local_items:
local_item = instance_pools.local_items[i] # type: Item
instance_pools.get_local(local_item)
name_attribute = 'instance_pool_name'
if local_item.content.get(name_attribute) != Local.get_file_name(local_item.path):
_log.error('%s')
print(local_item.content)
_log.info('All done!')
def get_local(self, local_item: Item, overwrite=False):
if overwrite or local_item.content is None:
local_item.content = Local.load_json(local_item.path)
for attribute in self._c.conf.instance_pools.strip_attributes:
local_item.content.pop(attribute, None)
local_item.content['instance_pool_name'] = self.remote_path(
local_item.content['instance_pool_name'])
def get_local(self, local_item: Item, overwrite=False):
if overwrite or local_item.content is None:
local_item.content = Local.load_json(local_item.path)
for attribute in self._c.conf.clusters.strip_attributes:
local_item.content.pop(attribute, None)
c = local_item.content
if c.get('instance_pool_name') and self._instance_pools:
ip = self._instance_pools.get_single_item(
c['instance_pool_name'])
assert ip is not None, f'Instance pool "{c["instance_pool_name"]}", ' \
f'referenced in cluster "{c["cluster_name"]}" not found'
c['instance_pool_id'] = ip.path
c.pop('instance_pool_name', None)
c['cluster_name'] = self.remote_path(c['cluster_name'])
def get_local(local_item: Item, overwrite=False):
if overwrite or local_item.content is None:
local_item.content = Local.load_binary(local_item.path)
def _ls_local(self):
self.local_items = Local.dbfs_ls(
op.join(self._c.conf.local_path, self._c.conf.dbfs.local_sub_dir))
def _ls_local(self):
self.local_items = Local.files_ls(
op.join(self._c.conf.local_path, self._c.conf.jobs.local_sub_dir),
['.json'], 'job')
def _ls_local(self):
self.local_items = Local.files_ls(
op.join(self._c.conf.local_path,
self._c.conf.instance_pools.local_sub_dir), ['.json'],
'instance pool')
def _ls_local(self):
self.local_items = Local.workspace_ls(
op.join(self._c.conf.local_path,
self._c.conf.workspace.local_sub_dir))