def get_local(self, local_item: Item, overwrite=False):
if overwrite or local_item.content is None:
local_item.content = Local.load_json(local_item.path)
for attribute in self._c.conf.jobs.strip_attributes:
local_item.content.pop(attribute, None)
c = local_item.content
# apply default values
c['timeout_seconds'] = c.get('timeout_seconds', 0)
# find the cluster
if c.get('existing_cluster_name') and self._clusters:
ec = self._clusters.get_single_item(c['existing_cluster_name'])
assert ec is not None, f'Cluster "{c["existing_cluster_name"]}", ' \
f'referenced in job "{c["name"]}" not found'
c['existing_cluster_id'] = ec.path
c.pop('existing_cluster_name', None)
# find the right notebook
notebook_path = c.get('notebook_task', {}).get('notebook_path')
if notebook_path:
remote_notebook_path = self._workspace.find_notebook(
notebook_path)
assert remote_notebook_path is not None, \
f'Notebook "{notebook_path}" referenced in job "{c["name"]}" not found'
c['notebook_task']['notebook_path'] = remote_notebook_path
c['name'] = self.remote_path(c['name'])
def get_local(self, local_item: Item, overwrite=False):
if overwrite or local_item.content is None:
local_item.content = Local.load_json(local_item.path)
for attribute in self._c.conf.instance_pools.strip_attributes:
local_item.content.pop(attribute, None)
local_item.content['instance_pool_name'] = self.remote_path(
local_item.content['instance_pool_name'])
def _get_remote(self, remote_item: Item, overwrite=False):
if overwrite or remote_item.content is None:
response = self._c.api.call(Endpoints.workspace_export,
body={
'path': remote_item.path,
'format': 'SOURCE'
})
remote_item.content = base64.b64decode(response.json()['content'])
def get_local(self, local_item: Item, overwrite=False):
if overwrite or local_item.content is None:
local_item.content = Local.load_json(local_item.path)
for attribute in self._c.conf.clusters.strip_attributes:
local_item.content.pop(attribute, None)
c = local_item.content
if c.get('instance_pool_name') and self._instance_pools:
ip = self._instance_pools.get_single_item(
c['instance_pool_name'])
assert ip is not None, f'Instance pool "{c["instance_pool_name"]}", ' \
f'referenced in cluster "{c["cluster_name"]}" not found'
c['instance_pool_id'] = ip.path
c.pop('instance_pool_name', None)
c['cluster_name'] = self.remote_path(c['cluster_name'])
def get_local(local_item: Item, overwrite=False):
if overwrite or local_item.content is None:
local_item.content = Local.load_binary(local_item.path)
def _get_remote(self, remote_item: Item, overwrite=False):
if overwrite or remote_item.content is None:
response = self._c.api.call(Endpoints.dbfs_read,
body={'path': remote_item.path})
remote_item.content = base64.b64decode(response.json()['content'])