def mkdirs_cli(api_client, dbfs_path):
"""
Make directories in DBFS.
Mkdirs will create directories along the path to the argument directory.
"""
DbfsApi(api_client).mkdirs(dbfs_path)
def rm_cli(api_client, recursive, dbfs_path):
"""
Remove files from dbfs.
To remove a directory you must provide the --recursive flag.
"""
DbfsApi(api_client).delete(dbfs_path, recursive)
def __init__(self, user, token, workspaceUrl):
self.dbcli_apiclient = ApiClient(user,
password=token,
host=workspaceUrl,
verify=True,
command_name='Python Client')
self.dbfs_api_client = DbfsApi(self.dbcli_apiclient)
def deploy(client: ApiClient, job_conf: Dict[str, Any], task_args: Dict[str, Any]):
dbfs_new_jar_name = job_conf['libraries'][0]['jar']
logging.info("Submitting job with configuration %s and jar file %s" % (job_conf, dbfs_new_jar_name))
dbfs_api = DbfsApi(client)
dbfs_api.cp(recursive=False, overwrite=True, src=task_args["jar"], dst=dbfs_new_jar_name)
job_data = client.perform_query('POST', '/jobs/create', data=job_conf, headers=None)
logging.info("Job creation data %s" % job_data)
if task_args["run_now"]:
logging.info("Requested to launch job immediately")
run_data = client.perform_query('POST', '/jobs/run-now', data=job_data, headers=None)
logging.info("Job launched with run data: %s" % run_data)
if task_args["trace"]:
logging.info("Requested to trace the job status")
run_finised = False
while not run_finised:
time.sleep(4)
run_status = client.perform_query('GET', '/jobs/runs/get',
data={"run_id": run_data["run_id"]},
headers=None)
logging.info(run_status)
result_state = run_status["state"].get("result_state", None)
if result_state:
run_finised = True
if result_state == "SUCCESS":
logging.info("Job successfully finished!")
else:
exception_text = "Job finished with result state %s. Please check run UI!" % result_state
raise Exception(exception_text)
logging.info("All deployment actions successfully performed")
def cp_cli(api_client, recursive, overwrite, src, dst):
"""
Copy files to and from DBFS.
Note that this function will fail if the src and dst are both on the local filesystem
or if they are both DBFS paths.
For non-recursive copies, if the dst is a directory, the file will be placed inside the
directory. For example ``dbfs cp dbfs:/apple.txt .`` will create a file at `./apple.txt`.
For recursive copies, files inside of the src directory will be copied inside the dst directory
with the same name. If the dst path does not exist, a directory will be created. For example
``dbfs cp -r dbfs:/foo foo`` will create a directory foo and place the files ``dbfs:/foo/a`` at
``foo/a``. If ``foo/a`` already exists, the file will not be overriden unless the --overwrite
flag is provided -- however, dbfs cp --recursive will continue to try and copy other files.
"""
# Copy to DBFS in this case
dbfs_api = DbfsApi(api_client)
if not DbfsPath.is_valid(src) and DbfsPath.is_valid(dst):
if not os.path.exists(src):
error_and_quit('The local file {} does not exist.'.format(src))
if not recursive:
if os.path.isdir(src):
error_and_quit((
'The local file {} is a directory. You must provide --recursive'
).format(src))
copy_to_dbfs_non_recursive(dbfs_api, src, DbfsPath(dst), overwrite)
else:
if not os.path.isdir(src):
copy_to_dbfs_non_recursive(dbfs_api, src, DbfsPath(dst),
overwrite)
return
copy_to_dbfs_recursive(dbfs_api, src, DbfsPath(dst), overwrite)
# Copy from DBFS in this case
elif DbfsPath.is_valid(src) and not DbfsPath.is_valid(dst):
if not recursive:
copy_from_dbfs_non_recursive(dbfs_api, DbfsPath(src), dst,
overwrite)
else:
dbfs_path_src = DbfsPath(src)
if not dbfs_api.get_status(dbfs_path_src).is_dir:
copy_from_dbfs_non_recursive(dbfs_api, dbfs_path_src, dst,
overwrite)
copy_from_dbfs_recursive(dbfs_api, dbfs_path_src, dst, overwrite)
elif not DbfsPath.is_valid(src) and not DbfsPath.is_valid(dst):
error_and_quit(
'Both paths provided are from your local filesystem. '
'To use this utility, one of the src or dst must be prefixed '
'with dbfs:/')
elif DbfsPath.is_valid(src) and DbfsPath.is_valid(dst):
error_and_quit(
'Both paths provided are from the DBFS filesystem. '
'To copy between the DBFS filesystem, you currently must copy the '
'file from DBFS to your local filesystem and then back.')
else:
assert False, 'not reached'
def main():
ENDPOINT = od.getenv('ENDPOINT')
TOKEN = os.getenv('TOKEN')
databricks_client = ApiClient(host=config.ENDPOINT, token=TOKEN)
dbfs_client = DbfsApi(self.databricks_client)
src_path = "~/dev/deploy_whl_cluster_test/dist/test_package-0.0.1-py3-none-any.whl"
dbfs_client.cp(src=src_path, dst=dst_path, overwrite=True, recursive=False)
def ls_cli(api_client, l, absolute, dbfs_path): # NOQA
"""
List files in DBFS.
"""
if len(dbfs_path) == 0:
dbfs_path = DbfsPath('dbfs:/')
elif len(dbfs_path) == 1:
dbfs_path = dbfs_path[0]
else:
error_and_quit('ls can take a maximum of one path.')
files = DbfsApi(api_client).list_files(dbfs_path)
table = tabulate([f.to_row(is_long_form=l, is_absolute=absolute) for f in files],
tablefmt='plain')
click.echo(table)
def prepare_for_operationalization(cluster_id, api_client, dbfs_path,
overwrite, spark_version):
"""
Installs appropriate versions of several libraries to support operationalization.
Args:
cluster_id (str): cluster_id representing the cluster to prepare for operationalization
api_client (ApiClient): the ApiClient object used to authenticate to the workspace
dbfs_path (str): the path on dbfs to upload libraries to
overwrite (bool): whether to overwrite existing files on dbfs with new files of the same name
spark_version (str): str version indicating which version of spark is installed on the databricks cluster
Returns:
A dictionary of libraries installed
"""
print("Preparing for operationlization...")
cosmosdb_jar_url = COSMOSDB_JAR_FILE_OPTIONS[spark_version]
# download the cosmosdb jar
local_jarname = os.path.basename(cosmosdb_jar_url)
# only download if you need it:
if overwrite or not os.path.exists(local_jarname):
print("Downloading {}...".format(cosmosdb_jar_url))
local_jarname, _ = urlretrieve(cosmosdb_jar_url, local_jarname)
else:
print("File {} already downloaded.".format(local_jarname))
# upload jar to dbfs:
upload_path = Path(dbfs_path, local_jarname).as_posix()
print("Uploading CosmosDB driver to databricks at {}".format(upload_path))
if dbfs_file_exists(api_client, upload_path) and overwrite:
print("Overwriting file at {}".format(upload_path))
DbfsApi(api_client).cp(recursive=False,
src=local_jarname,
dst=upload_path,
overwrite=overwrite)
# setup the list of libraries to install:
# jar library setup
libs2install = [{"jar": upload_path}]
# setup libraries to install:
libs2install.extend([{"pypi": {"package": i}} for i in PYPI_O16N_LIBS])
print(
"Installing jar and pypi libraries required for operationalization...")
LibrariesApi(api_client).install_libraries(cluster_id, libs2install)
return libs2install
def dbfs_file_exists(api_client, dbfs_path):
"""
Checks to determine whether a file exists.
Args:
api_client (ApiClient object): Object used for authenticating to the workspace
dbfs_path (str): Path to check
Returns:
True if file exists on dbfs, False otherwise.
"""
try:
DbfsApi(api_client).list_files(dbfs_path=DbfsPath(dbfs_path))
file_exists = True
except:
file_exists = False
return file_exists
def cp_cli(api_client, recursive, overwrite, src, dst):
"""
Copy files to and from DBFS.
Note that this function will fail if the src and dst are both on the local filesystem.
For non-recursive copies, if the dst is a directory, the file will be placed inside the
directory. For example ``dbfs cp dbfs:/apple.txt .`` will create a file at `./apple.txt`.
For recursive copies, files inside of the src directory will be copied inside the dst directory
with the same name. If the dst path does not exist, a directory will be created. For example
``dbfs cp -r dbfs:/foo foo`` will create a directory foo and place the files ``dbfs:/foo/a`` at
``foo/a``. If ``foo/a`` already exists, the file will not be overriden unless the --overwrite
flag is provided -- however, dbfs cp --recursive will continue to try and copy other files.
"""
# Copy to DBFS in this case
DbfsApi(api_client).cp(recursive, overwrite, src, dst)
def mv_cli(api_client, src, dst):
"""
Moves a file between two DBFS paths.
"""
DbfsApi(api_client).move(src, dst)
upload_path = Path(args.dbfs_path, args.eggname).as_posix()
# Check if file exists to alert user.
print("Uploading {} to databricks at {}".format(args.eggname, upload_path))
if dbfs_file_exists(my_api_client, upload_path):
if args.overwrite:
print("Overwriting file at {}".format(upload_path))
else:
raise IOError("""
{} already exists on databricks cluster.
This is likely an older version of the library.
Please use the '--overwrite' flag to proceed.
""".format(upload_path))
DbfsApi(my_api_client).cp(recursive=False,
src=myegg,
dst=upload_path,
overwrite=args.overwrite)
# steps below require the cluster to be running. Check status
try:
status = ClusterApi(my_api_client).get_cluster(args.cluster_id)
except HTTPError as e:
print(e)
print(textwrap.dedent(CLUSTER_NOT_FOUND_MSG.format(args.cluster_id)))
raise
if status["state"] == "TERMINATED":
print(
textwrap.dedent(
CLUSTER_NOT_RUNNING_MSG.format(args.cluster_id,
status["state"])))
def __init__(self, profile=None):
api_client = get_api_client(profile)
self.dbfs_client = DbfsApi(api_client)
self.runs_client = RunsApi(api_client)
def cat_cli(api_client, src):
"""
Show the contents of a file. Does not work for directories.
"""
DbfsApi(api_client).cat(src)
def tgt_dbfs_api(tgt_api_client:ApiClient):
return DbfsApi(tgt_api_client)
def src_dbfs_api(src_api_client:ApiClient):
return DbfsApi(src_api_client)
def __init__(self, api_client):
self.client = DeltaPipelinesService(api_client)
self.dbfs_client = DbfsApi(api_client)
def __init__(self, api_client):
self.jobs_client = JobsApi(api_client)
self.workspace_client = WorkspaceApi(api_client)
self.dbfs_client = DbfsApi(api_client)