def _dockerize(args):
"""
:param args:
:return:
"""
dockerize.main(DisdatConfig.instance(), args)
def load():
"""
Load the data contexts described at meta_dir. Each of these is a "remote."
Args:
local_ctxt_dir: Directory of contexts, e.g., ~/.disdat
Returns:
(dict) of 'name':context pairs.
"""
ctxt_dir = DisdatConfig.instance().get_context_dir()
if ctxt_dir is None:
raise Exception(
"Unable to load context without a metadata directory argument")
contexts = {}
files = glob.glob(os.path.join(ctxt_dir, '*'))
for ctxt in files:
_logger.debug("Loading context {}...".format(ctxt))
meta_file = os.path.join(ctxt_dir, ctxt, META_CTXT_FILE)
if not os.path.isfile(meta_file):
_logger.debug("No disdat {} meta ctxt data file found.".format(
meta_file))
else:
with open(meta_file, 'r') as json_file:
dc_dict = json.loads(json_file.readline())
dc = DataContext(ctxt_dir, **dc_dict)
contexts[dc.local_ctxt] = dc
return contexts
def get_fq_docker_repo_name(is_sagemaker, pipeline_setup_file):
"""
Produce the fully qualified docker repo name.
Args:
is_sagemaker (bool): for sagemaker image
pipeline_setup_file (str): the path to the setup.py file used to dockerize this pipeline
Returns:
(str): The fully qualified docker image repository name
"""
disdat_config = DisdatConfig.instance()
repository_prefix = None
if disdat_config.parser.has_option('docker', 'repository_prefix'):
repository_prefix = disdat_config.parser.get('docker',
'repository_prefix')
if is_sagemaker:
repository_name = common.make_sagemaker_project_repository_name(
repository_prefix, pipeline_setup_file)
else:
repository_name = common.make_project_repository_name(
repository_prefix, pipeline_setup_file)
# Figure out the fully-qualified repository name, i.e., the name
# including the registry.
registry_name = disdat_config.parser.get('docker', 'registry').strip('/')
if registry_name == '*ECR*':
fq_repository_name = aws.ecr_get_fq_repository_name(repository_name)
else:
fq_repository_name = '{}/{}'.format(registry_name, repository_name)
return fq_repository_name
def _apply(args):
"""
:param args:
:return:
"""
apply.main(DisdatConfig.instance(), args)
def main():
"""
Main is the package entry point.
"""
if getattr(sys, 'frozen', False):
here = os.path.join(sys._MEIPASS, 'disdat')
else:
here = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
args = sys.argv[1:]
# General options
parser = argparse.ArgumentParser(prog='dsdt', description='DisDat (dsdt) -- distributed data science management')
parser.add_argument(
'--profile',
type=str,
default=None,
help="An AWS credential profile to use when performing AWS operations (default is to use the \'default\' profile)",
dest='aws_profile'
)
parser.add_argument("--verbose", action='store_true', help='Be verbose: Show extra debugging information')
parser.add_argument("--version", action='version', version='Running Disdat version {}'.format(__version__))
subparsers = parser.add_subparsers()
ls_p = subparsers.add_parser('init')
ls_p.set_defaults(func=lambda args: DisdatConfig.init())
# Add disdat core subparsers
fs.add_arg_parser(subparsers)
add.add_arg_parser(subparsers)
lineage.add_arg_parser(subparsers)
# Add additional parsers if we are imported
resolve_cli_extras(subparsers)
args = parser.parse_args(args)
log_level = logging.INFO
if args.verbose:
log_level = logging.DEBUG
log.enable(level=log_level) # TODO: Add configurable verbosity
if args.aws_profile is not None:
os.environ['AWS_PROFILE'] = args.aws_profile
if hasattr(args,'func'):
args.func(args)
else:
print("dsdt requires arguments, see `dsdt -h` for usage")
def _run_local(cli, pipeline_setup_file, arglist, backend):
"""
Run container locally or run sagemaker container locally
Args:
cli (bool): Whether we were called from the CLI or API
pipeline_setup_file (str): The FQ path to the setup.py used to dockerize the pipeline.
arglist:
backend:
Returns:
output (str): Returns None if there is a failure
"""
on_macos = False
if platform == "darwin":
on_macos = True
client = docker.from_env()
environment = {}
if 'AWS_PROFILE' in os.environ:
environment['AWS_PROFILE'] = os.environ['AWS_PROFILE']
environment[common.LOCAL_EXECUTION] = 'True'
# Todo: Local runs do not yet set resource limits, but when they do, we'll have to set this
#environment['DISDAT_CPU_COUNT'] = vcpus
volumes = {}
aws_config_dir = os.getenv('AWS_CONFIG_DIR',
os.path.join(os.environ['HOME'], '.aws'))
if aws_config_dir is not None and os.path.exists(aws_config_dir):
volumes[aws_config_dir] = {'bind': '/root/.aws', 'mode': 'rw'}
local_disdat_meta_dir = DisdatConfig.instance().get_meta_dir()
volumes[local_disdat_meta_dir] = {'bind': '/root/.disdat', 'mode': 'rw'}
try:
if backend == Backend.LocalSageMaker:
pipeline_image_name = common.make_sagemaker_project_image_name(
pipeline_setup_file)
tempdir = tempfile.mkdtemp()
with open(os.path.join(tempdir, 'hyperparameters.json'),
'w') as of:
json.dump(_sagemaker_hyperparameters_from_arglist(arglist), of)
args = ['train'] # rewrite to just 'train'
# On mac OS, tempdir returns /var, but is actually /private/var
# Add /private since it that dir is shared (and not /var) with Docker.
if on_macos:
localdir = os.path.join('/private', tempdir[1:])
else:
localdir = tempdir
volumes[localdir] = {
'bind': '/opt/ml/input/config/',
'mode': 'rw'
}
_logger.info("VOLUMES: {}".format(volumes))
else:
# Add the actual command to the arglist (for non-sagemaker runs)
arglist = [ENTRYPOINT_BIN] + arglist
pipeline_image_name = common.make_project_image_name(
pipeline_setup_file)
_logger.debug('Running image {} with arguments {}'.format(
pipeline_image_name, arglist))
stdout = client.containers.run(pipeline_image_name,
arglist,
detach=False,
environment=environment,
init=True,
stderr=True,
volumes=volumes)
stdout = six.ensure_str(stdout)
if cli: print(stdout)
return stdout
except docker.errors.ContainerError as ce:
_logger.error(
"Internal error running image {}".format(pipeline_image_name))
_logger.error("Error: {}".format(six.ensure_str(ce.stderr)))
return six.ensure_str(ce)
except docker.errors.ImageNotFound:
_logger.error(
"Unable to find the docker image {}".format(pipeline_image_name))
return None
def _run_aws_sagemaker(arglist, fq_repository_name, job_name):
"""
Runs a training job on AWS SageMaker. This uses the default machine type
in the disdat.cfg file.
Args:
arglist:
fq_repository_name (str): fully qualified repository name
job_name: instance job name
Returns:
TrainingJobArn (str)
"""
disdat_config = DisdatConfig.instance()
job_name = job_name.replace(
'_',
'-') # b/c SageMaker complains it must be ^[a-zA-Z0-9](-*[a-zA-Z0-9])*
hyperparameter_dict = _sagemaker_hyperparameters_from_arglist(arglist)
algorithm_specification = {
'TrainingImage': fq_repository_name,
'TrainingInputMode': 'File'
}
role_arn = disdat_config.parser.get(_MODULE_NAME, 'aws_sagemaker_role_arn')
input_channel_config = [
{
'ChannelName': 'disdat_sagemaker_input_blackhole',
'DataSource': {
'S3DataSource': {
'S3DataType':
'S3Prefix',
'S3Uri':
disdat_config.parser.get(_MODULE_NAME,
'aws_sagemaker_s3_input_uri'),
'S3DataDistributionType':
'FullyReplicated'
}
},
'ContentType': 'application/javascript',
'CompressionType': 'None', # | 'Gzip',
'RecordWrapperType': 'None' # | 'RecordIO'
},
]
output_data_config = {
'S3OutputPath':
os.path.join(
disdat_config.parser.get(_MODULE_NAME,
'aws_sagemaker_s3_output_uri'), job_name)
}
resource_config = {
'InstanceType':
disdat_config.parser.get(_MODULE_NAME, 'aws_sagemaker_instance_type'),
'InstanceCount':
int(
disdat_config.parser.get(_MODULE_NAME,
'aws_sagemaker_instance_count')),
'VolumeSizeInGB':
int(
disdat_config.parser.get(_MODULE_NAME,
'aws_sagemaker_volume_sizeGB'))
# 'VolumeKmsKeyId': 'string'
}
vpc_config = None #'SecurityGroupIds': [], 'Subnets': []}
stopping_condition = {
'MaxRuntimeInSeconds':
int(
disdat_config.parser.get(_MODULE_NAME,
'aws_sagemaker_max_runtime_sec'))
}
tags = [{
'Key': 'user',
'Value': 'disdat'
}, {
'Key': 'job',
'Value': job_name
}]
if False:
print("Disdat SageMaker configs")
print("job name: {}".format(job_name))
print("hparams: {}".format(hyperparameter_dict))
print("algorithm: {}".format(algorithm_specification))
print("Role ARN: {}".format(role_arn))
print("Input data conf: {}".format(input_channel_config))
print("Output data conf: {}".format(output_data_config))
print("Resource conf: {}".format(resource_config))
print("VPC conf: {}".format(vpc_config))
print("Stopping condition seconds: {}".format(stopping_condition))
print("Tags: {}".format(tags))
client = b3.client('sagemaker', region_name=aws.profile_get_region())
response = client.create_training_job(
TrainingJobName=job_name,
HyperParameters=hyperparameter_dict,
AlgorithmSpecification=algorithm_specification,
RoleArn=role_arn,
InputDataConfig=input_channel_config,
OutputDataConfig=output_data_config,
ResourceConfig=resource_config,
StoppingCondition=stopping_condition,
Tags=tags)
_logger.info(
"Disdat SageMaker create_training_job response {}".format(response))
return response['TrainingJobArn']
def _run_aws_batch(arglist, fq_repository_name, job_name, pipeline_image_name,
aws_session_token_duration, vcpus, memory, no_submit,
job_role_arn):
"""
Run job on AWS Batch. Sends to queue configured in disdat.cfg.
This assumes that you have already created a cluster that will run the jobs
that have been assigned to that queue.
Args:
arglist:
fq_repository_name (str): The fully qualified docker repository name
job_name:
pipeline_image_name:
aws_session_token_duration:
vcpus:
memory:
no_submit (bool): default False
job_role_arn (str): Can be None
Returns:
"""
def check_role_arn(job_dict, jra):
""" Check to see if the job desc dictionary contains the same job_role_arn (jra)
"""
if jra is None:
if 'jobRoleArn' not in job_dict['containerProperties']:
return True
else:
if 'jobRoleArn' in job_dict['containerProperties']:
if job_dict['containerProperties']['jobRoleArn'] == jra:
return True
return False
disdat_config = DisdatConfig.instance()
# Get the parameter values required to kick off an AWS Batch job.
# Every batch job must:
# 1. Have a name
# 2. Have a job definition that declares which ECR-hosted Docker
# image to use.
# 3. Have a queue that feeds jobs into a compute cluster.
# 4. The command to execute inside the Docker image; the command
# args are more-or-less the same as the ones used to execute
# locally using 'dsdt run'
# Create a Job Definition and upload it.
# We create per-user job definitions so multiple users do not clobber each other.
# In addition, we never re-use a job definition, since the user may update
# the vcpu or memory requirements and those are stuck in the job definition
job_definition_name = aws.batch_get_job_definition_name(
pipeline_image_name)
if disdat_config.parser.has_option(_MODULE_NAME,
'aws_batch_job_definition'):
job_definition_name = disdat_config.parser.get(
_MODULE_NAME, 'aws_batch_job_definition')
# TODO: Look through all of history to find one that matches?
# TODO: Delete old jobs here or let user do it?
job_definition_obj = aws.batch_get_latest_job_definition(
job_definition_name)
if (job_definition_obj is not None
and job_definition_obj['containerProperties']['image']
== fq_repository_name
and job_definition_obj['containerProperties']['vcpus'] == vcpus
and job_definition_obj['containerProperties']['memory'] == memory
and check_role_arn(job_definition_obj, job_role_arn)):
job_definition_fqn = aws.batch_extract_job_definition_fqn(
job_definition_obj)
_logger.info("Re-using prior AWS Batch run job definition : {}".format(
job_definition_obj))
else:
""" Whether None or doesn't match, make a new one """
job_definition_obj = aws.batch_register_job_definition(
job_definition_name,
fq_repository_name,
vcpus=vcpus,
memory=memory,
job_role_arn=job_role_arn)
job_definition_fqn = aws.batch_get_job_definition(job_definition_name)
_logger.info(
"New AWS Batch run job definition {}".format(job_definition_fqn))
if no_submit:
# Return the job description object
return job_definition_obj
job_queue = disdat_config.parser.get(_MODULE_NAME, 'aws_batch_queue')
container_overrides = {'command': arglist}
# Through the magic of boto3_session_cache, the client in our script
# here can get at AWS profiles and temporary AWS tokens created in
# part from MFA tokens generated through the user's shells; we don't
# have to write special code of our own to deal with authenticating
# with AWS.
client = b3.client('batch', region_name=aws.profile_get_region())
# A bigger problem might be that the IAM role executing the job on
# a batch EC2 instance might not have access to the S3 remote. To
# get around this, allow the user to create some temporary AWS
# credentials.
if aws_session_token_duration > 0 and job_role_arn is None:
sts_client = b3.client('sts')
try:
token = sts_client.get_session_token(
DurationSeconds=aws_session_token_duration)
credentials = token['Credentials']
container_overrides['environment'] = [{
'name':
'AWS_ACCESS_KEY_ID',
'value':
credentials['AccessKeyId']
}, {
'name':
'AWS_SECRET_ACCESS_KEY',
'value':
credentials['SecretAccessKey']
}, {
'name':
'AWS_SESSION_TOKEN',
'value':
credentials['SessionToken']
}]
except Exception as e:
_logger.debug(
"Unable to generate an STS token, instead trying users default credentials..."
)
credentials = b3.session.Session().get_credentials()
container_overrides['environment'] = [{
'name':
'AWS_ACCESS_KEY_ID',
'value':
credentials.access_key
}, {
'name':
'AWS_SECRET_ACCESS_KEY',
'value':
credentials.secret_key
}, {
'name': 'AWS_SESSION_TOKEN',
'value': credentials.token
}]
container_overrides['environment'].append({
'name': 'DISDAT_CPU_COUNT',
'value': str(vcpus)
})
job = client.submit_job(jobName=job_name,
jobDefinition=job_definition_fqn,
jobQueue=job_queue,
containerOverrides=container_overrides)
status = job['ResponseMetadata']['HTTPStatusCode']
if status == 200:
_logger.info(
'Job {} (ID {}) with definition {} submitted to AWS Batch queue {}'
.format(job['jobName'], job['jobId'], job_definition_fqn,
job_queue))
return job
else:
_logger.error('Job submission failed: HTTP Status {}'.format())
return None
def _run_local(cli, arglist, pipeline_class_name, backend):
"""
Run container locally or run sagemaker container locally
Args:
cli (bool): Whether we were called from the CLI or API
arglist:
pipeline_class_name:
backend:
Returns:
output (str): Returns None if there is a failure
"""
on_macos = False
if platform == "darwin":
on_macos = True
client = docker.from_env()
environment = {}
if 'AWS_PROFILE' in os.environ:
environment['AWS_PROFILE'] = os.environ['AWS_PROFILE']
environment[common.LOCAL_EXECUTION] = 'True'
volumes = {}
aws_config_dir = os.getenv('AWS_CONFIG_DIR',
os.path.join(os.environ['HOME'], '.aws'))
if aws_config_dir is not None and os.path.exists(aws_config_dir):
volumes[aws_config_dir] = {'bind': '/root/.aws', 'mode': 'rw'}
local_disdat_meta_dir = DisdatConfig.instance().get_meta_dir()
volumes[local_disdat_meta_dir] = {'bind': '/root/.disdat', 'mode': 'rw'}
try:
if backend == Backend.LocalSageMaker:
pipeline_image_name = common.make_sagemaker_pipeline_image_name(
pipeline_class_name)
tempdir = tempfile.mkdtemp()
with open(os.path.join(tempdir, 'hyperparameters.json'),
'w') as of:
json.dump(_sagemaker_hyperparameters_from_arglist(arglist), of)
args = ['train'] # rewrite to just 'train'
# On mac OS, tempdir returns /var, but is actually /private/var
# Add /private since it that dir is shared (and not /var) with Docker.
if on_macos:
localdir = os.path.join('/private', tempdir[1:])
else:
localdir = tempdir
volumes[localdir] = {
'bind': '/opt/ml/input/config/',
'mode': 'rw'
}
_logger.info("VOLUMES: {}".format(volumes))
else:
pipeline_image_name = common.make_pipeline_image_name(
pipeline_class_name)
_logger.debug('Running image {} with arguments {}'.format(
pipeline_image_name, arglist))
stdout = client.containers.run(pipeline_image_name,
arglist,
detach=False,
environment=environment,
init=True,
stderr=True,
volumes=volumes)
if cli: print stdout
return stdout
except docker.errors.ImageNotFound:
_logger.error(
"Unable to find the docker image {}".format(pipeline_image_name))
return None
def main():
"""
Main as a function for testing convenience and as a package entry point.
:return: (shape of input df, shape of pushed df)
"""
if getattr(sys, 'frozen', False):
here = sys._MEIPASS
else:
here = os.path.join(os.path.abspath(os.path.dirname(__file__)), '..')
with open(os.path.join(here, 'VERSION')) as version_file:
__version__ = version_file.read().strip()
args = sys.argv[1:]
# General options
parser = argparse.ArgumentParser(
prog='dsdt',
description='DisDat (dsdt) -- distributed data science management')
parser.add_argument(
'--profile',
type=str,
default=None,
help=
"An AWS credential profile to use when performing AWS operations (default is to use the \'default\' profile)",
dest='aws_profile')
parser.add_argument("--verbose",
action='store_true',
help='Be verbose: Show extra debugging information')
parser.add_argument(
"--version",
action='version',
version='Running Disdat version {}'.format(__version__))
subparsers = parser.add_subparsers()
ls_p = subparsers.add_parser('init')
ls_p.set_defaults(func=lambda args: DisdatConfig.init())
# autodock
dockerize_p = subparsers.add_parser(
'dockerize', description="Dockerizer a particular transform.")
dockerize_p.add_argument(
'--config-dir',
type=str,
default=None,
help=
"A directory containing configuration files for the operating system within the Docker image",
)
dockerize_p.add_argument(
'--os-type',
type=str,
default=None,
help='The base operating system type for the Docker image')
dockerize_p.add_argument(
'--os-version',
type=str,
default=None,
help='The base operating system version for the Docker image')
dockerize_p.add_argument(
'--push',
action='store_true',
help=
"Push the image to a remote Docker registry (default is to not push; must set 'docker_registry' in Disdat config)",
)
dockerize_p.add_argument(
'--no-build',
action='store_false',
help=
'Do not build an image (only copy files into the Docker build context)',
dest='build',
)
dockerize_p.add_argument(
"pipe_root",
type=str,
help=
"Root of the Python source tree containing the user-defined transform; must have a setuptools-style setup.py file"
)
dockerize_p.add_argument(
"pipe_cls",
type=str,
help="User-defined transform, e.g., module.PipeClass")
dockerize_p.set_defaults(func=lambda args: _dockerize(args))
# run
run_p = subparsers.add_parser(
'run', description="Run containerized version of transform.")
run_p.add_argument('--backend',
default=run.Backend.default(),
type=str,
choices=run.Backend.options(),
help='An optional batch execution back-end to use')
run_p.add_argument("--force",
action='store_true',
help="If there are dependencies, force re-computation.")
run_p.add_argument(
"--no-push-input",
action='store_false',
help=
"Do not push the current committed input bundle before execution (default is to push)",
dest='push_input_bundle')
run_p.add_argument(
'-it',
'--input-tag',
nargs=1,
type=str,
action='append',
help="Input bundle tags: '-it authoritative:True -it version:0.7.1'")
run_p.add_argument(
'-ot',
'--output-tag',
nargs=1,
type=str,
action='append',
help="Output bundle tags: '-ot authoritative:True -ot version:0.7.1'")
run_p.add_argument(
"input_bundle",
type=str,
help="Name of source data bundle. '-' means no input bundle.")
run_p.add_argument(
"output_bundle",
type=str,
help="Name of destination bundle. '-' means default output bundle.")
run_p.add_argument("pipe_cls",
type=str,
help="User-defined transform, e.g., module.PipeClass")
run_p.add_argument(
"pipeline_args",
type=str,
nargs=argparse.REMAINDER,
help="Optional set of parameters for this pipe '--parameter value'")
run_p.set_defaults(func=lambda args: _run(args))
# apply
apply_p = subparsers.add_parser(
'apply',
description=
"Apply a transform to an input bundle to produce an output bundle.")
apply_p.add_argument(
'-it',
'--input-tag',
nargs=1,
type=str,
action='append',
help="Input bundle tags: '-it authoritative:True -it version:0.7.1'")
apply_p.add_argument(
'-ot',
'--output-tag',
nargs=1,
type=str,
action='append',
help="Output bundle tags: '-ot authoritative:True -ot version:0.7.1'")
apply_p.add_argument(
"input_bundle",
type=str,
help="Name of source data bundle. '-' means no input bundle.")
apply_p.add_argument(
"output_bundle",
type=str,
help="Name of destination bundle. '-' means default output bundle.")
apply_p.add_argument("pipe_cls",
type=str,
help="User-defined transform, e.g., module.PipeClass")
apply_p.add_argument("--local",
action='store_true',
help="Run the class locally (even if dockered)")
apply_p.add_argument(
"--force",
action='store_true',
help="If there are dependencies, force re-computation.")
apply_p.add_argument(
"params",
type=str,
nargs=argparse.REMAINDER,
help="Optional set of parameters for this pipe '--parameter value'")
apply_p.set_defaults(func=lambda args: _apply(args))
# File system operations
init_fs_cl(subparsers)
args = parser.parse_args(args)
log_level = logging.WARN
if args.verbose:
log_level = logging.DEBUG
logging.basicConfig(level=log_level)
if args.aws_profile is not None:
os.environ['AWS_PROFILE'] = args.aws_profile
args.func(args)
def main():
"""
Main as a function for testing convenience and as a package entry point.
:return: (shape of input df, shape of pushed df)
"""
if getattr(sys, 'frozen', False):
here = os.path.join(sys._MEIPASS, 'disdat')
else:
here = os.path.abspath(os.path.dirname(__file__))
with open(os.path.join(here, 'VERSION')) as version_file:
__version__ = version_file.read().strip()
args = sys.argv[1:]
# General options
parser = argparse.ArgumentParser(prog='dsdt', description='DisDat (dsdt) -- distributed data science management')
parser.add_argument(
'--profile',
type=str,
default=None,
help="An AWS credential profile to use when performing AWS operations (default is to use the \'default\' profile)",
dest='aws_profile'
)
parser.add_argument("--verbose", action='store_true', help='Be verbose: Show extra debugging information')
parser.add_argument("--version", action='version', version='Running Disdat version {}'.format(__version__))
subparsers = parser.add_subparsers()
ls_p = subparsers.add_parser('init')
ls_p.set_defaults(func=lambda args: DisdatConfig.init())
# dockerize
subparsers = dockerize.add_arg_parser(subparsers)
# run
subparsers = run.add_arg_parser(subparsers)
# apply
apply_p = subparsers.add_parser('apply', description="Apply a transform to an input bundle to produce an output bundle.")
apply_p.add_argument('-cs', '--central-scheduler', action='store_true', default=False, help="Use a central Luigi scheduler (defaults to local scheduler)")
apply_p.add_argument('-w', '--workers', type=int, default=1, help="Number of Luigi workers on this node")
apply_p.add_argument('-it', '--input-tag', nargs=1, type=str, action='append',
help="Input bundle tags: '-it authoritative:True -it version:0.7.1'")
apply_p.add_argument('-ot', '--output-tag', nargs=1, type=str, action='append',
help="Output bundle tags: '-ot authoritative:True -ot version:0.7.1'")
apply_p.add_argument('-o', '--output-bundle', type=str, default='-',
help="Name output bundle: '-o my.output.bundle'. Default name is '<TaskName>_<param_hash>'")
apply_p.add_argument('-f', '--force', action='store_true', help="Force re-computation of only this task.")
apply_p.add_argument('--force-all', action='store_true', help="Force re-computation of ALL upstream tasks.")
apply_p.add_argument('--incremental-push', action='store_true', help="Commit and push each task's bundle as it is produced to the remote.")
apply_p.add_argument('--incremental-pull', action='store_true', help="Localize bundles as they are needed by downstream tasks from the remote.")
apply_p.add_argument('pipe_cls', type=load_class, help="User-defined transform, e.g., 'module.PipeClass'")
apply_p.add_argument('params', type=str, nargs=argparse.REMAINDER,
help="Optional set of parameters for this pipe '--parameter value'")
apply_p.set_defaults(func=lambda args: apply.cli_apply(args))
# File system operations
init_fs_cl(subparsers)
# add
init_add_cl(subparsers)
# add
init_lineage_cl(subparsers)
args = parser.parse_args(args)
log_level = logging.INFO
if args.verbose:
log_level = logging.DEBUG
log.enable(level=log_level) # TODO: Add configurable verbosity
if args.aws_profile is not None:
os.environ['AWS_PROFILE'] = args.aws_profile
args.func(args)