def test_decode_tile_key(self):
"""Test encoding an object key"""
b = BossBackend(self.example_config_data)
b.setup(self.api_token)
params = {"collection": 1,
"experiment": 2,
"channel": 3,
"resolution": 0,
"x_index": 5,
"y_index": 6,
"z_index": 1,
"t_index": 0,
}
proj = [str(params['collection']), str(params['experiment']), str(params['channel'])]
key = b.encode_tile_key(proj,
params['resolution'],
params['x_index'],
params['y_index'],
params['z_index'],
params['t_index'],
)
parts = b.decode_tile_key(key)
assert parts["collection"] == params['collection']
assert parts["experiment"] == params['experiment']
assert parts["channel"] == params['channel']
assert parts["resolution"] == params['resolution']
assert parts["x_index"] == params['x_index']
assert parts["y_index"] == params['y_index']
assert parts["z_index"] == params['z_index']
assert parts["t_index"] == params['t_index']
def test_encode_tile_key(self):
"""Test encoding an object key"""
b = BossBackend(self.example_config_data)
b.setup(self.api_token)
params = {"collection": 1,
"experiment": 2,
"channel": 3,
"resolution": 0,
"x_index": 5,
"y_index": 6,
"z_index": 1,
"t_index": 0,
}
proj = [str(params['collection']), str(params['experiment']), str(params['channel'])]
key = b.encode_tile_key(proj,
params['resolution'],
params['x_index'],
params['y_index'],
params['z_index'],
params['t_index'],
)
assert key == six.u("03ca58a12ec662954ac12e06517d4269&1&2&3&0&5&6&1&0")
def test_encode_chunk_key(self):
"""Test encoding an object key"""
b = BossBackend(self.example_config_data)
b.setup(self.api_token)
params = {"collection": 1,
"experiment": 2,
"channel": 3,
"resolution": 0,
"x_index": 5,
"y_index": 6,
"z_index": 1,
"t_index": 0,
"num_tiles": 16,
}
proj = [str(params['collection']), str(params['experiment']), str(params['channel'])]
key = b.encode_chunk_key(params['num_tiles'], proj,
params['resolution'],
params['x_index'],
params['y_index'],
params['z_index'],
params['t_index'],
)
assert key == six.u("77ff984241a0d6aa443d8724a816866d&16&1&2&3&0&5&6&1&0")
def test_get_task(self):
"""Test getting a task from the upload queue"""
b = BossBackend(self.example_config_data)
b.setup(self.api_token)
# Make sure queue is empty.
sqs = boto3.resource('sqs')
queue = sqs.Queue(self.upload_queue_url)
queue.purge()
# Put some stuff on the task queue
self.setup_helper.add_tasks(self.aws_creds["access_key"],
self.aws_creds['secret_key'],
self.upload_queue_url, b)
# Join and get a task
b.join(23)
msg_id, rx_handle, msg_body = b.get_task()
assert isinstance(msg_id, str)
assert isinstance(rx_handle, str)
assert msg_body == self.setup_helper.test_msg[0]
msg_id, rx_handle, msg_body = b.get_task()
assert isinstance(msg_id, str)
assert isinstance(rx_handle, str)
assert msg_body == self.setup_helper.test_msg[1]
def test_create(self):
"""Test creating an ingest job - mock server response"""
b = BossBackend(self.example_config_data)
b.setup(self.api_token)
id = b.create(self.example_config_data)
assert id == 23
def test_setup_upload_queue(self):
"""Test connecting the backend to the upload queue"""
b = BossBackend(self.example_config_data)
b.setup(self.api_token)
b.setup_upload_queue(self.aws_creds, self.queue_url)
assert b.queue.url == self.queue_url
def test_setup_queues(self):
"""Test connecting the backend to the upload and tile index queues"""
b = BossBackend(self.example_config_data)
b.setup(self.api_token)
b.setup_queues(self.aws_creds, self.upload_queue_url,
self.tile_index_queue_url)
assert b.upload_queue.url == self.upload_queue_url
assert b.tile_index_queue.url == self.tile_index_queue_url
def test_join(self):
"""Test joining an existing ingest job - mock server response"""
b = BossBackend(self.example_config_data)
b.setup(self.api_token)
status, creds, queue_url, tile_bucket, params, tile_count = b.join(23)
assert b.queue.url == self.queue_url
assert status == 1
assert creds == self.aws_creds
assert queue_url == self.queue_url
assert tile_bucket == self.tile_bucket_name
assert tile_count == 500
assert 'KVIO_SETTINGS' in params
assert 'OBJECTIO_CONFIG' in params
assert 'STATEIO_CONFIG' in params
assert 'ingest_queue' in params
def test_delete_task(self):
b = BossBackend(self.example_config_data)
b.setup(self.api_token)
# Make sure queue is empty.
sqs = boto3.resource('sqs')
queue = sqs.Queue(self.upload_queue_url)
queue.purge()
# Put some stuff on the task queue
self.setup_helper.add_tasks(self.aws_creds["access_key"],
self.aws_creds['secret_key'],
self.upload_queue_url, b)
# Join and get a task
b.join(23)
msg_id, rx_handle, msg_body = b.get_task()
assert b.delete_task(msg_id, rx_handle)
def test_setup(self):
"""Method to test setup instance"""
b = BossBackend(self.example_config_data)
b.setup(self.api_token)
assert b.host == "https://api.theboss.io"
def test_delete(self):
"""Test deleting an existing ingest job - mock server response"""
b = BossBackend(self.example_config_data)
b.setup(self.api_token)
b.cancel(23)
def main():
parser = argparse.ArgumentParser(description="Client for facilitating large-scale data ingest",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="Visit https://docs.theBoss.io for more details")
parser.add_argument("--api-token", "-a",
default=None,
help="Token for API authentication. If not provided and ndio is configured those credentials will automatically be used.")
parser.add_argument("--job-id", "-j",
default=None,
help="ID of the ingest job if joining an existing ingest job")
parser.add_argument("--log-file", "-l",
default=None,
help="Absolute path to the logfile to use")
parser.add_argument("--log-level", "-v",
default="warning",
help="Log level to use: critical, error, warning, info, debug")
parser.add_argument("--version",
action="store_true",
default=False,
help="Get the package version")
parser.add_argument("--cancel", "-c",
action="store_true",
default=None,
help="Flag indicating if you'd like to cancel (and remove) an ingest job. This will not delete data already ingested, but will prevent continuing this ingest job.")
parser.add_argument("--force", "-f",
action="store_true",
default=False,
help="Flag indicating if you'd like ignore all confirmation prompts.")
parser.add_argument("--processes_nb", "-p", type=int,
default=1,
help="The number of client processes that will upload the images of the ingest job.")
parser.add_argument("config_file", nargs='?', help="Path to the ingest job configuration file")
args = parser.parse_args()
# Get the version
if args.version:
check_version()
return
# Make sure you have a config file
if args.config_file is None:
if args.cancel:
# If no config is provided and you are deleting, the client defaults to the production Boss stack
boss_backend_params = {"client": {
"backend": {
"name": "boss",
"class": "BossBackend",
"host": "api.theboss.io",
"protocol": "https"}}}
backend = BossBackend(boss_backend_params)
backend.setup(args.api_token)
# Trying to cancel
if args.job_id is None:
parser.print_usage()
print("Error: You must provide an ingest job ID to cancel")
sys.exit(1)
if not get_confirmation("Are you sure you want to cancel ingest job {}? ".format(args.job_id), args.force):
print("Command ignored. Job not cancelled")
sys.exit(0)
backend.cancel(args.job_id)
print("Ingest job {} successfully cancelled.".format(args.job_id))
sys.exit(0)
else:
# Not deleting, so you need a config file
parser.print_usage()
print("Error: Ingest Job Configuration File is required")
sys.exit(1)
# Setup logging
log_level = logging.getLevelName(args.log_level.upper())
if not args.log_file:
# Using default log path
log_path = os.path.expanduser("~/.boss-ingest")
log_file = os.path.join(log_path,
'ingest_log{}_pid{}.log'.format(datetime.datetime.now().strftime("%Y%m%d-%H%M%S"),
os.getpid()))
# Make sure the logs dir exists if using the default log path
if not os.path.exists(log_path):
os.makedirs(log_path)
else:
log_file = args.log_file
logging.basicConfig(level=log_level,
format='%(asctime)s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename=log_file,
filemode='a')
logging.getLogger('ingest-client').addHandler(logging.StreamHandler(sys.stdout))
# Create an engine instance
try:
engine = Engine(args.config_file, args.api_token, args.job_id)
except ConfigFileError as err:
print("ERROR: {}".format(err))
sys.exit(1)
if args.cancel:
# Trying to cancel
if args.job_id is None:
parser.print_usage()
print("Error: You must provide an ingest job ID to cancel")
sys.exit(1)
if not get_confirmation("Are you sure you want to cancel ingest job {}? ".format(args.job_id), args.force):
print("Command ignored. Job not cancelled")
sys.exit(0)
always_log_info("Attempting to cancel Ingest Job {}.".format(args.job_id))
engine.cancel()
always_log_info("Ingest job {} successfully cancelled.".format(args.job_id))
sys.exit(0)
else:
# Trying to create or join an ingest
if args.job_id is None:
# Creating a new session - make sure the user wants to do this.
print_estimated_job(args.config_file)
print("\n")
if not get_confirmation("Would you like to create a NEW ingest job?", args.force):
# Don't want to create a new job
print("Exiting")
sys.exit(0)
else:
# Resuming a session - make sure the user wants to do this.
if not get_confirmation("Are you sure you want to resume ingest job {}?".format(args.job_id), args.force):
# Don't want to resume
print("Exiting")
sys.exit(0)
# Setup engine instance. Prompt user to confirm things if needed
question_msgs = engine.setup()
if question_msgs:
for msg in question_msgs:
if not get_confirmation(msg, args.force):
print("Ingest job cancelled")
sys.exit(0)
if args.job_id is None:
# Create job
engine.create_job()
always_log_info("Successfully Created Ingest Job ID: {}".format(engine.ingest_job_id))
always_log_info("Note: You need this ID to continue this job later!")
if not get_confirmation("\nDo you want to start uploading now?", args.force):
print("OK - Your job is waiting for you. You can resume by providing Ingest Job ID '{}' to the client".format(engine.ingest_job_id))
sys.exit(0)
# Join job
engine.join()
else:
# Join job
engine.join()
# Create worker processes
workers = []
for i in range(args.processes_nb):
new_pipe = mp.Pipe(False)
new_process = mp.Process(target=worker_process_run, args=(args.config_file, args.api_token,
engine.ingest_job_id, new_pipe[0]))
workers.append((new_process, new_pipe[1]))
new_process.start()
# Sleep to slowly ramp up load on lambda
time.sleep(.25)
# Start the main process engine
start_time = time.time()
should_run = True
job_complete = False
while should_run:
try:
engine.monitor(workers)
# run will end if no more jobs are available, join other processes
should_run = False
job_complete = True
except KeyboardInterrupt:
# Make sure they want to stop this client
while True:
quit_uploading = input("Are you sure you want to quit uploading? (y/n)")
if quit_uploading.lower() == "y":
always_log_info("Stopping upload engine.")
should_run = False
break
elif quit_uploading.lower() == "n":
print("Continuing...")
break
else:
print("Enter 'y' or 'n' for 'yes' or 'no'")
# notify the worker processes that they should stop execution
for _, worker_pipe in workers:
worker_pipe.send(should_run)
always_log_info("Waiting for worker processes to close...")
time.sleep(1) # Make sure workers have cleaned up
for worker_process, worker_pipe in workers:
worker_process.join()
worker_pipe.close()
if job_complete:
always_log_info("Job Complete - No more tasks remaining.")
always_log_info("Upload finished after {} minutes.".format((time.time() - start_time) / 60))
else:
always_log_info("Client exiting")
always_log_info("Run time: {} minutes.".format((time.time() - start_time) / 60))
def main(configuration=None, parser_args=None):
"""Client UI main
Args:
configuration(ingestclient.core.config.Configuration): A pre-loaded configuration instance
parser_args(argparse.ArgumentParser): A pre-loaded ArgumentParser instance
Returns:
"""
parser = get_parser()
if parser_args is None:
args = parser.parse_args()
else:
args = parser_args
# Get the version
if args.version:
check_version()
return
# Make sure you have a config file
if args.config_file is None and configuration is None:
if args.cancel:
# If no config is provided and you are deleting, the client defaults to the production Boss stack
boss_backend_params = {
"client": {
"backend": {
"name": "boss",
"class": "BossBackend",
"host": "api.theboss.io",
"protocol": "https"
}
}
}
backend = BossBackend(boss_backend_params)
backend.setup(args.api_token)
# Trying to cancel
if args.job_id is None:
parser.print_usage()
print("Error: You must provide an ingest job ID to cancel")
sys.exit(1)
if not get_confirmation(
"Are you sure you want to cancel ingest job {}? ".format(
args.job_id), args.force):
print("Command ignored. Job not cancelled")
sys.exit(0)
backend.cancel(args.job_id)
print("Ingest job {} successfully cancelled.".format(args.job_id))
sys.exit(0)
else:
# Not deleting, so you need a config file
parser.print_usage()
print("Error: Ingest Job Configuration File is required")
sys.exit(1)
# Setup logging
log_level = logging.getLevelName(args.log_level.upper())
if not args.log_file:
# Using default log path
log_path = os.path.expanduser("~/.boss-ingest")
log_file = os.path.join(
log_path, 'ingest_log{}_pid{}.log'.format(
datetime.datetime.now().strftime("%Y%m%d-%H%M%S"),
os.getpid()))
# Make sure the logs dir exists if using the default log path
if not os.path.exists(log_path):
os.makedirs(log_path)
else:
log_file = args.log_file
logging.basicConfig(level=log_level,
format='%(asctime)s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename=log_file,
filemode='a')
logging.getLogger('ingest-client').addHandler(
logging.StreamHandler(sys.stdout))
# Create an engine instance
try:
engine = Engine(config_file=args.config_file,
backend_api_token=args.api_token,
ingest_job_id=args.job_id,
configuration=configuration)
except ConfigFileError as err:
print("ERROR: {}".format(err))
sys.exit(1)
if args.cancel:
# Trying to cancel
if args.job_id is None:
parser.print_usage()
print("Error: You must provide an ingest job ID to cancel")
sys.exit(1)
if not get_confirmation(
"Are you sure you want to cancel ingest job {}? ".format(
args.job_id), args.force):
print("Command ignored. Job not cancelled")
sys.exit(0)
always_log_info("Attempting to cancel Ingest Job {}.".format(
args.job_id))
engine.cancel()
always_log_info("Ingest job {} successfully cancelled.".format(
args.job_id))
sys.exit(0)
else:
# Trying to create or join an ingest
if args.job_id is None:
# Creating a new session - make sure the user wants to do this.
print_estimated_job(config_file=args.config_file,
configuration=configuration)
print("\n")
if not get_confirmation(
"Would you like to create a NEW ingest job?", args.force):
# Don't want to create a new job
print("Exiting")
sys.exit(0)
else:
# Resuming a session - make sure the user wants to do this.
if not get_confirmation(
"Are you sure you want to resume ingest job {}?".format(
args.job_id), args.force):
# Don't want to resume
print("Exiting")
sys.exit(0)
# Setup engine instance. Prompt user to confirm things if needed
question_msgs = engine.setup()
if question_msgs:
for msg in question_msgs:
if not get_confirmation(msg, args.force):
print("Ingest job cancelled")
sys.exit(0)
if args.job_id is None:
# Create job
engine.create_job()
always_log_info("Successfully Created Ingest Job ID: {}".format(
engine.ingest_job_id))
always_log_info("Note: You need this ID to continue this job later!")
if not get_confirmation("\nDo you want to start uploading now?",
args.force):
print(
"OK - Your job is waiting for you. You can resume by providing Ingest Job ID '{}' to the client"
.format(engine.ingest_job_id))
sys.exit(0)
# Join job
engine.join()
else:
# Join job
engine.join()
# Create worker processes
workers = []
for i in range(args.processes_nb):
new_pipe = mp.Pipe(False)
new_process = mp.Process(target=worker_process_run,
args=(args.api_token, engine.ingest_job_id,
new_pipe[0]),
kwargs={
'config_file': args.config_file,
'configuration': configuration
})
workers.append((new_process, new_pipe[1]))
new_process.start()
# Sleep to slowly ramp up load on lambda
time.sleep(.5)
# Start the main process engine
start_time = time.time()
should_run = True
job_complete = False
while should_run:
try:
engine.monitor(workers)
# run will end if no more jobs are available, join other processes
should_run = False
job_complete = True
except KeyboardInterrupt:
# Make sure they want to stop this client
while True:
quit_uploading = input(
"Are you sure you want to quit uploading? (y/n)")
if quit_uploading.lower() == "y":
always_log_info("Stopping upload engine.")
should_run = False
break
elif quit_uploading.lower() == "n":
print("Continuing...")
break
else:
print("Enter 'y' or 'n' for 'yes' or 'no'")
# notify the worker processes that they should stop execution
for _, worker_pipe in workers:
worker_pipe.send(should_run)
always_log_info("Waiting for worker processes to close...\n")
time.sleep(1) # Make sure workers have cleaned up
for worker_process, worker_pipe in workers:
worker_process.join()
worker_pipe.close()
if job_complete:
# If auto-complete, mark the job as complete and cleanup
always_log_info("All upload tasks completed in {:.2f} minutes.".format(
(time.time() - start_time) / 60))
if not args.manual_complete:
always_log_info(
" - Marking Ingest Job as complete and cleaning up. Please wait."
)
engine.complete()
always_log_info(" - Cleanup Done")
else:
always_log_info(
" - Auto-complete disabled. This ingest job will remain in the 'Uploading' state until you manually mark it as complete"
)
else:
always_log_info("Client exiting")
always_log_info("Run time: {:.2f} minutes.".format(
(time.time() - start_time) / 60))
def main(configuration=None, parser_args=None):
"""Client UI main
Args:
configuration(ingestclient.core.config.Configuration): A pre-loaded configuration instance
parser_args(argparse.ArgumentParser): A pre-loaded ArgumentParser instance
Returns:
"""
parser = get_parser()
if parser_args is None:
args = parser.parse_args()
else:
args = parser_args
# Get the version
if args.version:
check_version()
return
# Make sure you have a config file
if args.config_file is None and configuration is None:
if args.cancel:
# If no config is provided and you are deleting, the client defaults to the production Boss stack
boss_backend_params = {
"client": {
"backend": {
"name": "boss",
"class": "BossBackend",
"host": "api.theboss.io",
"protocol": "https"
}
}
}
backend = BossBackend(boss_backend_params)
backend.setup(args.api_token)
# Trying to cancel
if args.job_id is None:
parser.print_usage()
print("Error: You must provide an ingest job ID to cancel")
sys.exit(1)
if not get_confirmation(
"Are you sure you want to cancel ingest job {}? ".format(
args.job_id), args.force):
print("Command ignored. Job not cancelled")
sys.exit(0)
backend.cancel(args.job_id)
print("Ingest job {} successfully cancelled.".format(args.job_id))
sys.exit(0)
else:
# Not deleting, so you need a config file
parser.print_usage()
print("Error: Ingest Job Configuration File is required")
sys.exit(1)
# Setup logging
log_level = logging.getLevelName(args.log_level.upper())
if not args.log_file:
# Using default log path
log_path = os.path.expanduser("~/.boss-ingest")
log_file = os.path.join(
log_path, 'ingest_log{}_pid{}.log'.format(
datetime.datetime.now().strftime("%Y%m%d-%H%M%S"),
os.getpid()))
# Make sure the logs dir exists if using the default log path
if not os.path.exists(log_path):
os.makedirs(log_path)
else:
log_file = args.log_file
logging.basicConfig(level=log_level,
format='%(asctime)s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename=log_file,
filemode='a')
logging.getLogger('ingest-client').addHandler(
logging.StreamHandler(sys.stdout))
# Create an engine instance
try:
engine = Engine(config_file=args.config_file,
backend_api_token=args.api_token,
ingest_job_id=args.job_id,
configuration=configuration)
except ConfigFileError as err:
print("ERROR: {}".format(err))
sys.exit(1)
if args.cancel:
# Trying to cancel
if args.job_id is None:
parser.print_usage()
print("Error: You must provide an ingest job ID to cancel")
sys.exit(1)
if not get_confirmation(
"Are you sure you want to cancel ingest job {}? ".format(
args.job_id), args.force):
print("Command ignored. Job not cancelled")
sys.exit(0)
always_log_info("Attempting to cancel Ingest Job {}.".format(
args.job_id))
engine.cancel()
always_log_info("Ingest job {} successfully cancelled.".format(
args.job_id))
sys.exit(0)
else:
# Trying to create or join an ingest
if args.job_id is None:
# Creating a new session - make sure the user wants to do this.
print_estimated_job(config_file=args.config_file,
configuration=configuration)
print("\n")
if not get_confirmation(
"Would you like to create a NEW ingest job?", args.force):
# Don't want to create a new job
print("Exiting")
sys.exit(0)
else:
# Resuming a session - make sure the user wants to do this.
if not get_confirmation(
"Are you sure you want to resume ingest job {}?".format(
args.job_id), args.force):
# Don't want to resume
print("Exiting")
sys.exit(0)
# Setup engine instance. Prompt user to confirm things if needed
question_msgs = engine.setup()
if question_msgs:
for msg in question_msgs:
if not get_confirmation(msg, args.force):
print("Ingest job cancelled")
sys.exit(0)
if args.job_id is None:
# Create job
engine.create_job()
always_log_info("Successfully Created Ingest Job ID: {}".format(
engine.ingest_job_id))
always_log_info("Note: You need this ID to continue this job later!")
if not get_confirmation("\nDo you want to start uploading now?",
args.force):
print(
"OK - Your job is waiting for you. You can resume by providing Ingest Job ID '{}' to the client"
.format(engine.ingest_job_id))
sys.exit(0)
# Join job
engine.join()
start_time = time.time()
while upload(engine, args, configuration, start_time):
pass