def get_request_payer():
# Import here to avoid circular reference.
from rastervision.rv_config import RVConfig
rv_config = RVConfig.get_instance()
s3_config = rv_config.get_subconfig('AWS_S3')
# 'None' needs the quotes because boto3 cannot handle None.
return ('requester' if s3_config(
'requester_pays', parser=bool, default='False') else 'None')
def setup_plugin_logger(root_name):
plugin_logger = logging.getLogger(root_name)
sh = logging.StreamHandler()
sh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s:%(name)s: %(levelname)s - %(message)s',
'%Y-%m-%d %H:%M:%S')
sh.setFormatter(formatter)
plugin_logger.addHandler(sh)
plugin_logger.setLevel(RVConfig.get_instance().verbosity)
def _batch_submit(cmd,
debug=False,
profile=False,
attempts=5,
parent_job_ids=None,
num_array_jobs=None,
use_gpu=False):
rv_config = RVConfig.get_instance()
batch_config = rv_config.get_subconfig('AWS_BATCH')
job_queue = batch_config('cpu_job_queue')
job_def = batch_config('cpu_job_definition')
if use_gpu:
job_queue = batch_config('job_queue')
job_def = batch_config('job_definition')
import boto3
client = boto3.client('batch')
job_name = 'ffda-{}'.format(uuid.uuid4())
cmd_list = cmd.split(' ')
if debug:
cmd_list = [
'python', '-m', 'ptvsd', '--host', '0.0.0.0', '--port', '6006',
'--wait', '-m'
] + cmd_list
if profile:
cmd_list = ['kernprof', '-v', '-l'] + cmd_list
kwargs = {
'jobName': job_name,
'jobQueue': job_queue,
'jobDefinition': job_def,
'containerOverrides': {
'command': cmd_list
},
'retryStrategy': {
'attempts': attempts
},
}
if parent_job_ids:
kwargs['dependsOn'] = [{'jobId': id} for id in parent_job_ids]
if num_array_jobs:
kwargs['arrayProperties'] = {'size': num_array_jobs}
job_id = client.submit_job(**kwargs)['jobId']
msg = 'submitted job with jobName={} and jobId={}'.format(job_name, job_id)
print(cmd_list)
print(msg)
return job_id
def with_model_defaults(self, model_defaults_key):
"""Sets the backend configuration and pretrained model defaults
according to the model defaults configuration.
"""
model_defaults = RVConfig.get_instance().get_model_defaults()
if self.backend_type in model_defaults:
backend_defaults = model_defaults[self.backend_type]
if model_defaults_key in backend_defaults:
return self._load_model_defaults(
backend_defaults[model_defaults_key])
else:
raise rv.ConfigError('No defaults found for model key {}'
.format(model_defaults_key))
else:
raise rv.ConfigError('No model defaults for backend {}'
.format(self.backend_type))
return self
def __init__(self):
super().__init__()
rv_config = RVConfig.get_instance()
batch_config = rv_config.get_subconfig('AWS_BATCH')
self.attempts = batch_config('attempts', parser=int, default='1')
self.gpu = batch_config('gpu', parser=bool, default='true')
job_queue = batch_config('job_queue', default='')
if not job_queue:
if self.gpu:
job_queue = 'raster-vision-gpu'
else:
job_queue = 'raster-vision-cpu'
self.job_queue = job_queue
cpu_job_queue = batch_config('cpu_job_queue', default='')
if not cpu_job_queue:
if self.gpu:
cpu_job_queue = 'raster-vision-cpu'
else:
cpu_job_queue = job_queue
self.cpu_job_queue = cpu_job_queue
job_definition = batch_config('job_definition', default='')
if not job_definition:
if self.gpu:
job_definition = 'raster-vision-gpu'
else:
job_definition = 'raster-vision-cpu'
self.job_definition = job_definition
cpu_job_definition = batch_config('cpu_job_definition', default='')
if not cpu_job_definition:
if self.gpu:
cpu_job_definition = 'raster-vision-cpu'
else:
cpu_job_definition = job_definition
self.cpu_job_definition = cpu_job_definition
self.submit = self.batch_submit
self.execution_environment = 'Batch'
