def get_request_payer(): # Import here to avoid circular reference. from rastervision.rv_config import RVConfig rv_config = RVConfig.get_instance() s3_config = rv_config.get_subconfig('AWS_S3') # 'None' needs the quotes because boto3 cannot handle None. return ('requester' if s3_config( 'requester_pays', parser=bool, default='False') else 'None')
def setup_plugin_logger(root_name): plugin_logger = logging.getLogger(root_name) sh = logging.StreamHandler() sh.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s:%(name)s: %(levelname)s - %(message)s', '%Y-%m-%d %H:%M:%S') sh.setFormatter(formatter) plugin_logger.addHandler(sh) plugin_logger.setLevel(RVConfig.get_instance().verbosity)
def _batch_submit(cmd, debug=False, profile=False, attempts=5, parent_job_ids=None, num_array_jobs=None, use_gpu=False): rv_config = RVConfig.get_instance() batch_config = rv_config.get_subconfig('AWS_BATCH') job_queue = batch_config('cpu_job_queue') job_def = batch_config('cpu_job_definition') if use_gpu: job_queue = batch_config('job_queue') job_def = batch_config('job_definition') import boto3 client = boto3.client('batch') job_name = 'ffda-{}'.format(uuid.uuid4()) cmd_list = cmd.split(' ') if debug: cmd_list = [ 'python', '-m', 'ptvsd', '--host', '0.0.0.0', '--port', '6006', '--wait', '-m' ] + cmd_list if profile: cmd_list = ['kernprof', '-v', '-l'] + cmd_list kwargs = { 'jobName': job_name, 'jobQueue': job_queue, 'jobDefinition': job_def, 'containerOverrides': { 'command': cmd_list }, 'retryStrategy': { 'attempts': attempts }, } if parent_job_ids: kwargs['dependsOn'] = [{'jobId': id} for id in parent_job_ids] if num_array_jobs: kwargs['arrayProperties'] = {'size': num_array_jobs} job_id = client.submit_job(**kwargs)['jobId'] msg = 'submitted job with jobName={} and jobId={}'.format(job_name, job_id) print(cmd_list) print(msg) return job_id
def with_model_defaults(self, model_defaults_key): """Sets the backend configuration and pretrained model defaults according to the model defaults configuration. """ model_defaults = RVConfig.get_instance().get_model_defaults() if self.backend_type in model_defaults: backend_defaults = model_defaults[self.backend_type] if model_defaults_key in backend_defaults: return self._load_model_defaults( backend_defaults[model_defaults_key]) else: raise rv.ConfigError('No defaults found for model key {}' .format(model_defaults_key)) else: raise rv.ConfigError('No model defaults for backend {}' .format(self.backend_type)) return self
def __init__(self): super().__init__() rv_config = RVConfig.get_instance() batch_config = rv_config.get_subconfig('AWS_BATCH') self.attempts = batch_config('attempts', parser=int, default='1') self.gpu = batch_config('gpu', parser=bool, default='true') job_queue = batch_config('job_queue', default='') if not job_queue: if self.gpu: job_queue = 'raster-vision-gpu' else: job_queue = 'raster-vision-cpu' self.job_queue = job_queue cpu_job_queue = batch_config('cpu_job_queue', default='') if not cpu_job_queue: if self.gpu: cpu_job_queue = 'raster-vision-cpu' else: cpu_job_queue = job_queue self.cpu_job_queue = cpu_job_queue job_definition = batch_config('job_definition', default='') if not job_definition: if self.gpu: job_definition = 'raster-vision-gpu' else: job_definition = 'raster-vision-cpu' self.job_definition = job_definition cpu_job_definition = batch_config('cpu_job_definition', default='') if not cpu_job_definition: if self.gpu: cpu_job_definition = 'raster-vision-cpu' else: cpu_job_definition = job_definition self.cpu_job_definition = cpu_job_definition self.submit = self.batch_submit self.execution_environment = 'Batch'