def clone(self, params: JSONDict, workspace: Workspace) -> LocalStateResourceMixin: """Instantiate a local copy of the resource that came from the remote origin. We don't yet have local params, since this resource is not yet on the local machine. If not in batch mode, this method can ask the user for any additional information needed (e.g. a local path). In batch mode, should either come up with a reasonable default or error out if not enough information is available.""" workspace._get_local_scratch_space_for_resource( params["name"], create_if_not_present=True) return ApiResource(params["name"], params["role"], workspace)
def from_command_line(self, role: str, name: str, workspace: Workspace, *args, **kwargs) -> Resource: """Instantiate a resource object from the add command's arguments""" if role not in (ResourceRoles.SOURCE_DATA_SET, ResourceRoles.INTERMEDIATE_DATA): raise ConfigurationError( "API resources only supported for %s and %s roles" % (ResourceRoles.SOURCE_DATA_SET, ResourceRoles.INTERMEDIATE_DATA)) workspace._get_local_scratch_space_for_resource( name, create_if_not_present=True) return ApiResource(name, role, workspace)
def __init__( self, resource_type: str, name: str, role: str, workspace: Workspace, bucket_name: str, #region: str, ): if role == ResourceRoles.RESULTS: raise RESULTS_ROLE_NOT_SUPPORTED super().__init__(resource_type, name, role, workspace) self.param_defs.define( "bucket_name", default_value=None, optional=False, is_global=True, help="Name of the bucket", ptype=StringType(), ) self.bucket_name = self.param_defs.get("bucket_name", bucket_name) # type: str # local scratch space for resource is where we store the current snapshot and # the cache. self.local_scratch_dir = workspace._get_local_scratch_space_for_resource( self.name, create_if_not_present=True) self.current_snapshot_file = join(self.local_scratch_dir, 'current_snapshot.txt') self.current_snapshot = None # type: Optional[str] self.snapshot_fs = None # type: Optional[S3Snapshot] # we cache snapshot files in a subdirectory of the scratch dir self.snapshot_cache_dir = join(self.local_scratch_dir, "snapshot_cache") # Make sure it exists. if not exists(self.snapshot_cache_dir): os.makedirs(self.snapshot_cache_dir) if exists(self.current_snapshot_file): with open(self.current_snapshot_file, 'r') as f: self.current_snapshot = f.read().strip() self.fs = S3FileSystem(version_aware=True) self.snapshot_fs = self._load_snapshot(self.current_snapshot) else: self.fs = S3FileSystem()