def clone(self, params: JSONDict,
workspace: Workspace) -> LocalStateResourceMixin:
"""Instantiate a local copy of the resource
that came from the remote origin. We don't yet have local params,
since this resource is not yet on the local machine. If not in batch
mode, this method can ask the user for any additional information needed
(e.g. a local path). In batch mode, should either come up with a reasonable
default or error out if not enough information is available."""
workspace._get_local_scratch_space_for_resource(
params["name"], create_if_not_present=True)
return ApiResource(params["name"], params["role"], workspace)
def from_command_line(self, role: str, name: str, workspace: Workspace,
*args, **kwargs) -> Resource:
"""Instantiate a resource object from the add command's
arguments"""
if role not in (ResourceRoles.SOURCE_DATA_SET,
ResourceRoles.INTERMEDIATE_DATA):
raise ConfigurationError(
"API resources only supported for %s and %s roles" %
(ResourceRoles.SOURCE_DATA_SET,
ResourceRoles.INTERMEDIATE_DATA))
workspace._get_local_scratch_space_for_resource(
name, create_if_not_present=True)
return ApiResource(name, role, workspace)
def __init__(
self,
resource_type: str,
name: str,
role: str,
workspace: Workspace,
bucket_name: str,
#region: str,
):
if role == ResourceRoles.RESULTS:
raise RESULTS_ROLE_NOT_SUPPORTED
super().__init__(resource_type, name, role, workspace)
self.param_defs.define(
"bucket_name",
default_value=None,
optional=False,
is_global=True,
help="Name of the bucket",
ptype=StringType(),
)
self.bucket_name = self.param_defs.get("bucket_name",
bucket_name) # type: str
# local scratch space for resource is where we store the current snapshot and
# the cache.
self.local_scratch_dir = workspace._get_local_scratch_space_for_resource(
self.name, create_if_not_present=True)
self.current_snapshot_file = join(self.local_scratch_dir,
'current_snapshot.txt')
self.current_snapshot = None # type: Optional[str]
self.snapshot_fs = None # type: Optional[S3Snapshot]
# we cache snapshot files in a subdirectory of the scratch dir
self.snapshot_cache_dir = join(self.local_scratch_dir,
"snapshot_cache")
# Make sure it exists.
if not exists(self.snapshot_cache_dir):
os.makedirs(self.snapshot_cache_dir)
if exists(self.current_snapshot_file):
with open(self.current_snapshot_file, 'r') as f:
self.current_snapshot = f.read().strip()
self.fs = S3FileSystem(version_aware=True)
self.snapshot_fs = self._load_snapshot(self.current_snapshot)
else:
self.fs = S3FileSystem()