Python DatasetFile.from_s3_path示例

示例#1

文件： forecast_etl.py 项目： emmanuellim/improving-forecast-accuracy-with-machine-learning

    def _get_datasets(
        self,
    ) -> (
            Union[None, DatasetFileDataset],
            Union[None, DatasetFileDataset],
            Union[None, DatasetFileDataset],
    ):
        """
        Gets the datasets and dataset files associated with this forecast
        :return: (ts, rts, md)
        """
        datasets = self.config.datasets(self.dataset_file)
        prefix = f"s3://{self.dataset_file.bucket}/train/{self.dataset_file.prefix}"

        ts, rts, md = None, None, None

        for dataset in datasets:
            if dataset.dataset_type == DatasetType.TARGET_TIME_SERIES:
                ts = DatasetFileDataset(
                    dataset,
                    DatasetFile.from_s3_path(prefix +
                                             dataset.dataset_type.suffix),
                )
            elif dataset.dataset_type == DatasetType.RELATED_TIME_SERIES:
                rts = DatasetFileDataset(
                    dataset,
                    DatasetFile.from_s3_path(prefix +
                                             dataset.dataset_type.suffix),
                )
            elif dataset.dataset_type == DatasetType.ITEM_METADATA:
                md = DatasetFileDataset(
                    dataset,
                    DatasetFile.from_s3_path(prefix +
                                             dataset.dataset_type.suffix),
                )

        return (ts, rts, md)

示例#2

文件： test_dataset_file.py 项目： emmanuellim/improving-forecast-accuracy-with-machine-learning

def test_dataset_file_from_s3_path(path, bucket, key):
    dsf = DatasetFile.from_s3_path(s3_path=path)
    assert dsf.bucket == bucket
    assert dsf.key == key

示例#3

文件： test_dataset_file.py 项目： emmanuellim/improving-forecast-accuracy-with-machine-learning

def test_dataset_file_from_s3_path(path):
    dsf = DatasetFile.from_s3_path(s3_path=path)
    assert dsf.s3_url == path.split("?")[0]

示例#4

文件： test_dataset_file.py 项目： emmanuellim/improving-forecast-accuracy-with-machine-learning

def test_dataset_file_from_s3_path(path):
    dsf = DatasetFile.from_s3_path(s3_path=path)
    assert dsf.s3_prefix == "s3://some_bucket/"