Пример #1
0
    def test_cascading_inheritence(self):
        MLSchema.populate_registry()

        mlobject = MLObject()
        mlobject.set_type("0.0.1", "data_version_control")
        mlobject.run_id = uuid.uuid4()
        mlobject.step_id = uuid.uuid4()
        mlobject.run_date = datetime.datetime.now()
        mlobject.data_store = "I_am_a_datastore"
        mlobject.storage_connection_type = "AWS_BLOB"
        mlobject.connection.endpoint = "con_endpoint"
        mlobject.connection.access_key_id = "AKIAIOSFODNN7EXAMPLE"
        mlobject.connection.secret_access_key = (
            "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")

        mlobject.dvc_hash = "923caceea54b38177505632f5612cc569a49b22246e346a7"
        mlobject.validate()
Пример #2
0
    def test_load_and_save_file(self):
        run_id = uuid.uuid4()

        save_path = Path(self.test_dir.name) / str(run_id)
        save_path.mkdir()

        datapath_object = MLObject()
        datapath_object.set_type('0.0.1', MLSchemaTypes.DATAPATH)

        datapath_object.run_id = run_id
        datapath_object.step_id = uuid.uuid4()
        datapath_object.run_date = datetime.datetime.now()
        datapath_object.data_store = None  # This is an intentional bug

        # This is an intentional bug (Should be AWS_BLOB)
        datapath_object.storage_connection_type = 'AWS_BLOB_OBJECT'
        datapath_object.connection.endpoint = None  # Another intentional bug
        datapath_object.connection.access_key_id = 'AKIAIOSFODNN7EXAMPLE'
        datapath_object.connection.secret_access_key = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'

        response, errors = datapath_object.save(save_path)

        self.assertFalse(response)
        self.assertTrue(len(errors) == 3)
        self.assertTrue(len(list(Path(save_path).glob('*'))) == 0)

        datapath_object.storage_connection_type = 'AWS_BLOB'

        response, errors = datapath_object.save(save_path)

        self.assertFalse(response)
        self.assertTrue(len(errors) == 2)
        self.assertTrue(len(list(Path(save_path).glob('*'))) == 0)

        datapath_object.connection.endpoint = 'http://s3.amazon.com/BUCKET'

        response, errors = datapath_object.save(save_path)

        self.assertFalse(response)
        self.assertTrue(len(errors) == 1)
        self.assertTrue(len(list(Path(save_path).glob('*'))) == 0)

        datapath_object.data_store = 'BUCKET NAME'

        response, errors = datapath_object.save(save_path)

        self.assertTrue(response)
        self.assertTrue(len(errors) == 0)
        path = Path(save_path)
        all_files = list(path.glob('*'))
        self.assertTrue(len(all_files) == 1)

        ml_object, errors = MLObject.create_object_from_file(all_files[0])
        self.assertTrue(len(ml_object) == 13)
        self.assertTrue(len(errors) == 0)

        self.assertTrue(datapath_object.data_store == ml_object.data_store)
        self.assertTrue(datapath_object.storage_connection_type ==
                        ml_object.storage_connection_type)
        self.assertTrue(datapath_object.connection.endpoint ==
                        ml_object.connection.endpoint)