Ejemplo n.º 1
0
def test_library_object_serialization_deserialization():
    libraries = [{
        'jar': '/absolute/path/abc.ext'
    }, {
        'jar': 'relative/path.ext'
    }, {
        'jar': 'file:/file/scheme/relative/path.ext'
    }, {
        'jar': 'FILE:/all/caps.ext'
    }, {
        'egg': 'FiLe:/weird/case.ext'
    }, {
        'whl': 'file.ext'
    }, {
        'whl': 's3:/s3/path/file.ext'
    }, {
        'jar': 'dbfs:/dbfs/path/file.ext'
    }, {
        'maven': {
            'coordinates': 'com.org.name:package:0.1.0'
        }
    }, {
        'maven': {
            'coordinates': 'com.org.name:package:0.1.0',
            'exclusions': ['slf4j:slf4j', '*:hadoop-client']
        }
    }]
    library_objects = [
        LibraryObject('jar', '/absolute/path/abc.ext'),
        LibraryObject('jar', 'relative/path.ext'),
        LibraryObject('jar', 'file:/file/scheme/relative/path.ext'),
        LibraryObject('jar', 'FILE:/all/caps.ext'),
        LibraryObject('egg', 'FiLe:/weird/case.ext'),
        LibraryObject('whl', 'file.ext'),
        LibraryObject('whl', 's3:/s3/path/file.ext'),
        LibraryObject('jar', 'dbfs:/dbfs/path/file.ext'),
        LibraryObject('maven', {'coordinates': 'com.org.name:package:0.1.0'}),
        LibraryObject(
            'maven', {
                'coordinates': 'com.org.name:package:0.1.0',
                'exclusions': ['slf4j:slf4j', '*:hadoop-client']
            })
    ]
    llo = LibraryObject.from_json(libraries)
    assert llo == library_objects

    libs = LibraryObject.to_json(library_objects)
    assert libs == libraries
Ejemplo n.º 2
0
def test_partition_local_remote(pipelines_api):
    libraries = [
        # local files
        LibraryObject('jar', '/absolute/path/abc.ext'),
        LibraryObject('jar', 'relative/path.ext'),
        LibraryObject('jar', 'file:///file/scheme/abs/path.ext'),
        LibraryObject('jar', 'FILE:/all/caps.ext'),
        LibraryObject('jar', 'FiLe:/weird/case.ext'),
        LibraryObject('jar', 'file.ext'),
        LibraryObject('whl', 'rel/path.ext'),
        # shouldn't be uploaded
        LibraryObject('jar', 'dbfs:/absolute/path/abc.ext'),
        LibraryObject('jar', 's3:file:/file/scheme/abs/path.ext'),
        LibraryObject('jar', 'scheme:file.ext'),
        LibraryObject('jar', 'scheme:/abs/path.ext'),
        LibraryObject('jar', 'scheme://abs/path.ext'),
        LibraryObject('egg', 'file:/abs/path.ext'),
        LibraryObject('maven', {'coordinates': 'com.org.name:package:0.1.0'})
    ]
    expected_llo = [
        LibraryObject('jar', '/absolute/path/abc.ext'),
        LibraryObject('jar', 'relative/path.ext'),
        LibraryObject('jar', '/file/scheme/abs/path.ext'),
        LibraryObject('jar', '/all/caps.ext'),
        LibraryObject('jar', '/weird/case.ext'),
        LibraryObject('jar', 'file.ext'),
        LibraryObject('whl', 'rel/path.ext')
    ]
    expected_external = [
        LibraryObject('jar', 'dbfs:/absolute/path/abc.ext'),
        LibraryObject('jar', 's3:file:/file/scheme/abs/path.ext'),
        LibraryObject('jar', 'scheme:file.ext'),
        LibraryObject('jar', 'scheme:/abs/path.ext'),
        LibraryObject('jar', 'scheme://abs/path.ext'),
        LibraryObject('egg', 'file:/abs/path.ext'),
        LibraryObject('maven', {'coordinates': 'com.org.name:package:0.1.0'})
    ]
    llo, external = pipelines_api._identify_local_libraries(libraries)
    assert llo == expected_llo
    assert external == expected_external