def test_library_object_serialization_deserialization():
libraries = [{
'jar': '/absolute/path/abc.ext'
}, {
'jar': 'relative/path.ext'
}, {
'jar': 'file:/file/scheme/relative/path.ext'
}, {
'jar': 'FILE:/all/caps.ext'
}, {
'egg': 'FiLe:/weird/case.ext'
}, {
'whl': 'file.ext'
}, {
'whl': 's3:/s3/path/file.ext'
}, {
'jar': 'dbfs:/dbfs/path/file.ext'
}, {
'maven': {
'coordinates': 'com.org.name:package:0.1.0'
}
}, {
'maven': {
'coordinates': 'com.org.name:package:0.1.0',
'exclusions': ['slf4j:slf4j', '*:hadoop-client']
}
}]
library_objects = [
LibraryObject('jar', '/absolute/path/abc.ext'),
LibraryObject('jar', 'relative/path.ext'),
LibraryObject('jar', 'file:/file/scheme/relative/path.ext'),
LibraryObject('jar', 'FILE:/all/caps.ext'),
LibraryObject('egg', 'FiLe:/weird/case.ext'),
LibraryObject('whl', 'file.ext'),
LibraryObject('whl', 's3:/s3/path/file.ext'),
LibraryObject('jar', 'dbfs:/dbfs/path/file.ext'),
LibraryObject('maven', {'coordinates': 'com.org.name:package:0.1.0'}),
LibraryObject(
'maven', {
'coordinates': 'com.org.name:package:0.1.0',
'exclusions': ['slf4j:slf4j', '*:hadoop-client']
})
]
llo = LibraryObject.from_json(libraries)
assert llo == library_objects
libs = LibraryObject.to_json(library_objects)
assert libs == libraries
def test_partition_local_remote(pipelines_api):
libraries = [
# local files
LibraryObject('jar', '/absolute/path/abc.ext'),
LibraryObject('jar', 'relative/path.ext'),
LibraryObject('jar', 'file:///file/scheme/abs/path.ext'),
LibraryObject('jar', 'FILE:/all/caps.ext'),
LibraryObject('jar', 'FiLe:/weird/case.ext'),
LibraryObject('jar', 'file.ext'),
LibraryObject('whl', 'rel/path.ext'),
# shouldn't be uploaded
LibraryObject('jar', 'dbfs:/absolute/path/abc.ext'),
LibraryObject('jar', 's3:file:/file/scheme/abs/path.ext'),
LibraryObject('jar', 'scheme:file.ext'),
LibraryObject('jar', 'scheme:/abs/path.ext'),
LibraryObject('jar', 'scheme://abs/path.ext'),
LibraryObject('egg', 'file:/abs/path.ext'),
LibraryObject('maven', {'coordinates': 'com.org.name:package:0.1.0'})
]
expected_llo = [
LibraryObject('jar', '/absolute/path/abc.ext'),
LibraryObject('jar', 'relative/path.ext'),
LibraryObject('jar', '/file/scheme/abs/path.ext'),
LibraryObject('jar', '/all/caps.ext'),
LibraryObject('jar', '/weird/case.ext'),
LibraryObject('jar', 'file.ext'),
LibraryObject('whl', 'rel/path.ext')
]
expected_external = [
LibraryObject('jar', 'dbfs:/absolute/path/abc.ext'),
LibraryObject('jar', 's3:file:/file/scheme/abs/path.ext'),
LibraryObject('jar', 'scheme:file.ext'),
LibraryObject('jar', 'scheme:/abs/path.ext'),
LibraryObject('jar', 'scheme://abs/path.ext'),
LibraryObject('egg', 'file:/abs/path.ext'),
LibraryObject('maven', {'coordinates': 'com.org.name:package:0.1.0'})
]
llo, external = pipelines_api._identify_local_libraries(libraries)
assert llo == expected_llo
assert external == expected_external