def test_package_to_zip_source_remote(tmpdir):
# Write
path = BASE_URL % "data/table.csv"
target = os.path.join(tmpdir, "package.zip")
package = Package(name="name", resources=[{"name": "name", "path": path}])
package.to_zip(target)
# Read
package = Package(target)
assert package == {
"name": "name",
"resources": [{
"name": "name",
"path": path
}],
}
assert package.get_resource("name").read_rows() == [
{
"id": 1,
"name": "english"
},
{
"id": 2,
"name": "中国人"
},
]
def test_package_to_zip(tmpdir):
# Write
target = os.path.join(tmpdir, "package.zip")
package = Package("data/package.json")
package.to_zip(target)
# Read
package = Package(target)
assert package == {
"name": "name",
"resources": [{
"name": "name",
"path": "table.csv"
}],
}
assert package.get_resource("name").read_rows() == [
{
"id": 1,
"name": "english"
},
{
"id": 2,
"name": "中国人"
},
]
def test_package_to_zip_source_inline(tmpdir):
# Read
target = os.path.join(tmpdir, "package.zip")
data = [["id", "name"], ["1", "english"], ["2", "中国人"]]
package = Package(name="name", resources=[{"name": "name", "data": data}])
package.to_zip(target)
# Write
package = Package(target)
assert package == {
"name": "name",
"resources": [{
"name": "name",
"data": data
}],
}
assert package.get_resource("name").read_rows() == [
{
"id": 1,
"name": "english"
},
{
"id": 2,
"name": "中国人"
},
]
def test_package_to_zip_resolve_inline_and_remote(tmpdir):
# Write
target = os.path.join(tmpdir, "package.zip")
resource1 = Resource(name="name1",
data=[["id", "name"], [1, "english"], [2, "中国人"]])
resource2 = Resource(name="name2", path=BASE_URL % "data/table.csv")
package = Package(resources=[resource1, resource2])
package.to_zip(target, resolve=["inline", "remote"])
# Read
package = Package(target)
assert package.get_resource("name1").path == "name1.csv"
assert package.get_resource("name1").read_rows() == [
{
"id": 1,
"name": "english"
},
{
"id": 2,
"name": "中国人"
},
]
assert package.get_resource("name2").path == "name2.csv"
assert package.get_resource("name2").read_rows() == [
{
"id": 1,
"name": "english"
},
{
"id": 2,
"name": "中国人"
},
]
def test_package_to_zip_resource_multipart(tmpdir, database_url):
path = os.path.join(tmpdir, "package.zip")
source = Package(resources=[Resource(path=["data/chunk1.csv", "data/chunk2.csv"])])
source.to_zip(path)
target = Package.from_zip(path)
assert target.get_resource("chunk").path == ["data/chunk1.csv", "data/chunk2.csv"]
assert target.get_resource("chunk").read_rows() == [
{"id": 1, "name": "english"},
{"id": 2, "name": "中国人"},
]
def test_package_to_zip_resource_remote_path(tmpdir):
path = os.path.join(tmpdir, "package.zip")
source = Package(resources=[Resource(path=BASEURL % "data/table.csv")])
source.to_zip(path)
target = Package.from_zip(path)
assert target.get_resource("table").path == BASEURL % "data/table.csv"
assert target.get_resource("table").read_rows() == [
{"id": 1, "name": "english"},
{"id": 2, "name": "中国人"},
]
def test_package_to_zip_resource_sql(tmpdir, database_url):
path = os.path.join(tmpdir, "package.zip")
dialect = SqlDialect(table="table")
source = Package(resources=[Resource(database_url, name="table", dialect=dialect)])
source.to_zip(path)
target = Package.from_zip(path)
assert target.get_resource("table").path == database_url
assert target.get_resource("table").read_rows() == [
{"id": 1, "name": "english"},
{"id": 2, "name": "中国人"},
]
def test_package_to_zip_resource_memory_inline(tmpdir):
path = os.path.join(tmpdir, "package.zip")
data = [["id", "name"], [1, "english"], [2, "中国人"]]
source = Package(resources=[Resource(name="table", data=data)])
source.to_zip(path)
target = Package.from_zip(path)
assert target.get_resource("table").data == data
assert target.get_resource("table").read_rows() == [
{"id": 1, "name": "english"},
{"id": 2, "name": "中国人"},
]
def test_package_to_zip_resource_memory_function(tmpdir):
path = os.path.join(tmpdir, "package.zip")
data = lambda: [["id", "name"], [1, "english"], [2, "中国人"]]
source = Package(resources=[Resource(name="table", data=data)])
if IS_UNIX:
source.to_zip(path)
target = Package.from_zip(path)
assert target.get_resource("table").path == "table.csv"
assert target.get_resource("table").read_rows() == [
{"id": 1, "name": "english"},
{"id": 2, "name": "中国人"},
]
def test_package_to_zip(tmpdir):
path = os.path.join(tmpdir, "package.zip")
source = Package("data/package.json")
source.to_zip(path)
target = Package.from_zip(path)
assert target.name == "name"
assert target.get_resource("name").name == "name"
assert target.get_resource("name").path == "table.csv"
assert target.get_resource("name").read_rows() == [
{"id": 1, "name": "english"},
{"id": 2, "name": "中国人"},
]
def test_package_to_zip_resolve_inline_sql(tmpdir, database_url):
# Write
target = os.path.join(tmpdir, "package.zip")
resource = Resource.from_sql(name="table", url=database_url)
package = Package(resources=[resource])
package.to_zip(target, resolve=["inline"])
# Read
package = Package(target)
assert package.get_resource("table").path == "table.csv"
assert package.get_resource("table").read_rows() == [
{"id": 1, "name": "english"},
{"id": 2, "name": "中国人"},
]
def test_package_to_zip_resolve_remote(tmpdir):
# Write
target = os.path.join(tmpdir, "package.zip")
resource = Resource(path=BASE_URL % "data/table.csv")
package = Package(resources=[resource])
package.to_zip(target, resolve=["remote"])
# Read
package = Package(target)
assert package.get_resource("table").path == "table.csv"
assert package.get_resource("table").read_rows() == [
{"id": 1, "name": "english"},
{"id": 2, "name": "中国人"},
]
def test_package_to_zip_absolute_path(tmpdir):
# Write
target = os.path.join(tmpdir, "package.zip")
resource = Resource(path=os.path.abspath("data/table.csv"), trusted=True)
package = Package(resources=[resource], trusted=True)
package.to_zip(target)
# Read
package = Package(target)
assert package.get_resource("table").path == "table.csv"
assert package.get_resource("table").read_rows() == [
{"id": 1, "name": "english"},
{"id": 2, "name": "中国人"},
]
def test_package_to_zip_resolve_memory_sql(tmpdir, database_url):
# Write
target = os.path.join(tmpdir, "package.zip")
resource = Resource(database_url, dialect={"table": "table"})
package = Package(resources=[resource])
package.to_zip(target, resolve=["memory"])
# Read
package = Package.from_zip(target)
assert package.get_resource("table").path == "table.csv"
assert package.get_resource("table").read_rows() == [
{
"id": 1,
"name": "english"
},
{
"id": 2,
"name": "中国人"
},
]
def test_package_to_zip_withdir_path(tmpdir):
# Write
target = os.path.join(tmpdir, "package.zip")
resource = Resource(path="data/table.csv")
package = Package(resources=[resource])
package.to_zip(target)
# Read
package = Package.from_zip(target)
assert package.get_resource("table").path == "data/table.csv"
assert package.get_resource("table").read_rows() == [
{
"id": 1,
"name": "english"
},
{
"id": 2,
"name": "中国人"
},
]
def test_package_to_zip_resolve_memory(tmpdir):
# Write
target = os.path.join(tmpdir, "package.zip")
resource = Resource(name="table",
data=[["id", "name"], [1, "english"], [2, "中国人"]])
package = Package(resources=[resource])
package.to_zip(target, resolve=["memory"])
# Read
package = Package.from_zip(target)
assert package.get_resource("table").path == "table.csv"
assert package.get_resource("table").read_rows() == [
{
"id": 1,
"name": "english"
},
{
"id": 2,
"name": "中国人"
},
]
def package_event(event, format):
if format not in ['zip', 'json']:
return "Format not supported"
# Set up a data package
package = Package(
name='event-%d' % event.id,
title=event.name,
description="Event and project details collected with dribdat",
keywords=["dribdat", "hackathon", "co-creation"],
sources=[{
"title": "dribdat",
"path": "https://dribdat.cc"
}],
licenses=[{
"name":
"ODC-PDDL-1.0",
"path":
"http://opendatacommons.org/licenses/pddl/",
"title":
"Open Data Commons Public Domain Dedication & License 1.0"
}],
contributors=[{
"title": current_user.username,
"path": current_user.webpage_url or '',
"role": "author"
}],
homepage=request.host_url,
created=format_date(datetime.now(), '%Y-%m-%dT%H:%M'),
version="0.1.0",
)
# if False: # as CSV
# fp_projects = tempfile.NamedTemporaryFile(
# mode='w+t', prefix='projects-', suffix='.csv')
# # print("Writing to temp CSV file", fp_projects.name)
# fp_projects.write(gen_csv(project_list(event.id)))
# resource = Resource(fp_projects.name)
# if False:
# # print("Generating in-memory rowset")
# project_rows = gen_rows(project_list(event.id))
# resource = Resource(
# name='projects',
# data=project_rows,
# )
# Generate resources
# print("Generating in-memory JSON of event")
package.add_resource(
Resource(
name='events',
data=[event.get_full_data()],
))
# print("Generating in-memory JSON of projects")
package.add_resource(
Resource(
name='projects',
data=project_list(event.id, True),
))
if format == 'zip':
# print("Generating in-memory JSON of participants")
package.add_resource(
Resource(
name='users',
data=get_event_users(event),
))
# print("Generating in-memory JSON of activities")
package.add_resource(
Resource(
name='activities',
data=get_event_activities(event.id, 500),
))
# print("Generating in-memory JSON of activities")
package.add_resource(
Resource(
name='categories',
data=get_event_categories(event.id),
))
# print("Adding supplementary README")
package.add_resource(Resource(
name='readme',
path='PACKAGE.txt',
))
# Generate data package
fp_package = tempfile.NamedTemporaryFile(prefix='datapackage-',
suffix='.zip')
# print("Saving at", fp_package.name)
if format == 'json':
return jsonify(package)
elif format == 'zip':
package.to_zip(fp_package.name)
return send_file(fp_package.name, as_attachment=True)