def test_package_to_zip_source_remote(tmpdir): # Write path = BASE_URL % "data/table.csv" target = os.path.join(tmpdir, "package.zip") package = Package(name="name", resources=[{"name": "name", "path": path}]) package.to_zip(target) # Read package = Package(target) assert package == { "name": "name", "resources": [{ "name": "name", "path": path }], } assert package.get_resource("name").read_rows() == [ { "id": 1, "name": "english" }, { "id": 2, "name": "中国人" }, ]
def test_package_to_zip(tmpdir): # Write target = os.path.join(tmpdir, "package.zip") package = Package("data/package.json") package.to_zip(target) # Read package = Package(target) assert package == { "name": "name", "resources": [{ "name": "name", "path": "table.csv" }], } assert package.get_resource("name").read_rows() == [ { "id": 1, "name": "english" }, { "id": 2, "name": "中国人" }, ]
def test_package_to_zip_source_inline(tmpdir): # Read target = os.path.join(tmpdir, "package.zip") data = [["id", "name"], ["1", "english"], ["2", "中国人"]] package = Package(name="name", resources=[{"name": "name", "data": data}]) package.to_zip(target) # Write package = Package(target) assert package == { "name": "name", "resources": [{ "name": "name", "data": data }], } assert package.get_resource("name").read_rows() == [ { "id": 1, "name": "english" }, { "id": 2, "name": "中国人" }, ]
def test_package_to_zip_resolve_inline_and_remote(tmpdir): # Write target = os.path.join(tmpdir, "package.zip") resource1 = Resource(name="name1", data=[["id", "name"], [1, "english"], [2, "中国人"]]) resource2 = Resource(name="name2", path=BASE_URL % "data/table.csv") package = Package(resources=[resource1, resource2]) package.to_zip(target, resolve=["inline", "remote"]) # Read package = Package(target) assert package.get_resource("name1").path == "name1.csv" assert package.get_resource("name1").read_rows() == [ { "id": 1, "name": "english" }, { "id": 2, "name": "中国人" }, ] assert package.get_resource("name2").path == "name2.csv" assert package.get_resource("name2").read_rows() == [ { "id": 1, "name": "english" }, { "id": 2, "name": "中国人" }, ]
def test_package_to_zip_resource_multipart(tmpdir, database_url): path = os.path.join(tmpdir, "package.zip") source = Package(resources=[Resource(path=["data/chunk1.csv", "data/chunk2.csv"])]) source.to_zip(path) target = Package.from_zip(path) assert target.get_resource("chunk").path == ["data/chunk1.csv", "data/chunk2.csv"] assert target.get_resource("chunk").read_rows() == [ {"id": 1, "name": "english"}, {"id": 2, "name": "中国人"}, ]
def test_package_to_zip_resource_remote_path(tmpdir): path = os.path.join(tmpdir, "package.zip") source = Package(resources=[Resource(path=BASEURL % "data/table.csv")]) source.to_zip(path) target = Package.from_zip(path) assert target.get_resource("table").path == BASEURL % "data/table.csv" assert target.get_resource("table").read_rows() == [ {"id": 1, "name": "english"}, {"id": 2, "name": "中国人"}, ]
def test_package_to_zip_resource_sql(tmpdir, database_url): path = os.path.join(tmpdir, "package.zip") dialect = SqlDialect(table="table") source = Package(resources=[Resource(database_url, name="table", dialect=dialect)]) source.to_zip(path) target = Package.from_zip(path) assert target.get_resource("table").path == database_url assert target.get_resource("table").read_rows() == [ {"id": 1, "name": "english"}, {"id": 2, "name": "中国人"}, ]
def test_package_to_zip_resource_memory_inline(tmpdir): path = os.path.join(tmpdir, "package.zip") data = [["id", "name"], [1, "english"], [2, "中国人"]] source = Package(resources=[Resource(name="table", data=data)]) source.to_zip(path) target = Package.from_zip(path) assert target.get_resource("table").data == data assert target.get_resource("table").read_rows() == [ {"id": 1, "name": "english"}, {"id": 2, "name": "中国人"}, ]
def test_package_to_zip_resource_memory_function(tmpdir): path = os.path.join(tmpdir, "package.zip") data = lambda: [["id", "name"], [1, "english"], [2, "中国人"]] source = Package(resources=[Resource(name="table", data=data)]) if IS_UNIX: source.to_zip(path) target = Package.from_zip(path) assert target.get_resource("table").path == "table.csv" assert target.get_resource("table").read_rows() == [ {"id": 1, "name": "english"}, {"id": 2, "name": "中国人"}, ]
def test_package_to_zip(tmpdir): path = os.path.join(tmpdir, "package.zip") source = Package("data/package.json") source.to_zip(path) target = Package.from_zip(path) assert target.name == "name" assert target.get_resource("name").name == "name" assert target.get_resource("name").path == "table.csv" assert target.get_resource("name").read_rows() == [ {"id": 1, "name": "english"}, {"id": 2, "name": "中国人"}, ]
def test_package_to_zip_resolve_inline_sql(tmpdir, database_url): # Write target = os.path.join(tmpdir, "package.zip") resource = Resource.from_sql(name="table", url=database_url) package = Package(resources=[resource]) package.to_zip(target, resolve=["inline"]) # Read package = Package(target) assert package.get_resource("table").path == "table.csv" assert package.get_resource("table").read_rows() == [ {"id": 1, "name": "english"}, {"id": 2, "name": "中国人"}, ]
def test_package_to_zip_resolve_remote(tmpdir): # Write target = os.path.join(tmpdir, "package.zip") resource = Resource(path=BASE_URL % "data/table.csv") package = Package(resources=[resource]) package.to_zip(target, resolve=["remote"]) # Read package = Package(target) assert package.get_resource("table").path == "table.csv" assert package.get_resource("table").read_rows() == [ {"id": 1, "name": "english"}, {"id": 2, "name": "中国人"}, ]
def test_package_to_zip_absolute_path(tmpdir): # Write target = os.path.join(tmpdir, "package.zip") resource = Resource(path=os.path.abspath("data/table.csv"), trusted=True) package = Package(resources=[resource], trusted=True) package.to_zip(target) # Read package = Package(target) assert package.get_resource("table").path == "table.csv" assert package.get_resource("table").read_rows() == [ {"id": 1, "name": "english"}, {"id": 2, "name": "中国人"}, ]
def test_package_to_zip_resolve_memory_sql(tmpdir, database_url): # Write target = os.path.join(tmpdir, "package.zip") resource = Resource(database_url, dialect={"table": "table"}) package = Package(resources=[resource]) package.to_zip(target, resolve=["memory"]) # Read package = Package.from_zip(target) assert package.get_resource("table").path == "table.csv" assert package.get_resource("table").read_rows() == [ { "id": 1, "name": "english" }, { "id": 2, "name": "中国人" }, ]
def test_package_to_zip_withdir_path(tmpdir): # Write target = os.path.join(tmpdir, "package.zip") resource = Resource(path="data/table.csv") package = Package(resources=[resource]) package.to_zip(target) # Read package = Package.from_zip(target) assert package.get_resource("table").path == "data/table.csv" assert package.get_resource("table").read_rows() == [ { "id": 1, "name": "english" }, { "id": 2, "name": "中国人" }, ]
def test_package_to_zip_resolve_memory(tmpdir): # Write target = os.path.join(tmpdir, "package.zip") resource = Resource(name="table", data=[["id", "name"], [1, "english"], [2, "中国人"]]) package = Package(resources=[resource]) package.to_zip(target, resolve=["memory"]) # Read package = Package.from_zip(target) assert package.get_resource("table").path == "table.csv" assert package.get_resource("table").read_rows() == [ { "id": 1, "name": "english" }, { "id": 2, "name": "中国人" }, ]
def package_event(event, format): if format not in ['zip', 'json']: return "Format not supported" # Set up a data package package = Package( name='event-%d' % event.id, title=event.name, description="Event and project details collected with dribdat", keywords=["dribdat", "hackathon", "co-creation"], sources=[{ "title": "dribdat", "path": "https://dribdat.cc" }], licenses=[{ "name": "ODC-PDDL-1.0", "path": "http://opendatacommons.org/licenses/pddl/", "title": "Open Data Commons Public Domain Dedication & License 1.0" }], contributors=[{ "title": current_user.username, "path": current_user.webpage_url or '', "role": "author" }], homepage=request.host_url, created=format_date(datetime.now(), '%Y-%m-%dT%H:%M'), version="0.1.0", ) # if False: # as CSV # fp_projects = tempfile.NamedTemporaryFile( # mode='w+t', prefix='projects-', suffix='.csv') # # print("Writing to temp CSV file", fp_projects.name) # fp_projects.write(gen_csv(project_list(event.id))) # resource = Resource(fp_projects.name) # if False: # # print("Generating in-memory rowset") # project_rows = gen_rows(project_list(event.id)) # resource = Resource( # name='projects', # data=project_rows, # ) # Generate resources # print("Generating in-memory JSON of event") package.add_resource( Resource( name='events', data=[event.get_full_data()], )) # print("Generating in-memory JSON of projects") package.add_resource( Resource( name='projects', data=project_list(event.id, True), )) if format == 'zip': # print("Generating in-memory JSON of participants") package.add_resource( Resource( name='users', data=get_event_users(event), )) # print("Generating in-memory JSON of activities") package.add_resource( Resource( name='activities', data=get_event_activities(event.id, 500), )) # print("Generating in-memory JSON of activities") package.add_resource( Resource( name='categories', data=get_event_categories(event.id), )) # print("Adding supplementary README") package.add_resource(Resource( name='readme', path='PACKAGE.txt', )) # Generate data package fp_package = tempfile.NamedTemporaryFile(prefix='datapackage-', suffix='.zip') # print("Saving at", fp_package.name) if format == 'json': return jsonify(package) elif format == 'zip': package.to_zip(fp_package.name) return send_file(fp_package.name, as_attachment=True)