async def test_folder_delete(self):
db: Session = self.app["db"]
resp = await self.client.delete("/folder.json", params={})
self.assertEqual(resp.status, 400)
# return "not found" on bad id
resp = await self.client.delete("/folder.json", params={"id": 2348})
self.assertEqual(resp.status, 404)
# return "not found" on bad path
resp = await self.client.delete("/folder.json",
params={"path": "/bad/path"})
self.assertEqual(resp.status, 404)
# cannot delete folders with children unless recursive is set
f = folder.find("/a/new/path", db, create=True)
resp = await self.client.delete("/folder.json", params={"path": "/a"})
self.assertEqual(resp.status, 400)
self.assertTrue('recursive' in await resp.text())
self.assertIsNotNone(db.query(Folder).get(f.id))
# cannot delete locked folders
my_folder = folder.find("/top/middle", db)
my_folder.children[0].data_streams[0].is_configured = True
resp = await self.client.delete("/folder.json",
params={"path": "/top/middle"})
self.assertEqual(resp.status, 400)
self.assertIn("locked", await resp.text())
async def test_folder_delete_by_path(self):
db: Session = self.app["db"]
f = folder.find("/top/leaf", db)
payload = {'path': "/top/leaf"}
resp = await self.client.delete("/folder.json", params=payload)
self.assertEqual(resp.status, 200)
self.assertIsNone(folder.find("/top/leaf", db))
# deletes the streams
self.assertIsNone(folder.find_stream_by_path("/top/leaf/stream1", db))
# keeps the parent folders
self.assertIsNotNone(folder.find("/top", db))
async def test_folder_move_by_path(self):
db: Session = self.app["db"]
# move stream1 into folder3
payload = {
"src_path": "/top/leaf",
"dest_path": "/top/other"
}
resp = await self.client.put("/folder/move.json", json=payload)
self.assertEqual(resp.status, 200)
f = folder.find("/top/other/leaf", db)
self.assertEqual(f.data_streams[0].name, "stream1")
self.assertIsNone(folder.find("/top/leaf", db))
async def test_folder_delete_by_id(self):
db: Session = self.app["db"]
f_count = db.query(Folder).count()
f = folder.find("/an/empty/folder", db, create=True)
self.assertEqual(db.query(Folder).count(), f_count + 3)
payload = {'id': f.id}
resp = await self.client.delete("/folder.json", params=payload)
self.assertEqual(resp.status, 200)
self.assertIsNone(folder.find("/an/empty/folder", db))
# keeps the parent folders
self.assertEqual(f_count + 2, db.query(Folder).count())
self.assertIsNotNone(folder.find("/an/empty", db))
async def test_folder_move_by_id(self):
db: Session = self.app["db"]
# move stream1 into folder3
dest_folder = folder.find("/top/middle/leaf", db)
src_folder = folder.find("/top/leaf", db)
payload = {
"src_id": src_folder.id,
"dest_id": dest_folder.id
}
resp = await self.client.put("/folder/move.json", json=payload)
self.assertEqual(resp.status, 200)
self.assertEqual(src_folder.id,
folder.find("/top/middle/leaf/leaf", db).id)
self.assertIsNone(folder.find("/top/leaf", db))
async def delete(request):
db: Session = request.app["db"]
data_store: DataStore = request.app["data-store"]
event_store: EventStore = request.app["event-store"]
# find the requested folder
if 'path' in request.query:
my_folder: Folder = folder.find(request.query['path'], db)
elif 'id' in request.query:
my_folder = db.query(Folder).get(request.query["id"])
else:
return web.Response(text="specify an id or a path", status=400)
if my_folder is None:
return web.Response(text="folder does not exist", status=404)
if my_folder.locked:
return web.Response(text="folder is locked", status=400)
if 'recursive' in request.query and request.query["recursive"] == '1':
await _recursive_delete(my_folder.children, db, data_store, event_store)
elif len(my_folder.children) > 0:
return web.Response(text="specify [recursive] or remove child folders first", status=400)
for stream in my_folder.data_streams:
if stream.locked: # pragma: no cover
# shouldn't ever get here because of the check above, but just in case
return web.Response(text="cannot delete folder with locked streams", status=400)
await data_store.destroy(stream)
db.delete(stream)
for stream in my_folder.event_streams:
await event_store.destroy(stream)
db.delete(my_folder)
db.commit()
return web.Response(text="ok")
async def move(request):
db: Session = request.app["db"]
if request.content_type != 'application/json':
return web.Response(text='content-type must be application/json', status=400)
body = await request.json()
# find the folder
if 'src_path' in body:
my_folder = folder.find(body['src_path'], db)
elif 'src_id' in body:
my_folder = db.query(folder.Folder).get(body["src_id"])
else:
return web.Response(text="specify an id or a path", status=400)
if my_folder is None:
return web.Response(text="folder does not exist", status=404)
# make sure the folder is not locked
if my_folder.locked:
return web.Response(text="folder is locked", status=400)
# find or create the destination folder
if 'dest_path' in body:
try:
destination = folder.find(body['dest_path'], db, create=True)
except ConfigurationError as e:
return web.Response(text="Destination error: %s" % str(e), status=400)
elif 'dest_id' in body:
destination = db.query(Folder).get(body["dest_id"])
else:
return web.Response(text="specify a destination", status=400)
# make sure name is unique in the folder
peer_names = [f.name for f in destination.children]
if my_folder.name in peer_names:
db.rollback()
return web.Response(text="a folder with this name is in the destination folder",
status=400)
try:
# make sure this does not create a circular graph
parent = destination.parent
while parent is not None:
if parent == my_folder:
raise ValueError()
parent = parent.parent
destination.children.append(my_folder)
db.commit()
except (CircularDependencyError, ValueError):
db.rollback()
return web.Response(text="cannot place a parent in a child folder",
status=400)
return web.json_response({"stream": my_folder.to_json()})
async def test_folder_recursive_delete(self):
db: Session = self.app["db"]
f = folder.find("/top", db)
payload = {'path': "/top", 'recursive': "1"}
resp = await self.client.delete("/folder.json", params=payload)
self.assertEqual(resp.status, 200)
# this is the top folder so everything should be gone
self.assertEqual(1, db.query(Folder).count())
self.assertEqual(0, db.query(DataStream).count())
self.assertEqual(0, db.query(Element).count())
async def info(request: web.Request):
db: Session = request.app["db"]
if 'path' in request.query:
my_folder = folder.find(request.query['path'], db)
elif 'id' in request.query:
my_folder = db.query(Folder).get(request.query["id"])
else:
return web.Response(text="specify an id or path", status=400)
if my_folder is None:
return web.Response(text="folder does not exist", status=404)
return web.json_response(my_folder.to_json())
async def test_folder_update(self):
db: Session = self.app["db"]
my_folder = folder.find("/top/middle/leaf", db)
# change the stream name
payload = {
"id": my_folder.id,
"folder": {"name": "new name", "description": "new description"}
}
resp = await self.client.put("/folder.json", json=payload)
self.assertEqual(200, resp.status)
my_folder: DataStream = db.query(Folder).get(my_folder.id)
self.assertEqual("new name", my_folder.name)
self.assertEqual("new description", my_folder.description)
async def create(request):
db: Session = request.app["db"]
if request.content_type != 'application/json':
return web.Response(text='content-type must be application/json',
status=400)
body = await request.json()
if 'stream' not in body:
return web.Response(text="provide a stream", status=400)
# find or create the destination folder
if 'dest_path' in body:
try:
destination = folder.find(body['dest_path'], db, create=True)
except ConfigurationError as e:
return web.Response(text="Destination error: %s" % str(e),
status=400)
elif 'dest_id' in body:
destination = db.query(Folder).get(body["dest_id"])
else:
return web.Response(text="specify a destination", status=400)
try:
new_stream = event_stream.from_json(body['stream'])
# clear out the id's
new_stream.id = None
# make sure name is unique in this destination
existing_names = [
s.name
for s in destination.data_streams + destination.event_streams
]
if new_stream.name in existing_names:
raise ConfigurationError(
"stream with the same name exists in the folder")
destination.event_streams.append(new_stream)
db.commit()
except (TypeError, ValueError) as e:
db.rollback()
return web.Response(text="Invalid stream JSON: %r" % e, status=400)
except ConfigurationError as e:
db.rollback()
return web.Response(text="Invalid stream specification: %s" % e,
status=400)
except KeyError as e:
db.rollback()
return web.Response(text="Invalid or missing stream attribute: %s" % e,
status=400)
return web.json_response(data=new_stream.to_json())
def run(pipe_config: str, db: Session) -> DataStream:
# check for a remote stream config
(pipe_config, node_name) = strip_remote_config(pipe_config)
local = node_name is None
# separate the configuration pieces
(path, name, inline_config) = parse_pipe_config(pipe_config)
name = data_stream.validate_name(name)
# parse the inline configuration
(datatype, element_names) = parse_inline_config(inline_config)
# if the stream is local, check for it in the database
if local:
my_folder = folder.find(path, db, create=True)
# check if the stream exists in the database
existing_stream: DataStream = db.query(DataStream). \
filter_by(folder=my_folder, name=name). \
one_or_none()
if existing_stream is not None:
if len(inline_config) > 0:
_validate_config_match(existing_stream, datatype,
element_names)
return existing_stream
else: # make sure the remote node is a follower
if db.query(Follower).filter_by(name=node_name).one_or_none() is None:
raise ConfigurationError("Remote node [%s] is not a follower" %
node_name)
# if the stream doesn't exist it or its remote, it *must* have inline configuration
if len(inline_config) == 0:
if local:
msg = "add inline config or *.conf file for stream [%s]" % pipe_config
else:
msg = "remote streams must have inline config"
raise ConfigurationError(msg)
# build the stream from inline config
my_stream = data_stream.DataStream(name=name, datatype=datatype)
i = 0
for e in element_names:
my_stream.elements.append(Element(name=e, index=i))
i += 1
if local:
my_folder.data_streams.append(my_stream)
db.add(my_stream)
else:
my_stream.set_remote(node_name, path + '/' + my_stream.name)
return my_stream
async def move(request: web.Request):
db: Session = request.app["db"]
if request.content_type != 'application/json':
return web.Response(text='content-type must be application/json',
status=400)
body = await request.json()
# find the stream
if 'src_path' in body:
my_stream = folder.find_stream_by_path(body['src_path'],
db,
stream_type=DataStream)
elif 'src_id' in body:
my_stream = db.query(DataStream).get(body["src_id"])
else:
return web.Response(text="specify a source id or a path", status=400)
if my_stream is None:
return web.Response(text="stream does not exist", status=404)
if my_stream.locked:
return web.Response(text="locked streams cannot be moved", status=400)
# find or create the destination folder
if 'dest_path' in body:
try:
destination = folder.find(body['dest_path'], db, create=True)
except ConfigurationError as e:
return web.Response(text="Destination error: %s" % str(e),
status=400)
elif 'dest_id' in body:
destination = db.query(Folder).get(body["dest_id"])
else:
return web.Response(text="specify a destination", status=400)
# make sure name is unique in this destination
existing_names = [
s.name for s in destination.data_streams + destination.event_streams
]
if my_stream.name in existing_names:
db.rollback()
return web.Response(
text="stream with the same name exists in the destination folder",
status=400)
destination.data_streams.append(my_stream)
db.commit()
return web.json_response({"stream": my_stream.to_json()})
async def test_folder_move(self):
db: Session = self.app["db"]
# must be a json request
resp = await self.client.put("/folder/move.json",
data={"dest_path": "/new/location"})
self.assertEqual(resp.status, 400)
self.assertIn("json", await resp.text())
# must specify an id or a path
resp = await self.client.put("/folder/move.json",
json={"dest_path": "/new/location"})
self.assertEqual(resp.status, 400)
# must specify a destination
resp = await self.client.put("/folder/move.json",
json={"src_path": "/top/leaf"})
self.assertEqual(resp.status, 400)
# return "not found" on bad id
resp = await self.client.put("/folder/move.json",
json={
"src_id": 2348,
"dest_path": "/x/y"
})
self.assertEqual(resp.status, 404)
# return "not found" on bad path
resp = await self.client.put("/folder/move.json",
json={
"src_path": "/missing/folder",
"dest_patbh": "/x/y"
})
self.assertEqual(resp.status, 404)
# error on bad destination
resp = await self.client.put("/folder/move.json",
json={
"src_path": "/top/leaf",
"dest_path": "malformed"
})
self.assertEqual(resp.status, 400)
# cannot conflict with an existing folder in the destination
resp = await self.client.put("/folder/move.json",
json={
"src_path": "/top/leaf",
"dest_path": "/top/middle"
})
self.assertEqual(resp.status, 400)
# cannot move a folder into its children
resp = await self.client.put("/folder/move.json",
json={
"src_path": "/top",
"dest_path": "/top/leaf"
})
self.assertEqual(resp.status, 400)
self.assertTrue('parent' in await resp.text())
self.assertIsNotNone(folder.find('/top/middle/leaf', db))
# cannot move folders with locked streams
my_stream: DataStream = db.query(DataStream).filter_by(
name="stream1").one()
my_stream.is_configured = True
resp = await self.client.put("/folder/move.json",
json={
"src_path": "/top/leaf",
"dest_path": "/top/middle"
})
self.assertEqual(resp.status, 400)
self.assertTrue('locked' in await resp.text())
async def test_folder_update(self):
db: Session = self.app["db"]
my_folder = folder.find("/top/middle", db)
# must be a json request
resp = await self.client.put("/folder.json", data={"bad": "notjson"})
self.assertEqual(resp.status, 400)
self.assertIn("json", await resp.text())
# invalid name, nothing should be saved
payload = {
"id": my_folder.id,
"folder": {
"name": "",
"description": "new description"
}
}
resp = await self.client.put("/folder.json", json=payload)
self.assertEqual(resp.status, 400)
self.assertTrue('name' in await resp.text())
my_folder = folder.find("/top/middle", db)
self.assertEqual("middle", my_folder.name)
# request must specify an id
payload = {"folder": json.dumps({"name": "new name"})}
resp = await self.client.put("/folder.json", json=payload)
self.assertEqual(resp.status, 400)
self.assertTrue('id' in await resp.text())
# request must have folder attr
payload = {"id": my_folder.id}
resp = await self.client.put("/folder.json", json=payload)
self.assertEqual(resp.status, 400)
self.assertTrue('folder' in await resp.text())
# folder attr must be valid json
payload = {"id": my_folder.id, "folder": "notjson"}
resp = await self.client.put("/folder.json", json=payload)
self.assertEqual(resp.status, 400)
self.assertTrue('JSON' in await resp.text())
# cannot modify locked streams
my_folder.children[0].data_streams[0].is_configured = True
payload = {
"id": my_folder.id,
"folder": json.dumps({"name": "new name"})
}
resp = await self.client.put("/folder.json", json=payload)
self.assertEqual(resp.status, 400)
self.assertTrue('locked' in await resp.text())
my_folder.children[0].data_streams[0].is_configured = False
# folder must exist
payload = {"id": 4898, "stream": json.dumps({"name": "new name"})}
resp = await self.client.put("/folder.json", json=payload)
self.assertEqual(resp.status, 404)
self.assertTrue('exist' in await resp.text())
# folder name cannot conflict with a peer
payload = {
"id": my_folder.id,
"folder": {
"name": "same_name",
"description": "new description"
}
}
resp = await self.client.put("/folder.json", json=payload)
self.assertEqual(resp.status, 400)
self.assertIn('folder with this name', await resp.text())
my_folder = folder.find("/top/middle", db)
self.assertEqual("middle", my_folder.name)
async def run(src_db: 'Session', dest_db: 'Session',
src_datastore: 'DataStore', dest_datastore: 'DataStore',
stream_map: Optional[List], confirmed: bool,
start: Optional[int], end: Optional[int]):
from joule.models import DataStream, folder, data_stream
from joule.services import parse_pipe_config
src_streams = src_db.query(DataStream).all()
dest_streams = dest_db.query(DataStream).all()
await src_datastore.initialize(src_streams)
await dest_datastore.initialize(dest_streams)
if stream_map is None:
src_streams = src_db.query(DataStream).all()
src_paths = map(folder.get_stream_path, src_streams)
stream_map = map(lambda _path: [_path, _path], src_paths)
# create the copy map array
copy_maps = []
for item in stream_map:
# get the source stream
source = folder.find_stream_by_path(item[0], src_db)
if source is None:
raise errors.ConfigurationError(
"source stream [%s] does not exist" % item[0])
src_intervals = await src_datastore.intervals(source, start, end)
# get or create the destination stream
dest = folder.find_stream_by_path(item[1], dest_db)
if dest is None:
(path, name, _) = parse_pipe_config.parse_pipe_config(item[1])
dest_folder = folder.find(path, dest_db, create=True)
dest = data_stream.from_json(source.to_json())
# set the attributes on the new stream
dest.name = name
dest.keep_us = dest.KEEP_ALL
dest.is_configured = False
dest.is_source = False
dest.is_destination = False
dest.id = None
for e in dest.elements:
e.id = None
dest_folder.streams.append(dest)
dest_intervals = None
else:
# make sure the destination is compatible
if dest.layout != source.layout:
raise errors.ConfigurationError(
"source stream [%s] is not compatible with destination stream [%s]"
% (item[0], item[1]))
dest_intervals = await dest_datastore.intervals(dest, start, end)
# figure out the time bounds to copy
if dest_intervals is None:
copy_intervals = src_intervals
else:
copy_intervals = utilities.interval_difference(
src_intervals, dest_intervals)
copy_maps.append(CopyMap(source, dest, copy_intervals))
# display the copy table
rows = []
copy_required = False
for item in copy_maps:
if item.start is None:
start = "\u2014"
end = "\u2014"
else:
start = utilities.timestamp_to_human(item.start)
end = utilities.timestamp_to_human(item.end)
copy_required = True
rows.append([item.source_path, item.dest_path, start, end])
click.echo(
tabulate(rows,
headers=["Source", "Destination", "From", "To"],
tablefmt="fancy_grid"))
if not copy_required:
click.echo("No data needs to be copied")
return
if not confirmed and not click.confirm("Start data copy?"):
click.echo("cancelled")
return
dest_db.commit()
# execute the copy
for item in copy_maps:
await copy(item, src_datastore, dest_datastore, src_db, dest_db)
