async def test_folder_delete(self): db: Session = self.app["db"] resp = await self.client.delete("/folder.json", params={}) self.assertEqual(resp.status, 400) # return "not found" on bad id resp = await self.client.delete("/folder.json", params={"id": 2348}) self.assertEqual(resp.status, 404) # return "not found" on bad path resp = await self.client.delete("/folder.json", params={"path": "/bad/path"}) self.assertEqual(resp.status, 404) # cannot delete folders with children unless recursive is set f = folder.find("/a/new/path", db, create=True) resp = await self.client.delete("/folder.json", params={"path": "/a"}) self.assertEqual(resp.status, 400) self.assertTrue('recursive' in await resp.text()) self.assertIsNotNone(db.query(Folder).get(f.id)) # cannot delete locked folders my_folder = folder.find("/top/middle", db) my_folder.children[0].data_streams[0].is_configured = True resp = await self.client.delete("/folder.json", params={"path": "/top/middle"}) self.assertEqual(resp.status, 400) self.assertIn("locked", await resp.text())
async def test_folder_delete_by_path(self): db: Session = self.app["db"] f = folder.find("/top/leaf", db) payload = {'path': "/top/leaf"} resp = await self.client.delete("/folder.json", params=payload) self.assertEqual(resp.status, 200) self.assertIsNone(folder.find("/top/leaf", db)) # deletes the streams self.assertIsNone(folder.find_stream_by_path("/top/leaf/stream1", db)) # keeps the parent folders self.assertIsNotNone(folder.find("/top", db))
async def test_folder_move_by_path(self): db: Session = self.app["db"] # move stream1 into folder3 payload = { "src_path": "/top/leaf", "dest_path": "/top/other" } resp = await self.client.put("/folder/move.json", json=payload) self.assertEqual(resp.status, 200) f = folder.find("/top/other/leaf", db) self.assertEqual(f.data_streams[0].name, "stream1") self.assertIsNone(folder.find("/top/leaf", db))
async def test_folder_delete_by_id(self): db: Session = self.app["db"] f_count = db.query(Folder).count() f = folder.find("/an/empty/folder", db, create=True) self.assertEqual(db.query(Folder).count(), f_count + 3) payload = {'id': f.id} resp = await self.client.delete("/folder.json", params=payload) self.assertEqual(resp.status, 200) self.assertIsNone(folder.find("/an/empty/folder", db)) # keeps the parent folders self.assertEqual(f_count + 2, db.query(Folder).count()) self.assertIsNotNone(folder.find("/an/empty", db))
async def test_folder_move_by_id(self): db: Session = self.app["db"] # move stream1 into folder3 dest_folder = folder.find("/top/middle/leaf", db) src_folder = folder.find("/top/leaf", db) payload = { "src_id": src_folder.id, "dest_id": dest_folder.id } resp = await self.client.put("/folder/move.json", json=payload) self.assertEqual(resp.status, 200) self.assertEqual(src_folder.id, folder.find("/top/middle/leaf/leaf", db).id) self.assertIsNone(folder.find("/top/leaf", db))
async def delete(request): db: Session = request.app["db"] data_store: DataStore = request.app["data-store"] event_store: EventStore = request.app["event-store"] # find the requested folder if 'path' in request.query: my_folder: Folder = folder.find(request.query['path'], db) elif 'id' in request.query: my_folder = db.query(Folder).get(request.query["id"]) else: return web.Response(text="specify an id or a path", status=400) if my_folder is None: return web.Response(text="folder does not exist", status=404) if my_folder.locked: return web.Response(text="folder is locked", status=400) if 'recursive' in request.query and request.query["recursive"] == '1': await _recursive_delete(my_folder.children, db, data_store, event_store) elif len(my_folder.children) > 0: return web.Response(text="specify [recursive] or remove child folders first", status=400) for stream in my_folder.data_streams: if stream.locked: # pragma: no cover # shouldn't ever get here because of the check above, but just in case return web.Response(text="cannot delete folder with locked streams", status=400) await data_store.destroy(stream) db.delete(stream) for stream in my_folder.event_streams: await event_store.destroy(stream) db.delete(my_folder) db.commit() return web.Response(text="ok")
async def move(request): db: Session = request.app["db"] if request.content_type != 'application/json': return web.Response(text='content-type must be application/json', status=400) body = await request.json() # find the folder if 'src_path' in body: my_folder = folder.find(body['src_path'], db) elif 'src_id' in body: my_folder = db.query(folder.Folder).get(body["src_id"]) else: return web.Response(text="specify an id or a path", status=400) if my_folder is None: return web.Response(text="folder does not exist", status=404) # make sure the folder is not locked if my_folder.locked: return web.Response(text="folder is locked", status=400) # find or create the destination folder if 'dest_path' in body: try: destination = folder.find(body['dest_path'], db, create=True) except ConfigurationError as e: return web.Response(text="Destination error: %s" % str(e), status=400) elif 'dest_id' in body: destination = db.query(Folder).get(body["dest_id"]) else: return web.Response(text="specify a destination", status=400) # make sure name is unique in the folder peer_names = [f.name for f in destination.children] if my_folder.name in peer_names: db.rollback() return web.Response(text="a folder with this name is in the destination folder", status=400) try: # make sure this does not create a circular graph parent = destination.parent while parent is not None: if parent == my_folder: raise ValueError() parent = parent.parent destination.children.append(my_folder) db.commit() except (CircularDependencyError, ValueError): db.rollback() return web.Response(text="cannot place a parent in a child folder", status=400) return web.json_response({"stream": my_folder.to_json()})
async def test_folder_recursive_delete(self): db: Session = self.app["db"] f = folder.find("/top", db) payload = {'path': "/top", 'recursive': "1"} resp = await self.client.delete("/folder.json", params=payload) self.assertEqual(resp.status, 200) # this is the top folder so everything should be gone self.assertEqual(1, db.query(Folder).count()) self.assertEqual(0, db.query(DataStream).count()) self.assertEqual(0, db.query(Element).count())
async def info(request: web.Request): db: Session = request.app["db"] if 'path' in request.query: my_folder = folder.find(request.query['path'], db) elif 'id' in request.query: my_folder = db.query(Folder).get(request.query["id"]) else: return web.Response(text="specify an id or path", status=400) if my_folder is None: return web.Response(text="folder does not exist", status=404) return web.json_response(my_folder.to_json())
async def test_folder_update(self): db: Session = self.app["db"] my_folder = folder.find("/top/middle/leaf", db) # change the stream name payload = { "id": my_folder.id, "folder": {"name": "new name", "description": "new description"} } resp = await self.client.put("/folder.json", json=payload) self.assertEqual(200, resp.status) my_folder: DataStream = db.query(Folder).get(my_folder.id) self.assertEqual("new name", my_folder.name) self.assertEqual("new description", my_folder.description)
async def create(request): db: Session = request.app["db"] if request.content_type != 'application/json': return web.Response(text='content-type must be application/json', status=400) body = await request.json() if 'stream' not in body: return web.Response(text="provide a stream", status=400) # find or create the destination folder if 'dest_path' in body: try: destination = folder.find(body['dest_path'], db, create=True) except ConfigurationError as e: return web.Response(text="Destination error: %s" % str(e), status=400) elif 'dest_id' in body: destination = db.query(Folder).get(body["dest_id"]) else: return web.Response(text="specify a destination", status=400) try: new_stream = event_stream.from_json(body['stream']) # clear out the id's new_stream.id = None # make sure name is unique in this destination existing_names = [ s.name for s in destination.data_streams + destination.event_streams ] if new_stream.name in existing_names: raise ConfigurationError( "stream with the same name exists in the folder") destination.event_streams.append(new_stream) db.commit() except (TypeError, ValueError) as e: db.rollback() return web.Response(text="Invalid stream JSON: %r" % e, status=400) except ConfigurationError as e: db.rollback() return web.Response(text="Invalid stream specification: %s" % e, status=400) except KeyError as e: db.rollback() return web.Response(text="Invalid or missing stream attribute: %s" % e, status=400) return web.json_response(data=new_stream.to_json())
def run(pipe_config: str, db: Session) -> DataStream: # check for a remote stream config (pipe_config, node_name) = strip_remote_config(pipe_config) local = node_name is None # separate the configuration pieces (path, name, inline_config) = parse_pipe_config(pipe_config) name = data_stream.validate_name(name) # parse the inline configuration (datatype, element_names) = parse_inline_config(inline_config) # if the stream is local, check for it in the database if local: my_folder = folder.find(path, db, create=True) # check if the stream exists in the database existing_stream: DataStream = db.query(DataStream). \ filter_by(folder=my_folder, name=name). \ one_or_none() if existing_stream is not None: if len(inline_config) > 0: _validate_config_match(existing_stream, datatype, element_names) return existing_stream else: # make sure the remote node is a follower if db.query(Follower).filter_by(name=node_name).one_or_none() is None: raise ConfigurationError("Remote node [%s] is not a follower" % node_name) # if the stream doesn't exist it or its remote, it *must* have inline configuration if len(inline_config) == 0: if local: msg = "add inline config or *.conf file for stream [%s]" % pipe_config else: msg = "remote streams must have inline config" raise ConfigurationError(msg) # build the stream from inline config my_stream = data_stream.DataStream(name=name, datatype=datatype) i = 0 for e in element_names: my_stream.elements.append(Element(name=e, index=i)) i += 1 if local: my_folder.data_streams.append(my_stream) db.add(my_stream) else: my_stream.set_remote(node_name, path + '/' + my_stream.name) return my_stream
async def move(request: web.Request): db: Session = request.app["db"] if request.content_type != 'application/json': return web.Response(text='content-type must be application/json', status=400) body = await request.json() # find the stream if 'src_path' in body: my_stream = folder.find_stream_by_path(body['src_path'], db, stream_type=DataStream) elif 'src_id' in body: my_stream = db.query(DataStream).get(body["src_id"]) else: return web.Response(text="specify a source id or a path", status=400) if my_stream is None: return web.Response(text="stream does not exist", status=404) if my_stream.locked: return web.Response(text="locked streams cannot be moved", status=400) # find or create the destination folder if 'dest_path' in body: try: destination = folder.find(body['dest_path'], db, create=True) except ConfigurationError as e: return web.Response(text="Destination error: %s" % str(e), status=400) elif 'dest_id' in body: destination = db.query(Folder).get(body["dest_id"]) else: return web.Response(text="specify a destination", status=400) # make sure name is unique in this destination existing_names = [ s.name for s in destination.data_streams + destination.event_streams ] if my_stream.name in existing_names: db.rollback() return web.Response( text="stream with the same name exists in the destination folder", status=400) destination.data_streams.append(my_stream) db.commit() return web.json_response({"stream": my_stream.to_json()})
async def test_folder_move(self): db: Session = self.app["db"] # must be a json request resp = await self.client.put("/folder/move.json", data={"dest_path": "/new/location"}) self.assertEqual(resp.status, 400) self.assertIn("json", await resp.text()) # must specify an id or a path resp = await self.client.put("/folder/move.json", json={"dest_path": "/new/location"}) self.assertEqual(resp.status, 400) # must specify a destination resp = await self.client.put("/folder/move.json", json={"src_path": "/top/leaf"}) self.assertEqual(resp.status, 400) # return "not found" on bad id resp = await self.client.put("/folder/move.json", json={ "src_id": 2348, "dest_path": "/x/y" }) self.assertEqual(resp.status, 404) # return "not found" on bad path resp = await self.client.put("/folder/move.json", json={ "src_path": "/missing/folder", "dest_patbh": "/x/y" }) self.assertEqual(resp.status, 404) # error on bad destination resp = await self.client.put("/folder/move.json", json={ "src_path": "/top/leaf", "dest_path": "malformed" }) self.assertEqual(resp.status, 400) # cannot conflict with an existing folder in the destination resp = await self.client.put("/folder/move.json", json={ "src_path": "/top/leaf", "dest_path": "/top/middle" }) self.assertEqual(resp.status, 400) # cannot move a folder into its children resp = await self.client.put("/folder/move.json", json={ "src_path": "/top", "dest_path": "/top/leaf" }) self.assertEqual(resp.status, 400) self.assertTrue('parent' in await resp.text()) self.assertIsNotNone(folder.find('/top/middle/leaf', db)) # cannot move folders with locked streams my_stream: DataStream = db.query(DataStream).filter_by( name="stream1").one() my_stream.is_configured = True resp = await self.client.put("/folder/move.json", json={ "src_path": "/top/leaf", "dest_path": "/top/middle" }) self.assertEqual(resp.status, 400) self.assertTrue('locked' in await resp.text())
async def test_folder_update(self): db: Session = self.app["db"] my_folder = folder.find("/top/middle", db) # must be a json request resp = await self.client.put("/folder.json", data={"bad": "notjson"}) self.assertEqual(resp.status, 400) self.assertIn("json", await resp.text()) # invalid name, nothing should be saved payload = { "id": my_folder.id, "folder": { "name": "", "description": "new description" } } resp = await self.client.put("/folder.json", json=payload) self.assertEqual(resp.status, 400) self.assertTrue('name' in await resp.text()) my_folder = folder.find("/top/middle", db) self.assertEqual("middle", my_folder.name) # request must specify an id payload = {"folder": json.dumps({"name": "new name"})} resp = await self.client.put("/folder.json", json=payload) self.assertEqual(resp.status, 400) self.assertTrue('id' in await resp.text()) # request must have folder attr payload = {"id": my_folder.id} resp = await self.client.put("/folder.json", json=payload) self.assertEqual(resp.status, 400) self.assertTrue('folder' in await resp.text()) # folder attr must be valid json payload = {"id": my_folder.id, "folder": "notjson"} resp = await self.client.put("/folder.json", json=payload) self.assertEqual(resp.status, 400) self.assertTrue('JSON' in await resp.text()) # cannot modify locked streams my_folder.children[0].data_streams[0].is_configured = True payload = { "id": my_folder.id, "folder": json.dumps({"name": "new name"}) } resp = await self.client.put("/folder.json", json=payload) self.assertEqual(resp.status, 400) self.assertTrue('locked' in await resp.text()) my_folder.children[0].data_streams[0].is_configured = False # folder must exist payload = {"id": 4898, "stream": json.dumps({"name": "new name"})} resp = await self.client.put("/folder.json", json=payload) self.assertEqual(resp.status, 404) self.assertTrue('exist' in await resp.text()) # folder name cannot conflict with a peer payload = { "id": my_folder.id, "folder": { "name": "same_name", "description": "new description" } } resp = await self.client.put("/folder.json", json=payload) self.assertEqual(resp.status, 400) self.assertIn('folder with this name', await resp.text()) my_folder = folder.find("/top/middle", db) self.assertEqual("middle", my_folder.name)
async def run(src_db: 'Session', dest_db: 'Session', src_datastore: 'DataStore', dest_datastore: 'DataStore', stream_map: Optional[List], confirmed: bool, start: Optional[int], end: Optional[int]): from joule.models import DataStream, folder, data_stream from joule.services import parse_pipe_config src_streams = src_db.query(DataStream).all() dest_streams = dest_db.query(DataStream).all() await src_datastore.initialize(src_streams) await dest_datastore.initialize(dest_streams) if stream_map is None: src_streams = src_db.query(DataStream).all() src_paths = map(folder.get_stream_path, src_streams) stream_map = map(lambda _path: [_path, _path], src_paths) # create the copy map array copy_maps = [] for item in stream_map: # get the source stream source = folder.find_stream_by_path(item[0], src_db) if source is None: raise errors.ConfigurationError( "source stream [%s] does not exist" % item[0]) src_intervals = await src_datastore.intervals(source, start, end) # get or create the destination stream dest = folder.find_stream_by_path(item[1], dest_db) if dest is None: (path, name, _) = parse_pipe_config.parse_pipe_config(item[1]) dest_folder = folder.find(path, dest_db, create=True) dest = data_stream.from_json(source.to_json()) # set the attributes on the new stream dest.name = name dest.keep_us = dest.KEEP_ALL dest.is_configured = False dest.is_source = False dest.is_destination = False dest.id = None for e in dest.elements: e.id = None dest_folder.streams.append(dest) dest_intervals = None else: # make sure the destination is compatible if dest.layout != source.layout: raise errors.ConfigurationError( "source stream [%s] is not compatible with destination stream [%s]" % (item[0], item[1])) dest_intervals = await dest_datastore.intervals(dest, start, end) # figure out the time bounds to copy if dest_intervals is None: copy_intervals = src_intervals else: copy_intervals = utilities.interval_difference( src_intervals, dest_intervals) copy_maps.append(CopyMap(source, dest, copy_intervals)) # display the copy table rows = [] copy_required = False for item in copy_maps: if item.start is None: start = "\u2014" end = "\u2014" else: start = utilities.timestamp_to_human(item.start) end = utilities.timestamp_to_human(item.end) copy_required = True rows.append([item.source_path, item.dest_path, start, end]) click.echo( tabulate(rows, headers=["Source", "Destination", "From", "To"], tablefmt="fancy_grid")) if not copy_required: click.echo("No data needs to be copied") return if not confirmed and not click.confirm("Start data copy?"): click.echo("cancelled") return dest_db.commit() # execute the copy for item in copy_maps: await copy(item, src_datastore, dest_datastore, src_db, dest_db)