class WorkerLevelDBBackend(WorkerBackend): name = 'leveldb' version = SymVer(0, 0, 1) config_class = WorkerLevelDBConfig def __init__(self, backend_config: dict): super().__init__(backend_config) self.tasks_db = LevelDB(self.config.db_path) def read_task_info(self, task_id: str) -> TaskInfo: return TaskInfo.create(self.tasks_db.get(task_id)) def write_task_info(self, task_id: str, task_info: TaskInfo): return self.tasks_db.put(task_id, task_info.to_json()) def list_tasks(self, with_info: bool = False) -> 'Iterator[Tuple[str, Optional[TaskInfo]]]': for task_id, task_info in self.tasks_db.iterate_all(include_value=with_info): yield task_id, TaskInfo.create(task_info) if task_info else None
def __init__(self, backend_config: dict): super().__init__(backend_config) self.db = LevelDB(self.config.db_path) self.graphs = self.db.collection_view('graphs') self.schedule = self.db.collection_view('schedule') self.instances = self.db.collection_view('instances')
class MasterLevelDBBackend(MasterBackend): name = 'leveldb' version = SymVer(0, 0, 1) config_class = MasterLevelDBConfig def __init__(self, backend_config: dict): super().__init__(backend_config) self.db = LevelDB(self.config.db_path) self.graphs = self.db.collection_view('graphs') self.schedule = self.db.collection_view('schedule') self.instances = self.db.collection_view('instances') def read_graph_instance_info(self, instance_id: str) -> GraphInstanceInfo: return GraphInstanceInfo.create(self.instances.get(instance_id)) def write_graph_instance_info(self, instance_id: str, instance_info: GraphInstanceInfo): return self.instances.put(instance_id, instance_info.to_json()) def list_graph_instance_info(self, with_info: bool = False) -> Iterator[Tuple[str, Optional[GraphInstanceInfo]]]: for instance_id, instance_info in self.instances.iterate_all(include_value=with_info): yield instance_id, GraphInstanceInfo.create(instance_info) if instance_info else None def read_graph_struct(self, graph_name: str, revision: int = -1) -> GraphStruct: graph_view = self.graphs.collection_view(graph_name) if revision == -1: last_revision_struct = max( map(lambda _: GraphStruct.create(_[1]), graph_view.iterate_all(include_value=True)), key=lambda x: x.revision, default=None ) if last_revision_struct is None: raise GraphStructureNotFound(graph_name) return last_revision_struct return GraphStruct.create(graph_view.get(str(revision))) def add_graph_struct(self, graph_name: str, graph_struct: GraphStruct) -> int: graph_view = self.graphs.collection_view(graph_name) graph_struct.graph_name = graph_name try: new_revision = self.read_graph_struct(graph_name).revision + 1 except: new_revision = 0 graph_struct.revision = new_revision # FIXME(luckygeck): possible race condition graph_view.put(str(new_revision), graph_struct.to_json()) return new_revision def list_graph_struct(self, graph_name: Optional[str] = None, with_info: bool = False) -> Iterator[ Tuple[str, int, Optional[GraphStruct]]]: db = self.graphs.collection_view(graph_name) if graph_name else self.graphs for key, graph_struct in db.iterate_all(include_value=with_info): name, revision = (graph_name, key) if graph_name else key.split('=', 1) yield name, revision, GraphStruct.create(graph_struct) if graph_struct else None def read_schedule(self, graph_name: str) -> ScheduledGraph: return ScheduledGraph.create(self.schedule.get(graph_name)) def write_schedule(self, graph_name: str, schedule: str): schedule_json = ScheduledGraph().init(graph_name, schedule).to_json() graph_versions = self.list_graph_struct(graph_name, with_info=False) if next(graph_versions, None) is None: raise GraphStructureNotFound(graph_name) return self.schedule.put(graph_name, schedule_json) def list_schedules(self) -> Iterator[ScheduledGraph]: return (schedule for graph_name, schedule in self.schedule.iterate_all(include_value=True))
def __init__(self, backend_config: dict): super().__init__(backend_config) self.tasks_db = LevelDB(self.config.db_path)