def rm(key: str): if key in user_cache_frame: del user_cache_frame[key] if key in user_cache_obj: del user_cache_obj[key] am = get_address_manager() am.delete.remote(key)
def put_resource(self, key: Union[RunID, Tuple[str, str], str]): resource = self.get_resource(key) address_manager = get_address_manager() if isinstance(resource, ObjectID): address_manager.put.remote((key, resource, False)) return is_none = resource is None address = ray.put(resource) address_manager.put.remote((key, address, is_none))
def supervise(self, report=None): if report is None: report = SilentFeatureComputingReport() try: extra_iterations = 1 while True: signals = self.new_signals() syncs = filter_signals(signals, Sync) for sync in syncs: self.sync(**sync.get_contents()) resource_requests = filter_signals(signals, ResourceRequest) for rr in resource_requests: key = rr.get_contents() address_manager = get_address_manager() if ray.get(address_manager.has.remote(key)): if ray.get(address_manager.isnone.remote(key)): self.put_resource(key) else: address_manager.confirm.remote(key) else: self.put_resource(key) pid_signals = self.filter_map_id(signals, RunPID) for rid, pid_signal in pid_signals: self.scheduled[rid].pid = pid_signal.get_contents() progress_signals = self.filter_map_id(signals, ProgressSignal) for rid, ps in progress_signals: payload = ps.get_contents() title = payload.pop('title') rid = payload.pop('run_id', rid) if title is not None: rid = copy(rid) rid.function_name += f" - {title}" report.update(**payload, run_id=rid, autorefresh=False) text_chunks = self.filter_map_id(signals, TextChunk) for rid, tc in text_chunks: payload = tc.get_contents() rid = payload.pop('run_id', rid) report.update_text(rid, **payload, autorefresh=False) errors = self.filter_map_id(signals, rs.ErrorSignal) for rid, err in errors: report.update_text(rid, text='task failed', timestamp=time.time(), autorefresh=False) report.update_text(rid, text=err.error.strip(), timestamp=time.time(), autorefresh=False) report.refresh(force=True) report.refresh() time.sleep(0.01) extra_iterations -= self.completed() if extra_iterations < 0: break report.refresh(force=True) except KeyboardInterrupt: self.kill_scheduled()
def safe_put(kf: KTSFrame): address_manager = get_address_manager() h = kf.hash() if ray.get(address_manager.has.remote(h)): oid = ray.get(address_manager.get.remote(h)) else: oid = ray.put(kf) address_manager.put.remote((h, oid, False)) return oid
def request_resource(self, key, df): if in_worker(): request_time = time.time() rs.send(ResourceRequest(key)) address_manager = get_address_manager() while not ray.get(address_manager.has.remote(key)) or ray.get(address_manager.timestamp.remote(key)) < request_time: time.sleep(0.01) address = ray.get(address_manager.get.remote(key)) resource = ray.get(address) return resource else: rm = df.__meta__['run_manager'] resource = rm.get_resource(key) if isinstance(resource, ObjectID): resource = ray.get(resource) return resource
import time from itertools import product import numpy as np import pandas as pd import pytest from kts.core.backend.address_manager import get_address_manager from kts.core.backend.progress import pbar from kts.core.backend.signal import get_signal_manager from kts.core.feature_constructor.user_defined import FeatureConstructor am = get_address_manager() sm = get_signal_manager() @pytest.mark.parametrize('remote', [False, True]) def test_fc_run(clear_caches, int_frame, run_manager, report, remote): am.clear.remote() def func(df): return df**2 fc = FeatureConstructor(func) fc.parallel = remote assert len(run_manager.scheduled) == 0 res = run_manager.run([fc], frame=int_frame, train=True, fold='preview', ret=True,