def rm(key: str):
if key in user_cache_frame:
del user_cache_frame[key]
if key in user_cache_obj:
del user_cache_obj[key]
am = get_address_manager()
am.delete.remote(key)
def put_resource(self, key: Union[RunID, Tuple[str, str], str]):
resource = self.get_resource(key)
address_manager = get_address_manager()
if isinstance(resource, ObjectID):
address_manager.put.remote((key, resource, False))
return
is_none = resource is None
address = ray.put(resource)
address_manager.put.remote((key, address, is_none))
def supervise(self, report=None):
if report is None:
report = SilentFeatureComputingReport()
try:
extra_iterations = 1
while True:
signals = self.new_signals()
syncs = filter_signals(signals, Sync)
for sync in syncs:
self.sync(**sync.get_contents())
resource_requests = filter_signals(signals, ResourceRequest)
for rr in resource_requests:
key = rr.get_contents()
address_manager = get_address_manager()
if ray.get(address_manager.has.remote(key)):
if ray.get(address_manager.isnone.remote(key)):
self.put_resource(key)
else:
address_manager.confirm.remote(key)
else:
self.put_resource(key)
pid_signals = self.filter_map_id(signals, RunPID)
for rid, pid_signal in pid_signals:
self.scheduled[rid].pid = pid_signal.get_contents()
progress_signals = self.filter_map_id(signals, ProgressSignal)
for rid, ps in progress_signals:
payload = ps.get_contents()
title = payload.pop('title')
rid = payload.pop('run_id', rid)
if title is not None:
rid = copy(rid)
rid.function_name += f" - {title}"
report.update(**payload, run_id=rid, autorefresh=False)
text_chunks = self.filter_map_id(signals, TextChunk)
for rid, tc in text_chunks:
payload = tc.get_contents()
rid = payload.pop('run_id', rid)
report.update_text(rid, **payload, autorefresh=False)
errors = self.filter_map_id(signals, rs.ErrorSignal)
for rid, err in errors:
report.update_text(rid, text='task failed', timestamp=time.time(), autorefresh=False)
report.update_text(rid, text=err.error.strip(), timestamp=time.time(), autorefresh=False)
report.refresh(force=True)
report.refresh()
time.sleep(0.01)
extra_iterations -= self.completed()
if extra_iterations < 0:
break
report.refresh(force=True)
except KeyboardInterrupt:
self.kill_scheduled()
def safe_put(kf: KTSFrame):
address_manager = get_address_manager()
h = kf.hash()
if ray.get(address_manager.has.remote(h)):
oid = ray.get(address_manager.get.remote(h))
else:
oid = ray.put(kf)
address_manager.put.remote((h, oid, False))
return oid
def request_resource(self, key, df):
if in_worker():
request_time = time.time()
rs.send(ResourceRequest(key))
address_manager = get_address_manager()
while not ray.get(address_manager.has.remote(key)) or ray.get(address_manager.timestamp.remote(key)) < request_time:
time.sleep(0.01)
address = ray.get(address_manager.get.remote(key))
resource = ray.get(address)
return resource
else:
rm = df.__meta__['run_manager']
resource = rm.get_resource(key)
if isinstance(resource, ObjectID):
resource = ray.get(resource)
return resource
import time
from itertools import product
import numpy as np
import pandas as pd
import pytest
from kts.core.backend.address_manager import get_address_manager
from kts.core.backend.progress import pbar
from kts.core.backend.signal import get_signal_manager
from kts.core.feature_constructor.user_defined import FeatureConstructor
am = get_address_manager()
sm = get_signal_manager()
@pytest.mark.parametrize('remote', [False, True])
def test_fc_run(clear_caches, int_frame, run_manager, report, remote):
am.clear.remote()
def func(df):
return df**2
fc = FeatureConstructor(func)
fc.parallel = remote
assert len(run_manager.scheduled) == 0
res = run_manager.run([fc],
frame=int_frame,
train=True,
fold='preview',
ret=True,