def local_worker(self, *args, kf: KTSFrame):
run_id = RunID(kf._scope, kf._fold, kf.hash())
return_state = kf._train # default for cached FCs or first calls of not cached FCs
if not self.cache and bool(kf._state):
# second call of not cached FC does not return state, as it is saved previously
# refer to https://github.com/konodyuk/kts/tree/master/kts/core#caching-policy
return_state = False
stats = Stats(kf)
if in_worker() and self.verbose:
report = None
io = self.remote_io(run_id)
rs.send(ProgressSignal(0, 1, None, None, None, run_id))
elif not in_worker() and self.verbose:
report = kf.__meta__['report']
io = self.local_io(report, run_id)
report.update(run_id, 0, 1)
else:
report = None
io = self.suppress_io()
with stats, io, self.suppress_stderr(), pbar.local_mode(report, run_id):
res_kf = self.compute(*args, kf)
if 'columns' in dir(res_kf) and '__columns' not in kf._state:
kf._state['__columns'] = list(res_kf.columns)
if return_state:
res_state = kf._state
else:
res_state = None
if in_worker() and self.verbose:
rs.send(ProgressSignal(1, 1, stats.data['took'], None, None, run_id))
elif not in_worker() and self.verbose:
report = kf.__meta__['report']
report.update(run_id, 1, 1, stats.data['took'])
return res_kf, res_state, stats.data
def __init__(self, iterable: Iterable, total: Optional[int] = None, title: Optional[str] = None):
if in_worker():
cls = RemoteProgressBar
else:
cls = LocalProgressBar
cls.report = self.report
cls.run_id = self.run_id
self.internal = cls(iterable, total, title)
def sync(self, run_id, res_df, res_state, stats, df):
if not self.cache:
# result frame is not saved to cache if FC is not cached
# refer to https://github.com/konodyuk/kts/tree/master/kts/core#caching-policy
res_df = None
if in_worker():
if not isinstance(res_df, ObjectID):
res_df = ray.put(res_df)
if not isinstance(res_state, ObjectID):
res_state = ray.put(res_state)
if not isinstance(stats, ObjectID):
stats = ray.put(stats)
rs.send(Sync(run_id, res_df, res_state, stats))
else:
rm = df.__meta__['run_manager']
rm.sync(run_id, res_df, res_state, stats)
def request_resource(self, key, df):
if in_worker():
request_time = time.time()
rs.send(ResourceRequest(key))
address_manager = get_address_manager()
while not ray.get(address_manager.has.remote(key)) or ray.get(address_manager.timestamp.remote(key)) < request_time:
time.sleep(0.01)
address = ray.get(address_manager.get.remote(key))
resource = ray.get(address)
return resource
else:
rm = df.__meta__['run_manager']
resource = rm.get_resource(key)
if isinstance(resource, ObjectID):
resource = ray.get(resource)
return resource
def compute(self, kf: KTSFrame):
assert not in_worker()
result = pd.DataFrame(index=kf.index.copy(),
columns=self.result_columns)
known_index = kf.index.intersection(self.oof.index)
unknown_index = kf.index.difference(self.oof.index)
if len(known_index) > 0:
result.loc[known_index] = self.oof.loc[known_index]
if len(unknown_index) > 0:
pred = self.experiment.predict(kf.loc[unknown_index])
if len(pred.shape) == 1:
pred = pred.reshape((-1, 1))
result.loc[unknown_index] = pred
if kf._train and self.noise_level:
rng = np.random.RandomState(self.random_state)
noise = rng.rand(*result.shape)
noise = (noise - 0.5) * self.noise_level
result += noise
return result
from kts.__version__ import __version__
from kts.core.backend.util import in_cli, in_worker, in_pytest
if not in_cli() and not in_worker() and not in_pytest():
from kts.core.init import init
init()
import kts.stl as stl
from kts.api.decorators import preview, feature, helper, generic, delete
from kts.core.backend.progress import pbar
from kts.core.cache import save, load, rm, ls
from kts.core.feature_set import FeatureSet
from kts.core.frame import KTSFrame
from kts.core.lists import feature_list as features, helper_list as helpers
from kts.feature_selection import *
from kts.modelling.custom_model import CustomModel, custom_model
from kts.modelling.mixins import NormalizeFillNAMixin
from kts.ui.dashboard import dashboard
from kts.ui.settings import set_highlighting, set_theme, set_animation
from kts.validation.leaderboard import leaderboard, leaderboard_list
from kts.validation.validator import Validator
from kts.validation.split import Refiner
lb = leaderboard
lbs = leaderboard_list