def local_worker(self, *args, kf: KTSFrame): run_id = RunID(kf._scope, kf._fold, kf.hash()) return_state = kf._train # default for cached FCs or first calls of not cached FCs if not self.cache and bool(kf._state): # second call of not cached FC does not return state, as it is saved previously # refer to https://github.com/konodyuk/kts/tree/master/kts/core#caching-policy return_state = False stats = Stats(kf) if in_worker() and self.verbose: report = None io = self.remote_io(run_id) rs.send(ProgressSignal(0, 1, None, None, None, run_id)) elif not in_worker() and self.verbose: report = kf.__meta__['report'] io = self.local_io(report, run_id) report.update(run_id, 0, 1) else: report = None io = self.suppress_io() with stats, io, self.suppress_stderr(), pbar.local_mode(report, run_id): res_kf = self.compute(*args, kf) if 'columns' in dir(res_kf) and '__columns' not in kf._state: kf._state['__columns'] = list(res_kf.columns) if return_state: res_state = kf._state else: res_state = None if in_worker() and self.verbose: rs.send(ProgressSignal(1, 1, stats.data['took'], None, None, run_id)) elif not in_worker() and self.verbose: report = kf.__meta__['report'] report.update(run_id, 1, 1, stats.data['took']) return res_kf, res_state, stats.data
def __init__(self, iterable: Iterable, total: Optional[int] = None, title: Optional[str] = None): if in_worker(): cls = RemoteProgressBar else: cls = LocalProgressBar cls.report = self.report cls.run_id = self.run_id self.internal = cls(iterable, total, title)
def sync(self, run_id, res_df, res_state, stats, df): if not self.cache: # result frame is not saved to cache if FC is not cached # refer to https://github.com/konodyuk/kts/tree/master/kts/core#caching-policy res_df = None if in_worker(): if not isinstance(res_df, ObjectID): res_df = ray.put(res_df) if not isinstance(res_state, ObjectID): res_state = ray.put(res_state) if not isinstance(stats, ObjectID): stats = ray.put(stats) rs.send(Sync(run_id, res_df, res_state, stats)) else: rm = df.__meta__['run_manager'] rm.sync(run_id, res_df, res_state, stats)
def request_resource(self, key, df): if in_worker(): request_time = time.time() rs.send(ResourceRequest(key)) address_manager = get_address_manager() while not ray.get(address_manager.has.remote(key)) or ray.get(address_manager.timestamp.remote(key)) < request_time: time.sleep(0.01) address = ray.get(address_manager.get.remote(key)) resource = ray.get(address) return resource else: rm = df.__meta__['run_manager'] resource = rm.get_resource(key) if isinstance(resource, ObjectID): resource = ray.get(resource) return resource
def compute(self, kf: KTSFrame): assert not in_worker() result = pd.DataFrame(index=kf.index.copy(), columns=self.result_columns) known_index = kf.index.intersection(self.oof.index) unknown_index = kf.index.difference(self.oof.index) if len(known_index) > 0: result.loc[known_index] = self.oof.loc[known_index] if len(unknown_index) > 0: pred = self.experiment.predict(kf.loc[unknown_index]) if len(pred.shape) == 1: pred = pred.reshape((-1, 1)) result.loc[unknown_index] = pred if kf._train and self.noise_level: rng = np.random.RandomState(self.random_state) noise = rng.rand(*result.shape) noise = (noise - 0.5) * self.noise_level result += noise return result
from kts.__version__ import __version__ from kts.core.backend.util import in_cli, in_worker, in_pytest if not in_cli() and not in_worker() and not in_pytest(): from kts.core.init import init init() import kts.stl as stl from kts.api.decorators import preview, feature, helper, generic, delete from kts.core.backend.progress import pbar from kts.core.cache import save, load, rm, ls from kts.core.feature_set import FeatureSet from kts.core.frame import KTSFrame from kts.core.lists import feature_list as features, helper_list as helpers from kts.feature_selection import * from kts.modelling.custom_model import CustomModel, custom_model from kts.modelling.mixins import NormalizeFillNAMixin from kts.ui.dashboard import dashboard from kts.ui.settings import set_highlighting, set_theme, set_animation from kts.validation.leaderboard import leaderboard, leaderboard_list from kts.validation.validator import Validator from kts.validation.split import Refiner lb = leaderboard lbs = leaderboard_list