def test_task_metrics_simple(self, tmpdir, pandas_data_frame): metrics_folder = target(str(tmpdir)) task_run = Mock() task_run.meta_files = TaskRunMetaFiles(metrics_folder) t = FileTrackingStore() tr_tracker = TaskRunTracker(task_run=task_run, tracking_store=t) tr_tracker.settings.features.get_value_meta_conf = Mock( return_value=ValueMetaConf.enabled() ) tr_tracker.log_metric("a", 1) tr_tracker.log_metric("a_string", "1") tr_tracker.log_metric("a_list", [1, 3]) tr_tracker.log_metric("a_tuple", (1, 2)) user_metrics = TaskRunMetricsFileStoreReader( metrics_folder ).get_all_metrics_values(MetricSource.user) assert user_metrics == { "a": 1.0, "a_list": "[1, 3]", "a_string": 1.0, "a_tuple": "(1, 2)", }
def test_task_metrics_histograms(self, tmpdir, pandas_data_frame): metrics_folder = target(str(tmpdir)) task_run = Mock() task_run.meta_files = TaskRunMetaFiles(metrics_folder) t = FileTrackingStore() tr_tracker = TaskRunTracker(task_run=task_run, tracking_store=t) tr_tracker.settings.tracking.get_value_meta_conf = Mock( return_value=ValueMetaConf.enabled() ) tr_tracker.log_data("df", pandas_data_frame, meta_conf=ValueMetaConf.enabled()) hist_metrics = TaskRunMetricsFileStoreReader( metrics_folder ).get_all_metrics_values(MetricSource.histograms) expected_preview = ( " Names Births Married\n" " Bob 968 True\n" " Jessica 155 False\n" " Mary 77 True\n" " John 578 False\n" " Mel 973 True" ) # std value varies in different py versions due to float precision fluctuation df_births_std = hist_metrics["df.Births.std"] assert df_births_std == pytest.approx(428.4246)
def test_task_run_url(self): stub_task_run = Mock() stub_task_run.run.tracker.databand_url = "http://example.com" stub_task_run.run.job_name = "test_job" stub_task_run.run.run_uid = "00000000-0000-0000-0000-000000000001" stub_task_run.task_run_uid = "00000000-0000-0000-0000-000000000002" tracker = TaskRunTracker(stub_task_run, {}) expected = "http://example.com/app/jobs/test_job/00000000-0000-0000-0000-000000000001/00000000-0000-0000-0000-000000000002" assert tracker.task_run_url() == expected
def try_get_or_create_task_run(): # type: ()-> TaskRunTracker task_run = try_get_current_task_run() if task_run: return task_run try: from dbnd._core.task_run.task_run_tracker import TaskRunTracker from dbnd._core.configuration.environ_config import DBND_TASK_RUN_ATTEMPT_UID tra_uid = os.environ.get(DBND_TASK_RUN_ATTEMPT_UID) if tra_uid: task_run = TaskRunMock(tra_uid) from dbnd import config from dbnd._core.settings import CoreConfig with config({CoreConfig.tracker_raise_on_error: False}, source="ondemand_tracking"): tracking_store = CoreConfig().get_tracking_store() trt = TaskRunTracker(task_run, tracking_store) task_run.tracker = trt return task_run # let's check if we are in airflow env from dbnd._core.inplace_run.airflow_dag_inplace_tracking import ( try_get_airflow_context, ) airflow_context = try_get_airflow_context() if airflow_context: from dbnd._core.inplace_run.airflow_dag_inplace_tracking import ( get_airflow_tracking_manager, ) atm = get_airflow_tracking_manager(airflow_context) if atm: return atm.airflow_operator__task_run from dbnd._core.inplace_run.inplace_run_manager import is_inplace_run if is_inplace_run(): return dbnd_run_start() except Exception: logger.info("Failed during dbnd inplace tracking init.", exc_info=True) return None
def _get_task_run_mock(tra_uid): """ We need better implementation for this, currently in use only for spark """ try: from dbnd._core.task_run.task_run_tracker import TaskRunTracker task_run = TaskRunMock(tra_uid) from dbnd import config from dbnd._core.settings import CoreConfig with config( {CoreConfig.tracker_raise_on_error: False}, source="on_demand_tracking" ): tracking_store = CoreConfig().get_tracking_store() trt = TaskRunTracker(task_run, tracking_store) task_run.tracker = trt return task_run except Exception: logger.info("Failed during dbnd inplace tracking init.", exc_info=True) return None
def __init__( self, task, run, task_af_id=None, try_number=1, is_dynamic=None, task_engine=None, ): # type: (Task, DatabandRun, str, int, bool, EngineConfig)-> None # actually this is used as Task uid self.task = task # type: Task self.run = run # type: DatabandRun self.task_engine = task_engine self.try_number = try_number self.is_dynamic = is_dynamic if is_dynamic is not None else task.task_is_dynamic self.is_system = task.task_is_system self.task_af_id = task_af_id or self.task.task_id if task.ctrl.force_task_run_uid: self.task_run_uid = tr_uid = task.ctrl.force_task_run_uid if isinstance(tr_uid, TaskRunUidGen): self.task_run_uid = tr_uid.generate_task_run_uid( run=run, task=task, task_af_id=self.task_af_id ) else: self.task_run_uid = get_uuid() # used by all kind of submission controllers self.job_name = clean_job_name(self.task_af_id).lower() self.job_id = self.job_name + "_" + str(self.task_run_uid)[:8] # DNS-1123 subdomain name (k8s) self.job_id__dns1123 = clean_job_name_dns1123( "dbnd.{task_family}.{task_name}".format( task_family=self.task.task_meta.task_family, task_name=self.task.task_meta.task_name, ), postfix=".%s" % str(self.task_run_uid)[:8], ) # custom per task engine , or just use one from global env dbnd_local_root = ( self.task_engine.dbnd_local_root or self.run.env.dbnd_local_root ) self.local_task_run_root = ( dbnd_local_root.folder(run.run_folder_prefix) .folder("tasks") .folder(self.task.task_id) ) self._attempt_number = 1 self.task_run_attempt_uid = get_uuid() self.attempt_folder = None self.meta_files = None self.log = None self.init_attempt() # TODO: inherit from parent task if disabled self.is_tracked = task._conf__tracked if self.is_tracked and self.run.is_tracked: tracking_store = self.run.context.tracking_store else: tracking_store = ConsoleStore() self.tracking_store = tracking_store self.tracker = TaskRunTracker(task_run=self, tracking_store=tracking_store) self.runner = TaskRunRunner(task_run=self) self.deploy = TaskSyncCtrl(task_run=self) self.task_tracker_url = self.tracker.task_run_url() self.external_resource_urls = dict() self.errors = [] self.is_root = False self.is_reused = False self.is_skipped = False # Task can be skipped as it's not required by any other task scheduled to run self.is_skipped_as_not_required = False self._airflow_context = None self._task_run_state = None self.start_time = None self.finished_time = None
class TaskRun(object): def __init__( self, task, run, task_af_id=None, try_number=1, is_dynamic=None, task_engine=None, ): # type: (Task, DatabandRun, str, int, bool, EngineConfig)-> None # actually this is used as Task uid self.task = task # type: Task self.run = run # type: DatabandRun self.task_engine = task_engine self.try_number = try_number self.is_dynamic = is_dynamic if is_dynamic is not None else task.task_is_dynamic self.is_system = task.task_is_system self.task_af_id = task_af_id or self.task.task_id if task.ctrl.force_task_run_uid: self.task_run_uid = tr_uid = task.ctrl.force_task_run_uid if isinstance(tr_uid, TaskRunUidGen): self.task_run_uid = tr_uid.generate_task_run_uid( run=run, task=task, task_af_id=self.task_af_id ) else: self.task_run_uid = get_uuid() # used by all kind of submission controllers self.job_name = clean_job_name(self.task_af_id).lower() self.job_id = self.job_name + "_" + str(self.task_run_uid)[:8] # DNS-1123 subdomain name (k8s) self.job_id__dns1123 = clean_job_name_dns1123( "dbnd.{task_family}.{task_name}".format( task_family=self.task.task_meta.task_family, task_name=self.task.task_meta.task_name, ), postfix=".%s" % str(self.task_run_uid)[:8], ) # custom per task engine , or just use one from global env dbnd_local_root = ( self.task_engine.dbnd_local_root or self.run.env.dbnd_local_root ) self.local_task_run_root = ( dbnd_local_root.folder(run.run_folder_prefix) .folder("tasks") .folder(self.task.task_id) ) self._attempt_number = 1 self.task_run_attempt_uid = get_uuid() self.attempt_folder = None self.meta_files = None self.log = None self.init_attempt() # TODO: inherit from parent task if disabled self.is_tracked = task._conf__tracked if self.is_tracked and self.run.is_tracked: tracking_store = self.run.context.tracking_store else: tracking_store = ConsoleStore() self.tracking_store = tracking_store self.tracker = TaskRunTracker(task_run=self, tracking_store=tracking_store) self.runner = TaskRunRunner(task_run=self) self.deploy = TaskSyncCtrl(task_run=self) self.task_tracker_url = self.tracker.task_run_url() self.external_resource_urls = dict() self.errors = [] self.is_root = False self.is_reused = False self.is_skipped = False # Task can be skipped as it's not required by any other task scheduled to run self.is_skipped_as_not_required = False self._airflow_context = None self._task_run_state = None self.start_time = None self.finished_time = None @property def task_run_env(self): return self.run.context.task_run_env def task_run_attempt_file(self, *path): return target(self.attempt_folder, *path) @property def last_error(self): return self.errors[-1] if self.errors else None def _get_log_files(self): log_local = None if self.log.local_log_file: log_local = self.log.local_log_file.path log_remote = None if self.log.remote_log_file: log_remote = self.log.remote_log_file.path return log_local, log_remote @property def task_run_state(self): return self._task_run_state @task_run_state.setter def task_run_state(self, value): raise AttributeError("Please use explicit .set_task_run_state()") def set_task_run_state(self, state, track=True, error=None): # type: (TaskRunState, bool, Any) -> bool # Optional bool track param - will send tracker.set_task_run_state() by default if not state or self._task_run_state == state: return False if error: self.errors.append(error) if state == TaskRunState.RUNNING: self.start_time = utcnow() self._task_run_state = state if track: self.tracking_store.set_task_run_state( task_run=self, state=state, error=error ) return True def set_task_reused(self): self._task_run_state = TaskRunState.SUCCESS self.tracking_store.set_task_reused(task_run=self) def set_external_resource_urls(self, links_dict): # if value is None skip if links_dict is None: # TODO: Throw exception? return for link in links_dict: if not links_dict[link]: # Dict has empty fields # TODO: Throw exception? return self.external_resource_urls.update(links_dict) self.tracking_store.save_external_links( task_run=self, external_links_dict=links_dict ) @property def attempt_number(self): return self._attempt_number @attempt_number.setter def attempt_number(self, value): if not value: raise DatabandRuntimeError("cannot set None as the attempt number") if value != self._attempt_number: self._attempt_number = value self.init_attempt() self.run.tracker.tracking_store.add_task_runs( run=self.run, task_runs=[self] ) @property def airflow_context(self): return self._airflow_context @airflow_context.setter def airflow_context(self, value): self._airflow_context = value if self._airflow_context: self.attempt_number = self._airflow_context["ti"].try_number def init_attempt(self): self.task_run_attempt_uid = get_uuid() self.attempt_folder = self.task._meta_output.folder( "attempt_%s_%s" % (self.attempt_number, self.task_run_attempt_uid), extension=None, ) self.meta_files = TaskRunMetaFiles(self.attempt_folder) self.log = TaskRunLogManager(task_run=self) def __repr__(self): return "TaskRun(%s, %s)" % (self.task.task_name, self.task_run_state)
def test_task_metrics_simple(self, tmpdir, pandas_data_frame): metrics_folder = target(str(tmpdir)) task_run = Mock() task_run.meta_files = TaskRunMetaFiles(metrics_folder) t = FileTrackingStore() tr_tracker = TaskRunTracker(task_run=task_run, tracking_store=t) tr_tracker.log_metric("a", 1) tr_tracker.log_metric("a_string", "1") tr_tracker.log_metric("a_list", [1, 3]) tr_tracker.log_metric("a_tuple", (1, 2)) tr_tracker.log_dataframe("df", pandas_data_frame) actual = TaskRunMetricsFileStoreReader( metrics_folder).get_all_metrics_values() print(actual) assert actual == { "a": 1.0, "a_list": "[1, 3]", "a_string": 1.0, "a_tuple": "(1, 2)", "df.preview": "Names Births", "df.schema": "{", "df.shape": "[5, 2]", "df.shape_0_": 5.0, "df.shape_1_": 2.0, }
def test_task_metrics_histograms(self, tmpdir, pandas_data_frame): metrics_folder = target(str(tmpdir)) task_run = Mock() task_run.meta_files = TaskRunMetaFiles(metrics_folder) t = FileTrackingStore() tr_tracker = TaskRunTracker(task_run=task_run, tracking_store=t) tr_tracker.settings.features.get_value_meta_conf = Mock( return_value=ValueMetaConf.enabled() ) tr_tracker.log_data( "df", pandas_data_frame, meta_conf=ValueMetaConf.enabled(), ) hist_metrics = TaskRunMetricsFileStoreReader( metrics_folder ).get_all_metrics_values(MetricSource.histograms) expected_preview = ( " Names Births Married\n" " Bob 968 True\n" " Jessica 155 False\n" " Mary 77 True\n" " John 578 False\n" " Mel 973 True" ) # std value varies in different py versions due to float precision fluctuation df_births_std = hist_metrics["df.Births.std"] assert df_births_std == pytest.approx(428.4246) hist_metrics["df.histograms"].pop("Names") hist_metrics["df.histograms"].pop("Births") hist_metrics.pop("df.Married.top") hist_metrics.pop("df.Names.top") hist_metrics["df.stats"]["Names"].pop("top") hist_metrics["df.stats"]["Married"].pop("top") assert hist_metrics == { "df.Births.type": "int64", "df.Births.25%": 155.0, "df.Births.50%": 578.0, "df.Births.75%": 968.0, "df.Births.count": 5.0, "df.Births.distinct": 5, "df.Births.std": df_births_std, "df.Births.max": 973.0, "df.Births.mean": 550.2, "df.Births.min": 77.0, "df.Births.non-null": 5, "df.Births.null-count": 0, "df.Married.count": 5, "df.Married.distinct": 2, "df.Married.freq": 3, "df.Married.non-null": 5, "df.Married.null-count": 0, "df.Married.type": "bool", "df.Married.unique": 2, "df.Names.count": 5, "df.Names.distinct": 5, "df.Names.freq": 1, "df.Names.non-null": 5, "df.Names.null-count": 0, "df.Names.type": "object", "df.Names.unique": 5, "df.histograms": {"Married": [[3, 2], [True, False]],}, "df.preview": expected_preview, "df.schema": { "columns": ["Names", "Births", "Married"], "dtypes": {"Births": "int64", "Names": "object", "Married": "bool"}, "shape": [5, 3], "size": 15, "type": "DataFrame", }, "df.shape": [5, 3], "df.shape0": 5, "df.shape1": 3, "df.stats": { "Births": { "type": "int64", "25%": 155.0, "50%": 578.0, "75%": 968.0, "count": 5.0, "distinct": 5, "max": 973.0, "mean": 550.2, "min": 77.0, "non-null": 5, "null-count": 0, "std": df_births_std, }, "Married": { "count": 5, "distinct": 2, "freq": 3, "non-null": 5, "null-count": 0, "type": "bool", "unique": 2, }, "Names": { "count": 5, "distinct": 5, "freq": 1, "non-null": 5, "null-count": 0, "type": "object", "unique": 5, }, }, }
class TaskRun(object): def __init__( self, task, run, task_af_id=None, try_number=1, is_dynamic=None, task_engine=None, ): # type: (Task, DatabandRun, str, int, bool, EngineConfig)-> None # actually this is used as Task uid self.task = task # type: Task self.run = run # type: DatabandRun self.task_engine = task_engine self.is_dynamic = is_dynamic if is_dynamic is not None else task.task_is_dynamic self.is_system = task.task_is_system self.task_af_id = task_af_id or self.task.task_id if task.ctrl.force_task_run_uid: self.task_run_uid = tr_uid = task.ctrl.force_task_run_uid if isinstance(tr_uid, TaskRunUidGen): self.task_run_uid = tr_uid.generate_task_run_uid( run=run, task=task, task_af_id=self.task_af_id ) else: self.task_run_uid = get_uuid() # used by all kind of submission controllers self.job_name = clean_job_name(self.task_af_id).lower() self.job_id = self.job_name + "_" + str(self.task_run_uid)[:8] # custom per task engine , or just use one from global env dbnd_local_root = ( self.task_engine.dbnd_local_root or self.run.env.dbnd_local_root ) self.local_task_run_root = ( dbnd_local_root.folder(run.run_folder_prefix) .folder("tasks") .folder(self.task.task_id) ) self.attempt_number = try_number self.task_run_attempt_uid = None self.attempt_folder = None self.meta_files = None self.log = None self.init_new_task_run_attempt() # TODO: inherit from parent task if disabled self.is_tracked = task._conf__tracked if self.is_tracked and self.run.is_tracked: tracking_store = self.run.context.tracking_store else: tracking_store = get_tracking_store( self.run.context, tracking_store_names=["console"], api_channel_name=None, max_retires=1, tracker_raise_on_error=False, remove_failed_store=True, ) self.tracking_store = tracking_store self.tracker = TaskRunTracker(task_run=self, tracking_store=tracking_store) self.runner = TaskRunRunner(task_run=self) self.deploy = TaskSyncCtrl(task_run=self) self.sync_local = TaskRunLocalSyncer(task_run=self) self.task_tracker_url = self.tracker.task_run_url() self.external_resource_urls = dict() self.errors = [] self.is_root = False self.is_reused = False self.is_skipped = False # Task can be skipped as it's not required by any other task scheduled to run self.is_skipped_as_not_required = False self.airflow_context = None self._task_run_state = None self.start_time = None self.finished_time = None def __getstate__(self): d = self.__dict__.copy() if "airflow_context" in d: # airflow context contains "auto generated code" that can not be pickled (Vars class) # we don't need to pickle it as we pickle DAGs separately d = self.__dict__.copy() del d["airflow_context"] return d @property def task_run_env(self): return self.run.context.task_run_env def task_run_attempt_file(self, *path): return target(self.attempt_folder, *path) @property def last_error(self): return self.errors[-1] if self.errors else None def _get_log_files(self): log_local = None if self.log.local_log_file: log_local = self.log.local_log_file.path log_remote = None if self.log.remote_log_file: log_remote = self.log.remote_log_file.path return log_local, log_remote @property def task_run_state(self): return self._task_run_state @task_run_state.setter def task_run_state(self, value): raise AttributeError("Please use explicit .set_task_run_state()") def set_task_run_state(self, state, track=True, error=None): # type: (TaskRunState, bool, Any) -> bool # Optional bool track param - will send tracker.set_task_run_state() by default if not state or self._task_run_state == state: return False if error: self.errors.append(error) if state == TaskRunState.RUNNING: self.start_time = utcnow() self._task_run_state = state if track: self.tracking_store.set_task_run_state( task_run=self, state=state, error=error ) return True def set_task_reused(self): self._task_run_state = TaskRunState.SUCCESS self.tracking_store.set_task_reused(task_run=self) def set_external_resource_urls(self, links_dict): # if value is None skip if links_dict is None: # TODO: Throw exception? return for link in links_dict: if not links_dict[link]: # Dict has empty fields # TODO: Throw exception? return self.external_resource_urls.update(links_dict) self.tracking_store.save_external_links( task_run=self, external_links_dict=links_dict ) def update_task_run_attempt(self, attempt_number): if attempt_number is None: raise DatabandRuntimeError("cannot set None as the attempt number") if self.attempt_number != attempt_number: self.attempt_number = attempt_number self.init_new_task_run_attempt() def init_new_task_run_attempt(self): # trying to find if we should use attempt_uid that been set from external process. # if so - the attempt_uid is uniquely for this task_run_attempt, and that why we pop. self.task_run_attempt_uid = get_task_run_attempt_uid_by_task_run(self) self.attempt_folder = self.task._meta_output.folder( "attempt_%s_%s" % (self.attempt_number, self.task_run_attempt_uid), extension=None, ) self.attempt_folder_local = self.local_task_run_root.folder( "attempt_%s_%s" % (self.attempt_number, self.task_run_attempt_uid), extension=None, ) self.attemp_folder_local_cache = self.attempt_folder_local.folder("cache") self.meta_files = TaskRunMetaFiles(self.attempt_folder) self.log = TaskRunLogManager(task_run=self) def __repr__(self): return "TaskRun(id=%s, af_id=%s)" % (self.task.task_id, self.task_af_id)