def _fixup(self): self.is_mr2 = True jobid = self.id if self.state in ('FINISHED', 'FAILED', 'KILLED'): setattr(self, 'status', self.finalStatus) else: setattr(self, 'status', self.state) setattr(self, 'jobId', jobid) setattr(self, 'jobId_short', re.sub('(application|job)_', '', self.jobId)) setattr(self, 'jobName', self.name) setattr(self, 'is_retired', False) setattr(self, 'maps_percent_complete', self.progress) setattr(self, 'reduces_percent_complete', self.progress) setattr(self, 'queueName', self.queue) setattr(self, 'priority', '') if self.finishedTime == 0: finishTime = int(time.time() * 1000) else: finishTime = self.finishedTime setattr(self, 'durationInMillis', finishTime - self.startedTime) setattr(self, 'startTimeMs', self.startedTime) setattr(self, 'startTimeFormatted', format_unixtime_ms(self.startedTime)) setattr(self, 'finishTimeFormatted', format_unixtime_ms(finishTime)) setattr(self, 'finishedMaps', None) setattr(self, 'desiredMaps', None) setattr(self, 'finishedReduces', None) setattr(self, 'desiredReduces', None) setattr(self, 'durationFormatted', format_duration_in_millis(self.durationInMillis)) if not hasattr(self, 'acls'): setattr(self, 'acls', {})
def _fixup(self): jobid = self.id setattr(self, 'status', self.state) setattr(self, 'jobName', self.name) setattr(self, 'jobId', jobid) setattr(self, 'jobId_short', self.jobId.replace('job_', '')) setattr(self, 'is_retired', False) setattr(self, 'maps_percent_complete', None) setattr(self, 'reduces_percent_complete', None) if self.finishTime == 0: finishTime = int(time.time() * 1000) else: finishTime = self.finishTime if self.startTime == 0: durationInMillis = None else: durationInMillis = finishTime - self.startTime setattr(self, 'duration', durationInMillis) setattr(self, 'durationFormatted', self.duration and format_duration_in_millis(self.duration)) setattr(self, 'finishTimeFormatted', format_unixtime_ms(self.finishTime)) setattr(self, 'startTimeFormatted', format_unixtime_ms(self.startTime)) setattr(self, 'finishedMaps', self.mapsCompleted) setattr(self, 'desiredMaps', 0) setattr(self, 'finishedReduces', self.reducesCompleted) setattr(self, 'desiredReduces', 0) setattr(self, 'applicationType', 'MR2')
def _fixup(self): setattr(self, 'jobId', self.job.jobId) setattr(self, 'taskId', self.id) setattr(self, 'taskId_short', self.id) setattr(self, 'taskType', self.type) setattr(self, 'execStartTimeMs', self.startTime) setattr(self, 'mostRecentState', self.state) setattr(self, 'execStartTimeFormatted', format_unixtime_ms(self.startTime)) setattr(self, 'execFinishTimeFormatted', format_unixtime_ms(self.finishTime)) setattr(self, 'startTimeFormatted', self.startTime)
def _fixup(self): setattr(self, "jobId", self.job.jobId) setattr(self, "taskId", self.id) setattr(self, "taskId_short", self.id) setattr(self, "taskType", self.type) setattr(self, "execStartTimeMs", self.startTime) setattr(self, "mostRecentState", self.state) setattr(self, "execStartTimeFormatted", format_unixtime_ms(self.startTime)) setattr(self, "execFinishTimeFormatted", format_unixtime_ms(self.finishTime)) setattr(self, "startTimeFormatted", format_unixtime_ms(self.startTime)) setattr(self, "progress", self.progress / 100)
def _fixup(self): self.is_mr2 = True jobid = self.id self.yarnStatus = self.state if self.state in ('FINISHED', 'FAILED', 'KILLED'): setattr(self, 'status', self.finalStatus) else: setattr(self, 'status', self.state) setattr(self, 'jobId', jobid) setattr(self, 'jobId_short', re.sub('(application|job)_', '', self.jobId)) setattr(self, 'jobName', self.name) setattr(self, 'applicationType', self.applicationType) setattr(self, 'is_retired', False) setattr(self, 'maps_percent_complete', self.progress) setattr(self, 'reduces_percent_complete', self.progress) setattr(self, 'queueName', self.queue) setattr(self, 'priority', '') if self.finishedTime == 0: finishTime = int(time.time() * 1000) else: finishTime = self.finishedTime if self.startedTime == 0: durationInMillis = None else: durationInMillis = finishTime - self.startedTime setattr(self, 'durationInMillis', durationInMillis) setattr( self, 'durationFormatted', durationInMillis and format_duration_in_millis(self.durationInMillis)) setattr(self, 'startTimeMs', self.startedTime) setattr(self, 'startTimeFormatted', format_unixtime_ms(self.startedTime)) setattr(self, 'finishTimeFormatted', format_unixtime_ms(finishTime)) setattr(self, 'finishedMaps', None) setattr(self, 'desiredMaps', None) setattr(self, 'finishedReduces', None) setattr(self, 'desiredReduces', None) for attr in [ 'preemptedResourceVCores', 'vcoreSeconds', 'memorySeconds', 'diagnostics' ]: if not hasattr(self, attr): setattr(self, attr, 'N/A') if not hasattr(self, 'acls'): setattr(self, 'acls', {}) # YARN returns a N/A url if it's not set. if not hasattr(self, 'trackingUrl') or self.trackingUrl == 'http://N/A': self.trackingUrl = None
def _fixup(self): setattr(self, 'attemptId', self.id) setattr(self, 'attemptId_short', self.id) setattr(self, 'taskTrackerId', getattr(self, 'assignedContainerId', None)) setattr(self, 'startTimeFormatted', format_unixtime_ms(self.startTime)) setattr(self, 'finishTimeFormatted', format_unixtime_ms(self.finishTime)) setattr(self, 'outputSize', None) setattr(self, 'phase', None) setattr(self, 'shuffleFinishTimeFormatted', None) setattr(self, 'sortFinishTimeFormatted', None) setattr(self, 'mapFinishTimeFormatted', None) if not hasattr(self, 'diagnostics'): self.diagnostics = '' if not hasattr(self, 'assignedContainerId'): setattr(self, 'assignedContainerId', '')
def _fixup(self): jobid = self.id if self.state in ('FINISHED', 'FAILED', 'KILLED'): setattr(self, 'status', self.finalStatus) else: setattr(self, 'status', self.state) setattr(self, 'type', self.applicationType) setattr(self, 'applicationType', 'YarnV2') setattr(self, 'jobName', self.name) setattr(self, 'jobId', jobid) setattr(self, 'jobId_short', self.jobId.replace('job_', '')) setattr(self, 'is_retired', False) setattr(self, 'is_mr2', True) setattr(self, 'maps_percent_complete', None) setattr(self, 'reduces_percent_complete', None) setattr(self, 'finishedMaps', 0) setattr(self, 'desiredMaps', 0) setattr(self, 'finishedReduces', 0) setattr(self, 'desiredReduces', 0) if self.finishedTime == 0: finishTime = int(time.time() * 1000) else: finishTime = self.finishedTime if self.startedTime == 0: durationInMillis = None else: durationInMillis = finishTime - self.startedTime setattr(self, 'duration', durationInMillis) setattr(self, 'durationInMillis', durationInMillis) setattr(self, 'durationFormatted', self.duration and format_duration_in_millis(self.duration)) setattr(self, 'finishTimeFormatted', format_unixtime_ms(finishTime)) setattr(self, 'startTimeFormatted', format_unixtime_ms(self.startedTime)) setattr(self, 'startTimeMs', self.startTimeFormatted) setattr(self, 'startTime', self.startedTime) setattr(self, 'finishTime', finishTime) try: setattr( self, 'assignedContainerId', urllib.parse.urlsplit(self.amContainerLogs).path.split( '/node/containerlogs/')[1].split('/')[0]) except Exception: setattr(self, 'assignedContainerId', '')
def _fixup(self): jobid = self.id setattr(self, "status", self.state) setattr(self, "jobId", jobid) setattr(self, "jobId_short", self.jobId.replace("job_", "")) setattr(self, "is_retired", True) setattr(self, "maps_percent_complete", None) setattr(self, "reduces_percent_complete", None) setattr(self, "duration", self.finishTime - self.startTime) setattr(self, "finishTimeFormatted", format_unixtime_ms(self.finishTime)) setattr(self, "startTimeFormatted", format_unixtime_ms(self.startTime)) setattr(self, "finishedMaps", self.mapsCompleted) setattr(self, "desiredMaps", None) setattr(self, "finishedReduces", self.reducesCompleted) setattr(self, "desiredReduces", None)
def _fixup(self): self.is_mr2 = True jobid = self.id if self.state in ("FINISHED", "FAILED", "KILLED"): setattr(self, "status", self.finalStatus) else: setattr(self, "status", self.state) setattr(self, "jobId", jobid) setattr(self, "jobId_short", re.sub("(application|job)_", "", self.jobId)) setattr(self, "jobName", self.name) setattr(self, "is_retired", False) setattr(self, "maps_percent_complete", self.progress) setattr(self, "reduces_percent_complete", self.progress) setattr(self, "queueName", self.queue) setattr(self, "priority", "") if self.finishedTime == 0: finishTime = int(time.time() * 1000) else: finishTime = self.finishedTime setattr(self, "durationInMillis", finishTime - self.startedTime) setattr(self, "startTimeMs", self.startedTime) setattr(self, "startTimeFormatted", format_unixtime_ms(self.startedTime)) setattr(self, "finishedMaps", None) setattr(self, "desiredMaps", None) setattr(self, "finishedReduces", None) setattr(self, "desiredReduces", None) setattr(self, "durationFormatted", format_duration_in_millis(self.durationInMillis))
def _fixup(self): setattr(self, "attemptId", self.id) setattr(self, "attemptId_short", self.id) setattr(self, "taskTrackerId", getattr(self, "assignedContainerId", None)) setattr(self, "startTimeFormatted", format_unixtime_ms(self.startTime)) setattr(self, "finishTimeFormatted", format_unixtime_ms(self.finishTime)) setattr(self, "outputSize", None) setattr(self, "phase", None) setattr(self, "shuffleFinishTimeFormatted", None) setattr(self, "sortFinishTimeFormatted", None) setattr(self, "mapFinishTimeFormatted", None) setattr(self, "progress", self.progress / 100) if not hasattr(self, "diagnostics"): self.diagnostics = "" if not hasattr(self, "assignedContainerId"): setattr(self, "assignedContainerId", "")
def _fixup(self): jobid = self.id setattr(self, 'status', self.state) setattr(self, 'jobId', jobid) setattr(self, 'jobId_short', self.jobId.replace('job_', '')) setattr(self, 'is_retired', True) setattr(self, 'maps_percent_complete', None) setattr(self, 'reduces_percent_complete', None) setattr(self, 'duration', self.finishTime - self.startTime) setattr(self, 'finishTimeFormatted', format_unixtime_ms(self.finishTime)) setattr(self, 'startTimeFormatted', format_unixtime_ms(self.startTime)) setattr(self, 'finishedMaps', self.mapsCompleted) setattr(self, 'desiredMaps', None) setattr(self, 'finishedReduces', self.reducesCompleted) setattr(self, 'desiredReduces', None)
def _fixup(self): self.is_mr2 = True jobid = self.id self.yarnStatus = self.state if self.state in ('FINISHED', 'FAILED', 'KILLED'): setattr(self, 'status', self.finalStatus) else: setattr(self, 'status', self.state) setattr(self, 'jobId', jobid) setattr(self, 'jobId_short', re.sub('(application|job)_', '', self.jobId)) setattr(self, 'jobName', self.name) setattr(self, 'applicationType', self.applicationType) setattr(self, 'is_retired', False) setattr(self, 'maps_percent_complete', self.progress) setattr(self, 'reduces_percent_complete', self.progress) setattr(self, 'queueName', self.queue) setattr(self, 'priority', '') if self.finishedTime == 0: finishTime = int(time.time() * 1000) else: finishTime = self.finishedTime if self.startedTime == 0: durationInMillis = None else: durationInMillis = finishTime - self.startedTime setattr(self, 'durationInMillis', durationInMillis) setattr(self, 'durationFormatted', durationInMillis and format_duration_in_millis(self.durationInMillis)) setattr(self, 'startTimeMs', self.startedTime) setattr(self, 'startTimeFormatted', format_unixtime_ms(self.startedTime)) setattr(self, 'finishTimeFormatted', format_unixtime_ms(finishTime)) setattr(self, 'finishedMaps', None) setattr(self, 'desiredMaps', None) setattr(self, 'finishedReduces', None) setattr(self, 'desiredReduces', None) for attr in ['preemptedResourceVCores', 'vcoreSeconds', 'memorySeconds', 'diagnostics']: if not hasattr(self, attr): setattr(self, attr, 'N/A') if not hasattr(self, 'acls'): setattr(self, 'acls', {}) # YARN returns a N/A url if it's not set. if not hasattr(self, 'trackingUrl') or self.trackingUrl == 'http://N/A': self.trackingUrl = None
def _fixup(self): jobid = self.id if self.state in ('FINISHED', 'FAILED', 'KILLED'): setattr(self, 'status', self.finalStatus) else: setattr(self, 'status', self.state) setattr(self, 'type', self.applicationType) setattr(self, 'applicationType', 'YarnV2') setattr(self, 'jobName', self.name) setattr(self, 'jobId', jobid) setattr(self, 'jobId_short', self.jobId.replace('job_', '')) setattr(self, 'is_retired', False) setattr(self, 'is_mr2', True) setattr(self, 'maps_percent_complete', None) setattr(self, 'reduces_percent_complete', None) setattr(self, 'finishedMaps', 0) setattr(self, 'desiredMaps', 0) setattr(self, 'finishedReduces', 0) setattr(self, 'desiredReduces', 0) if self.finishedTime == 0: finishTime = int(time.time() * 1000) else: finishTime = self.finishedTime if self.startedTime == 0: durationInMillis = None else: durationInMillis = finishTime - self.startedTime setattr(self, 'duration', durationInMillis) setattr(self, 'durationInMillis', durationInMillis) setattr(self, 'durationFormatted', self.duration and format_duration_in_millis(self.duration)) setattr(self, 'finishTimeFormatted', format_unixtime_ms(finishTime)) setattr(self, 'startTimeFormatted', format_unixtime_ms(self.startedTime)) setattr(self, 'startTimeMs', self.startTimeFormatted) setattr(self, 'startTime', self.startedTime) setattr(self, 'finishTime', finishTime) try: setattr(self, 'assignedContainerId', urlparse.urlsplit(self.amContainerLogs).path.split('/node/containerlogs/')[1].split('/')[0]) except Exception: setattr(self, 'assignedContainerId', '')
def _fixup(self): if not hasattr(self, 'diagnostics'): self.diagnostics = '' setattr(self, 'type', 'YarnV2') if self.finishedTime == 0: finishTime = int(time.time() * 1000) else: finishTime = self.finishedTime if self.startTime == 0: durationInMillis = None else: durationInMillis = finishTime - self.startTime setattr(self, 'duration', durationInMillis) setattr(self, 'durationInMillis', durationInMillis) setattr(self, 'durationFormatted', self.duration and format_duration_in_millis(self.duration)) setattr(self, 'finishTimeFormatted', format_unixtime_ms(finishTime)) setattr(self, 'startTimeFormatted', format_unixtime_ms(self.startTime)) setattr(self, 'status', 'RUNNING' if self.finishedTime == 0 else 'SUCCEEDED') setattr(self, 'properties', {})
def _fixup(self): jobid = self.id setattr(self, 'status', self.state) setattr(self, 'jobId', jobid) setattr(self, 'jobId_short', self.jobId.replace('job_', '')) setattr(self, 'is_retired', False) setattr(self, 'maps_percent_complete', None) setattr(self, 'reduces_percent_complete', None) if self.state in ('FINISHED', 'FAILED', 'KILLED'): setattr(self, 'finishedTime', self.finishTime) setattr(self, 'startedTime', self.startTime) setattr(self, 'duration', self.finishTime - self.startTime) setattr(self, 'durationFormatted', format_duration_in_millis(self.duration)) setattr(self, 'finishTimeFormatted', format_unixtime_ms(self.finishTime)) setattr(self, 'startTimeFormatted', format_unixtime_ms(self.startTime)) setattr(self, 'finishedMaps', self.mapsCompleted) setattr(self, 'desiredMaps', None) setattr(self, 'finishedReduces', self.reducesCompleted) setattr(self, 'desiredReduces', None)
def _fixup(self): jobid = self.id if self.state in ('SUCCEEDED', 'FAILED', 'KILL_WAIT', 'KILLED', 'ERROR'): setattr(self, 'status', self.state) # When a job is finished, just use 'job' instead of 'application' jobid = jobid.replace('application', 'job') else: jobid = jobid.replace('job', 'application') setattr(self, 'status', self.state) setattr(self, 'jobId', jobid) setattr(self, 'jobId_short', self.jobId.replace('job_', '')) setattr(self, 'is_retired', True) setattr(self, 'maps_percent_complete', None) setattr(self, 'reduces_percent_complete', None) setattr(self, 'duration', self.finishTime - self.startTime) setattr(self, 'finishTimeFormatted', format_unixtime_ms(self.finishTime)) setattr(self, 'startTimeFormatted', format_unixtime_ms(self.startTime)) setattr(self, 'finishedMaps', self.mapsCompleted) setattr(self, 'desiredMaps', None) setattr(self, 'finishedReduces', self.reducesCompleted) setattr(self, 'desiredReduces', None)
def _fixup(self): jobid = self.id setattr(self, 'status', self.state) setattr(self, 'jobId', jobid) setattr(self, 'jobId_short', self.jobId.replace('job_', '')) setattr(self, 'is_retired', False) setattr(self, 'maps_percent_complete', None) setattr(self, 'reduces_percent_complete', None) if self.state in ('FINISHED', 'FAILED', 'KILLED'): setattr(self, 'finishTime', self.finishedTime) setattr(self, 'startTime', self.startedTime) setattr(self, 'duration', self.finishTime - self.startTime) setattr(self, 'finishTimeFormatted', format_unixtime_ms(self.finishTime)) setattr(self, 'startTimeFormatted', format_unixtime_ms(self.startTime)) setattr(self, 'finishedMaps', self.mapsCompleted) setattr(self, 'desiredMaps', None) setattr(self, 'finishedReduces', self.reducesCompleted) setattr(self, 'desiredReduces', None) if not hasattr(self, 'acls'): setattr(self, 'acls', {})