def get_preview(cls, task, single=False): result = None if not task: pass elif task.use_frames: if single: # path to the most recently updated preview try: # previews that were updated at least once iterator = ifilter(lambda p: bool(p.last_update_time), task.preview_updaters) # find the max timestamp updater = max(iterator, key=lambda p: p.last_update_time) return to_unicode(updater.preview_file_path) except StopIteration: return None else: # paths for all frames return [ to_unicode(p.preview_file_path) for p in task.preview_updaters ] else: result = to_unicode(task.preview_updater.preview_file_path) return cls._preview_result(result, single=single)
def payments(sort, status): deferred = payments.client.get_payments_list() result = sync_wait(deferred) or [] values = [] if status is not None: result = filter_by_status(result, status) for payment in result: payment_value = float(payment["value"]) payment_fee = payment["fee"] or "" if payment_fee: payment_fee = __value(payment_fee, "ETH") entry = [ to_unicode(payment["subtask"]), to_unicode(payment["payee"]), to_unicode(payment["status"]), __value(payment_value, "GNT"), to_unicode(payment_fee) ] values.append(entry) return CommandResult.to_tabular(payments_table_headers, values, sort=sort)
def to_dictionary(self): return { u'id': to_unicode(self.header.task_id), u'name': to_unicode(self.task_definition.task_name), u'type': to_unicode(self.task_definition.task_type), u'subtasks': self.get_total_tasks(), u'progress': self.get_progress() }
def _cache_response(self, resource_name, response, task_id): if isinstance(response, list): for entry in response: self._cache_response(resource_name, entry, task_id) elif response and 'Hash' in response and 'Name' in response: if os.path.basename(response.get('Name')) != os.path.basename(resource_name): raise Exception("Resource manager: Invalid response {}".format(response)) res_path = to_unicode(resource_name) res_hash = to_unicode(response.get('Hash')) resource = self._wrap_resource((res_path, res_hash), task_id) self._cache_resource(resource)
def to_dictionary(self): preview = self.extra_data.get('result_preview') if isinstance(preview, basestring): preview = to_unicode(preview) elif isinstance(preview, collections.Iterable): preview = [to_unicode(entry) for entry in preview] return { u'time_remaining': self.remaining_time, u'status': to_unicode(self.status), u'preview': preview }
def _update_frame_status(self, frame): frame_key = to_unicode(frame) state = self.frames_state[frame_key] subtask_ids = self.frames_subtasks[frame_key] parts = max(1, int(self.total_tasks / len(self.frames))) counters = defaultdict(lambda: 0, dict()) # Count the number of occurrences of each subtask state for subtask_id in filter(bool, subtask_ids): subtask = self.subtasks_given[subtask_id] counters[subtask['status']] += 1 # Count statuses different from 'finished' and 'failure' computing = len([ x for x in counters.keys() if x not in [SubtaskStatus.finished, SubtaskStatus.failure] ]) # Finished if at least n subtasks >= parts were finished if counters[SubtaskStatus.finished] >= parts: state.status = TaskStatus.finished # Computing if at least one subtask did not fail elif computing > 0: state.status = TaskStatus.computing # Failure if the only known subtask status is 'failure' elif counters[SubtaskStatus.failure] > 0: state.status = TaskStatus.aborted
def _command(cls, command: List[str], vm_name: Optional[str] = None, args: Optional[Union[Tuple, List[str]]] = None, shell: bool = False) -> str: if args: command += list(args) if vm_name: command += [vm_name] if logger.isEnabledFor(logging.DEBUG): command.insert(1, '-D') logger.debug('Docker command: %s', command) try: output = subprocess.check_output( command, startupinfo=SUBPROCESS_STARTUP_INFO, shell=shell, stdin=subprocess.DEVNULL, stderr=subprocess.STDOUT) except FileNotFoundError as exc: raise subprocess.CalledProcessError(127, str(exc)) except subprocess.CalledProcessError as exc: logger.debug('Docker command output: %s', exc.output) raise exc logger.debug('Docker command output: %s', output) return to_unicode(output)
def _sort_dict(self, dictionary, filter_properties=False): result = dict() for k, v in dictionary.iteritems(): if filter_properties and (k.startswith('_') or callable(v)): continue result[to_unicode(k)] = self._sort_obj(v) return sorted(result.items())
def incomes(sort, status): deferred = incomes.client.get_incomes_list() result = sync_wait(deferred) or [] values = [] if status is not None: result = filter_by_status(result, status) for income in result: entry = [ to_unicode(income["payer"]), to_unicode(income["status"]), __value(float(income["value"]), "GNT"), ] values.append(entry) return CommandResult.to_tabular(incomes_table_headers, values, sort=sort)
def test_unicode(self): source = str("test string") result = to_unicode(source) assert result is source source = "\xd0\xd1\xd2\xd3" result = to_unicode(source) assert result is source source = "test string" result = to_unicode(source) assert isinstance(result, str) assert result == source source = None result = to_unicode(source) assert result is None
def test_unicode(self): source = unicode("test string") result = to_unicode(source) assert result is source source = "\xd0\xd1\xd2\xd3" result = to_unicode(source) assert result is source source = "test string" result = to_unicode(source) assert type(result) is unicode assert result is not source assert result == source source = None result = to_unicode(source) assert result is None
def get_preview(cls, task, single=False): result = None if not task: pass elif task.use_frames: if single: return to_unicode(task.last_preview_path) else: previews = [to_unicode(p) for p in task.preview_task_file_path] result = {} for i, f in enumerate(task.frames): try: result[to_unicode(f)] = previews[i] except IndexError: result[to_unicode(f)] = None else: result = to_unicode(task.preview_task_file_path or task.preview_file_path) return cls._preview_result(result, single=single)
def parse_rpc_address(ctx, param, value): del ctx, param value = to_unicode(value) if value: try: return SocketAddress.parse(value) except ipaddress.AddressValueError as e: raise click.BadParameter( "Invalid RPC address specified: {}".format(e)) return None
def _from_dict_traverse_obj(cls, obj): if isinstance(obj, dict): if cls._is_class(obj): return cls.obj_from_dict(obj) return cls._from_dict_traverse_dict(obj) elif isinstance(obj, basestring): return to_unicode(obj) elif isinstance(obj, collections.Iterable): return obj.__class__([cls._from_dict_traverse_obj(o) for o in obj]) return obj
def to_dictionary(self): return { u'subtask_id': to_unicode(self.subtask_id), u'node_name': to_unicode(self.computer.node_name), u'node_id': to_unicode(self.computer.node_id), u'node_performance': to_unicode(self.computer.performance), u'node_ip_address': to_unicode(self.computer.ip_address), u'node_port': self.computer.port, u'status': to_unicode(self.subtask_status), u'progress': self.subtask_progress, u'time_started': self.time_started, u'time_remaining': self.subtask_rem_time, u'results': [to_unicode(r) for r in self.results], u'stderr': to_unicode(self.stderr), u'stdout': to_unicode(self.stdout) }
def _sort_obj(self, v): if isinstance(v, dict): return self._sort_dict(v) # treat objects as dictionaries elif hasattr(v, '__dict__'): return self._sort_dict(v.__dict__, filter_properties=True) elif isinstance(v, basestring): return to_unicode(v) elif isinstance(v, collections.Iterable): return v.__class__([self._sort_obj(_v) for _v in v]) return v
def publish(cls, component, method, stage, data=None): """ Convenience function for publishing the execution stage event. :param component: Component name :param method: Method name :param stage: Execution stage to report at: pre, post (defined in golem.report.Stage). Exceptions are always reported. If not specified, both 'pre' and 'post' are used. :param data: Payload (optional) :return: None """ if cls._rpc_publisher: from twisted.internet import reactor cls._last_status = (to_unicode(component), to_unicode(method), to_unicode(stage), data) reactor.callFromThread(cls._rpc_publisher.publish, Golem.evt_golem_status, *cls._last_status)
def _to_dict_traverse_obj(cls, obj, typed=True): if isinstance(obj, dict): return cls._to_dict_traverse_dict(obj, typed) elif isinstance(obj, basestring): return to_unicode(obj) elif isinstance(obj, collections.Iterable): if isinstance(obj, (set, frozenset)): logger.warning('set/frozenset have known problems with umsgpack: %r', obj) return obj.__class__([cls._to_dict_traverse_obj(o, typed) for o in obj]) elif cls.deep_serialization: if hasattr(obj, '__dict__') and not cls._is_builtin(obj): return cls.obj_to_dict(obj, typed) return obj
def get_subtasks_borders(self, task_id, part=1): task = self.tasks[task_id] task_type_name = task.task_definition.task_type.lower() task_type = self.task_types[task_type_name] subtasks_count = task.get_total_tasks() return { to_unicode(subtask_id): task_type.get_task_border(subtask, task.task_definition, subtasks_count, as_path=True) for subtask_id, subtask in task.get_subtasks(part).items() }
def build_minimal_definition(cls, task_type, dictionary): parent = super(FrameRenderingTaskBuilder, cls) options = dictionary.get('options') or dict() frames_string = to_unicode(options.get('frames', 1)) frames = cls.string_to_frames(frames_string) use_frames = options.get('use_frames', len(frames) > 1) definition = parent.build_minimal_definition(task_type, dictionary) definition.options.frames_string = frames_string definition.options.frames = frames definition.options.use_frames = use_frames return definition
def get_list_of_all_payments(self): # This data is used by UI. return [{ "subtask": to_unicode(payment.subtask), "payee": to_unicode(encode_hex(payment.payee)), "value": to_unicode(payment.value), "status": to_unicode(payment.status.name), "fee": to_unicode(payment.details.fee), "block_number": to_unicode(payment.details.block_number), "transaction": to_unicode(payment.details.tx), "created": datetime_to_timestamp_utc(payment.created_date), "modified": datetime_to_timestamp_utc(payment.modified_date) } for payment in self.db.get_newest_payment()]
def get_subtasks_borders(self, task_id): task = self.tasks[task_id] task_type_name = task.task_definition.task_type.lower() task_type = self.task_types[task_type_name] task_state = self.tasks_states[task_id] total_subtasks = task.get_total_tasks() return { to_unicode(subtask.subtask_id): task_type.get_task_border(subtask, task.task_definition, total_subtasks, as_path=True) for subtask in task_state.subtask_states.values() }
def make_log_analyses(log_content, return_data): _get_warnings(log_content, return_data) rendering_time = find_rendering_time(log_content) if rendering_time: return_data["rendering_time"] = to_unicode(rendering_time) output_path = find_filepath(log_content) if output_path: return_data["output_path"] = to_unicode(output_path) frames = find_frames(log_content) if frames: return_data["frames"] = frames resolution = find_resolution(log_content) if resolution: return_data["resolution"] = resolution file_format = find_file_format(log_content) if file_format: return_data["file_format"] = to_unicode(file_format) engine_type = find_engine_type(log_content) if engine_type: return_data['engine_type'] = to_unicode(engine_type) if engine_type == "CYCLES": samples_pp = find_samples_for_scenes(log_content) if samples_pp: return_data['samples'] = samples_pp
def get_subtasks(self, frame): if self.task_definition.options.use_frames: subtask_ids = self.frames_subtasks.get(to_unicode(frame), []) subtask_ids = filter(None, subtask_ids) else: subtask_ids = self.subtasks_given.keys() subtasks = dict() # Convert to SubtaskState in order to match parent's return type for subtask_id in subtask_ids: state = SubtaskState() state.extra_data = self.subtasks_given[subtask_id] subtasks[subtask_id] = state return subtasks
def to_dictionary(self): return { 'subtask_id': to_unicode(self.subtask_id), 'node_id': to_unicode(self.node_id), 'node_name': to_unicode(self.node_name), 'status': self.subtask_status.value, 'progress': self.subtask_progress, 'time_started': self.time_started, 'time_remaining': self.subtask_rem_time, 'results': [to_unicode(r) for r in self.results], 'stderr': to_unicode(self.stderr), 'stdout': to_unicode(self.stdout), 'description': self.subtask_definition, }
def build_dictionary(cls, definition): task_timeout = timeout_to_string(definition.full_task_timeout) subtask_timeout = timeout_to_string(definition.subtask_timeout) output_path = cls.build_output_path(definition) return { u'type': to_unicode(definition.task_type), u'name': to_unicode(definition.task_name), u'timeout': to_unicode(task_timeout), u'subtask_timeout': to_unicode(subtask_timeout), u'subtask_count': definition.total_subtasks, u'bid': float(definition.max_price) / denoms.ether, u'resources': [to_unicode(r) for r in definition.resources], u'options': { u'output_path': to_unicode(output_path) } }
def _map_payment(cls, obj): obj["payee"] = to_unicode(obj["payee"]) obj["value"] = to_unicode(obj["value"]) obj["fee"] = to_unicode(obj["fee"]) return obj
def _ip_from_str(ip_address): return ipaddress.ip_address(to_unicode(ip_address))
def query_extra_data(self, perf_index: float, num_cores: int = 0, node_id: Optional[str] = None, node_name: Optional[str] = None) \ -> FrameRenderingTask.ExtraData: start_task, end_task = self._get_next_task() scene_file = self._get_scene_file_rel_path() if self.use_frames: frames, parts = self._choose_frames(self.frames, start_task, self.total_tasks) else: frames = self.frames or [1] parts = 1 if not self.use_frames: min_y, max_y = self._get_min_max_y(start_task) elif parts > 1: min_y = (parts - self._count_part(start_task, parts)) * (1.0 / parts) max_y = (parts - self._count_part(start_task, parts) + 1) * (1.0 / parts) else: min_y = 0.0 max_y = 1.0 # Blender is using single precision math, we use numpy to emulate this. # Send already converted values to blender. min_y = numpy.float32(min_y) max_y = numpy.float32(max_y) script_src = generate_blender_crop_file( resolution=(self.res_x, self.res_y), borders_x=(0.0, 1.0), borders_y=(min_y, max_y), use_compositing=self.compositing, samples=self.samples) extra_data = { "path_root": self.main_scene_dir, "start_task": start_task, "end_task": end_task, "total_tasks": self.total_tasks, "outfilebasename": self.outfilebasename, "scene_file": scene_file, "script_src": script_src, "frames": frames, "output_format": self.output_format, } subtask_id = self.create_subtask_id() logger.debug( 'Created new subtask for task. ' 'task_id=%s, subtask_id=%s, node_id=%s', self.header.task_id, subtask_id, short_node_id(node_id or '')) self.subtasks_given[subtask_id] = copy(extra_data) self.subtasks_given[subtask_id]['subtask_id'] = subtask_id self.subtasks_given[subtask_id]['status'] = SubtaskStatus.starting self.subtasks_given[subtask_id]['node_id'] = node_id self.subtasks_given[subtask_id]['parts'] = parts self.subtasks_given[subtask_id]['res_x'] = self.res_x self.subtasks_given[subtask_id]['res_y'] = self.res_y self.subtasks_given[subtask_id]['samples'] = self.samples self.subtasks_given[subtask_id]['use_frames'] = self.use_frames self.subtasks_given[subtask_id]['all_frames'] = self.frames self.subtasks_given[subtask_id]['crop_window'] = (0.0, 1.0, min_y, max_y) self.subtasks_given[subtask_id]['subtask_timeout'] = \ self.header.subtask_timeout self.subtasks_given[subtask_id]['tmp_dir'] = self.tmp_dir # FIXME issue #1955 part = self._count_part(start_task, parts) for frame in frames: frame_key = to_unicode(frame) state = self.frames_state[frame_key] state.status = TaskStatus.computing state.started = state.started or time.time() self.frames_subtasks[frame_key][part - 1] = subtask_id if not self.use_frames: self._update_task_preview() else: self._update_frame_task_preview() ctd = self._new_compute_task_def(subtask_id, extra_data, perf_index=perf_index) self.subtasks_given[subtask_id]['ctd'] = ctd return self.ExtraData(ctd=ctd)
def get_preview(cls, task, single=False): result = to_unicode(task.preview_file_path) if task else None return cls._preview_result(result, single=single)