def get_preview(cls, task, single=False):
result = None
if not task:
pass
elif task.use_frames:
if single:
# path to the most recently updated preview
try:
# previews that were updated at least once
iterator = ifilter(lambda p: bool(p.last_update_time),
task.preview_updaters)
# find the max timestamp
updater = max(iterator, key=lambda p: p.last_update_time)
return to_unicode(updater.preview_file_path)
except StopIteration:
return None
else:
# paths for all frames
return [
to_unicode(p.preview_file_path)
for p in task.preview_updaters
]
else:
result = to_unicode(task.preview_updater.preview_file_path)
return cls._preview_result(result, single=single)
def payments(sort, status):
deferred = payments.client.get_payments_list()
result = sync_wait(deferred) or []
values = []
if status is not None:
result = filter_by_status(result, status)
for payment in result:
payment_value = float(payment["value"])
payment_fee = payment["fee"] or ""
if payment_fee:
payment_fee = __value(payment_fee, "ETH")
entry = [
to_unicode(payment["subtask"]),
to_unicode(payment["payee"]),
to_unicode(payment["status"]),
__value(payment_value, "GNT"),
to_unicode(payment_fee)
]
values.append(entry)
return CommandResult.to_tabular(payments_table_headers, values, sort=sort)
def to_dictionary(self):
return {
u'id': to_unicode(self.header.task_id),
u'name': to_unicode(self.task_definition.task_name),
u'type': to_unicode(self.task_definition.task_type),
u'subtasks': self.get_total_tasks(),
u'progress': self.get_progress()
}
def _cache_response(self, resource_name, response, task_id):
if isinstance(response, list):
for entry in response:
self._cache_response(resource_name, entry, task_id)
elif response and 'Hash' in response and 'Name' in response:
if os.path.basename(response.get('Name')) != os.path.basename(resource_name):
raise Exception("Resource manager: Invalid response {}".format(response))
res_path = to_unicode(resource_name)
res_hash = to_unicode(response.get('Hash'))
resource = self._wrap_resource((res_path, res_hash), task_id)
self._cache_resource(resource)
def to_dictionary(self):
preview = self.extra_data.get('result_preview')
if isinstance(preview, basestring):
preview = to_unicode(preview)
elif isinstance(preview, collections.Iterable):
preview = [to_unicode(entry) for entry in preview]
return {
u'time_remaining': self.remaining_time,
u'status': to_unicode(self.status),
u'preview': preview
}
def _update_frame_status(self, frame):
frame_key = to_unicode(frame)
state = self.frames_state[frame_key]
subtask_ids = self.frames_subtasks[frame_key]
parts = max(1, int(self.total_tasks / len(self.frames)))
counters = defaultdict(lambda: 0, dict())
# Count the number of occurrences of each subtask state
for subtask_id in filter(bool, subtask_ids):
subtask = self.subtasks_given[subtask_id]
counters[subtask['status']] += 1
# Count statuses different from 'finished' and 'failure'
computing = len([
x for x in counters.keys()
if x not in [SubtaskStatus.finished, SubtaskStatus.failure]
])
# Finished if at least n subtasks >= parts were finished
if counters[SubtaskStatus.finished] >= parts:
state.status = TaskStatus.finished
# Computing if at least one subtask did not fail
elif computing > 0:
state.status = TaskStatus.computing
# Failure if the only known subtask status is 'failure'
elif counters[SubtaskStatus.failure] > 0:
state.status = TaskStatus.aborted
def _command(cls,
command: List[str],
vm_name: Optional[str] = None,
args: Optional[Union[Tuple, List[str]]] = None,
shell: bool = False) -> str:
if args:
command += list(args)
if vm_name:
command += [vm_name]
if logger.isEnabledFor(logging.DEBUG):
command.insert(1, '-D')
logger.debug('Docker command: %s', command)
try:
output = subprocess.check_output(
command,
startupinfo=SUBPROCESS_STARTUP_INFO,
shell=shell,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT)
except FileNotFoundError as exc:
raise subprocess.CalledProcessError(127, str(exc))
except subprocess.CalledProcessError as exc:
logger.debug('Docker command output: %s', exc.output)
raise exc
logger.debug('Docker command output: %s', output)
return to_unicode(output)
def _sort_dict(self, dictionary, filter_properties=False):
result = dict()
for k, v in dictionary.iteritems():
if filter_properties and (k.startswith('_') or callable(v)):
continue
result[to_unicode(k)] = self._sort_obj(v)
return sorted(result.items())
def incomes(sort, status):
deferred = incomes.client.get_incomes_list()
result = sync_wait(deferred) or []
values = []
if status is not None:
result = filter_by_status(result, status)
for income in result:
entry = [
to_unicode(income["payer"]),
to_unicode(income["status"]),
__value(float(income["value"]), "GNT"),
]
values.append(entry)
return CommandResult.to_tabular(incomes_table_headers, values, sort=sort)
def test_unicode(self):
source = str("test string")
result = to_unicode(source)
assert result is source
source = "\xd0\xd1\xd2\xd3"
result = to_unicode(source)
assert result is source
source = "test string"
result = to_unicode(source)
assert isinstance(result, str)
assert result == source
source = None
result = to_unicode(source)
assert result is None
def test_unicode(self):
source = unicode("test string")
result = to_unicode(source)
assert result is source
source = "\xd0\xd1\xd2\xd3"
result = to_unicode(source)
assert result is source
source = "test string"
result = to_unicode(source)
assert type(result) is unicode
assert result is not source
assert result == source
source = None
result = to_unicode(source)
assert result is None
def get_preview(cls, task, single=False):
result = None
if not task:
pass
elif task.use_frames:
if single:
return to_unicode(task.last_preview_path)
else:
previews = [to_unicode(p) for p in task.preview_task_file_path]
result = {}
for i, f in enumerate(task.frames):
try:
result[to_unicode(f)] = previews[i]
except IndexError:
result[to_unicode(f)] = None
else:
result = to_unicode(task.preview_task_file_path
or task.preview_file_path)
return cls._preview_result(result, single=single)
def parse_rpc_address(ctx, param, value):
del ctx, param
value = to_unicode(value)
if value:
try:
return SocketAddress.parse(value)
except ipaddress.AddressValueError as e:
raise click.BadParameter(
"Invalid RPC address specified: {}".format(e))
return None
def _from_dict_traverse_obj(cls, obj):
if isinstance(obj, dict):
if cls._is_class(obj):
return cls.obj_from_dict(obj)
return cls._from_dict_traverse_dict(obj)
elif isinstance(obj, basestring):
return to_unicode(obj)
elif isinstance(obj, collections.Iterable):
return obj.__class__([cls._from_dict_traverse_obj(o) for o in obj])
return obj
def to_dictionary(self):
return {
u'subtask_id': to_unicode(self.subtask_id),
u'node_name': to_unicode(self.computer.node_name),
u'node_id': to_unicode(self.computer.node_id),
u'node_performance': to_unicode(self.computer.performance),
u'node_ip_address': to_unicode(self.computer.ip_address),
u'node_port': self.computer.port,
u'status': to_unicode(self.subtask_status),
u'progress': self.subtask_progress,
u'time_started': self.time_started,
u'time_remaining': self.subtask_rem_time,
u'results': [to_unicode(r) for r in self.results],
u'stderr': to_unicode(self.stderr),
u'stdout': to_unicode(self.stdout)
}
def _sort_obj(self, v):
if isinstance(v, dict):
return self._sort_dict(v)
# treat objects as dictionaries
elif hasattr(v, '__dict__'):
return self._sort_dict(v.__dict__, filter_properties=True)
elif isinstance(v, basestring):
return to_unicode(v)
elif isinstance(v, collections.Iterable):
return v.__class__([self._sort_obj(_v) for _v in v])
return v
def publish(cls, component, method, stage, data=None):
"""
Convenience function for publishing the execution stage event.
:param component: Component name
:param method: Method name
:param stage: Execution stage to report at: pre, post (defined in
golem.report.Stage). Exceptions are always reported. If not specified,
both 'pre' and 'post' are used.
:param data: Payload (optional)
:return: None
"""
if cls._rpc_publisher:
from twisted.internet import reactor
cls._last_status = (to_unicode(component), to_unicode(method),
to_unicode(stage), data)
reactor.callFromThread(cls._rpc_publisher.publish,
Golem.evt_golem_status, *cls._last_status)
def _to_dict_traverse_obj(cls, obj, typed=True):
if isinstance(obj, dict):
return cls._to_dict_traverse_dict(obj, typed)
elif isinstance(obj, basestring):
return to_unicode(obj)
elif isinstance(obj, collections.Iterable):
if isinstance(obj, (set, frozenset)):
logger.warning('set/frozenset have known problems with umsgpack: %r', obj)
return obj.__class__([cls._to_dict_traverse_obj(o, typed) for o in obj])
elif cls.deep_serialization:
if hasattr(obj, '__dict__') and not cls._is_builtin(obj):
return cls.obj_to_dict(obj, typed)
return obj
def get_subtasks_borders(self, task_id, part=1):
task = self.tasks[task_id]
task_type_name = task.task_definition.task_type.lower()
task_type = self.task_types[task_type_name]
subtasks_count = task.get_total_tasks()
return {
to_unicode(subtask_id):
task_type.get_task_border(subtask,
task.task_definition,
subtasks_count,
as_path=True)
for subtask_id, subtask in task.get_subtasks(part).items()
}
def build_minimal_definition(cls, task_type, dictionary):
parent = super(FrameRenderingTaskBuilder, cls)
options = dictionary.get('options') or dict()
frames_string = to_unicode(options.get('frames', 1))
frames = cls.string_to_frames(frames_string)
use_frames = options.get('use_frames', len(frames) > 1)
definition = parent.build_minimal_definition(task_type, dictionary)
definition.options.frames_string = frames_string
definition.options.frames = frames
definition.options.use_frames = use_frames
return definition
def get_list_of_all_payments(self):
# This data is used by UI.
return [{
"subtask": to_unicode(payment.subtask),
"payee": to_unicode(encode_hex(payment.payee)),
"value": to_unicode(payment.value),
"status": to_unicode(payment.status.name),
"fee": to_unicode(payment.details.fee),
"block_number": to_unicode(payment.details.block_number),
"transaction": to_unicode(payment.details.tx),
"created": datetime_to_timestamp_utc(payment.created_date),
"modified": datetime_to_timestamp_utc(payment.modified_date)
} for payment in self.db.get_newest_payment()]
def get_subtasks_borders(self, task_id):
task = self.tasks[task_id]
task_type_name = task.task_definition.task_type.lower()
task_type = self.task_types[task_type_name]
task_state = self.tasks_states[task_id]
total_subtasks = task.get_total_tasks()
return {
to_unicode(subtask.subtask_id):
task_type.get_task_border(subtask,
task.task_definition,
total_subtasks,
as_path=True)
for subtask in task_state.subtask_states.values()
}
def make_log_analyses(log_content, return_data):
_get_warnings(log_content, return_data)
rendering_time = find_rendering_time(log_content)
if rendering_time:
return_data["rendering_time"] = to_unicode(rendering_time)
output_path = find_filepath(log_content)
if output_path:
return_data["output_path"] = to_unicode(output_path)
frames = find_frames(log_content)
if frames:
return_data["frames"] = frames
resolution = find_resolution(log_content)
if resolution:
return_data["resolution"] = resolution
file_format = find_file_format(log_content)
if file_format:
return_data["file_format"] = to_unicode(file_format)
engine_type = find_engine_type(log_content)
if engine_type:
return_data['engine_type'] = to_unicode(engine_type)
if engine_type == "CYCLES":
samples_pp = find_samples_for_scenes(log_content)
if samples_pp:
return_data['samples'] = samples_pp
def get_subtasks(self, frame):
if self.task_definition.options.use_frames:
subtask_ids = self.frames_subtasks.get(to_unicode(frame), [])
subtask_ids = filter(None, subtask_ids)
else:
subtask_ids = self.subtasks_given.keys()
subtasks = dict()
# Convert to SubtaskState in order to match parent's return type
for subtask_id in subtask_ids:
state = SubtaskState()
state.extra_data = self.subtasks_given[subtask_id]
subtasks[subtask_id] = state
return subtasks
def to_dictionary(self):
return {
'subtask_id': to_unicode(self.subtask_id),
'node_id': to_unicode(self.node_id),
'node_name': to_unicode(self.node_name),
'status': self.subtask_status.value,
'progress': self.subtask_progress,
'time_started': self.time_started,
'time_remaining': self.subtask_rem_time,
'results': [to_unicode(r) for r in self.results],
'stderr': to_unicode(self.stderr),
'stdout': to_unicode(self.stdout),
'description': self.subtask_definition,
}
def build_dictionary(cls, definition):
task_timeout = timeout_to_string(definition.full_task_timeout)
subtask_timeout = timeout_to_string(definition.subtask_timeout)
output_path = cls.build_output_path(definition)
return {
u'type': to_unicode(definition.task_type),
u'name': to_unicode(definition.task_name),
u'timeout': to_unicode(task_timeout),
u'subtask_timeout': to_unicode(subtask_timeout),
u'subtask_count': definition.total_subtasks,
u'bid': float(definition.max_price) / denoms.ether,
u'resources': [to_unicode(r) for r in definition.resources],
u'options': {
u'output_path': to_unicode(output_path)
}
}
def _map_payment(cls, obj):
obj["payee"] = to_unicode(obj["payee"])
obj["value"] = to_unicode(obj["value"])
obj["fee"] = to_unicode(obj["fee"])
return obj
def _ip_from_str(ip_address):
return ipaddress.ip_address(to_unicode(ip_address))
def query_extra_data(self, perf_index: float, num_cores: int = 0,
node_id: Optional[str] = None,
node_name: Optional[str] = None) \
-> FrameRenderingTask.ExtraData:
start_task, end_task = self._get_next_task()
scene_file = self._get_scene_file_rel_path()
if self.use_frames:
frames, parts = self._choose_frames(self.frames, start_task,
self.total_tasks)
else:
frames = self.frames or [1]
parts = 1
if not self.use_frames:
min_y, max_y = self._get_min_max_y(start_task)
elif parts > 1:
min_y = (parts - self._count_part(start_task, parts)) * (1.0 /
parts)
max_y = (parts - self._count_part(start_task, parts) + 1) * (1.0 /
parts)
else:
min_y = 0.0
max_y = 1.0
# Blender is using single precision math, we use numpy to emulate this.
# Send already converted values to blender.
min_y = numpy.float32(min_y)
max_y = numpy.float32(max_y)
script_src = generate_blender_crop_file(
resolution=(self.res_x, self.res_y),
borders_x=(0.0, 1.0),
borders_y=(min_y, max_y),
use_compositing=self.compositing,
samples=self.samples)
extra_data = {
"path_root": self.main_scene_dir,
"start_task": start_task,
"end_task": end_task,
"total_tasks": self.total_tasks,
"outfilebasename": self.outfilebasename,
"scene_file": scene_file,
"script_src": script_src,
"frames": frames,
"output_format": self.output_format,
}
subtask_id = self.create_subtask_id()
logger.debug(
'Created new subtask for task. '
'task_id=%s, subtask_id=%s, node_id=%s', self.header.task_id,
subtask_id, short_node_id(node_id or ''))
self.subtasks_given[subtask_id] = copy(extra_data)
self.subtasks_given[subtask_id]['subtask_id'] = subtask_id
self.subtasks_given[subtask_id]['status'] = SubtaskStatus.starting
self.subtasks_given[subtask_id]['node_id'] = node_id
self.subtasks_given[subtask_id]['parts'] = parts
self.subtasks_given[subtask_id]['res_x'] = self.res_x
self.subtasks_given[subtask_id]['res_y'] = self.res_y
self.subtasks_given[subtask_id]['samples'] = self.samples
self.subtasks_given[subtask_id]['use_frames'] = self.use_frames
self.subtasks_given[subtask_id]['all_frames'] = self.frames
self.subtasks_given[subtask_id]['crop_window'] = (0.0, 1.0, min_y,
max_y)
self.subtasks_given[subtask_id]['subtask_timeout'] = \
self.header.subtask_timeout
self.subtasks_given[subtask_id]['tmp_dir'] = self.tmp_dir
# FIXME issue #1955
part = self._count_part(start_task, parts)
for frame in frames:
frame_key = to_unicode(frame)
state = self.frames_state[frame_key]
state.status = TaskStatus.computing
state.started = state.started or time.time()
self.frames_subtasks[frame_key][part - 1] = subtask_id
if not self.use_frames:
self._update_task_preview()
else:
self._update_frame_task_preview()
ctd = self._new_compute_task_def(subtask_id,
extra_data,
perf_index=perf_index)
self.subtasks_given[subtask_id]['ctd'] = ctd
return self.ExtraData(ctd=ctd)
def get_preview(cls, task, single=False):
result = to_unicode(task.preview_file_path) if task else None
return cls._preview_result(result, single=single)
