def parse_and_execute(self, args=None):
"""Parses the given argument list and executes the command.
@note will catch exceptions and translates them into exit codes and logging output
@param args tuple or list of argument strings. e.g. sys.argv[1:]. If None, sys.argv[1:0] is used automatically
@return exit code as integer between 0 and 255. If an unhandled exception occurred, 255 will be returned
@note If we implement subcommands, giving no arguments will print usage information and set an error code
"""
if args is None:
args = sys.argv[1:]
# end handle args default
parser = self.argparser()
try:
if self._has_subcommands() and not args:
parser.print_usage()
return self.ERROR
# print usage if nothing was specified
parsed_args, remaining_args = parser.parse_known_args(args)
if remaining_args:
# traverse the subcommand chain and check if the last one actually allows unknown args
level = self._level
cmd = self
unknown_allowed = False
while cmd and not unknown_allowed:
cmd = getattr(parsed_args, self._subcommand_slot_name(level), None)
unknown_allowed |= cmd and cmd.allow_unknown_args or False
level += 1
# end while there is no one to allow unknowns
if not unknown_allowed:
sys.stderr.write("The following arguments could not be parsed: '%s'\n" % ' '.join(remaining_args))
return self.ERROR
# end abort if no one allowed them
# end handle remaining
return self.execute(parsed_args, remaining_args)
except ArgparserHandledCall as info:
# Happens on help or version - exit with error anyway as we didn't do anything useful
if info.message:
sys.stdout.write(info.message + '/n')
return self.ARGUMENT_HANDLED
except ParserError as err:
self.log().error(str(err))
return self.ARGUMENT_ERROR
except InputError as err:
cmd = getattr(parsed_args, self._subcommand_slot_name(), None)
(cmd and cmd.log() or self.log()).error(str(err))
return self.ARGUMENT_ERROR
except (ArgumentError, ArgumentTypeError) as err:
parser.print_usage(sys.stderr)
self.log().error(str(err))
return self.ARGUMENT_ERROR
except SuccessfulBreak:
return self.SUCCESS
except KeyboardInterrupt:
# Signal 15, or Ctrl+C
self.log().error("Interrupted by user")
return self.KEYBOARD_INTERRUPT
except Exception as err:
self.log().error("An unhandled exception occurred", exc_info=True)
return self.UNHANDLED_ERROR
def _node_attr(self, node, attr, depth, mandatory):
value = getattr(node, attr)
if value is None or not value:
return
# end handle no-value
if isinstance(value, JobDate):
value = "%s %s %s:%s" % (value.month, value.day, value.hour, value.minute)
elif isinstance(value, list):
value = ', '.join(str(val) for val in value)
elif isinstance(value, Ref):
value = value.id
elif isinstance(node, Cmd) and attr == 'appname':
# needs special handling, as args is a variables argument list we have parsed ourselves
if node.args:
value += ' ' + ' '.join(str(arg) for arg in node.args)
# end handle node args
else:
value = str(value)
# end handle different types
if ' ' in value or '\n' in value:
value = "{ %s }" % value
# end handle brackets
if not mandatory:
self._writer('-%s ' % attr)
# end handle arg prefix
self._writer(value + ' ')
def _expandvars_deep(cls, path, env=os.environ):
"""As above, but recursively expands as many variables as possible"""
rval = cls._expandvars(path, env)
while str(rval) != str(path):
path = rval
rval = cls._expandvars(path)
# END expansion loop
return rval
def _insert_limits(self, max_mem_gb, max_cores, args, env, blade_data):
"""Setup prman limits using environment variables"""
# We have a total of 2GB for the system to split up on all slots
total_mem_relative = (blade_data.memFree - self.reserved_system_memory_gb) / blade_data.memFree
max_mem_relative = total_mem_relative / blade_data.slotsMax
env['RMAN_CPU_COUNT'] = str(max_cores)
env['RMAN_MEM_LIMIT'] = str(max_mem_relative)
log.log(logging.TRACE, 'rman_mem_limit = %s = %s / data.slotsMax', max_mem_relative, total_mem_relative)
def expand_or_raise(self):
"""@return Copy of self with all variables expanded ( using `expand` )
non-recursively !
:raise ValueError: If we could not expand all environment variables as
their values where missing in the environment"""
rval = self.expand()
if str(rval) == str(self) and rval.containsvars():
raise ValueError("Failed to expand all environment variables in %r, got %r" % (self, rval))
return rval
def pformat(self):
"""Display the contents of the Context primarily for debugging purposes
@return string indicating the human-readable contents
@todo: revise printing"""
otp = str(self)
otp += "\t* registered instances|types:\n"
for item in self._registry:
otp += "\t%s\n" % item
# Could list supported interfaces here
otp += "\t* store:\n"
otp += re.sub(r"(^|\n)", r"\1\t", str(self.settings())) + "\n"
return otp
def login_name():
"""
Cross-platform way to get the current user login
@attention this uses an environment variable in Windows, technically allows
users to impersonate others quite easily.
"""
# getuser is linux only !
# py2: str conversion required to get unicode
if sys.platform == 'win32':
return str(os.environ['USERNAME'])
else:
return str(getpass.getuser())
def _insert_limits(self, max_mem_gb, max_cores, args, env, blade_data):
"""Setup limits for the maya render commands. No matter which renderer is used, we will just set up
all the limits we know about"""
max_mem_MB = int(max_mem_gb * 1024)
args[0:0] = [
'-mr:memory', str(max_mem_MB),
'-mr:renderThreads', str(max_cores),
'-sw:mm', str(min(max_mem_MB, 2048)), # yes, there is a limit of the attribute, incredible
# The idea would be to sooner than later just use our own batch scripts for rendering, dealing
# with settings and assets properly right away.
#'-sw:n', str(max_cores) # even though this is given with -r sw -help, it doesnt work
# Additionally, make sure we are verbose enough to get progress information
'-mr:verbose', str(5),
]
def text(self, encoding=None, errors='strict'):
r""" Open this file, read it in, return the content as a string.
This uses "U" mode in Python 2.3 and later, so "\\r\\n" and "\\r"
are automatically translated to '\n'.
Optional arguments:
* encoding - The Unicode encoding (or character set) of
the file. If present, the content of the file is
decoded and returned as a unicode object; otherwise
it is returned as an 8-bit str.
* errors - How to handle Unicode errors; see help(str.decode)
for the options. Default is 'strict'.
"""
mode = 'U' # we are in python 2.4 at least
f = None
if encoding is None:
f = self.open(mode)
else:
f = codecs.open(self, 'r', encoding, errors)
# END handle encoding
try:
if sys.version_info[0] < 3:
return str(f.read())
else:
return f.read()
# end assure type is correct
finally:
f.close()
def _tank_instance(cls, env, paths, settings):
"""@return the initialized tank package that exists at TANK_STUDIO_INSTALL_TREE, and the context path which created
the instance
@param env enviornment of the to-be-started process
@param paths from which to pull the context. They should be sorted from most specialized to to least
specialized
@param settings matching the tank_engine_schema
@throws EnvironmentError if we couldn't find it"""
sgtk = cls._sgtk_module(env)
if settings.force_tank_from_entity:
if not (settings.entity_type and settings.entity_id):
msg = "Was forced to create tank from entity, but didn't get entity information"
raise AssertionError(msg)
# end
return sgtk.tank_from_entity(settings.entity_type, settings.entity_id), 'context_path_not_set'
else:
errors = list()
for path in paths:
try:
return sgtk.tank_from_path(path), path
except Exception as err:
errors.append(err)
# end for each path to try
# end handle tank instantiation mode
raise EnvironmentError("Failed to initialize tank from any of the given context paths: %s\nErrors: %s"
% (', '.join(paths), '\n'.join(str(err) for err in errors)))
def run(self):
try:
self._result = self._fun()
except Exception as exc:
self._exc = exc
if self._log is not None:
self._log.critical("%s failed" % str(self._fun), exc_info=1)
def test_ordered_dict_special_api(self):
"""Assert on custom API methods"""
odict = OrderedDict()
odict.foo = "bar"
assert odict.foo is odict["foo"], "setattr should work, as well as getattr"
assert isinstance(str(odict), str)
def test_move_fs_op(self, base_dir):
for dry_run in range(2):
source_item = base_dir / "directory_to_move"
dest_item = base_dir / "move_destination"
for creator in (source_item.mkdir, source_item.touch):
for dest_is_dir in range(2):
if source_item.isdir():
source_item.rmdir()
elif source_item.isfile():
source_item.remove()
# END handle removal of existing one
# prep sandbox
if dest_item.isdir():
dest_item.rmdir()
if dest_is_dir:
dest_item.mkdir()
creator()
t = Transaction(log, dry_run=dry_run)
mo = MoveFSItemOperation(t, source_item, str(dest_item))
assert t.apply().succeeded()
assert source_item.exists() == dry_run
assert mo.actual_destination().exists() != dry_run
assert not t.rollback().succeeded()
assert source_item.exists()
assert not mo.actual_destination().exists()
def test_create_op(self, base_dir):
destination = base_dir / "my_new_item"
for dry_run in range(2):
for content in (None, bytes(b"hello world")):
for mode in (0o755, None):
for gid in (None, os.getgid()):
for uid in (None, os.getuid()):
for dest_exists in range(2):
assert not destination.exists()
t = Transaction(log, dry_run=dry_run)
co = TestCreateFSItemOperation(
t, str(destination), content, mode=mode, uid=uid, gid=gid)
if dest_exists:
# Will ignore existing items, but cares about the type
destination.mkdir()
assert t.apply().succeeded() == (content is None)
destination.rmdir()
else:
t.apply()
if not (gid or uid and os.getuid() != 0 and type(t.exception()) is OSError):
assert t.succeeded()
assert destination.exists() != dry_run
# end ignore non-root permissions issues
assert not t.rollback().succeeded()
assert not destination.exists()
def _tree_iterator(self, context):
"""@return a python iterator which yields one nuke task with the respective command
@note we don't set constraints, this is happening when rendering on the farm"""
data = self._context_value(context)
cmd = copy.deepcopy(self._cached_wrapped_command(data.job.file))
# SETUP LICENSE
# The base-class took care of selecting tags, we just check if we have to set up the
# interactive mode
additional_args = list()
if self.limit_nuke_interactive in cmd.tags:
additional_args.append('-i')
# make sure there is only one license - our overrides are additive, and this is a fix
# to duplicate counts
if self.limit_nuke_render in cmd.tags:
cmd.tags.remove(self.limit_nuke_render)
# end handle license
# end handle interactive license
cmd.args = cmd.args + [
'-f', # render full size, no proxy
'-F %i-%i' % (data.frame.chunk.first, data.frame.chunk.last), # can do stepping as well: A-BxC
'-V', '1', # be verbose, but not too verbose
'-x', # execute script (rather than edit)
str(data.job.file)
] + additional_args
## allow retries in special situations, can be multiple ones
cmd.retryrc = NukeTractorDelegate.read_error_return_code
yield Task(title='nuke render', cmds = cmd)
def wrapper(self, *args, **kwargs):
path = Path(_maketemp(prefix=func.__name__))
path.mkdir()
keep = False
prev_val = os.environ.get('RW_DIR')
os.environ['RW_DIR'] = str(path)
prev_cwd = os.getcwd()
os.chdir(path)
try:
try:
return func(self, path, *args, **kwargs)
except Exception as err:
print(("Test %s.%s failed with error %s: '%s', output is at %r"
% (type(self).__name__, type(err), err, func.__name__, path)), file=sys.stderr)
keep = True
raise
# end be informed about failure
finally:
if prev_val is not None:
os.environ['RW_DIR'] = prev_val
# end restore state
os.chdir(prev_cwd)
# Need to collect here to be sure all handles have been closed. It appears
# a windows-only issue. In fact things should be deleted, as well as
# memory maps closed, once objects go out of scope. For some reason
# though this is not the case here unless we collect explicitly.
if not keep:
gc.collect()
shutil.rmtree(path)
def _resolve_scalar_value(self, key, value):
"""@return a resolved single scalar string value"""
# Actually, all of the values we see should be strings
# however, the caller is and may be 'stupid', so we handle it here
if not isinstance(value, string_types):
return value
# end ignore non-string types
formatter = self.StringFormatterType()
try:
last_value = ''
count = 0
while last_value != value:
count += 1
last_value = value
new_value = formatter.vformat(value, [], self._data)
# we could have string-like types, and format degenerates them to just strings
if type(new_value) is not type(value):
new_value = type(value)(new_value)
value = new_value
if count > MAX_ITERATIONS:
raise AssertionError(
"Value at '%s' could not be resolved after %i iterations - recursive values detected, last value was '%s', new value was '%s'" % (key, count, last_value, new_value))
# end
# end recursive resolution
return value
except (KeyError, AttributeError, ValueError, TypeError) as err:
msg = "Failed to resolve value '%s' at key '%s' with error: %s"
self._log.warn(msg, value, key, str(err))
# if we can't resolve, we have to resolve substitute to an empty value. Otherwise
# the application might continue using a format string, which it can't check for
# validity at all. Default values (like empty strings) can though
return type(value)()
def _force_removal(self, destination):
"""Forcefully delete given directory or file, linux only.
@throws OSError"""
self.log.info("about to remove directory at %s ... " % destination)
rval = subprocess.call([self.rm_path, "-Rf", str(destination)])
if rval != 0:
raise OSError("Failed to remove file or directory that we managed to copy previously: %s" % destination)
self.log.info("... done removing destination path")
def append(self, value):
"""Assured proper type and case"""
if not isinstance(value, str):
value = str(value)
value = value.lower()
if value in self:
return
super(Tags, self).append(value)
def _from_data(self, job):
self.ui.title.setText(job.title)
self.ui.file.setText(str(job.file))
self.ui.comment.setText(job.comment)
self.ui.service.setText(job.service)
if job.comment or job.service:
self.ui.advanced.setChecked(True)
self.ui.advanced_widget.setVisible(True)
def _to_string_key(cls, key):
"""Assure the key is not mal-formed, convert None to ''"""
if key is RootKey:
return str()
# end convert None to ''
# py2/3: comparing string is difficult, as they need to be unicode
# Here we easily get deserialized strings, which are not necessarily unicode in py2
assert isinstance(key, (str, __builtins__["str"])) and cls.key_separator not in key
return key
def _transform(cls, value):
if cls._is_valid_member(value):
return value
# end handle value is valid already
try:
return cls.MemberType(value)
except Exception as err:
msg = "Conversion of '%s' (%s) to type %s failed with error: %s"
log.error(msg, value, type(value).__name__, cls.MemberType.__name__, str(err))
return cls.MemberType()
def _search_re(self):
if self.__search_re is None:
try:
self.__search_re = re.compile(self._expression, self._re_flags | re.DOTALL)
except AssertionError:
e = sys.exc_info()[1] # to keep py3k and backward compat
if str(e).endswith('this version only supports 100 named groups'):
raise TooManyFields('sorry, you are attempting to parse too '
'many complex fields')
return self.__search_re
def pformat(self):
""" print a comprehensive representation of the stack
@todo convert this into returning a data structure which would be useful and printable
"""
otp = str()
for idx, env in enumerate(self._stack):
otp += "### Context %i - %s ###############\n\n" % (idx, env.name())
otp += env.pformat()
# for each env on stack
return otp
def __str__(self):
"""@return ourselves as list of element names, separated with the actual separator"""
out = str()
if self:
for index in range(len(self) - 1):
elm = self[index]
out += elm.name() + elm.child_separator
# end for each node
out += self[-1].name()
# end if we have at least an item
return out
def pre_start(self, executable, env, args, cwd, resolve):
executable, env, new_args, cwd = super(TankCommandDelegate, self).pre_start(executable, env, args, cwd, resolve)
# and the second argument must be the tank install root ... lets make it happy
if len(new_args) > 2 and not os.path.isabs(new_args[1]):
install_root = Path(new_args[0]).dirname().dirname().dirname()
assert install_root.basename() == 'install', "Expected first argument '%s' to be tank_cmd.py right in the install root" % new_args[0]
new_args.insert(1, install_root.dirname())
# end handle install root
last_arg = new_args[-1]
if not last_arg.startswith(self.tank_pc_arg):
# we assume to be in the right spot, but a check can't hurt until
# we are able to do more ourselves
actual_executable = self._actual_executable()
base = actual_executable.dirname()
assert (base / 'tank').exists(), "Currently '%s' must be right next to the 'tank' executable" % executable
new_args.append(str(self.tank_pc_arg + base))
# end setup context
#######################
# Process Arguments ##
#####################
if len(new_args) > 6 and new_args[3].startswith(self.launch_prefix):
# now we could go crazy and try to find asset paths in order to provide context to bprocess
# We could also use the shotgun context in some way, to feed data to our own asset management
# However, for now using the project itself should just be fine, but this is certainly
# to be improved
# Additinally, what we really want is to start any supported program, and enforce tank support by
# fixing up delegates. For that, we will create a new process controller, which uses our Application
# instance, and the delegate that it defined so far.
# However, we are currently unable to truly provide the information we have to a new process controller,
# unless it's communicated via the context.
# It should be one of ours (e.g. TankEngineDelegate derivative) if there is tank support, which
# requires proper configuration.
# For that to work, we will override the entire start procedure, as in pre-start we can't and should not
# swap in the entire delegate
def set_overrides(schema, value):
value.host_app_name = new_args[3][len(self.launch_prefix):]
value.entity_type = new_args[4]
value.entity_id = int(new_args[5])
# end overrides setter
# This call will also push the context onto the stack, nothing more to be done here
self.ApplyChangeContextType('tank-engine-information').setup(self._app.context(),
set_overrides,
tank_engine_schema)
#end handle particular command mode
return (executable, env, new_args, cwd)
def _assert_element_node_list(self, nlist):
"""verify functionality of element node list"""
assert len(nlist) > 0, "nlist should not be empty"
self.failUnless(nlist.is_leaf() == (len(nlist[-1].children()) == 0))
assert nlist.find_first_if(lambda elm: 'Root' in elm.type()), "Every node list should have a type"
assert not str(nlist).startswith(nlist[-1].child_separator)
# just make the call to execute code - it can complain about a few things that subclasses may implement
# differently
index = dict()
for elm in nlist:
elm.validate(index)
def _sgtk_module(cls, env):
"""@return the sgtk package, as demoninated in the environment
@throws EnvironmentError if we couldn't find it"""
root = env.get('TANK_STUDIO_INSTALL_TREE')
if not root:
raise EnvironmentError("Expected TANK_STUDIO_INSTALL_TREE environment variable to be set")
root = Path(root) / 'core' / 'python'
sys.path.append(str(root))
try:
import sgtk
except ImportError as err:
raise EnvironmentError("Failed to import tank from '%s' with error: %s", root, str(err))
# end
return sgtk
def _match_re(self):
if self.__match_re is None:
expression = '^%s$' % self._expression
try:
self.__match_re = re.compile(expression, self._re_flags | re.DOTALL)
except AssertionError:
e = sys.exc_info()[1] # to keep py3k and backward compat
if str(e).endswith('this version only supports 100 named groups'):
raise TooManyFields('sorry, you are attempting to parse too '
'many complex fields')
except re.error:
raise NotImplementedError(
"Group names (e.g. (?P<name>) can cause failure, as they are not esacped properly: '%s'" % expression)
return self.__match_re
def apply_overrides(self, schema, args):
"""Parse overrides and set them into a new context
@param schema KeyValueStoreSchema of your command
@param all override values as 'key=value' string
@note to be called in execute() method"""
if args.overrides:
env = self.application().context().push('user overrides')
kvstore = env._kvstore
for kvstring in args.overrides:
k, v = parse_key_value_string(kvstring, self.key_value_separator)
kvstore.set_value('%s.%s' % (schema.key(), k), v)
# end for each string to apply
# end handle overrides
if getattr(args, 'show_settings', None):
sys.stdout.write("%s.*\n" % schema.key())
sys.stdout.write(str(self.application().context().settings().value_by_schema(schema)))
raise SuccessfulBreak
