def run_shell(self):
"""Run shell applications."""
from pypeapp.lib.Terminal import Terminal
from pypeapp import execute
self._initialize()
t = Terminal()
t.echo(">>> Running pype shell ...")
if sys.platform == 'win32':
execute(['powershell', '-NoLogo'])
else:
execute(['bash'])
def run_pype_setup_tests(self, keyword=None, id=None):
"""Run pytest on `tests` directory."""
from pypeapp.lib.Terminal import Terminal
import pytest
self._initialize()
t = Terminal()
t.echo(">>> Running test on pype-setup ...")
args = [
'-x', '--capture=sys', '--print', '-W',
'ignore::DeprecationWarning'
]
if keyword:
t.echo(" - selecting [ {} ]".format(keyword))
args.append('-k')
args.append(keyword)
args.append(os.path.join(os.getenv('PYPE_SETUP_PATH'), 'tests'))
elif id:
t.echo(" - selecting test ID [ {} ]".format(id[0]))
args.append(id[0])
else:
args.append(os.path.join(os.getenv('PYPE_SETUP_PATH'), 'tests'))
pytest.main(args)
def publish(self, gui=False, paths=None):
""" Starts headless publishing.
Publish collects json from current working directory
or supplied paths argument.
:param gui: launch Pyblish gui or not
:type gui: bool
:param paths: paths to jsons
:type paths: list
"""
from pypeapp.lib.Terminal import Terminal
t = Terminal()
error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}"
self._initialize()
# find path from different platforms in environment and remap it to
# current platform paths. Only those paths specified in Storage
# will be remapped.
# os.environ.update(self.path_remapper())
from pype import install, uninstall
# Register target and host
import pyblish.api
install()
pyblish.api.register_target("filesequence")
pyblish.api.register_host("shell")
self._update_python_path()
if not any(paths):
t.echo("No publish paths specified")
return False
remapped_path = self.path_remapper(
{
"PYPE_PUBLISH_DATA": os.pathsep.join(paths)
})
os.environ.update(remapped_path)
if gui:
import pyblish_qml
pyblish_qml.show(modal=True)
else:
import pyblish.util
t.echo(">>> Running publish ...")
# Error exit as soon as any error occurs.
for result in pyblish.util.publish_iter():
if result["error"]:
t.echo(error_format.format(**result))
uninstall()
sys.exit(1)
uninstall()
def print_info(self):
"""Print additional information to console."""
from pypeapp.lib.Terminal import Terminal
from pypeapp.lib.log import _mongo_settings
t = Terminal()
(
host, port, database, username, password, collection, auth_db
) = _mongo_settings()
infos = []
if os.getenv('PYPE_DEV'):
infos.append(("Pype variant", "staging"))
else:
infos.append(("Pype variant", "production"))
infos.append(("Running pype from", os.environ.get('PYPE_SETUP_PATH')))
infos.append(("Using config at", os.environ.get('PYPE_CONFIG')))
infos.append(("Using mongodb", os.environ.get("AVALON_MONGO")))
if os.environ.get("FTRACK_SERVER"):
infos.append(("Using FTrack at",
os.environ.get("FTRACK_SERVER")))
if os.environ.get('DEADLINE_REST_URL'):
infos.append(("Using Deadline webservice at",
os.environ.get("DEADLINE_REST_URL")))
if os.environ.get('MUSTER_REST_URL'):
infos.append(("Using Muster at",
os.environ.get("MUSTER_REST_URL")))
if host:
infos.append(("Logging to mongodb", "{}/{}".format(
host, database)))
if port:
infos.append((" - port", port))
if username:
infos.append((" - user", username))
if collection:
infos.append((" - collection", collection))
if auth_db:
infos.append((" - auth source", auth_db))
maximum = max([len(i[0]) for i in infos])
for info in infos:
padding = (maximum - len(info[0])) + 1
t.echo("... {}:{}[ {} ]".format(info[0], " " * padding, info[1]))
print('\n')
def _initialize(self):
from pypeapp.deployment import Deployment
from pypeapp.lib.Terminal import Terminal
try:
import configparser
except Exception:
import ConfigParser as configparser
cur_dir = os.path.dirname(os.path.abspath(__file__))
config_file_path = os.path.join(cur_dir, "config.ini")
if os.path.exists(config_file_path):
config = configparser.ConfigParser()
config.read(config_file_path)
try:
value = config["DEFAULT"]["dev"]
if value.lower() == "true":
os.environ["PYPE_DEV"] = "1"
except KeyError:
pass
# if not called, console coloring will get mangled in python.
Terminal()
pype_setup = os.getenv('PYPE_SETUP_PATH')
d = Deployment(pype_setup)
tools, config_path = d.get_environment_data()
os.environ['PYPE_CONFIG'] = config_path
os.environ['TOOL_ENV'] = os.path.normpath(
os.path.join(config_path, 'environments'))
self._add_modules()
self._load_default_environments(tools=tools)
self.print_info()
def pype_setup_coverage(self, pype):
"""Generate code coverage on pype-setup."""
from pypeapp.lib.Terminal import Terminal
import pytest
self._initialize()
t = Terminal()
t.echo(">>> Generating coverage on pype-setup ...")
pytest.main([
'-v', '-x', '--color=yes', '--cov={}'.format(pype), '--cov-config',
'.coveragerc', '--cov-report=html', '--ignore={}'.format(
os.path.join(os.environ.get("PYPE_SETUP_PATH"), "vendor")),
'--ignore={}'.format(
os.path.join(os.environ.get("PYPE_SETUP_PATH"), "repos"))
])
def emit(self, record):
if not self.enable:
return
try:
msg = self.format(record)
msg = Terminal.log(msg)
stream = self.stream
fs = "%s\n"
if not _unicode: # if no unicode support...
stream.write(fs % msg)
else:
try:
if (isinstance(msg, unicode) and # noqa: F821
getattr(stream, 'encoding', None)):
ufs = u'%s\n'
try:
stream.write(ufs % msg)
except UnicodeEncodeError:
stream.write((ufs % msg).encode(stream.encoding))
else:
if (getattr(stream, 'encoding', 'utf-8')):
ufs = u'%s\n'
stream.write(ufs % unicode(msg)) # noqa: F821
else:
stream.write(fs % msg)
except UnicodeError:
stream.write(fs % msg.encode("UTF-8"))
self.flush()
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
print(repr(record))
self.handleError(record)
def localize_package(self, path):
"""
Copy package directory to pype environment "localized" folder.
Useful for storing binaries that are not accessible when calling over
UNC paths or similar scenarios.
:param path: source
:type path: str
"""
package_name = path.split(os.path.sep)[-1]
destination = os.path.join(os.environ.get("PYPE_ENV"), "localized",
package_name)
if os.path.isdir(destination):
term = Terminal()
term.echo("*** destination already exists "
"[ {} ], removing".format(destination))
shutil.rmtree(destination)
shutil.copytree(path, destination)
def print_info(self):
"""Print additional information to console."""
from pypeapp.lib.Terminal import Terminal
from pypeapp.lib.mongo import get_default_components
from pypeapp.lib.log import LOG_DATABASE_NAME, LOG_COLLECTION_NAME
t = Terminal()
components = get_default_components()
infos = []
if os.getenv('PYPE_DEV'):
infos.append(("Pype variant", "staging"))
else:
infos.append(("Pype variant", "production"))
infos.append(("Running pype from", os.environ.get('PYPE_SETUP_PATH')))
infos.append(("Using config at", os.environ.get('PYPE_CONFIG')))
infos.append(("Using mongodb", components["host"]))
if os.environ.get("FTRACK_SERVER"):
infos.append(("Using FTrack at", os.environ.get("FTRACK_SERVER")))
if os.environ.get('DEADLINE_REST_URL'):
infos.append(("Using Deadline webservice at",
os.environ.get("DEADLINE_REST_URL")))
if os.environ.get('MUSTER_REST_URL'):
infos.append(
("Using Muster at", os.environ.get("MUSTER_REST_URL")))
if components["host"]:
infos.append(("Logging to MongoDB", components["host"]))
infos.append((" - port", components["port"] or "<N/A>"))
infos.append((" - database", LOG_DATABASE_NAME))
infos.append((" - collection", LOG_COLLECTION_NAME))
infos.append((" - user", components["username"] or "<N/A>"))
if components["auth_db"]:
infos.append((" - auth source", components["auth_db"]))
maximum = max([len(i[0]) for i in infos])
for info in infos:
padding = (maximum - len(info[0])) + 1
t.echo("... {}:{}[ {} ]".format(info[0], " " * padding, info[1]))
print('\n')
def launch_local_mongodb(self):
""" This will run local instance of mongodb.
:returns: process return code
:rtype: int
"""
import subprocess
from pypeapp.lib.Terminal import Terminal
self._initialize()
t = Terminal()
# Get database location.
try:
location = os.environ["AVALON_DB_DATA"]
except KeyError:
location = os.path.join(os.path.expanduser("~"), "data", "db")
# Create database directory.
if not os.path.exists(location):
os.makedirs(location)
# Start server.
if platform.system().lower() == "linux":
t.echo("Local mongodb is running...")
t.echo("Using port {} and db at {}".format(
os.environ["AVALON_MONGO_PORT"], location))
p = subprocess.Popen(
["mongod", "--dbpath", location, "--port",
os.environ["AVALON_MONGO_PORT"]], close_fds=True
)
elif platform.system().lower() == "windows":
t.echo("Local mongodb is running...")
t.echo("Using port {} and db at {}".format(
os.environ["AVALON_MONGO_PORT"], location))
p = subprocess.Popen(
["start", "Avalon MongoDB", "call", "mongod", "--dbpath",
location, "--port", os.environ["AVALON_MONGO_PORT"]],
shell=True
)
return p.returncode
def get_logger(self, name=None, host=None):
logger = logging.getLogger(name or '__main__')
if self.PYPE_DEBUG > 1:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
global _mongo_logging
add_mongo_handler = _mongo_logging
add_console_handler = True
for handler in logger.handlers:
if isinstance(handler, MongoHandler):
add_mongo_handler = False
elif isinstance(handler, PypeStreamHandler):
add_console_handler = False
if add_console_handler:
logger.addHandler(self._get_console_handler())
if add_mongo_handler:
try:
logger.addHandler(self._get_mongo_handler())
except MongoEnvNotSet:
# Skip if mongo environments are not set yet
_mongo_logging = False
except Exception:
lines = traceback.format_exception(*sys.exc_info())
for line in lines:
if line.endswith("\n"):
line = line[:-1]
Terminal.echo(line)
_mongo_logging = False
return logger
def _initialize(self):
from pypeapp.storage import Storage
from pypeapp.deployment import Deployment
from pypeapp.lib.Terminal import Terminal
# if not called, console coloring will get mangled in python.
Terminal()
pype_setup = os.getenv('PYPE_ROOT')
d = Deployment(pype_setup)
tools, config_path = d.get_environment_data()
os.environ['PYPE_CONFIG'] = config_path
os.environ['TOOL_ENV'] = os.path.normpath(
os.path.join(
config_path,
'environments'))
self._add_modules()
Storage().update_environment()
self._load_default_environments(tools=tools)
self.print_info()
def run_application(self, app, project, asset, task, tools, arguments):
"""Run application in project/asset/task context.
With default or specified tools enviornment. This uses pre-defined
launcher in `pype-config/launchers` where there must be *toml*
file with definition and in platform directory its launcher shell
script or binary executables. Arguments will be passed to this script
or executable.
:param app: Full application name (`maya_2018`)
:type app: Str
:param project: Project name
:type project: Str
:param asset: Asset name
:type asset: Str
:param task: Task name
:type task: Str
:param tools: Comma separated list of tools (`"mtoa_2.1.0,yeti_4"`)
:type tools: Str
:param arguments: List of other arguments passed to app
:type: List
:rtype: None
"""
import toml
import subprocess
from pypeapp.lib.Terminal import Terminal
from pypeapp import Anatomy
t = Terminal()
self._initialize()
self._update_python_path()
import acre
from avalon import lib
from pype import lib as pypelib
abspath = lib.which_app(app)
if abspath is None:
t.echo("!!! Application [ {} ] is not registered.".format(app))
t.echo("*** Please define its toml file.")
return
app_toml = toml.load(abspath)
executable = app_toml['executable']
app_dir = app_toml['application_dir']
# description = app_toml.get('description', None)
# preactions = app_toml.get('preactions', [])
launchers_path = os.path.join(os.environ["PYPE_CONFIG"], "launchers")
database = pypelib.get_avalon_database()
avalon_project = database[project].find_one({"type": "project"})
if avalon_project is None:
t.echo(
"!!! Project [ {} ] doesn't exists in Avalon.".format(project))
return False
# get asset from db
avalon_asset = database[project].find_one({
"type": "asset",
"name": asset
})
avalon_tools = avalon_project["data"]["tools_env"]
if tools:
avalon_tools = tools.split(",") or []
hierarchy = ""
parents = avalon_asset["data"]["parents"] or []
if parents:
hierarchy = os.path.join(*parents)
data = {
"project": {
"name": project,
"code": avalon_project['data']['code']
},
"task": task,
"asset": asset,
"app": app_dir,
"hierarchy": hierarchy,
}
anatomy = Anatomy(project)
anatomy_filled = anatomy.format(data)
workdir = os.path.normpath(anatomy_filled["work"]["folder"])
# set PYPE_ROOT_* environments
anatomy.set_root_environments()
# set environments for Avalon
os.environ["AVALON_PROJECT"] = project
os.environ["AVALON_SILO"] = None
os.environ["AVALON_ASSET"] = asset
os.environ["AVALON_TASK"] = task
os.environ["AVALON_APP"] = app.split("_")[0]
os.environ["AVALON_APP_NAME"] = app
os.environ["AVALON_WORKDIR"] = workdir
os.environ["AVALON_HIERARCHY"] = hierarchy
try:
os.makedirs(workdir)
except FileExistsError:
pass
tools_attr = [os.environ["AVALON_APP"], os.environ["AVALON_APP_NAME"]]
tools_attr += avalon_tools
print("TOOLS: {}".format(tools_attr))
tools_env = acre.get_tools(tools_attr)
env = acre.compute(tools_env)
env = acre.merge(env, current_env=dict(os.environ))
env = {k: str(v) for k, v in env.items()}
# sanitize slashes in path
env["PYTHONPATH"] = env["PYTHONPATH"].replace("/", "\\")
env["PYTHONPATH"] = env["PYTHONPATH"].replace("\\\\", "\\")
launchers_path = os.path.join(launchers_path,
platform.system().lower())
execfile = None
if sys.platform == "win32":
# test all avaliable executable format, find first and use it
for ext in os.environ["PATHEXT"].split(os.pathsep):
fpath = os.path.join(launchers_path.strip('"'),
executable + ext)
if os.path.isfile(fpath) and os.access(fpath, os.X_OK):
execfile = fpath
break
# Run SW if was found executable
if execfile is not None:
try:
t.echo(">>> Running [ {} {} ]".format(
executable, " ".join(arguments)))
args = [execfile]
args.extend(arguments)
subprocess.run(args, env=env)
except ValueError as e:
t.echo("!!! Error while launching application:")
t.echo(e)
return
else:
t.echo(
"!!! cannot find application launcher [ {} ]".format(app))
return
if sys.platform.startswith('linux'):
execfile = os.path.join(launchers_path.strip('"'), executable)
if os.path.isfile(execfile):
try:
fp = open(execfile)
except PermissionError as p:
t.echo("!!! Access denied on launcher [ {} ]".format(app))
t.echo(p)
return
fp.close()
else:
t.echo("!!! Launcher doesn\'t exist [ {} ]".format(execfile))
return
# Run SW if was found executable
if execfile is not None:
args = ['/usr/bin/env', 'bash', execfile]
args.extend(arguments)
t.echo(">>> Running [ {} ]".format(" ".join(args)))
try:
subprocess.run(args, env=env)
except ValueError as e:
t.echo("!!! Error while launching application:")
t.echo(e)
return
else:
t.echo(
"!!! cannot find application launcher [ {} ]".format(app))
return
def make_docs(self):
"""Generate documentation using Sphinx.
Documentation is generated for both **pype-setup** and **pype**.
"""
from pypeapp.lib.Terminal import Terminal
from pypeapp import execute
self._initialize()
t = Terminal()
source_dir_setup = os.path.join(os.environ.get("PYPE_SETUP_PATH"),
"docs", "source")
build_dir_setup = os.path.join(os.environ.get("PYPE_SETUP_PATH"),
"docs", "build")
source_dir_pype = os.path.join(os.environ.get("PYPE_SETUP_PATH"),
"repos", "pype", "docs", "source")
build_dir_pype = os.path.join(os.environ.get("PYPE_SETUP_PATH"),
"repos", "pype", "docs", "build")
t.echo(">>> Generating documentation ...")
t.echo(" - Cleaning up ...")
execute(
['sphinx-build', '-M', 'clean', source_dir_setup, build_dir_setup],
shell=True)
execute(
['sphinx-build', '-M', 'clean', source_dir_pype, build_dir_pype],
shell=True)
t.echo(" - generating sources ...")
execute([
'sphinx-apidoc', '-M', '-f', '-d', '4', '--ext-autodoc',
'--ext-intersphinx', '--ext-viewcode', '-o', source_dir_setup,
'pypeapp'
],
shell=True)
vendor_ignore = os.path.join(os.environ.get("PYPE_SETUP_PATH"),
"repos", "pype", "pype", "vendor")
execute([
'sphinx-apidoc', '-M', '-f', '-d', '6', '--ext-autodoc',
'--ext-intersphinx', '--ext-viewcode', '-o', source_dir_pype,
'pype', '{}{}*'.format(vendor_ignore, os.path.sep)
],
shell=True)
t.echo(" - Building html ...")
execute(
['sphinx-build', '-M', 'html', source_dir_setup, build_dir_setup],
shell=True)
execute(
['sphinx-build', '-M', 'html', source_dir_pype, build_dir_pype],
shell=True)
t.echo(">>> Done. Documentation id generated:")
t.echo("*** For pype-setup: [ {} ]".format(build_dir_setup))
t.echo("*** For pype: [ {} ]".format(build_dir_pype))
def validate(self, skip=False) -> bool:
""" Do deployment setting validation.
First, deployment settings is determined. It can be default
provided *deploy.json* or overrided one in
*deploy/somedir/deploy.json*. This file is then validated against
json schema. Then it will check, if stuff defined in settings is
present and deployed.
:param skip: if True skip if directory not exists. Used during
installation where some directories will be
installed nevertheless.
:type skip: bool
:return: True if validated, otherwise throw exception
:rtype: bool
:raises: :class:`DeployException` With info on what is wrong
"""
import git
settings = self._determine_deployment_file()
deploy = self._read_deployment_file(settings)
term = Terminal()
if (not self._validate_schema(deploy)):
raise DeployException(
"Invalid deployment file [ {} ]".format(settings), 200)
# go throught repositories
for ritem in deploy.get('repositories'):
test_path = os.path.join(self._pype_root, "repos",
ritem.get('name'))
term.echo(" - validating [ {} ]".format(ritem.get('name')))
# does repo directory exist?
if not self._validate_is_directory(test_path):
if skip:
continue
raise DeployException(
"Repo path doesn't exist [ {} ]".format(test_path), 130)
if not self._validate_is_repo(test_path):
raise DeployException(
"Path {} exists but it is not valid repository".format(
test_path))
# bare repo isn't allowed
if self._validate_is_bare(test_path):
raise DeployException(
"Repo on path [ {} ] is bare".format(test_path), 300)
# check origin
if not self._validate_origin(test_path, ritem.get('url')):
raise DeployException(
"Repo {} origin {} should be {}.".format(
test_path,
git.Repo(test_path).remotes.origin.url,
ritem.get('url')), 300)
# check we are on branch
if ritem.get('branch'):
if not self._validate_is_branch(test_path,
ritem.get('branch')):
raise DeployException(
'repo {0} head is not on {1}(!={2}) branch'.format(
ritem.get('name'), ritem.get('branch'),
git.Repo(test_path).heads[0].name), 210)
# check we are on ref
if ritem.get('ref'):
if not self._validate_is_ref(test_path, ritem.get('ref')):
raise DeployException(
'repo {0} head is not on {1}(!={2}) ref'.format(
ritem.get('name'), ritem.get('ref'),
git.Repo(test_path).heads[0].commit.hexsha), 220)
# check tag
if ritem.get('tag'):
if not self._validate_is_tag(test_path, ritem.get('tag')):
raise DeployException(
'repo {0} head is not on tag {1}'.format(
ritem.get('name'), ritem.get('tag')), 230)
# Go through archive files.
if deploy.get('archive_files'):
for item in deploy.get('archive_files'):
test_path = os.path.join(self._pype_root,
item.get('extract_path'))
# does repo directory exist?
if not self._validate_is_directory(test_path):
if skip:
continue
raise DeployException(
"Vendor path doesn't exist [ {} ]".format(test_path),
130 # noqa: E501
)
return True
def deploy(self, force=False):
""" Do repositories deployment and install python dependencies.
Go throught deployment file and install repositories specified
there. Also add additional python dependencies with pip.
:param force: overwrite existng repos if it's working tree is
dirty.
:type force: bool
:raises: :class:`DeployException`
"""
import git
from pypeapp.lib.git_progress import _GitProgress
settings = self._determine_deployment_file()
deploy = self._read_deployment_file(settings)
term = Terminal()
# go throught repositories
term.echo(">>> Deploying repositories ...")
for ritem in deploy.get('repositories'):
path = os.path.join(self._pype_root, "repos", ritem.get('name'))
term.echo(" -- processing [ {} / {} ]".format(
ritem.get('name'),
ritem.get('branch') or ritem.get('tag')))
if self._validate_is_directory(path):
# path exists
# is it repository?
if not self._validate_is_repo(path):
# no, remove dir no matter of content
term.echo(" - removing existing directory and cloning...")
self._recreate_repository(path, ritem)
else:
# dir is repository
repo = git.Repo(path)
# is it right one?
if not self._validate_origin(path, str(ritem.get('url'))):
# repository has different origin then specified
term.echo("!!! repository has different origin. ")
if (self._validate_is_dirty(path) is True
and force is False):
raise DeployException(('Invalid repository on '
'path {}'.format(path)))
term.echo(" -> recreating repository. ")
self._recreate_repository(path, ritem)
pass
if (self._validate_is_dirty(path) is True
and force is False):
raise DeployException(("Repo on path [ {} ] has dirty"
" worktree").format(path), 300)
# are we on correct branch?
if not ritem.get('tag'):
if not self._validate_is_branch(
path, ritem.get('branch')):
term.echo(" . switching to [ {} ] ...".format(
ritem.get('branch')))
branch = repo.create_head(ritem.get('branch'),
'HEAD')
branch.checkout(force=force)
# update repo
term.echo(" . updating ...")
repo.remotes.origin.fetch(tags=True, force=True)
# build refspec
if ritem.get('branch'):
refspec = "refs/heads/{}".format(ritem.get('branch'))
repo.remotes.origin.pull(refspec)
elif ritem.get('tag'):
tags = repo.tags
if ritem.get('tag') not in tags:
raise DeployException(
("Tag {} is missing on remote "
"origin").format(ritem.get('tag')))
t = tags[ritem.get('tag')]
term.echo(" . tag: [{}, {} / {}]".format(
t.name, t.commit, t.commit.committed_date))
repo.remotes.origin.pull(ritem.get('tag'))
else:
# path doesn't exist, clone
try:
git.Repo.clone_from(ritem.get('url'),
path,
progress=_GitProgress(),
env=None,
b=ritem.get('branch')
or ritem.get('tag'),
recursive=True)
except git.exc.GitCommandError as e:
raise DeployException("Git clone failed for {}".format(
ritem.get("url"))) from e
# Go through zip files.
term.echo(">>> Deploying archive files ...")
if deploy.get('archive_files'):
for item in deploy.get("archive_files"):
term.echo(" -- processing [ {} ]".format(
item.get("extract_path")))
path = os.path.normpath(
os.path.join(self._pype_root, item.get("extract_path")))
if self._validate_is_directory(path):
term.echo(" - removing existing directory.")
shutil.rmtree(path)
# Download archive file.
archive_type = item.get('archive_type')
basename = os.path.split(path)[-1]
filename = '.'.join([basename, archive_type])
archive_file_path = tempfile.mkdtemp(basename + '_archive')
archive_file_path = os.path.join(archive_file_path, filename)
if item.get("vendor"):
source = os.path.join(os.environ.get("PYPE_ROOT"),
'vendor', 'packages',
item.get("vendor"))
if not os.path.isfile(source):
raise DeployException(
"Local archive {} doesn't exist".format(source))
shutil.copyfile(source, archive_file_path)
if item.get("url"):
term.echo(" - downloading [ {} ]".format(item.get("url")))
success = self._download_file(item.get("url"),
archive_file_path)
if not success:
raise DeployException(
"Failed to download [ {} ]".format(
item.get("url")),
130 # noqa: E501
)
# checksum
if item.get('md5_url'):
response = urlopen(item.get('md5_url'))
md5 = response.read().decode('ascii').split(" ")[0]
calc = self.calculate_checksum(archive_file_path)
if md5 != calc:
raise DeployException(
"Checksum failed {} != {} on {}".format(
md5, calc, archive_file_path))
# Extract files from archive
if archive_type in ['zip']:
zip_file = zipfile.ZipFile(archive_file_path)
zip_file.extractall(path)
elif archive_type in [
'tar', 'tgz', 'tar.gz', 'tar.xz', 'tar.bz2'
]:
if archive_type == 'tar':
tar_type = 'r:'
elif archive_type.endswith('xz'):
tar_type = 'r:xz'
elif archive_type.endswith('gz'):
tar_type = 'r:gz'
elif archive_type.endswith('bz2'):
tar_type = 'r:bz2'
else:
tar_type = 'r:*'
try:
tar_file = tarfile.open(archive_file_path, tar_type)
except tarfile.ReadError:
raise DeployException(
"corrupted archive: also consider to download the "
"archive manually, add its path to the url, run "
"`./pype deploy`")
tar_file.extractall(path)
tar_file.close()
# Move folders/files if skip first subfolder is set
if item.get('skip_first_subfolder', False):
self.move_subfolders_to_main(path)
# install python dependencies
term.echo(">>> Adding python dependencies ...")
for pitem in deploy.get('pip'):
term.echo(" -- processing [ {} ]".format(pitem))
try:
subprocess.check_output(
[sys.executable, '-m', 'pip', 'install', pitem])
except subprocess.CalledProcessError as e:
raise DeployException('PIP command failed with {}'.format(
e.returncode)) from e
def launch_local_mongodb(self):
"""Run local instance of mongodb.
:returns: process return code
:rtype: int
"""
import subprocess
from pypeapp.lib.Terminal import Terminal
from pypeapp.lib.mongo import get_default_components
self._initialize()
t = Terminal()
# Get database location.
try:
location = os.environ["AVALON_DB_DATA"]
except KeyError:
location = os.path.join(os.path.expanduser("~"), "data", "db")
# Create database directory.
if not os.path.exists(location):
os.makedirs(location)
components = get_default_components()
_mongo_port = components["port"]
if _mongo_port is None:
_mongo_port = "N/A"
mongo_port = "{}".format(_mongo_port)
# Start server.
if (platform.system().lower() == "linux"
or platform.system().lower() == "darwin"):
t.echo(("*** You may need to allow mongod "
"to run in "
"[ System Settings / Security & Privacy ]"))
t.echo("Local mongodb is running...")
t.echo("Using port {} and db at {}".format(mongo_port, location))
p = subprocess.Popen(
["mongod", "--dbpath", location, "--port", mongo_port],
close_fds=True)
elif platform.system().lower() == "windows":
t.echo("Local mongodb is running...")
t.echo("Using port {} and db at {}".format(mongo_port, location))
p = subprocess.Popen([
"start", "Avalon MongoDB", "call", "mongod", "--dbpath",
location, "--port", mongo_port
],
shell=True)
else:
t.echo("!!! Unsupported platorm - [ {} ]".format(
platform.system().lower()))
return False
return p.returncode
def validate_jsons(self):
"""Validate configuration JSON files for syntax errors."""
import json
import glob
from pypeapp.lib.Terminal import Terminal
self._initialize()
t = Terminal()
t.echo(">>> validating ...")
files = [
f for f in glob.glob(os.environ.get("PYPE_CONFIG") + os.path.sep +
"**/*.json",
recursive=True)
] or []
files += [
f for f in glob.glob(os.environ.get("PYPE_PROJECT_CONFIGS") +
os.path.sep + "**/*.json",
recursive=True)
] or []
failures = 0
for f in files:
t.echo(" - {}".format(f))
with open(f, "r") as jf:
json_str = jf.read()
try:
json.loads(json_str)
except json.decoder.JSONDecodeError as e:
t.echo("!!! failed on [ {} ]".format(f))
t.echo(str(e))
failures += 1
if failures > 0:
t.echo("!!! Failed on [ {} ] file(s), "
"see log above.".format(failures))
else:
t.echo(">>> All OK.")
import os
import re
import click
from avalon import io, api
from pprint import pprint
from pypeapp.lib.Terminal import Terminal
from pypeapp import Anatomy
import shutil
import speedcopy
t = Terminal()
texture_extensions = [
'.tif', '.tiff', '.jpg', '.jpeg', '.tx', '.png', '.tga', '.psd', '.dpx',
'.hdr', '.hdri', '.exr', '.sxr', '.psb'
]
class TextureCopy:
def __init__(self):
if not io.Session:
io.install()
def _get_textures(self, path):
textures = []
for dir, subdir, files in os.walk(path):
textures.extend(
os.path.join(dir, x) for x in files
if os.path.splitext(x)[1].lower() in texture_extensions)
def print_info(self):
""" This will print additional information to console. """
from pypeapp.lib.Terminal import Terminal
from pypeapp.lib.log import _mongo_settings
t = Terminal()
host, port, database, username, password, collection, auth_db = _mongo_settings() # noqa: E501
t.echo("... Running pype from\t\t\t[ {} ]".format(
os.environ.get('PYPE_ROOT')))
t.echo("... Using config at\t\t\t[ {} ]".format(
os.environ.get('PYPE_CONFIG')))
t.echo("... Projects root\t\t\t[ {} ]".format(
os.environ.get('PYPE_STUDIO_PROJECTS_PATH')))
t.echo("... Using mongodb\t\t\t[ {} ]".format(
os.environ.get("AVALON_MONGO")))
if os.environ.get('FTRACK_SERVER'):
t.echo("... Using FTrack at\t\t\t[ {} ]".format(
os.environ.get("FTRACK_SERVER")))
if os.environ.get('DEADLINE_REST_URL'):
t.echo("... Using Deadline webservice at\t[ {} ]".format(
os.environ.get("DEADLINE_REST_URL")))
if os.environ.get('MUSTER_REST_URL'):
t.echo("... Using Muster at\t\t\t[ {} ]".format(
os.environ.get("MUSTER_REST_URL")))
if host:
t.echo("... Logging to mongodb\t\t\t[ {}/{} ]".format(
host, database))
if port:
t.echo(" - port\t\t\t\t[ {} ]".format(port))
if username:
t.echo(" - user\t\t\t\t[ {} ]".format(username))
if collection:
t.echo(" - collection\t\t\t\t[ {} ]".format(collection))
if auth_db:
t.echo(" - auth source\t\t\t\t[ {} ]".format(auth_db))
print('\n')