def count_lines(self, project_info):
try:
os.mkdir(TMP_DIR)
repository = project_info['repositories'][0]
tmp_path = os.path.join(TMP_DIR, project_info['name'])
if 'svn' in repository:
svn_repo = RemoteClient(repository)
svn_repo.checkout(tmp_path)
elif 'git' in repository:
Repo.clone_from(repository, tmp_path)
project_dir = tmp_path
trunk_path = os.path.join(tmp_path, 'trunk')
if os.path.isdir(trunk_path):
project_dir = trunk_path
subprocess.call(['./cloc.exe', '--json', '--out=%s' % CLOC_OUT, project_dir])
try:
with open(CLOC_OUT) as cloc_file:
cloc = json.load(cloc_file)
project_info['cloc'] = cloc
finally:
if os.path.isfile(CLOC_OUT):
os.remove(CLOC_OUT)
finally:
if os.path.isdir(TMP_DIR):
subprocess.call(['rm', '-rf', TMP_DIR])
def _pull_packages(self):
try:
rc = RemoteClient(self.request.POST.get("svn_url", ""),
username=deploy_conf.SVN_USERNAME,
password=deploy_conf.SVN_PASSWORD)
rc.checkout(str(self.SVN_CHECKOUT_DIR))
except Exception as e:
raise Exception("get packages failed!\n")
def test_checkout(self):
"""
Testing checkout
:return:
"""
RemoteClient(self.test_svn_url).checkout('trial')
self.assertTrue(os.path.exists('trial'))
def test_error_client_formation(self):
"""
Testing Value error while client formation
:return:
"""
with self.assertRaises(SvnException):
RemoteClient(self.test_fake_url).checkout('.')
def __init__(self, svn_url: str):
self.url = svn_url
self.repo = SvnRepo(svn_url)
self.svn_name = posixpath.basename(svn_url)
self.os_name = None
self.os_version = None
self.processor = None
self.bitness = None
self.build_environment = None
if self.svn_name.casefold().startswith("android".casefold()):
self.os_name = "Android"
elif self.svn_name.casefold().startswith("darwin".casefold()):
self.os_name = "Darwin"
darwin_platform_tokens = re.split(r"[\.-]", self.svn_name)
if len(darwin_platform_tokens) >= 4:
self.os_version = pkg_resources.parse_version(".".join(
darwin_platform_tokens[1:-1]))
self.processor = darwin_platform_tokens[-1]
elif self.svn_name.casefold().startswith("linux".casefold()):
self.os_name = "Linux"
elif self.svn_name.casefold().startswith("win".casefold()):
self.os_name = "Windows"
if "64".casefold() in self.svn_name.casefold():
self.bitness = 64
else:
self.bitness = 32
if "vc".casefold() in self.svn_name.casefold():
self.build_environment = "vc" + self.svn_name.split("vc")[-1]
else:
self.os_name = None
def __init__(self, svn_url: str):
self.url = svn_url
self.repo = SvnRepo(svn_url)
self.version = pkg_resources.parse_version(
os.path.basename(os.path.normpath(svn_url)).replace(
"blender-", "").replace("-release",
"").replace("-winfix",
"").replace("-", "."))
self._platforms = None
self._platforms_dict = None
def download(matching):
if 'master' in matching:
matching = matching.replace('master', 'test')
remote = RemoteClient(matching)
remote.export('fileout')
# DO NOT name this file as svn.py import sys from svn.remote import RemoteClient r = RemoteClient(sys.argv[1]) print(r.info()['entry_revision'])
def build_cmake(self, extension: Extension):
"""
The steps required to build the extension
"""
# We import the setup_requires modules here because if we import them
# at the top this script will always fail as they won't be present
from git import Repo as GitRepo
from svn.remote import RemoteClient as SvnRepo
self.announce("Preparing the build environment", level=3)
blender_dir = os.path.join(BLENDERPY_DIR, "blender")
build_dir = pathlib.Path(self.build_temp)
extension_path = pathlib.Path(self.get_ext_fullpath(extension.name))
os.makedirs(blender_dir, exist_ok=True)
os.makedirs(str(build_dir), exist_ok=True)
os.makedirs(str(extension_path.parent.absolute()), exist_ok=True)
# Now that the necessary directories are created, ensure that OS
# specific steps are performed; a good example is checking on linux
# that the required build libraries are in place.
os_build_args = []
# Have to find the correct release tag to checkout here, as potentially
# master may not be the correct one for this Python version. We use svn
# to find whether or not master, or a specific tag supports the
# current python version
if sys.platform == "win32": # Windows only steps
import winreg
vs_versions = []
for version in [12, 14, 15]:
try:
winreg.OpenKey(winreg.HKEY_CLASSES_ROOT,
f"VisualStudio.DTE.{version}.0")
except:
pass
else:
vs_versions.append(version)
if not vs_versions:
raise Exception("Windows users must have Visual Studio 2013 "
"or later installed")
if max(vs_versions) == 15:
os_build_args += ["-G", f"Visual Studio 15 2017"
f"{' Win64' if BITS == 64 else ''}"]
elif max(vs_versions) == 14:
os_build_args += ["-G", f"Visual Studio 14 2015"
f"{' Win64' if BITS == 64 else ''}"]
elif max(vs_versions) == 12:
os_build_args += ["-G", f"Visual Studio 12 2013"
f"{' Win64' if BITS == 64 else ''}"]
# TODO: Clean up here
svn_lib_options += [f"win{64 if BITS == 64 else 'dows'}_vc{version}"
for version in vs_versions]
blender_svn_repo = SvnRepo(svn_url)
os.makedirs(svn_dir, exist_ok=True)
self.announce(f"Checking out svn libs from {svn_url}", level=3)
try:
blender_svn_repo.checkout(svn_dir)
except Exception as e:
self.warn("Windows users must have the svn executable "
"available from the command line")
self.warn("Please install Tortoise SVN with \"command line "
"client tools\" as described here")
self.warn("https://stackoverflow.com/questions/1625406/using-"
"tortoisesvn-via-the-command-line")
raise e
elif sys.platform == "linux": # Linux only steps
# TODO: Test linux environment, issue #1
pass
elif sys.platform == "darwin": # MacOS only steps
# TODO: Test MacOS environment, issue #2
pass
# Perform relatively common build steps
# TODO: if blender desired version, then see if we can install that
# Otherwise fail, if no desired version, find the latest version that
# supports our python and install that
git_repo = GitRepo(GIT_BASE_URL)
svn_repo = SvnRepo(SVN_BASE_URL)
if BLENDER_DESIRED_VERSION:
match = BLENDER_VERSION_REGEX.match(BLENDER_DESIRED_VERSION)
if match:
# We have a blender version that conforms to the naming scheme
# now to see if it actually exists in git and svn
if match.group(0) in git_repo.tags:
# The version was tagged in the git repository
# now, format the version to match the svn versioning
# scheme...
svn_version_tag = (f"blender-{match.group(1)}"
f"{match.group(2) if not match.group(2).startswith("-rc")}-release")
svn_tag_repo = SvnRepo(os.path.join(SVN_BASE_URL, SVN_TAGS))
if svn_version_tag in svn_tag_repo.list():
# The version was released in svn and we found it
# Now, is it compatible with our OS and python version?
else:
raise Exception(f"{BLENDER_DESIRED_VERSION} was found "
f"in the git repository but not the "
f"svn repository.")
else:
raise Exception(f"The provided version "
f"{BLENDER_DESIRED_VERSION} does not "
f"exist; please check "
f"https://git.blender.org/gitweb/"
f"gitweb.cgi/blender.git/tags for a list "
f"of valid Blender releases")
else:
# The blender version did not conform to the naming scheme
# fail and notify the user how to list the version
raise Exception(f"The provided version "
f"{BLENDER_DESIRED_VERSION} did not match "
f"Blender's naming scheme. Please list your "
f"desired version as 'v' followed by a digit, "
f"followed by a period, followed by two "
f"digits and either 'a', 'b', 'c' or '-rc' "
f"(versions using '-rc' can optionally add "
f"a number which specifies which release "
f"candidate they want to install) such that "
f"the version looks like the following: "
f"v2.74-rc2")
else:
if sys.version_info >= (3, 6):
# we can get from svn and git master branch
else:
# we must find a compatible version
self.announce(f"Cloning Blender source from {BLENDER_GIT_REPO_URL}",
level=3)
try:
blender_git_repo = GitRepo(blender_dir)
except:
GitRepo.clone_from(BLENDER_GIT_REPO_URL, blender_dir)
blender_git_repo = GitRepo(blender_dir)
finally:
blender_git_repo.heads.master.checkout()
blender_git_repo.remotes.origin.pull()
self.announce(f"Updating Blender git submodules", level=3)
blender_git_repo.git.submodule('update', '--init', '--recursive')
for submodule in blender_git_repo.submodules:
submodule_repo = submodule.module()
submodule_repo.heads.master.checkout()
submodule_repo.remotes.origin.pull()
self.announce("Configuring cmake project", level=3)
self.spawn(['cmake', '-H'+blender_dir, '-B'+self.build_temp,
'-DWITH_PLAYER=OFF', '-DWITH_PYTHON_INSTALL=OFF',
'-DWITH_PYTHON_MODULE=ON',
f"-DPYTHON_VERSION="
f"{sys.version_info[0]}.{sys.version_info[1]}"]
+ os_build_args)
self.announce("Building binaries", level=3)
self.spawn(["cmake", "--build", self.build_temp, "--target", "INSTALL",
"--config", "Release"])
# Build finished, now copy the files into the copy directory
# The copy directory is the parent directory of the extension (.pyd)
self.announce("Moving Blender python module", level=3)
bin_dir = os.path.join(str(build_dir), 'bin', 'Release')
self.distribution.bin_dir = bin_dir
bpy_path = [os.path.join(bin_dir, _bpy) for _bpy in
os.listdir(bin_dir) if
os.path.isfile(os.path.join(bin_dir, _bpy)) and
os.path.splitext(_bpy)[0].startswith('bpy') and
os.path.splitext(_bpy)[1] in [".pyd", ".so"]][0]
shutil.move(str(bpy_path), str(extension_path))
def handle(self, *args, **options):
# Handle svn credentials
user = os.environ.get('SVN_USER')
password = os.environ.get('SVN_PASS')
if user is None or password is None:
logging.error("No credentials found.")
return
# Connect external repos and update
# Could've used local repos file:/// without user & pass
try:
r = RemoteClient(settings.REPOS_URL + '/tags/',
username=user,
password=password)
except SvnException as e:
logging.error("Error {} connecting to {}".format(
e, settings.REPOS_URL))
return
updates_available = False
folders = r.list(extended=True)
for folder in folders:
if folder['is_directory']:
tag = folder['name']
# Does this tag already exist?
if not Version_SVN.objects.filter(release=tag).exists():
# Get general info
t = RemoteClient(settings.REPOS_URL + '/tags/' + tag,
username=user,
password=password)
info = t.info()
revision = info['commit#revision']
log = list(t.log_default(stop_on_copy=True))[0].msg
if len(log) > 256: # Character limit on changelog field
log = log[:256]
url = settings.REPOS_URL + '/tags/' + tag
# Get revisions for all folders in the script folder
versions = {}
e = RemoteClient(settings.REPOS_URL + '/tags/' + tag +
'/scripts/',
username=user,
password=password)
entries = e.list(extended=True)
for entry in entries:
if entry['is_directory']:
versions[entry['name']] = entry['commit_revision']
# Create model
updates_available = True
logging.info("Creating tag {}".format(tag))
version = Version_SVN(release=tag,
revision=revision,
versions=versions,
url=url,
changelog=log)
version.save()
if not updates_available:
logging.info("No updates found.")
class SvnOSPlatform():
PYTHON_PATTERN = (r'(?<=python)(\d)(?:\.?)(\d)')
PYTHON_REGEX = re.compile(PYTHON_PATTERN)
def __init__(self, svn_url: str):
self.url = svn_url
self.repo = SvnRepo(svn_url)
self.svn_name = posixpath.basename(svn_url)
self.os_name = None
self.os_version = None
self.processor = None
self.bitness = None
self.build_environment = None
if self.svn_name.casefold().startswith("android".casefold()):
self.os_name = "Android"
elif self.svn_name.casefold().startswith("darwin".casefold()):
self.os_name = "Darwin"
darwin_platform_tokens = re.split(r"[\.-]", self.svn_name)
if len(darwin_platform_tokens) >= 4:
self.os_version = pkg_resources.parse_version(".".join(
darwin_platform_tokens[1:-1]))
self.processor = darwin_platform_tokens[-1]
elif self.svn_name.casefold().startswith("linux".casefold()):
self.os_name = "Linux"
elif self.svn_name.casefold().startswith("win".casefold()):
self.os_name = "Windows"
if "64".casefold() in self.svn_name.casefold():
self.bitness = 64
else:
self.bitness = 32
if "vc".casefold() in self.svn_name.casefold():
self.build_environment = "vc" + self.svn_name.split("vc")[-1]
else:
self.os_name = None
def python_versions(self) -> List[Tuple[int, int]]:
try:
versions = [
re.search(self.PYTHON_REGEX, version["name"])
for version in self.repo.list(extended=True,
rel_path="python/lib")
if re.search(self.PYTHON_REGEX, version["name"])
and version["kind"] == "file"
]
return [(int(version.group(1)), int(version.group(2)))
for version in versions]
except SvnException:
return []
class BlenderSvn(SourceVersionControl):
"""The Blender `svn` library index
Provides the set of required libraries to build Blender
"""
BASE_URL = "https://svn.blender.org/svnroot/bf-blender"
BASE_REPO = SvnRepo(BASE_URL)
def __init__(self, svn_url: str):
self.url = svn_url
self.repo = SvnRepo(svn_url)
self.version = pkg_resources.parse_version(
os.path.basename(os.path.normpath(svn_url)).replace(
"blender-", "").replace("-release",
"").replace("-winfix",
"").replace("-", "."))
self._platforms = None
self._platforms_dict = None
@classmethod
def tags(cls) -> List[str]:
"""The tags that `svn` found
Able to be run as a class method so that we can query `svn` and build
our list of svn supported versions
Returns:
List[str] -- List of strings, the full paths to the svn version
"""
return [
posixpath.join(cls.BASE_URL, "tags", _version)
for _version in cls.BASE_REPO.list(rel_path="/tags")
if _version.startswith("blender")
]
def platforms(self) -> List[SvnOSPlatform]:
if self._platforms is not None: # Don't perform expensive search twice
return self._platforms
results = self.get_platforms(self.url)
self._platforms = results
return results
def platforms_dict(self) -> Dict[str, List[Tuple[int, int]]]:
"""Provides a dictionary of os and Python version compatibility
Platforms are determined based on the os listed per release of Blender
API in the `svn` repos
"""
# SVN search and checkout is expensive; if this seems overkill it's
# because any other method was taking ~ 5 minutes to list a
# svn tag!
if self._platforms_dict is not None: # Don't perform expensive search twice
return self._platforms_dict
results = self.get_platforms_dict(self.url)
self._platforms_dict = results
return results
@staticmethod
def get_platforms(url: str) -> List[SvnOSPlatform]:
results = []
try:
for _os in SvnRepo(url).list(extended=True, rel_path="lib"):
if not any([
_os["name"] == x
for x in ["benchmarks", "package", "python", "tests"]
]):
results.append(
SvnOSPlatform(posixpath.join(url + "lib",
_os["name"])))
except SvnException: # This can happen when the "lib" path does not exist
pass
return results
@staticmethod
def get_platforms_dict(url: str) -> Dict[str, List[Tuple[int, int]]]:
results = {}
try:
_oss = [
SvnOSPlatform(posixpath.join(url + "lib", _os["name"]))
for _os in SvnRepo(url).list(extended=True, rel_path="lib")
if any([
_os["name"].startswith(x)
for x in ["android", "darwin", "linux", "win"]
])
]
for _os in _oss:
results[_os.svn_name] = _os.python_versions()
except SvnException: # This can happen when the "lib" path does not exist
pass
return results
def checkout(self, full_path: pathlib.Path):
"""Retrieve the `svn` sources
Arguments:
full_path {pathlib.Path} -- place to put svn sources
"""
self.repo.checkout(str(full_path))
def svnclient(url, username=None, password=None):
return RemoteClient(url,
username=username,
password=password,
svn_filepath=svncommad,
trust_cert=True)