def clean(ext='false'):
"""
Clean up the generated files and temp dirs.
:param ext:
Cleanup even the external, downloaded packages. Default is 'false'.
This should be only needed when the resource has been changed,
thus a complete rebuild is needed.
"""
# Clean dist dir
shutil.rmtree(DIST_DIR, ignore_errors=True)
# Clean build dir
shutil.rmtree(BUILD_DIR, ignore_errors=True)
for plugin_dir in PLUGIN_DIRS:
shutil.rmtree(os.path.join(PROJECT_DIR, plugin_dir, 'dist'), ignore_errors=True)
if get_bool_str(ext):
for res in ext_resources:
res_path = get_ext_path(res.name)
shutil.rmtree(res_path, ignore_errors=True)
def clean(ext='false'):
"""
Clean up the generated files and temp dirs.
:param ext:
Cleanup even the external, downloaded packages. Default is 'false'.
This should be only needed when the resource has been changed,
thus a complete rebuild is needed.
"""
# Clean dist dir
shutil.rmtree(DIST_DIR, ignore_errors=True)
# Clean build dir
shutil.rmtree(BUILD_DIR, ignore_errors=True)
for plugin_dir in PLUGIN_DIRS:
shutil.rmtree(os.path.join(PROJECT_DIR, plugin_dir, 'dist'),
ignore_errors=True)
if get_bool_str(ext):
for res in ext_resources:
res_path = get_ext_path(res.name)
shutil.rmtree(res_path, ignore_errors=True)
def buildext(allext='false',patch='true',branch='master'):
"""
Build and optionally patch the 3rd party modules and libraries.
The outcome (tar.gz/egg) files are placed in dist directory
:param allext:
Download also non-GitResources. Default is 'false'.
:param patch:
Patch those plugins having patches, currently, trac and gitosis.
:param branch:
For GitResources, selects the branch to be used. Default is 'master'.
.. NOTE::
If you want to include the external release into multiproject-all package, run the build
task with parameters::
fab dist.build:ext=true
"""
allext = get_bool_str(allext)
# Construct and create building directory for external resources
extbuild = build_join('ext')
shutil.rmtree(extbuild, ignore_errors=True)
os.makedirs(extbuild)
for res in ext_resources:
res_path = get_ext_path(res.name)
is_git = isinstance(res, GitResource)
if not is_git and not allext:
continue
logger.info('Starting to download / fetch resource %s' % res.name)
must_retrieve = False
resource_id_file = join(res_path, '.fabric_resource_id.txt')
res_lines = [line.strip() for line in str(res).split(',')]
res_lines.append('# This is a file used by fabric dist.buildext command.')
if not os.path.exists(res_path):
must_retrieve = True
else:
# Check folder contents.
# If the fetch identifier is missing, it is assumed to be the correct one.
if os.path.exists(resource_id_file):
prev_lines = [line.strip() for line in open(resource_id_file, 'r')]
if res_lines != prev_lines:
logger.warning('Resource %s has been changed, retrieving it.' % res.name)
logger.info('Previous resource: %s' % prev_lines)
logger.info('Current resource: %s' % res_lines)
must_retrieve = True
if not get_files(os.path.abspath(res_path), 'setup.py', recursive=True):
must_retrieve = True
if must_retrieve:
shutil.rmtree(res_path, ignore_errors=True)
os.makedirs(res_path)
res.retrieve(res_path)
outfile = open(resource_id_file, 'w')
outfile.writelines([line + '\n' for line in res_lines])
outfile.close()
else:
logger.warning('Resource %s was already retrieved.' % res.name)
if is_git:
# The GitResources are always updated
if not os.path.exists(join(res_path, '.git')):
raise Exception('GitResource in %s is invalid. Run `fab dist.clean:ext=true`' % res_path)
with lcd(res_path):
local('git fetch')
# Else, we assume that the resource has been already retrieved
# Copy the files into ext build dir
ext_build_dir = join(extbuild, res.name)
shutil.copytree(res_path, ext_build_dir)
if isinstance(res, GitResource):
with lcd(ext_build_dir):
logger.info('For %s, git checkout %s' % (res.name, branch))
local('git checkout %s' % branch)
local('git merge origin/%s' % branch)
# Now the plugin files are inside 'build/ext/', and we can continue
# Work in build directory
with lcd(extbuild):
# Retreive resource and place them to build directory
# Apply patches
if get_bool_str(patch) and allext:
# Patch Trac
logger.info('Patching Trac...')
with lcd(join(extbuild, 'trac')):
for patch in get_files(join(PROJECT_DIR, 'ext/patches/trac'), '*.patch', recursive=True):
local('patch --ignore-whitespace -p0 -i %s' % patch)
# Patch Gitosis
logger.info('Patching Gitosis...')
with lcd(join(extbuild, 'gitosis')):
for patch in get_files(join(PROJECT_DIR, 'ext/patches/gitosis'), '*.patch', recursive=True):
local('patch --ignore-whitespace -p0 -i %s' % patch)
# Build eggs (in build dir)
logger.info('Laying eggs and source dists...')
# Iterate folders where setup.py can be found
for setuppy_path in get_files(os.path.abspath(extbuild), 'setup.py', recursive=True):
plugin_dir = os.path.dirname(setuppy_path)
logger.info('Building package for %s' % os.path.basename(plugin_dir))
with lcd(plugin_dir):
local('python setup.py bdist_egg')
local('python setup.py sdist')
# Copy distributable files to dist
if not os.path.exists(DIST_DIR):
os.makedirs(DIST_DIR)
for egg in get_files(os.path.abspath(extbuild), '*.egg', recursive=True):
shutil.copy(egg, join(DIST_DIR, os.path.basename(egg)))
for targz in get_files(os.path.abspath(extbuild), '*.tar.gz', recursive=True):
shutil.copy(targz, join(DIST_DIR, os.path.basename(targz)))
def buildext(allext='false', patch='true', branch='master'):
"""
Build and optionally patch the 3rd party modules and libraries.
The outcome (tar.gz/egg) files are placed in dist directory
:param allext:
Download also non-GitResources. Default is 'false'.
:param patch:
Patch those plugins having patches, currently, trac and gitosis.
:param branch:
For GitResources, selects the branch to be used. Default is 'master'.
.. NOTE::
If you want to include the external release into multiproject-all package, run the build
task with parameters::
fab dist.build:ext=true
"""
allext = get_bool_str(allext)
# Construct and create building directory for external resources
extbuild = build_join('ext')
shutil.rmtree(extbuild, ignore_errors=True)
os.makedirs(extbuild)
for res in ext_resources:
res_path = get_ext_path(res.name)
is_git = isinstance(res, GitResource)
if not is_git and not allext:
continue
logger.info('Starting to download / fetch resource %s' % res.name)
must_retrieve = False
resource_id_file = join(res_path, '.fabric_resource_id.txt')
res_lines = [line.strip() for line in str(res).split(',')]
res_lines.append(
'# This is a file used by fabric dist.buildext command.')
if not os.path.exists(res_path):
must_retrieve = True
else:
# Check folder contents.
# If the fetch identifier is missing, it is assumed to be the correct one.
if os.path.exists(resource_id_file):
prev_lines = [
line.strip() for line in open(resource_id_file, 'r')
]
if res_lines != prev_lines:
logger.warning(
'Resource %s has been changed, retrieving it.' %
res.name)
logger.info('Previous resource: %s' % prev_lines)
logger.info('Current resource: %s' % res_lines)
must_retrieve = True
if not get_files(
os.path.abspath(res_path), 'setup.py', recursive=True):
must_retrieve = True
if must_retrieve:
shutil.rmtree(res_path, ignore_errors=True)
os.makedirs(res_path)
res.retrieve(res_path)
outfile = open(resource_id_file, 'w')
outfile.writelines([line + '\n' for line in res_lines])
outfile.close()
else:
logger.warning('Resource %s was already retrieved.' % res.name)
if is_git:
# The GitResources are always updated
if not os.path.exists(join(res_path, '.git')):
raise Exception(
'GitResource in %s is invalid. Run `fab dist.clean:ext=true`'
% res_path)
with lcd(res_path):
local('git fetch')
# Else, we assume that the resource has been already retrieved
# Copy the files into ext build dir
ext_build_dir = join(extbuild, res.name)
shutil.copytree(res_path, ext_build_dir)
if isinstance(res, GitResource):
with lcd(ext_build_dir):
logger.info('For %s, git checkout %s' % (res.name, branch))
local('git checkout %s' % branch)
local('git merge origin/%s' % branch)
# Now the plugin files are inside 'build/ext/', and we can continue
# Work in build directory
with lcd(extbuild):
# Retreive resource and place them to build directory
# Apply patches
if get_bool_str(patch) and allext:
# Patch Trac
logger.info('Patching Trac...')
with lcd(join(extbuild, 'trac')):
for patch in get_files(join(PROJECT_DIR, 'ext/patches/trac'),
'*.patch',
recursive=True):
local('patch --ignore-whitespace -p0 -i %s' % patch)
# Patch Gitosis
logger.info('Patching Gitosis...')
with lcd(join(extbuild, 'gitosis')):
for patch in get_files(join(PROJECT_DIR,
'ext/patches/gitosis'),
'*.patch',
recursive=True):
local('patch --ignore-whitespace -p0 -i %s' % patch)
# Build eggs (in build dir)
logger.info('Laying eggs and source dists...')
# Iterate folders where setup.py can be found
for setuppy_path in get_files(os.path.abspath(extbuild),
'setup.py',
recursive=True):
plugin_dir = os.path.dirname(setuppy_path)
logger.info('Building package for %s' % os.path.basename(plugin_dir))
with lcd(plugin_dir):
local('python setup.py bdist_egg')
local('python setup.py sdist')
# Copy distributable files to dist
if not os.path.exists(DIST_DIR):
os.makedirs(DIST_DIR)
for egg in get_files(os.path.abspath(extbuild), '*.egg', recursive=True):
shutil.copy(egg, join(DIST_DIR, os.path.basename(egg)))
for targz in get_files(os.path.abspath(extbuild),
'*.tar.gz',
recursive=True):
shutil.copy(targz, join(DIST_DIR, os.path.basename(targz)))