def __call__(self):
scm.tested_for_scms(('svn', 'gitsvn', 'git'), '.')
scm.require_package_root_cwd()
self.check_conditions()
package_name = scm.get_package_name('.')
if self.options.domain:
domain = self.options.domain
else:
domain = package_name
package_root = scm.get_package_root_path('.')
package_dir = os.path.join(package_root, *package_name.split('.'))
pot_path = os.path.join(self.locales_dir, '%s.pot' % domain)
output.part_title('Rebuilding pot file at %s' % pot_path)
# rebuild
cmd = ['%s rebuild-pot' % self.i18ndude]
cmd.append('--pot %s' % pot_path)
# manual file
manual_file = os.path.join(self.locales_dir, '%s-manual.pot' % domain)
if os.path.exists(manual_file):
print ' merging manual pot file:', manual_file
cmd.append('--merge %s' % manual_file)
cmd.append('--create %s %s' % (
domain,
package_dir,
))
cmd = ' \\\n'.join(cmd)
runcmd(cmd)
def __call__(self):
if len(self.args)<2:
output.error('At least 2 arguments are required', exit=1)
if len(self.args)>3:
output.error('At most 3 arguments are required', exit=1)
# extract and check arguments
if len(self.args)==2:
self.fromnr, self.action = self.args
self.tonr = self.fromnr
elif len(self.args)==3:
self.fromnr, self.tonr, self.action = self.args
self.fromnr = int(self.unalias(self.fromnr))
self.tonr = int(self.unalias(self.tonr))
if self.tonr < self.fromnr:
output.error('FROM (%s) must be lower than TO (%s)' % (
str(self.fromnr),
str(self.tonr),
))
# find instances
instance_names = ['instance' + str(self.alias(x))
for x in range(self.fromnr, self.tonr+1)]
instance_paths = []
for name in instance_names:
path = os.path.join(self.find_buildout_directory(), 'bin', name)
if os.path.exists(path):
instance_paths.append(path)
print ' * found %s' % path
else:
output.warning('%s not found: skipping' % path)
# call instances
for i, path in enumerate(instance_paths):
if i!=0 and self.options.delay:
print ' * waiting for %s second' % str(self.options.delay)
time.sleep(self.options.delay)
runcmd('%s %s' % (path, self.action))
def build_mo_files(self):
"""Build mo files in locales and i18n dirs
"""
output.part_title('Build .mo files')
po_files_to_build = []
for basedir, dirs, files in os.walk(os.path.abspath('.')):
if basedir.endswith('i18n') or basedir.endswith('LC_MESSAGES'):
for file_ in files:
if file_.endswith('.po'):
po_files_to_build.append(os.path.join(basedir, file_))
for path in po_files_to_build:
runcmd('msgfmt %s -o %s' % (path, path[:-3] + '.mo'),
log=True)
def get_svn_url(directory_or_url):
if sum([int(directory_or_url.startswith(x)) for x in ('https://', 'http://', 'svn:')])>0:
return directory_or_url
else:
directory = directory_or_url
if not is_git(directory):
raise NotAGitsvnRepository
return ''.join(runcmd('cd %s; git svn info | grep URL | cut -d " " -f 2' % directory, log=False, respond=True)).strip()
def pull_changes(path):
if is_git_svn(path):
cmd = 'cd %s ; git svn fetch ; git svn rebase' % path
elif is_git(path):
cmd = 'cd %s ; git pull' % path
else:
raise NotAGitsvnRepository
return runcmd(cmd, log=True, respond=True)
def get_svn_url(directory_or_url):
if sum([int(directory_or_url.startswith(x)) for x in ('https://', 'http://', 'svn:')])>0:
return directory_or_url
else:
directory = directory_or_url
if not is_subversion(directory):
raise NotASubversionCheckout
return ''.join(runcmd('svn info %s | grep URL | cut -d " " -f 2' % directory, log=False, respond=True)).strip()
def get_revision_for(self, package):
url = PackageSourceMemory().guess_url(package)
url = "/".join(url.split("/")[:-1])
cmd = "svn ls --xml %s" % url
xmldata = "".join(runcmd(cmd, log=False, respond=True))
doc = minidom.parseString(xmldata)
for entry in doc.getElementsByTagName("entry"):
if entry.getElementsByTagName("name")[0].firstChild.nodeValue.strip() == package:
return entry.getElementsByTagName("commit")[0].getAttribute("revision")
return None
def check_project_layout(directory_or_url, raise_exception=True, ask_for_creation=True):
"""
Checks if the project has a default svn layout with the folders
trunk, tags and branches.
"""
if not raise_exception:
try:
url = get_package_root_url(directory_or_url)
except InvalidProjectLayout:
return False
else:
url = get_package_root_url(directory_or_url)
dircontent = runcmd('svn ls %s' % url, log=False, respond=True)
dircontent = [x.strip()[:-1] for x in dircontent]
# check if there are the expected folders
excpected = ('trunk', 'tags', 'branches')
missing = []
for dir in excpected:
if dir not in dircontent:
missing.append(dir)
# ask what to do, if there are folders missing
if len(missing)>0:
if ask_for_creation:
output.error('[%s] Invalid project layout, folders missing: ' %
get_package_name(url) + ', '.join(missing))
if input.prompt_bool('Would you like to create the missing folders?'):
cmd = 'svn mkdir '
cmd += ' '.join([os.path.join(url, dir) for dir in missing])
cmd += ' -m "created folders: %s for package %s"' % (
', '.join(missing),
get_package_name(directory_or_url),
)
runcmd(cmd, log=True, respond=True)
# need to clean caches
flush_cache(runcmd)
flush_cache(runcmd_with_exitcode)
return check_project_layout(directory_or_url, raise_exception=raise_exception, ask_for_creation=False)
if raise_exception:
raise InvalidProjectLayout
else:
return False
return True
def get_existing_tags(path):
if '://' in path:
raise Exception('Not a directory: ' % path)
if is_git_svn(path):
root_svn = get_package_root_url('.')
return svn.get_existing_tags(root_svn)
elif is_git(path):
tags = runcmd('git tag', log=False, respond=True)
tags = [t.strip() for t in tags]
# make a dict
return dict(zip(tags, tags))
def release_egg(self):
output.part_title('Releasing agg to target %s' % self.pypi_target)
sdist_params = ''
# Workaround for broken tarfile implementation in python 2.4
# more infos at http://bugs.python.org/issue4218
# Only python 2.4.x is affected
if (2, 5) > sys.version_info > (2, 4):
# Probably sys.hexinfo would be the better solution
output.part_title(
'Python 2.4.x detected, use sdist with --formats=zip')
sdist_params = '--formats=zip'
cmd = '%s setup.py mregister sdist %s mupload -r %s' % (
sys.executable,
sdist_params,
self.pypi_target
)
runcmd(cmd)
runcmd('rm -rf dist build')
def check_conditions(self):
self.i18ndude = 'i18ndude'
# we should be in a local repository
try:
package_name = scm.get_package_name('.')
print ' using package name:', package_name
except scm.NotAScm:
output.error('Not in a SVN- or GIT-Checkout, unable to guess '
'package name', exit=1)
# get package root
package_root = scm.get_package_root_path('.')
print ' using package root path:', package_root
# check locales directory
self.locales_dir = os.path.abspath(os.path.join(
package_root,
'/'.join(package_name.split('.')),
'locales',
))
print ' using locales dir:', self.locales_dir
if not os.path.isdir(self.locales_dir):
runcmd('mkdir -p %s' % self.locales_dir)
def __call__(self):
path = ['.']
zopeFound = False
while not zopeFound:
dir = os.listdir('/'.join(path))
if '/' == os.path.abspath('/'.join(path)):
error('File system root reached: no zope instance found ..',
exit=True)
elif 'bin' in dir:
binContents = os.listdir('/'.join(path + ['bin']))
instances = filter(lambda x: x.startswith('instance'),
binContents)
if len(instances) > 0:
zopeFound = True
else:
path.append('..')
else:
path.append('..')
p = os.path.abspath('/'.join(path + ['bin', instances[0]]))
cmd = '%s %s' % (p, ' '.join(sys.argv[2:]))
runcmd(cmd, log=True)
def listdir(url):
"""
Returns list of elements in this directory (url)
e.g. ['trunk/', 'branches/', 'README.txt']
"""
xml_data = ''.join(runcmd('svn ls --xml %s' % url, log=False, respond=True))
dom = xml.dom.minidom.parseString(xml_data)
names = []
for entry in dom.getElementsByTagName('entry'):
node = entry.getElementsByTagName('name')[0]
name = ''.join([child.toxml() for child in node.childNodes])
names.append(name)
return name
def add_and_commit_files(message, files="*", push=True):
"""Adds and commits files to the scm. The repository
must be .
Use files='*' to commit all changed files
"""
commit_all_files = files in ("*", ".")
if not commit_all_files and type(files) in (unicode, str):
files = [files]
if is_subversion("."):
if commit_all_files:
runcmd_unmemoized("svn add *")
else:
for file_ in files:
runcmd_unmemoized("svn add %s" % file_)
elif is_git("."):
if commit_all_files:
runcmd_unmemoized("git add .")
else:
for file_ in files:
runcmd("git add %s" % file_)
else:
raise Exception("unknown scm")
commit_files(message, files=files, push=push)
def create_tag(self):
output.part_title('Creating subversion tag')
if scm.is_subversion('.'):
root_url = scm.get_package_root_url('.')
trunk_url = scm.get_svn_url('.')
tag_url = os.path.join(root_url, 'tags', self.new_tag_version)
cmd = 'svn cp %s %s -m "creating tag %s for package %s"' % (
trunk_url,
tag_url,
self.new_tag_version,
svn.get_package_name('.'),
)
runcmd(cmd)
elif scm.is_git_svn('.'):
cmd = 'git svn tag %s' % self.new_tag_version
runcmd(cmd)
git.pull_changes('.')
elif scm.is_git('.'):
runcmd('git tag -a %s -m "tagged by ftw.manager"' % self.new_tag_version, log=True)
runcmd('git push origin --tags', log=True)
def get_existing_tags(directory_or_url):
"""
Returns a dictionary of tags and the revision of the last commit
{
u'2.0.4' : u'15433',
u'2.3' : u'27827',
}
"""
tags_dir = os.path.join(get_package_root_url(directory_or_url), 'tags')
xml_data = ''.join(runcmd('svn ls --xml %s' % tags_dir, log=False, respond=True))
dom = xml.dom.minidom.parseString(xml_data)
tags = {}
for entry in dom.getElementsByTagName('entry'):
node = entry.getElementsByTagName('name')[0]
name = ''.join([child.toxml() for child in node.childNodes])
rev = entry.getElementsByTagName('commit')[0].getAttribute('revision')
tags[name] = rev
return tags
def __call__(self):
scm.tested_for_scms(('svn', 'gitsvn', 'git'), '.')
scm.require_package_root_cwd()
if len(self.args) < 1:
output.error('Language code is required', exit=1)
lang = self.args[0]
# check
self.check_conditions()
package_name = scm.get_package_name('.')
if self.options.domain:
domain = self.options.domain
else:
domain = package_name
# check pot file
pot_path = os.path.join(self.locales_dir, '%s.pot' % domain)
if not os.path.exists(pot_path):
output.error('Could not find pot file at: %s' % pot_path, exit=1)
# check language directory
lang_dir = os.path.join(self.locales_dir, lang, 'LC_MESSAGES')
if not os.path.isdir(lang_dir):
runcmd('mkdir -p %s' % lang_dir)
# touch po file
po_file = os.path.join(lang_dir, '%s.po' % domain)
if not os.path.isfile(po_file):
runcmd('touch %s' % po_file)
# sync
output.part_title('Syncing language "%s"' % lang)
cmd = '%s sync --pot %s %s' % (
self.i18ndude,
pot_path,
po_file,
)
runcmd(cmd)
# remove language
output.part_title('Removing language code from po file')
data = open(po_file).read().split('\n')
file = open(po_file, 'w')
for row in data:
if not row.startswith('"Language-Code') and \
not row.startswith('"Language-Name'):
file.write(row)
file.write('\n')
file.close()
def check_requires(self):
""" Checks, if there are missing dependencies
"""
self.notify_part('Check dependencies')
# get current requires
requires = self.egginfo.install_requires
# extend it with extra requires
if self.egginfo.extras_require:
for ename, erequires in self.egginfo.extras_require.items():
requires.extend(erequires)
print ' current requirements (including extras):'
for egg in requires:
print ' -', egg
print ''
# add the requirements without extras too
for egg in requires[:]:
if '[' in egg:
requires.append(egg.split('[')[0])
self.notify_check('Its not necessary to import some default plone / zope stuff')
failures = False
for egg in requires:
if egg in PACKAGE_REQUIREMENTS_INADVISABLE and egg not in TESTING_PACKAGES:
self.notify(False, 'Maybe you should remove the requirement ' +\
output.colorize(egg, output.ERROR) +\
output.colorize('. It seems to be in a python, ' +\
'zope or plone distribution and ' +\
'those packages should not be ' +\
'set as requirement.', output.WARNING),
problem_level=2)
failures = True
if not failures:
self.notify(True)
self.notify_check('Check imports on python files and zcml stuff')
propose_requires = []
# contains ipath:file mapping of python and zcml imports
ipath_file_mapping = {}
# SEARCH PYTHON FILES
# make a grep on python files
py_grep_results = runcmd("find . -name '*.py' -exec grep -Hr 'import ' {} \;",
respond=True)
# cleanup:
# - strip rows
# - remove the rows with spaces (they are not realle imports)
# - remove "as xxx"
py_grep_results = filter(lambda row: ' ' in row,
[row.strip().split(' as ')[0] for row in py_grep_results])
for row in py_grep_results:
file_, statement = row.split(':')
# make a import path
ipath = statement.replace('from ', '').replace(' import ',
'.').replace('import', '').strip()
ipath = ipath.replace('...', '').replace('>>>', '')
ipath_parts = ipath.split('.')
if '#' in ipath:
continue
# ignore namespace imports (python internals etc)
if len(ipath_parts) == 1:
continue
ipath_file_mapping[ipath] = file_
# SEARCH ZCML FILES
cmd = "find . -name '*.zcml' -exec grep -Hr '\(layer\|package\|for\)=' {} \;"
zcml_grep_results = runcmd(cmd, respond=True)
# cleanup results
zcml_xpr = re.compile('(for|layer)="(.*?)("|$)')
for row in zcml_grep_results:
file_, stmt = row.split(':', 1)
stmt = stmt.strip()
match = zcml_xpr.search(stmt)
if not match:
# maybe we have a more complicated statement (e.g. multiline)
break
ipath = match.groups()[1].strip()
ipath = ipath.replace('*', '').strip()
if '#' in ipath:
continue
if not ipath.startswith('.'):
ipath_file_mapping[ipath] = file_
# for later use
guessed_related_packages = self._get_guessed_related_packages()
# HANDLE ALL IMPORTS
for ipath, file_ in ipath_file_mapping.items():
ipath_parts = ipath.split('.')
# ignore local imports
if ipath.startswith(scm.get_package_name('.')):
continue
# is it already required?
found = False
for egg in requires:
if ipath.startswith(egg):
found = True
break
if not found:
# is it already proposed?
for egg in propose_requires:
if ipath.startswith(egg):
found = True
break
if not found:
# is it ignored?
for egg in PACKAGE_REQUIREMENTS_INADVISABLE + TESTING_PACKAGES:
if ipath.startswith(egg):
found = True
break
if found:
continue
# maybe we have a module which import relatively
module_path = os.path.join(os.path.dirname(file_),
ipath_parts[0])
if os.path.isfile(module_path + '.py') or \
os.path.isfile(module_path + '/__init__.py'):
continue
# start on level 2 and for searching egg
guessed_egg_names = ['.'.join(ipath_parts[:i])
for i, part in enumerate(ipath_parts)
if i > 1]
# does one of the eggs exist?
found = False
# is there package in our src directory, if we are in one?
for egg_name in guessed_egg_names:
if egg_name.strip() in guessed_related_packages:
if egg_name.strip() not in propose_requires:
propose_requires.append(egg_name.strip())
found = True
break
# .. in pypi
if not found:
for egg_name in guessed_egg_names:
if len(self.find_egg_in_index(egg_name)) > 0:
if egg_name.strip() not in propose_requires:
propose_requires.append(egg_name.strip())
found = True
break
# .. or do we have one in the svn cache?
if not found:
for egg_name in guessed_egg_names:
if scm.guess_package_url(egg_name):
if egg_name.strip() not in propose_requires:
propose_requires.append(egg_name.strip())
found = True
break
if not found:
print ' ', output.colorize(ipath, output.INFO), 'in', \
output.colorize(file_, output.INFO), \
output.colorize('is not covered by requirements '
'and I could find a egg with such a name',
output.WARNING)
if scm.get_package_name('.') in propose_requires:
propose_requires.remove(scm.get_package_name('.'))
if len(propose_requires)==0:
self.notify(True)
return
propose_requires.sort()
print ''
print ' There are some requirements missing. I propose to add these:'
for egg in propose_requires:
print ' ', egg
print ''
# try to add them automatically:
if input.prompt_bool('Should I try to add them?'):
rows = open('setup.py').read().split('\n')
# find "install_requires"
frows = filter(lambda row:row.strip().startswith('install_requires='),
rows)
if len(frows) != 1:
output.error('Somethings wrong with your setup.py: expected only '
'one row containing "install_requires=", but '
'got %i' % len(frows), exit=1)
insert_at = rows.index(frows[0]) + 1
for egg in propose_requires:
rows.insert(insert_at, ' ' * 8 + "'%s'," % egg)
file_ = open('setup.py', 'w')
file_.write('\n'.join(rows))
file_.close()
self._validate_setup_py()
scm.add_and_commit_files('setup.py: added missing dependencies',
'setup.py')
self.notify_fix_completed()
def __call__(self):
output.warning('GIT repositories are not supported yet ' +\
'(only svn or gitsvn)')
if len(self.args) == 0:
output.error('package_name is required', exit=1)
package_name = self.args[0]
git.setup_gitsvn_cache()
default_vcs = config.Configuration().default_vcs
if default_vcs == 'git':
# already checked out once?
cache_path = os.path.join(git.get_gitsvn_cache_path(),
package_name)
if not os.path.isdir(cache_path):
svn_url = self.get_svn_url(package_name)
git.checkout_gitsvn(svn_url)
else:
# cache_path existing ; just update and clone
runcmd('cd %s; git reset --hard' % cache_path)
runcmd('cd %s; git svn fetch' % cache_path)
runcmd('cd %s; git svn rebase' % cache_path)
runcmd('cp -r %s .' % cache_path)
runcmd('cd %s ; git checkout %s' % (
package_name,
'master',
))
git.apply_svn_ignores(package_name)
elif default_vcs == 'svn':
svn_url = self.get_svn_url(package_name)
runcmd('svn co %s %s' % (
svn_url,
package_name))
def has_local_changes(path):
cmd = 'cd %s ; git status | grep "\t"' % path
return len(runcmd(cmd, log=False, respond=True))>0
def has_local_changes(path):
cmd = 'svn st %s | grep -v ^X | grep -v ^Performing | grep -v ^$' % path
return len(runcmd(cmd, log=False, respond=True))>0
def pre_build_check(self):
""" Check if a build will work later. Check this before doing anything
by building and loading the egg.
"""
output.part_title('Make a test-build for preventing bad dists')
cwd = os.getcwd()
# make a sdist
runcmd('%s setup.py sdist' % sys.executable)
os.chdir('dist')
# switch dir
print output.colorize('cd dist', output.INFO)
# extract
runcmd('tar -xf *.tar.gz')
# find extracted dir / chdir
distdir = None
for file_ in os.listdir('.'):
if os.path.isdir(file_):
distdir = file_
break
if not distdir:
output.error('Something is wrong: could not find extracted dist directory',
exit=1)
os.chdir(distdir)
print output.colorize('cd %s' % distdir, output.INFO)
# test setup.py
cmd = '%s setup.py egg_info' % sys.executable
state, response, error = runcmd_with_exitcode(cmd,
respond=True,
respond_error=True)
# cd back to original dir
os.chdir(cwd)
# remove stuff
runcmd('rm -rf dist')
# did it work?
if state != 0:
output.error('Something\'s wrong: could not load setup.py on distribution, ' +\
'you may have a problem with your setup.py / MANIFEST.in:',
exit=(not error and True or False))
if response:
print output.colorize(response, output.INFO)
if error:
output.error(error, exit=True)
# check locales
locales_dir = os.path.join(scm.get_package_name('.').replace('.', '/'),
'locales')
if os.path.isdir(locales_dir):
for basedir, dirs, files in os.walk(locales_dir):
for file_ in files:
path = os.path.join(basedir, file_)
if path.endswith('.po'):
# check with msgfmt
exitcode, errors = runcmd_with_exitcode(
'msgfmt -o /dev/null %s' % path,
log=True,
respond_error=True)
if exitcode > 0:
output.error(errors, exit=True)
data = open(path).read()
if 'fuzzy' in data:
print path
output.error('You have "Fuzzy" entries in your '
'translations! I\'m not releasing '
'it like this.', exit=True)
def checkout_gitsvn(svn_url, location='.'):
svn_url = svn_url[-1]=='/' and svn_url[:-1] or svn_url
root_url = svn.get_package_root_url(svn_url)
if not svn.isdir(svn_url):
raise svn.InvalidSubversionURL
expected_dirs = ('trunk', 'tags', 'branches')
got_dirs = expected_dirs
try:
svn.check_project_layout(svn_url)
except svn.InvalidProjectLayout:
# check the directories and print a warning
dircontent = runcmd('svn ls %s' % svn_url, log=False, respond=True)
dircontent = [x.strip()[:-1] for x in dircontent]
got_dirs = []
missing_dirs = []
for dir in expected_dirs:
if dir in dircontent:
got_dirs.append(dir)
else:
missing_dirs.append(dir)
output.warning('Directory %s missing!' % dir)
package_name = svn.get_package_name(svn_url)
cache_path = os.path.join(get_gitsvn_cache_path(), package_name)
if os.path.exists(package_name):
raise Exception('%s already existing' % os.path.abspath(package_name))
gitbranch = svnurl_get_gitbranch(svn_url)
# clone it
if os.path.exists(cache_path):
runcmd('cd %s; git reset --hard' % cache_path)
runcmd('cd %s; git svn fetch' % cache_path)
runcmd('cd %s; git svn rebase' % cache_path)
else:
if got_dirs==expected_dirs:
# we have a standard layout
cmd = 'cd %s; git svn clone --stdlayout %s' % (
get_gitsvn_cache_path(),
root_url,
)
else:
# some dirs are missing
args = ['--%s=%s' % (d,d) for d in got_dirs]
cmd = 'cd %s; git svn clone %s %s' % (
get_gitsvn_cache_path(),
' '.join(args),
root_url,
)
runcmd(cmd)
runcmd('cp -r %s %s' % (cache_path, location))
co_path = os.path.join(location, package_name)
runcmd('cd %s ; git checkout %s' % (
co_path,
gitbranch,
))
runcmd('cd %s ; git reset --hard' % co_path)
runcmd('cd %s ; git svn rebase' % co_path)
