def __call__(self): scm.tested_for_scms(('svn', 'gitsvn', 'git'), '.') scm.require_package_root_cwd() self.check_conditions() package_name = scm.get_package_name('.') if self.options.domain: domain = self.options.domain else: domain = package_name package_root = scm.get_package_root_path('.') package_dir = os.path.join(package_root, *package_name.split('.')) pot_path = os.path.join(self.locales_dir, '%s.pot' % domain) output.part_title('Rebuilding pot file at %s' % pot_path) # rebuild cmd = ['%s rebuild-pot' % self.i18ndude] cmd.append('--pot %s' % pot_path) # manual file manual_file = os.path.join(self.locales_dir, '%s-manual.pot' % domain) if os.path.exists(manual_file): print ' merging manual pot file:', manual_file cmd.append('--merge %s' % manual_file) cmd.append('--create %s %s' % ( domain, package_dir, )) cmd = ' \\\n'.join(cmd) runcmd(cmd)
def __call__(self): if len(self.args)<2: output.error('At least 2 arguments are required', exit=1) if len(self.args)>3: output.error('At most 3 arguments are required', exit=1) # extract and check arguments if len(self.args)==2: self.fromnr, self.action = self.args self.tonr = self.fromnr elif len(self.args)==3: self.fromnr, self.tonr, self.action = self.args self.fromnr = int(self.unalias(self.fromnr)) self.tonr = int(self.unalias(self.tonr)) if self.tonr < self.fromnr: output.error('FROM (%s) must be lower than TO (%s)' % ( str(self.fromnr), str(self.tonr), )) # find instances instance_names = ['instance' + str(self.alias(x)) for x in range(self.fromnr, self.tonr+1)] instance_paths = [] for name in instance_names: path = os.path.join(self.find_buildout_directory(), 'bin', name) if os.path.exists(path): instance_paths.append(path) print ' * found %s' % path else: output.warning('%s not found: skipping' % path) # call instances for i, path in enumerate(instance_paths): if i!=0 and self.options.delay: print ' * waiting for %s second' % str(self.options.delay) time.sleep(self.options.delay) runcmd('%s %s' % (path, self.action))
def build_mo_files(self): """Build mo files in locales and i18n dirs """ output.part_title('Build .mo files') po_files_to_build = [] for basedir, dirs, files in os.walk(os.path.abspath('.')): if basedir.endswith('i18n') or basedir.endswith('LC_MESSAGES'): for file_ in files: if file_.endswith('.po'): po_files_to_build.append(os.path.join(basedir, file_)) for path in po_files_to_build: runcmd('msgfmt %s -o %s' % (path, path[:-3] + '.mo'), log=True)
def get_svn_url(directory_or_url): if sum([int(directory_or_url.startswith(x)) for x in ('https://', 'http://', 'svn:')])>0: return directory_or_url else: directory = directory_or_url if not is_git(directory): raise NotAGitsvnRepository return ''.join(runcmd('cd %s; git svn info | grep URL | cut -d " " -f 2' % directory, log=False, respond=True)).strip()
def pull_changes(path): if is_git_svn(path): cmd = 'cd %s ; git svn fetch ; git svn rebase' % path elif is_git(path): cmd = 'cd %s ; git pull' % path else: raise NotAGitsvnRepository return runcmd(cmd, log=True, respond=True)
def get_svn_url(directory_or_url): if sum([int(directory_or_url.startswith(x)) for x in ('https://', 'http://', 'svn:')])>0: return directory_or_url else: directory = directory_or_url if not is_subversion(directory): raise NotASubversionCheckout return ''.join(runcmd('svn info %s | grep URL | cut -d " " -f 2' % directory, log=False, respond=True)).strip()
def get_revision_for(self, package): url = PackageSourceMemory().guess_url(package) url = "/".join(url.split("/")[:-1]) cmd = "svn ls --xml %s" % url xmldata = "".join(runcmd(cmd, log=False, respond=True)) doc = minidom.parseString(xmldata) for entry in doc.getElementsByTagName("entry"): if entry.getElementsByTagName("name")[0].firstChild.nodeValue.strip() == package: return entry.getElementsByTagName("commit")[0].getAttribute("revision") return None
def check_project_layout(directory_or_url, raise_exception=True, ask_for_creation=True): """ Checks if the project has a default svn layout with the folders trunk, tags and branches. """ if not raise_exception: try: url = get_package_root_url(directory_or_url) except InvalidProjectLayout: return False else: url = get_package_root_url(directory_or_url) dircontent = runcmd('svn ls %s' % url, log=False, respond=True) dircontent = [x.strip()[:-1] for x in dircontent] # check if there are the expected folders excpected = ('trunk', 'tags', 'branches') missing = [] for dir in excpected: if dir not in dircontent: missing.append(dir) # ask what to do, if there are folders missing if len(missing)>0: if ask_for_creation: output.error('[%s] Invalid project layout, folders missing: ' % get_package_name(url) + ', '.join(missing)) if input.prompt_bool('Would you like to create the missing folders?'): cmd = 'svn mkdir ' cmd += ' '.join([os.path.join(url, dir) for dir in missing]) cmd += ' -m "created folders: %s for package %s"' % ( ', '.join(missing), get_package_name(directory_or_url), ) runcmd(cmd, log=True, respond=True) # need to clean caches flush_cache(runcmd) flush_cache(runcmd_with_exitcode) return check_project_layout(directory_or_url, raise_exception=raise_exception, ask_for_creation=False) if raise_exception: raise InvalidProjectLayout else: return False return True
def get_existing_tags(path): if '://' in path: raise Exception('Not a directory: ' % path) if is_git_svn(path): root_svn = get_package_root_url('.') return svn.get_existing_tags(root_svn) elif is_git(path): tags = runcmd('git tag', log=False, respond=True) tags = [t.strip() for t in tags] # make a dict return dict(zip(tags, tags))
def release_egg(self): output.part_title('Releasing agg to target %s' % self.pypi_target) sdist_params = '' # Workaround for broken tarfile implementation in python 2.4 # more infos at http://bugs.python.org/issue4218 # Only python 2.4.x is affected if (2, 5) > sys.version_info > (2, 4): # Probably sys.hexinfo would be the better solution output.part_title( 'Python 2.4.x detected, use sdist with --formats=zip') sdist_params = '--formats=zip' cmd = '%s setup.py mregister sdist %s mupload -r %s' % ( sys.executable, sdist_params, self.pypi_target ) runcmd(cmd) runcmd('rm -rf dist build')
def check_conditions(self): self.i18ndude = 'i18ndude' # we should be in a local repository try: package_name = scm.get_package_name('.') print ' using package name:', package_name except scm.NotAScm: output.error('Not in a SVN- or GIT-Checkout, unable to guess ' 'package name', exit=1) # get package root package_root = scm.get_package_root_path('.') print ' using package root path:', package_root # check locales directory self.locales_dir = os.path.abspath(os.path.join( package_root, '/'.join(package_name.split('.')), 'locales', )) print ' using locales dir:', self.locales_dir if not os.path.isdir(self.locales_dir): runcmd('mkdir -p %s' % self.locales_dir)
def __call__(self): path = ['.'] zopeFound = False while not zopeFound: dir = os.listdir('/'.join(path)) if '/' == os.path.abspath('/'.join(path)): error('File system root reached: no zope instance found ..', exit=True) elif 'bin' in dir: binContents = os.listdir('/'.join(path + ['bin'])) instances = filter(lambda x: x.startswith('instance'), binContents) if len(instances) > 0: zopeFound = True else: path.append('..') else: path.append('..') p = os.path.abspath('/'.join(path + ['bin', instances[0]])) cmd = '%s %s' % (p, ' '.join(sys.argv[2:])) runcmd(cmd, log=True)
def listdir(url): """ Returns list of elements in this directory (url) e.g. ['trunk/', 'branches/', 'README.txt'] """ xml_data = ''.join(runcmd('svn ls --xml %s' % url, log=False, respond=True)) dom = xml.dom.minidom.parseString(xml_data) names = [] for entry in dom.getElementsByTagName('entry'): node = entry.getElementsByTagName('name')[0] name = ''.join([child.toxml() for child in node.childNodes]) names.append(name) return name
def add_and_commit_files(message, files="*", push=True): """Adds and commits files to the scm. The repository must be . Use files='*' to commit all changed files """ commit_all_files = files in ("*", ".") if not commit_all_files and type(files) in (unicode, str): files = [files] if is_subversion("."): if commit_all_files: runcmd_unmemoized("svn add *") else: for file_ in files: runcmd_unmemoized("svn add %s" % file_) elif is_git("."): if commit_all_files: runcmd_unmemoized("git add .") else: for file_ in files: runcmd("git add %s" % file_) else: raise Exception("unknown scm") commit_files(message, files=files, push=push)
def create_tag(self): output.part_title('Creating subversion tag') if scm.is_subversion('.'): root_url = scm.get_package_root_url('.') trunk_url = scm.get_svn_url('.') tag_url = os.path.join(root_url, 'tags', self.new_tag_version) cmd = 'svn cp %s %s -m "creating tag %s for package %s"' % ( trunk_url, tag_url, self.new_tag_version, svn.get_package_name('.'), ) runcmd(cmd) elif scm.is_git_svn('.'): cmd = 'git svn tag %s' % self.new_tag_version runcmd(cmd) git.pull_changes('.') elif scm.is_git('.'): runcmd('git tag -a %s -m "tagged by ftw.manager"' % self.new_tag_version, log=True) runcmd('git push origin --tags', log=True)
def get_existing_tags(directory_or_url): """ Returns a dictionary of tags and the revision of the last commit { u'2.0.4' : u'15433', u'2.3' : u'27827', } """ tags_dir = os.path.join(get_package_root_url(directory_or_url), 'tags') xml_data = ''.join(runcmd('svn ls --xml %s' % tags_dir, log=False, respond=True)) dom = xml.dom.minidom.parseString(xml_data) tags = {} for entry in dom.getElementsByTagName('entry'): node = entry.getElementsByTagName('name')[0] name = ''.join([child.toxml() for child in node.childNodes]) rev = entry.getElementsByTagName('commit')[0].getAttribute('revision') tags[name] = rev return tags
def __call__(self): scm.tested_for_scms(('svn', 'gitsvn', 'git'), '.') scm.require_package_root_cwd() if len(self.args) < 1: output.error('Language code is required', exit=1) lang = self.args[0] # check self.check_conditions() package_name = scm.get_package_name('.') if self.options.domain: domain = self.options.domain else: domain = package_name # check pot file pot_path = os.path.join(self.locales_dir, '%s.pot' % domain) if not os.path.exists(pot_path): output.error('Could not find pot file at: %s' % pot_path, exit=1) # check language directory lang_dir = os.path.join(self.locales_dir, lang, 'LC_MESSAGES') if not os.path.isdir(lang_dir): runcmd('mkdir -p %s' % lang_dir) # touch po file po_file = os.path.join(lang_dir, '%s.po' % domain) if not os.path.isfile(po_file): runcmd('touch %s' % po_file) # sync output.part_title('Syncing language "%s"' % lang) cmd = '%s sync --pot %s %s' % ( self.i18ndude, pot_path, po_file, ) runcmd(cmd) # remove language output.part_title('Removing language code from po file') data = open(po_file).read().split('\n') file = open(po_file, 'w') for row in data: if not row.startswith('"Language-Code') and \ not row.startswith('"Language-Name'): file.write(row) file.write('\n') file.close()
def check_requires(self): """ Checks, if there are missing dependencies """ self.notify_part('Check dependencies') # get current requires requires = self.egginfo.install_requires # extend it with extra requires if self.egginfo.extras_require: for ename, erequires in self.egginfo.extras_require.items(): requires.extend(erequires) print ' current requirements (including extras):' for egg in requires: print ' -', egg print '' # add the requirements without extras too for egg in requires[:]: if '[' in egg: requires.append(egg.split('[')[0]) self.notify_check('Its not necessary to import some default plone / zope stuff') failures = False for egg in requires: if egg in PACKAGE_REQUIREMENTS_INADVISABLE and egg not in TESTING_PACKAGES: self.notify(False, 'Maybe you should remove the requirement ' +\ output.colorize(egg, output.ERROR) +\ output.colorize('. It seems to be in a python, ' +\ 'zope or plone distribution and ' +\ 'those packages should not be ' +\ 'set as requirement.', output.WARNING), problem_level=2) failures = True if not failures: self.notify(True) self.notify_check('Check imports on python files and zcml stuff') propose_requires = [] # contains ipath:file mapping of python and zcml imports ipath_file_mapping = {} # SEARCH PYTHON FILES # make a grep on python files py_grep_results = runcmd("find . -name '*.py' -exec grep -Hr 'import ' {} \;", respond=True) # cleanup: # - strip rows # - remove the rows with spaces (they are not realle imports) # - remove "as xxx" py_grep_results = filter(lambda row: ' ' in row, [row.strip().split(' as ')[0] for row in py_grep_results]) for row in py_grep_results: file_, statement = row.split(':') # make a import path ipath = statement.replace('from ', '').replace(' import ', '.').replace('import', '').strip() ipath = ipath.replace('...', '').replace('>>>', '') ipath_parts = ipath.split('.') if '#' in ipath: continue # ignore namespace imports (python internals etc) if len(ipath_parts) == 1: continue ipath_file_mapping[ipath] = file_ # SEARCH ZCML FILES cmd = "find . -name '*.zcml' -exec grep -Hr '\(layer\|package\|for\)=' {} \;" zcml_grep_results = runcmd(cmd, respond=True) # cleanup results zcml_xpr = re.compile('(for|layer)="(.*?)("|$)') for row in zcml_grep_results: file_, stmt = row.split(':', 1) stmt = stmt.strip() match = zcml_xpr.search(stmt) if not match: # maybe we have a more complicated statement (e.g. multiline) break ipath = match.groups()[1].strip() ipath = ipath.replace('*', '').strip() if '#' in ipath: continue if not ipath.startswith('.'): ipath_file_mapping[ipath] = file_ # for later use guessed_related_packages = self._get_guessed_related_packages() # HANDLE ALL IMPORTS for ipath, file_ in ipath_file_mapping.items(): ipath_parts = ipath.split('.') # ignore local imports if ipath.startswith(scm.get_package_name('.')): continue # is it already required? found = False for egg in requires: if ipath.startswith(egg): found = True break if not found: # is it already proposed? for egg in propose_requires: if ipath.startswith(egg): found = True break if not found: # is it ignored? for egg in PACKAGE_REQUIREMENTS_INADVISABLE + TESTING_PACKAGES: if ipath.startswith(egg): found = True break if found: continue # maybe we have a module which import relatively module_path = os.path.join(os.path.dirname(file_), ipath_parts[0]) if os.path.isfile(module_path + '.py') or \ os.path.isfile(module_path + '/__init__.py'): continue # start on level 2 and for searching egg guessed_egg_names = ['.'.join(ipath_parts[:i]) for i, part in enumerate(ipath_parts) if i > 1] # does one of the eggs exist? found = False # is there package in our src directory, if we are in one? for egg_name in guessed_egg_names: if egg_name.strip() in guessed_related_packages: if egg_name.strip() not in propose_requires: propose_requires.append(egg_name.strip()) found = True break # .. in pypi if not found: for egg_name in guessed_egg_names: if len(self.find_egg_in_index(egg_name)) > 0: if egg_name.strip() not in propose_requires: propose_requires.append(egg_name.strip()) found = True break # .. or do we have one in the svn cache? if not found: for egg_name in guessed_egg_names: if scm.guess_package_url(egg_name): if egg_name.strip() not in propose_requires: propose_requires.append(egg_name.strip()) found = True break if not found: print ' ', output.colorize(ipath, output.INFO), 'in', \ output.colorize(file_, output.INFO), \ output.colorize('is not covered by requirements ' 'and I could find a egg with such a name', output.WARNING) if scm.get_package_name('.') in propose_requires: propose_requires.remove(scm.get_package_name('.')) if len(propose_requires)==0: self.notify(True) return propose_requires.sort() print '' print ' There are some requirements missing. I propose to add these:' for egg in propose_requires: print ' ', egg print '' # try to add them automatically: if input.prompt_bool('Should I try to add them?'): rows = open('setup.py').read().split('\n') # find "install_requires" frows = filter(lambda row:row.strip().startswith('install_requires='), rows) if len(frows) != 1: output.error('Somethings wrong with your setup.py: expected only ' 'one row containing "install_requires=", but ' 'got %i' % len(frows), exit=1) insert_at = rows.index(frows[0]) + 1 for egg in propose_requires: rows.insert(insert_at, ' ' * 8 + "'%s'," % egg) file_ = open('setup.py', 'w') file_.write('\n'.join(rows)) file_.close() self._validate_setup_py() scm.add_and_commit_files('setup.py: added missing dependencies', 'setup.py') self.notify_fix_completed()
def __call__(self): output.warning('GIT repositories are not supported yet ' +\ '(only svn or gitsvn)') if len(self.args) == 0: output.error('package_name is required', exit=1) package_name = self.args[0] git.setup_gitsvn_cache() default_vcs = config.Configuration().default_vcs if default_vcs == 'git': # already checked out once? cache_path = os.path.join(git.get_gitsvn_cache_path(), package_name) if not os.path.isdir(cache_path): svn_url = self.get_svn_url(package_name) git.checkout_gitsvn(svn_url) else: # cache_path existing ; just update and clone runcmd('cd %s; git reset --hard' % cache_path) runcmd('cd %s; git svn fetch' % cache_path) runcmd('cd %s; git svn rebase' % cache_path) runcmd('cp -r %s .' % cache_path) runcmd('cd %s ; git checkout %s' % ( package_name, 'master', )) git.apply_svn_ignores(package_name) elif default_vcs == 'svn': svn_url = self.get_svn_url(package_name) runcmd('svn co %s %s' % ( svn_url, package_name))
def has_local_changes(path): cmd = 'cd %s ; git status | grep "\t"' % path return len(runcmd(cmd, log=False, respond=True))>0
def has_local_changes(path): cmd = 'svn st %s | grep -v ^X | grep -v ^Performing | grep -v ^$' % path return len(runcmd(cmd, log=False, respond=True))>0
def pre_build_check(self): """ Check if a build will work later. Check this before doing anything by building and loading the egg. """ output.part_title('Make a test-build for preventing bad dists') cwd = os.getcwd() # make a sdist runcmd('%s setup.py sdist' % sys.executable) os.chdir('dist') # switch dir print output.colorize('cd dist', output.INFO) # extract runcmd('tar -xf *.tar.gz') # find extracted dir / chdir distdir = None for file_ in os.listdir('.'): if os.path.isdir(file_): distdir = file_ break if not distdir: output.error('Something is wrong: could not find extracted dist directory', exit=1) os.chdir(distdir) print output.colorize('cd %s' % distdir, output.INFO) # test setup.py cmd = '%s setup.py egg_info' % sys.executable state, response, error = runcmd_with_exitcode(cmd, respond=True, respond_error=True) # cd back to original dir os.chdir(cwd) # remove stuff runcmd('rm -rf dist') # did it work? if state != 0: output.error('Something\'s wrong: could not load setup.py on distribution, ' +\ 'you may have a problem with your setup.py / MANIFEST.in:', exit=(not error and True or False)) if response: print output.colorize(response, output.INFO) if error: output.error(error, exit=True) # check locales locales_dir = os.path.join(scm.get_package_name('.').replace('.', '/'), 'locales') if os.path.isdir(locales_dir): for basedir, dirs, files in os.walk(locales_dir): for file_ in files: path = os.path.join(basedir, file_) if path.endswith('.po'): # check with msgfmt exitcode, errors = runcmd_with_exitcode( 'msgfmt -o /dev/null %s' % path, log=True, respond_error=True) if exitcode > 0: output.error(errors, exit=True) data = open(path).read() if 'fuzzy' in data: print path output.error('You have "Fuzzy" entries in your ' 'translations! I\'m not releasing ' 'it like this.', exit=True)
def checkout_gitsvn(svn_url, location='.'): svn_url = svn_url[-1]=='/' and svn_url[:-1] or svn_url root_url = svn.get_package_root_url(svn_url) if not svn.isdir(svn_url): raise svn.InvalidSubversionURL expected_dirs = ('trunk', 'tags', 'branches') got_dirs = expected_dirs try: svn.check_project_layout(svn_url) except svn.InvalidProjectLayout: # check the directories and print a warning dircontent = runcmd('svn ls %s' % svn_url, log=False, respond=True) dircontent = [x.strip()[:-1] for x in dircontent] got_dirs = [] missing_dirs = [] for dir in expected_dirs: if dir in dircontent: got_dirs.append(dir) else: missing_dirs.append(dir) output.warning('Directory %s missing!' % dir) package_name = svn.get_package_name(svn_url) cache_path = os.path.join(get_gitsvn_cache_path(), package_name) if os.path.exists(package_name): raise Exception('%s already existing' % os.path.abspath(package_name)) gitbranch = svnurl_get_gitbranch(svn_url) # clone it if os.path.exists(cache_path): runcmd('cd %s; git reset --hard' % cache_path) runcmd('cd %s; git svn fetch' % cache_path) runcmd('cd %s; git svn rebase' % cache_path) else: if got_dirs==expected_dirs: # we have a standard layout cmd = 'cd %s; git svn clone --stdlayout %s' % ( get_gitsvn_cache_path(), root_url, ) else: # some dirs are missing args = ['--%s=%s' % (d,d) for d in got_dirs] cmd = 'cd %s; git svn clone %s %s' % ( get_gitsvn_cache_path(), ' '.join(args), root_url, ) runcmd(cmd) runcmd('cp -r %s %s' % (cache_path, location)) co_path = os.path.join(location, package_name) runcmd('cd %s ; git checkout %s' % ( co_path, gitbranch, )) runcmd('cd %s ; git reset --hard' % co_path) runcmd('cd %s ; git svn rebase' % co_path)