def git_hash_time(hash, folders): changes = set() changes_file = get_rd_file('changes.txt') if os.path.exists(changes_file): with open(changes_file, 'r') as f: changes = set([line.strip() for line in f.readlines()]) mtime = git.log('-1', hash, '--pretty=format:%ct') if mtime == '': return mtime = int(mtime) for folder in folders: folder = folder.strip() source_folder = '%s/src/main/java' % folder if not os.path.isdir(source_folder): source_folder = '%s/docroot/WEB-INF/src' % folder if not os.path.isdir(source_folder): source_folder = '%s/src' % folder if not os.path.isdir(source_folder): continue for file in git.ls_files(source_folder).split('\n'): if file not in changes and os.path.isfile(file): os.utime(file, (mtime, mtime))
def updateInformationsAboutTests(self): """ Update informations about tests from Makefiles. """ git = self.__getGitCmd() # git ls-files --full-name *Makefile mkFiles = git.ls_files('--full-name', '*Makefile') for mkFile in mkFiles.splitlines(): folder = os.path.dirname(mkFile) info = self.__parseMakefile("%s/%s" % (self.path_absolute, mkFile)) if 'Name' not in info: self.__getLog().warning("The test '%s' doesn't contain" " Name in Makefile" % folder) continue owner = Author.parseAuthor(info.get('Owner')) name = re.sub('\s+.*', '', info.get('Name')) test = None tests = Test.objects.filter(folder=folder, git=self).order_by('id') if len(tests) > 0: test = tests[0] test.name = name if len(tests) > 1: # This is here, because we have got more records in DB for # the one test, when the name of test was changed. testH = TestHistory.objects.filter(test=test) for it in tests[1:]: for it2 in TestHistory.objects.filter(test=it): if it2 in testH: it2.delete() else: it2.test = test it2.save() deps = Test.objects.filter(dependencies=it) for it3 in deps: it3.dependencies.remove(it) if test not in it3.dependencies: it3.dependencies.append(test) it3.save() it.delete() else: test, status = Test.objects.get_or_create(name=name, defaults={ "git": self}) test.owner = owner test.folder = folder if 'Description' in info and \ test.description != info.get('Description'): test.description = info.get('Description') if 'TestTime' in info and test.time != info.get('TestTime'): test.time = info.get('TestTime') if 'Type' in info and test.type != info.get('Type'): test.type = info.get('Type') if 'RunFor' in info: self.__updateGroups(test, info.get('RunFor')) self.__updateDependences(test, info.get('RhtsRequires')) test.save()
def updateInformationsAboutTests(self): """ Update informations about tests from Makefiles. """ git = self.__getGitCmd() # git ls-files --full-name *Makefile mkFiles = git.ls_files('--full-name', '*Makefile') for mkFile in mkFiles.splitlines(): folder = os.path.dirname(mkFile) info = self.__parseMakefile("%s/%s" % (self.path_absolute, mkFile)) if 'Name' not in info: self.__getLog().warning("The test '%s' doesn't contain" " Name in Makefile" % folder) continue owner = Author.parseAuthor(info.get('Owner')) name = re.sub('\s+.*', '', info.get('Name')) test = None tests = Test.objects.filter(folder=folder, git=self).order_by('id') if len(tests) > 0: test = tests[0] test.name = name if len(tests) > 1: # This is here, because we have got more records in DB for # the one test, when the name of test was changed. testH = TestHistory.objects.filter(test=test) for it in tests[1:]: for it2 in TestHistory.objects.filter(test=it): if it2 in testH: it2.delete() else: it2.test = test it2.save() deps = Test.objects.filter(dependencies=it) for it3 in deps: it3.dependencies.remove(it) if test not in it3.dependencies: it3.dependencies.append(test) it3.save() it.delete() else: test, status = Test.objects.get_or_create( name=name, defaults={"git": self}) test.owner = owner test.folder = folder if 'Description' in info and \ test.description != info.get('Description'): test.description = info.get('Description') if 'TestTime' in info and test.time != info.get('TestTime'): test.time = info.get('TestTime') if 'Type' in info and test.type != info.get('Type'): test.type = info.get('Type') if 'RunFor' in info: self.__updateGroups(test, info.get('RunFor')) self.__updateDependences(test, info.get('RhtsRequires')) test.save()
def add_mt_disclaimers(repository, file_info): now = datetime.now() os.chdir(repository.github.git_root) root_folder = repository.github.single_folder \ if repository.github.single_folder is not None else repository.github.project_folder for file in get_eligible_files(repository, git.ls_files(root_folder), 'en'): crowdin_file = get_crowdin_file(repository, file) target_file = 'ja/' + file[3:] if file[0:3] == 'en/' else file.replace('/en/', '/ja/') if not os.path.isfile('%s/%s' % (repository.github.git_root, target_file)): if target_file[-9:] == '.markdown': target_file = target_file[:-9] + '.md' elif target_file[-3:] == '.md': target_file = target_file[:-3] + '.markdown' if not os.path.isfile('%s/%s' % (repository.github.git_root, target_file)): continue if crowdin_file not in file_info or file_info[crowdin_file]['translated'] == file_info[crowdin_file]['approved']: continue new_lines = [] with open(target_file, 'r') as f: new_lines = [ line for line in f.readlines() if line.find('<p class="alert alert-info"><span class="wysiwyg-color-blue120">') == -1 ] content = '%s\n%s' % (''.join(new_lines), disclaimer_text) with open(target_file, 'w') as f: f.write(content) git.add(target_file) git.commit('-m', 'Added machine translation disclaimer %s' % now.strftime("%Y-%m-%d %H:%M:%S")) os.chdir(initial_dir)
def generate_html_files(repository, file_info): now = datetime.now() os.chdir(repository.github.git_root) root_folder = repository.github.single_folder \ if repository.github.single_folder is not None else repository.github.project_folder titles = {} for file in get_eligible_files(repository, git.ls_files(root_folder), 'en'): crowdin_file = get_crowdin_file(repository, file) target_file = 'ja/' + file[3:] if file[0:3] == 'en/' else file.replace('/en/', '/ja/') if not os.path.isfile('%s/%s' % (repository.github.git_root, target_file)): if target_file[-9:] == '.markdown': target_file = target_file[:-9] + '.md' elif target_file[-3:] == '.md': target_file = target_file[:-3] + '.markdown' if not os.path.isfile('%s/%s' % (repository.github.git_root, target_file)): continue html_file = target_file[:target_file.rfind('.')] + '.html' titles[html_file] = get_title(file) _pandoc(target_file, html_file, '--from=gfm', '--to=html') git.add(html_file) git.commit('-m', 'Updated pandoc conversion %s' % now.strftime("%Y-%m-%d %H:%M:%S")) os.chdir(initial_dir) return titles
def git_find(haystack, needle, commit=None): haystack = relpath(haystack, git_root) if commit is None: file_list = set([ relpath(x, haystack).replace('\\', '/') for x in git.ls_files(haystack).split('\n') if is_project_file(needle, x) ]) else: file_list = set([ relpath(x, haystack).replace('\\', '/') for x in git.ls_files('--with-tree=%s' % commit, haystack).split('\n') if is_project_file(needle, x) ]) # First, assume that we're looking for a module root, so check for # bnd.bnd, ivy.xml, and package.json for pattern in ['/%s/%s', '%s/%s']: for module_marker in ['bnd.bnd', 'ivy.xml', 'package.json']: filter_needle = pattern % (needle, module_marker) filtered_list = [ file for file in file_list if file.find(filter_needle) > -1 ] if len(filtered_list) > 0: return (get_filtered_folders(filtered_list, needle), None) for pattern in ['%s/', '/%s', '%s']: for module_marker in ['bnd.bnd', 'ivy.xml', 'package.json']: filter_needle = pattern % needle filtered_list = [ file for file in file_list if file.find(filter_needle) > -1 and file.find(module_marker) > -1 ] if len(filtered_list) > 0: return (get_filtered_folders(filtered_list, needle), None) # Next, check for a folder that isn't a module root, which can either be # an exact match or a suffix match. Prefer in that order. for pattern in ['/%s/', '%s/']: filter_needle = pattern % needle filtered_list = [ file for file in file_list if file.find(filter_needle) > -1 ] if len(filtered_list) > 0: return (get_filtered_folders(filtered_list, needle), None) # Finally, assume that maybe the person is looking for a folder containing # a file and is using the file name as an abbreviated way at getting there. # Choose a prefix match first, then an anywhere match. for pattern in ['/%s', '%s']: filter_needle = pattern % needle filtered_list = [ file for file in file_list if file.find(filter_needle) > -1 ] if len(filtered_list) > 0: return (get_filtered_folders(filtered_list, needle), filtered_list) return (None, None)
def check_renamed_articles(repository, language, articles, section_paths): old_dir = os.getcwd() os.chdir(repository.github.git_root) old_article_paths = { get_article_id(file): file for file in git.ls_files('en/').split('\n') } new_article_paths = { str(article['id']): get_article_path(article, 'en', section_paths) for article in articles.values() if is_tracked_article(article, section_paths) } old_translated_paths = { get_article_id(file): file for file in git.ls_files(language + '/').split('\n') } new_translated_paths = { str(article['id']): get_article_path(article, language, section_paths) for article in articles.values() if is_tracked_article(article, section_paths) } for article_id, new_article_path in sorted(new_article_paths.items()): if article_id not in old_article_paths: continue old_article_path = old_article_paths[article_id] if old_article_path == new_article_path: continue if not os.path.exists(old_article_path): continue os.makedirs(os.path.dirname(new_article_path), exist_ok=True) os.rename(old_article_path, new_article_path) git.add(old_article_path) git.add(new_article_path) for article_id, new_article_path in sorted(new_translated_paths.items()): if article_id not in old_translated_paths: continue old_article_path = old_translated_paths[article_id] if old_article_path == new_article_path: continue if not os.path.exists(old_article_path): continue os.makedirs(os.path.dirname(new_article_path), exist_ok=True) os.rename(old_article_path, new_article_path) git.add(old_article_path) git.add(new_article_path) git.commit('-m', 'Renamed articles: %s' % datetime.now()) os.chdir(old_dir) return new_article_paths
def getparent(check_tags): if git_root is None: return current_branch # Find the current branch, accounting for detached head if current_branch in [ 'master', 'master-private', '7.0.x', '7.0.x-private', 'ee-6.2.x', 'ee-6.1.x', 'ee-6.0.x' ]: return current_branch # Extract the full version full_version = None if isfile(join(git_root, 'release.properties')): full_version = get_file_property(join(git_root, 'release.properties'), 'lp.version') elif isfile(join(git_root, 'build.properties')) and isfile( join(git_root, 'app.server.properties')): full_version = get_file_property(join(git_root, 'build.properties'), 'lp.version') elif isfile(join(git_root, 'git-commit-portal')): with open(join(git_root, 'git-commit-portal'), 'r') as file: commit = file.readlines()[0].strip() full_version = get_git_file_property(commit, 'release.properties', 'lp.version') else: return getparent_origin() # If the short version is 6.x, then we have a shortcut short_version = '.'.join(full_version.split('.')[0:2]) base_branch = None if short_version == '6.0': base_branch = 'ee-6.0.x' elif short_version == '6.1': base_branch = 'ee-6.1.x' elif short_version == '6.2': base_branch = 'ee-6.2.x' else: # Determine the base version using build.properties if isfile(join(git_root, 'build.properties')): base_branch = get_file_property(join(git_root, 'build.properties'), 'git.working.branch.name') if base_branch is None and isfile(join(git_root, 'git-commit-portal')): with open(join(git_root, 'git-commit-portal'), 'r') as file: commit = file.readlines()[0].strip() base_branch = get_git_file_property(commit, 'build.properties', 'git.working.branch.name') if base_branch is None: base_branch = current_branch if current_branch != 'HEAD' else '7.0.x' elif base_branch == 'ee-7.0.x': base_branch = '7.0.x' elif isdir(join(git_root, 'modules/private')) and len( git.ls_files('build.properties').strip() ) == 0 and base_branch.find('-private') == -1: base_branch = '%s-private' % base_branch # If this is master or master-private, or we've recently rebased to 7.0.x or 7.0.x-private, # then use the branch instead of the tag if not check_tags: return base_branch if base_branch != 'ee-7.0.x': exit_code = subprocess.call( ['git', 'merge-base', '--is-ancestor', base_branch, 'HEAD']) if exit_code == 0: return base_branch # Find the closest matching tag base_tag = '' if base_branch == '7.1.x' or base_branch == '7.1.x-private': base_tag = git.describe('--tags', 'HEAD', '--abbrev=0', '--match=fix-pack-dxp-*') if base_tag is None or len(base_tag) == 0: base_tag = git.describe('--tags', 'HEAD', '--abbrev=0', '--match=fix-pack-base-71*') if base_tag is None or len(base_tag) == 0: base_tag = git.describe('--tags', 'HEAD', '--abbrev=0', '--match=7.1.*-ga*') elif base_branch == '7.0.x' or base_branch == '7.0.x-private': base_tag = git.describe('--tags', 'HEAD', '--abbrev=0', '--match=fix-pack-de-*') if base_tag is None or len(base_tag) == 0: base_tag = git.describe('--tags', 'HEAD', '--abbrev=0', '--match=fix-pack-base-70*') if base_tag is None or len(base_tag) == 0: base_tag = git.describe('--tags', 'HEAD', '--abbrev=0', '--match=7.0.*-ga*') elif base_branch == 'ee-6.2.x': base_tag = git.describe('--tags', 'HEAD', '--abbrev=0', '--match=fix-pack-base-62*') elif base_branch == 'ee-6.1.x': base_tag = git.describe('--tags', 'HEAD', '--abbrev=0', '--match=fix-pack-base-6130') if base_tag.find('fix-pack-base-') == 0 or base_tag.find( 'fix-pack-de-') == 0 or base_tag.find( 'fix-pack-dxp-') == 0 or base_tag.find('-ga') > -1: return base_tag return base_branch
def getparent(check_tags): if git_root is None: return current_branch # Find the current branch, accounting for detached head ee_branches = ['6.2.x', '6.1.x', '6.0.x'] ee_branches = ee_branches + ['ee-%s' % branch for branch in ee_branches] de_branches = ['7.0.x', '7.0.x-private', 'ee-7.0.x'] dxp_branches = ['7.3.x', '7.2.x', '7.1.x'] dxp_branches = dxp_branches + [ '%s-private' % branch for branch in dxp_branches ] if current_branch == 'master' or current_branch in ee_branches + de_branches + dxp_branches: return current_branch # Extract the full version full_version = None if isfile(join(git_root, 'release.properties')): full_version = get_file_property(join(git_root, 'release.properties'), 'lp.version') elif isfile(join(git_root, 'build.properties')) and isfile( join(git_root, 'app.server.properties')): full_version = get_file_property(join(git_root, 'build.properties'), 'lp.version') elif isfile(join(git_root, 'git-commit-portal')): with open(join(git_root, 'git-commit-portal'), 'r') as file: commit = file.readlines()[0].strip() full_version = get_git_file_property(commit, 'release.properties', 'lp.version') else: return getparent_origin() # If the short version is 6.x, then we have a shortcut short_version = '.'.join(full_version.split('.')[0:2]) base_branch = None if short_version == '6.0': base_branch = 'ee-6.0.x' elif short_version == '6.1': base_branch = 'ee-6.1.x' elif short_version == '6.2': base_branch = 'ee-6.2.x' else: # Determine the base version using build.properties if isfile(join(git_root, 'build.properties')): base_branch = get_file_property(join(git_root, 'build.properties'), 'git.working.branch.name') if base_branch is None: if isfile(join(git_root, 'git-commit-portal')): with open(join(git_root, 'git-commit-portal'), 'r') as file: commit = file.readlines()[0].strip() base_branch = get_git_file_property( commit, 'build.properties', 'git.working.branch.name') elif base_branch == 'master': if git.for_each_ref( 'refs/remotes/upstream*/%s.x' % short_version) != '': base_branch = '%s.x' % short_version if base_branch is None: base_branch = current_branch if current_branch != 'HEAD' else '7.0.x' elif base_branch == 'ee-7.0.x': base_branch = '7.0.x' elif isdir(join(git_root, 'modules/private')) and len( git.ls_files('build.properties').strip() ) == 0 and base_branch.find('-private') == -1: base_branch = '%s-private' % base_branch # If this is master or master-private, or we've recently rebased to 7.0.x or 7.0.x-private, # then use the branch instead of the tag if not check_tags: return base_branch # Find the closest matching tag base_tag = None if base_branch in dxp_branches or base_branch in de_branches: base_tag = git.describe( 'HEAD', '--tags', '--abbrev=0', '--match=fix-pack-*-%s%s10*' % (base_branch[0], base_branch[2])) if base_tag is None or len(base_tag) == 0: base_tag = git.describe( 'HEAD', '--tags', '--abbrev=0', '--match=%s.%s.*-ga*' % (base_branch[0], base_branch[2])) elif base_branch in ee_branches: base_tag = git.describe('HEAD', '--tags', '--abbrev=0', '--match=fix-pack-base-6%s*' % base_branch[5]) if base_tag is None or len(base_tag) == 0: if base_branch.find('ee-') == 0: base_tag = git.describe( 'HEAD', '--tags', '--abbrev=0', '--match=%s.%s.*-ga*' % (base_branch[3], base_branch[5])) else: base_tag = git.describe( 'HEAD', '--tags', '--abbrev=0', '--match=%s.%s.*-ga*' % (base_branch[0], base_branch[2])) if base_tag is None: return base_branch if base_tag.find('fix-pack-base-') == 0 or base_tag.find( 'fix-pack-de-') == 0 or base_tag.find( 'fix-pack-dxp-') == 0 or base_tag.find('-ga') > -1: return base_tag return base_branch