def git_hash_time(hash, folders):
changes = set()
changes_file = get_rd_file('changes.txt')
if os.path.exists(changes_file):
with open(changes_file, 'r') as f:
changes = set([line.strip() for line in f.readlines()])
mtime = git.log('-1', hash, '--pretty=format:%ct')
if mtime == '':
return
mtime = int(mtime)
for folder in folders:
folder = folder.strip()
source_folder = '%s/src/main/java' % folder
if not os.path.isdir(source_folder):
source_folder = '%s/docroot/WEB-INF/src' % folder
if not os.path.isdir(source_folder):
source_folder = '%s/src' % folder
if not os.path.isdir(source_folder):
continue
for file in git.ls_files(source_folder).split('\n'):
if file not in changes and os.path.isfile(file):
os.utime(file, (mtime, mtime))
def updateInformationsAboutTests(self):
"""
Update informations about tests from Makefiles.
"""
git = self.__getGitCmd()
# git ls-files --full-name *Makefile
mkFiles = git.ls_files('--full-name', '*Makefile')
for mkFile in mkFiles.splitlines():
folder = os.path.dirname(mkFile)
info = self.__parseMakefile("%s/%s" % (self.path_absolute, mkFile))
if 'Name' not in info:
self.__getLog().warning("The test '%s' doesn't contain"
" Name in Makefile" % folder)
continue
owner = Author.parseAuthor(info.get('Owner'))
name = re.sub('\s+.*', '', info.get('Name'))
test = None
tests = Test.objects.filter(folder=folder, git=self).order_by('id')
if len(tests) > 0:
test = tests[0]
test.name = name
if len(tests) > 1:
# This is here, because we have got more records in DB for
# the one test, when the name of test was changed.
testH = TestHistory.objects.filter(test=test)
for it in tests[1:]:
for it2 in TestHistory.objects.filter(test=it):
if it2 in testH:
it2.delete()
else:
it2.test = test
it2.save()
deps = Test.objects.filter(dependencies=it)
for it3 in deps:
it3.dependencies.remove(it)
if test not in it3.dependencies:
it3.dependencies.append(test)
it3.save()
it.delete()
else:
test, status = Test.objects.get_or_create(name=name, defaults={
"git": self})
test.owner = owner
test.folder = folder
if 'Description' in info and \
test.description != info.get('Description'):
test.description = info.get('Description')
if 'TestTime' in info and test.time != info.get('TestTime'):
test.time = info.get('TestTime')
if 'Type' in info and test.type != info.get('Type'):
test.type = info.get('Type')
if 'RunFor' in info:
self.__updateGroups(test, info.get('RunFor'))
self.__updateDependences(test, info.get('RhtsRequires'))
test.save()
def updateInformationsAboutTests(self):
"""
Update informations about tests from Makefiles.
"""
git = self.__getGitCmd()
# git ls-files --full-name *Makefile
mkFiles = git.ls_files('--full-name', '*Makefile')
for mkFile in mkFiles.splitlines():
folder = os.path.dirname(mkFile)
info = self.__parseMakefile("%s/%s" % (self.path_absolute, mkFile))
if 'Name' not in info:
self.__getLog().warning("The test '%s' doesn't contain"
" Name in Makefile" % folder)
continue
owner = Author.parseAuthor(info.get('Owner'))
name = re.sub('\s+.*', '', info.get('Name'))
test = None
tests = Test.objects.filter(folder=folder, git=self).order_by('id')
if len(tests) > 0:
test = tests[0]
test.name = name
if len(tests) > 1:
# This is here, because we have got more records in DB for
# the one test, when the name of test was changed.
testH = TestHistory.objects.filter(test=test)
for it in tests[1:]:
for it2 in TestHistory.objects.filter(test=it):
if it2 in testH:
it2.delete()
else:
it2.test = test
it2.save()
deps = Test.objects.filter(dependencies=it)
for it3 in deps:
it3.dependencies.remove(it)
if test not in it3.dependencies:
it3.dependencies.append(test)
it3.save()
it.delete()
else:
test, status = Test.objects.get_or_create(
name=name, defaults={"git": self})
test.owner = owner
test.folder = folder
if 'Description' in info and \
test.description != info.get('Description'):
test.description = info.get('Description')
if 'TestTime' in info and test.time != info.get('TestTime'):
test.time = info.get('TestTime')
if 'Type' in info and test.type != info.get('Type'):
test.type = info.get('Type')
if 'RunFor' in info:
self.__updateGroups(test, info.get('RunFor'))
self.__updateDependences(test, info.get('RhtsRequires'))
test.save()
def add_mt_disclaimers(repository, file_info):
now = datetime.now()
os.chdir(repository.github.git_root)
root_folder = repository.github.single_folder \
if repository.github.single_folder is not None else repository.github.project_folder
for file in get_eligible_files(repository, git.ls_files(root_folder), 'en'):
crowdin_file = get_crowdin_file(repository, file)
target_file = 'ja/' + file[3:] if file[0:3] == 'en/' else file.replace('/en/', '/ja/')
if not os.path.isfile('%s/%s' % (repository.github.git_root, target_file)):
if target_file[-9:] == '.markdown':
target_file = target_file[:-9] + '.md'
elif target_file[-3:] == '.md':
target_file = target_file[:-3] + '.markdown'
if not os.path.isfile('%s/%s' % (repository.github.git_root, target_file)):
continue
if crowdin_file not in file_info or file_info[crowdin_file]['translated'] == file_info[crowdin_file]['approved']:
continue
new_lines = []
with open(target_file, 'r') as f:
new_lines = [
line for line in f.readlines()
if line.find('<p class="alert alert-info"><span class="wysiwyg-color-blue120">') == -1
]
content = '%s\n%s' % (''.join(new_lines), disclaimer_text)
with open(target_file, 'w') as f:
f.write(content)
git.add(target_file)
git.commit('-m', 'Added machine translation disclaimer %s' % now.strftime("%Y-%m-%d %H:%M:%S"))
os.chdir(initial_dir)
def generate_html_files(repository, file_info):
now = datetime.now()
os.chdir(repository.github.git_root)
root_folder = repository.github.single_folder \
if repository.github.single_folder is not None else repository.github.project_folder
titles = {}
for file in get_eligible_files(repository, git.ls_files(root_folder), 'en'):
crowdin_file = get_crowdin_file(repository, file)
target_file = 'ja/' + file[3:] if file[0:3] == 'en/' else file.replace('/en/', '/ja/')
if not os.path.isfile('%s/%s' % (repository.github.git_root, target_file)):
if target_file[-9:] == '.markdown':
target_file = target_file[:-9] + '.md'
elif target_file[-3:] == '.md':
target_file = target_file[:-3] + '.markdown'
if not os.path.isfile('%s/%s' % (repository.github.git_root, target_file)):
continue
html_file = target_file[:target_file.rfind('.')] + '.html'
titles[html_file] = get_title(file)
_pandoc(target_file, html_file, '--from=gfm', '--to=html')
git.add(html_file)
git.commit('-m', 'Updated pandoc conversion %s' % now.strftime("%Y-%m-%d %H:%M:%S"))
os.chdir(initial_dir)
return titles
def git_find(haystack, needle, commit=None):
haystack = relpath(haystack, git_root)
if commit is None:
file_list = set([
relpath(x, haystack).replace('\\', '/')
for x in git.ls_files(haystack).split('\n')
if is_project_file(needle, x)
])
else:
file_list = set([
relpath(x, haystack).replace('\\', '/')
for x in git.ls_files('--with-tree=%s' %
commit, haystack).split('\n')
if is_project_file(needle, x)
])
# First, assume that we're looking for a module root, so check for
# bnd.bnd, ivy.xml, and package.json
for pattern in ['/%s/%s', '%s/%s']:
for module_marker in ['bnd.bnd', 'ivy.xml', 'package.json']:
filter_needle = pattern % (needle, module_marker)
filtered_list = [
file for file in file_list if file.find(filter_needle) > -1
]
if len(filtered_list) > 0:
return (get_filtered_folders(filtered_list, needle), None)
for pattern in ['%s/', '/%s', '%s']:
for module_marker in ['bnd.bnd', 'ivy.xml', 'package.json']:
filter_needle = pattern % needle
filtered_list = [
file for file in file_list if file.find(filter_needle) > -1
and file.find(module_marker) > -1
]
if len(filtered_list) > 0:
return (get_filtered_folders(filtered_list, needle), None)
# Next, check for a folder that isn't a module root, which can either be
# an exact match or a suffix match. Prefer in that order.
for pattern in ['/%s/', '%s/']:
filter_needle = pattern % needle
filtered_list = [
file for file in file_list if file.find(filter_needle) > -1
]
if len(filtered_list) > 0:
return (get_filtered_folders(filtered_list, needle), None)
# Finally, assume that maybe the person is looking for a folder containing
# a file and is using the file name as an abbreviated way at getting there.
# Choose a prefix match first, then an anywhere match.
for pattern in ['/%s', '%s']:
filter_needle = pattern % needle
filtered_list = [
file for file in file_list if file.find(filter_needle) > -1
]
if len(filtered_list) > 0:
return (get_filtered_folders(filtered_list, needle), filtered_list)
return (None, None)
def check_renamed_articles(repository, language, articles, section_paths):
old_dir = os.getcwd()
os.chdir(repository.github.git_root)
old_article_paths = {
get_article_id(file): file
for file in git.ls_files('en/').split('\n')
}
new_article_paths = {
str(article['id']): get_article_path(article, 'en', section_paths)
for article in articles.values()
if is_tracked_article(article, section_paths)
}
old_translated_paths = {
get_article_id(file): file
for file in git.ls_files(language + '/').split('\n')
}
new_translated_paths = {
str(article['id']): get_article_path(article, language, section_paths)
for article in articles.values()
if is_tracked_article(article, section_paths)
}
for article_id, new_article_path in sorted(new_article_paths.items()):
if article_id not in old_article_paths:
continue
old_article_path = old_article_paths[article_id]
if old_article_path == new_article_path:
continue
if not os.path.exists(old_article_path):
continue
os.makedirs(os.path.dirname(new_article_path), exist_ok=True)
os.rename(old_article_path, new_article_path)
git.add(old_article_path)
git.add(new_article_path)
for article_id, new_article_path in sorted(new_translated_paths.items()):
if article_id not in old_translated_paths:
continue
old_article_path = old_translated_paths[article_id]
if old_article_path == new_article_path:
continue
if not os.path.exists(old_article_path):
continue
os.makedirs(os.path.dirname(new_article_path), exist_ok=True)
os.rename(old_article_path, new_article_path)
git.add(old_article_path)
git.add(new_article_path)
git.commit('-m', 'Renamed articles: %s' % datetime.now())
os.chdir(old_dir)
return new_article_paths
def getparent(check_tags):
if git_root is None:
return current_branch
# Find the current branch, accounting for detached head
if current_branch in [
'master', 'master-private', '7.0.x', '7.0.x-private', 'ee-6.2.x',
'ee-6.1.x', 'ee-6.0.x'
]:
return current_branch
# Extract the full version
full_version = None
if isfile(join(git_root, 'release.properties')):
full_version = get_file_property(join(git_root, 'release.properties'),
'lp.version')
elif isfile(join(git_root, 'build.properties')) and isfile(
join(git_root, 'app.server.properties')):
full_version = get_file_property(join(git_root, 'build.properties'),
'lp.version')
elif isfile(join(git_root, 'git-commit-portal')):
with open(join(git_root, 'git-commit-portal'), 'r') as file:
commit = file.readlines()[0].strip()
full_version = get_git_file_property(commit, 'release.properties',
'lp.version')
else:
return getparent_origin()
# If the short version is 6.x, then we have a shortcut
short_version = '.'.join(full_version.split('.')[0:2])
base_branch = None
if short_version == '6.0':
base_branch = 'ee-6.0.x'
elif short_version == '6.1':
base_branch = 'ee-6.1.x'
elif short_version == '6.2':
base_branch = 'ee-6.2.x'
else:
# Determine the base version using build.properties
if isfile(join(git_root, 'build.properties')):
base_branch = get_file_property(join(git_root, 'build.properties'),
'git.working.branch.name')
if base_branch is None and isfile(join(git_root, 'git-commit-portal')):
with open(join(git_root, 'git-commit-portal'), 'r') as file:
commit = file.readlines()[0].strip()
base_branch = get_git_file_property(commit, 'build.properties',
'git.working.branch.name')
if base_branch is None:
base_branch = current_branch if current_branch != 'HEAD' else '7.0.x'
elif base_branch == 'ee-7.0.x':
base_branch = '7.0.x'
elif isdir(join(git_root, 'modules/private')) and len(
git.ls_files('build.properties').strip()
) == 0 and base_branch.find('-private') == -1:
base_branch = '%s-private' % base_branch
# If this is master or master-private, or we've recently rebased to 7.0.x or 7.0.x-private,
# then use the branch instead of the tag
if not check_tags:
return base_branch
if base_branch != 'ee-7.0.x':
exit_code = subprocess.call(
['git', 'merge-base', '--is-ancestor', base_branch, 'HEAD'])
if exit_code == 0:
return base_branch
# Find the closest matching tag
base_tag = ''
if base_branch == '7.1.x' or base_branch == '7.1.x-private':
base_tag = git.describe('--tags', 'HEAD', '--abbrev=0',
'--match=fix-pack-dxp-*')
if base_tag is None or len(base_tag) == 0:
base_tag = git.describe('--tags', 'HEAD', '--abbrev=0',
'--match=fix-pack-base-71*')
if base_tag is None or len(base_tag) == 0:
base_tag = git.describe('--tags', 'HEAD', '--abbrev=0',
'--match=7.1.*-ga*')
elif base_branch == '7.0.x' or base_branch == '7.0.x-private':
base_tag = git.describe('--tags', 'HEAD', '--abbrev=0',
'--match=fix-pack-de-*')
if base_tag is None or len(base_tag) == 0:
base_tag = git.describe('--tags', 'HEAD', '--abbrev=0',
'--match=fix-pack-base-70*')
if base_tag is None or len(base_tag) == 0:
base_tag = git.describe('--tags', 'HEAD', '--abbrev=0',
'--match=7.0.*-ga*')
elif base_branch == 'ee-6.2.x':
base_tag = git.describe('--tags', 'HEAD', '--abbrev=0',
'--match=fix-pack-base-62*')
elif base_branch == 'ee-6.1.x':
base_tag = git.describe('--tags', 'HEAD', '--abbrev=0',
'--match=fix-pack-base-6130')
if base_tag.find('fix-pack-base-') == 0 or base_tag.find(
'fix-pack-de-') == 0 or base_tag.find(
'fix-pack-dxp-') == 0 or base_tag.find('-ga') > -1:
return base_tag
return base_branch
def getparent(check_tags):
if git_root is None:
return current_branch
# Find the current branch, accounting for detached head
ee_branches = ['6.2.x', '6.1.x', '6.0.x']
ee_branches = ee_branches + ['ee-%s' % branch for branch in ee_branches]
de_branches = ['7.0.x', '7.0.x-private', 'ee-7.0.x']
dxp_branches = ['7.3.x', '7.2.x', '7.1.x']
dxp_branches = dxp_branches + [
'%s-private' % branch for branch in dxp_branches
]
if current_branch == 'master' or current_branch in ee_branches + de_branches + dxp_branches:
return current_branch
# Extract the full version
full_version = None
if isfile(join(git_root, 'release.properties')):
full_version = get_file_property(join(git_root, 'release.properties'),
'lp.version')
elif isfile(join(git_root, 'build.properties')) and isfile(
join(git_root, 'app.server.properties')):
full_version = get_file_property(join(git_root, 'build.properties'),
'lp.version')
elif isfile(join(git_root, 'git-commit-portal')):
with open(join(git_root, 'git-commit-portal'), 'r') as file:
commit = file.readlines()[0].strip()
full_version = get_git_file_property(commit, 'release.properties',
'lp.version')
else:
return getparent_origin()
# If the short version is 6.x, then we have a shortcut
short_version = '.'.join(full_version.split('.')[0:2])
base_branch = None
if short_version == '6.0':
base_branch = 'ee-6.0.x'
elif short_version == '6.1':
base_branch = 'ee-6.1.x'
elif short_version == '6.2':
base_branch = 'ee-6.2.x'
else:
# Determine the base version using build.properties
if isfile(join(git_root, 'build.properties')):
base_branch = get_file_property(join(git_root, 'build.properties'),
'git.working.branch.name')
if base_branch is None:
if isfile(join(git_root, 'git-commit-portal')):
with open(join(git_root, 'git-commit-portal'), 'r') as file:
commit = file.readlines()[0].strip()
base_branch = get_git_file_property(
commit, 'build.properties', 'git.working.branch.name')
elif base_branch == 'master':
if git.for_each_ref(
'refs/remotes/upstream*/%s.x' % short_version) != '':
base_branch = '%s.x' % short_version
if base_branch is None:
base_branch = current_branch if current_branch != 'HEAD' else '7.0.x'
elif base_branch == 'ee-7.0.x':
base_branch = '7.0.x'
elif isdir(join(git_root, 'modules/private')) and len(
git.ls_files('build.properties').strip()
) == 0 and base_branch.find('-private') == -1:
base_branch = '%s-private' % base_branch
# If this is master or master-private, or we've recently rebased to 7.0.x or 7.0.x-private,
# then use the branch instead of the tag
if not check_tags:
return base_branch
# Find the closest matching tag
base_tag = None
if base_branch in dxp_branches or base_branch in de_branches:
base_tag = git.describe(
'HEAD', '--tags', '--abbrev=0',
'--match=fix-pack-*-%s%s10*' % (base_branch[0], base_branch[2]))
if base_tag is None or len(base_tag) == 0:
base_tag = git.describe(
'HEAD', '--tags', '--abbrev=0',
'--match=%s.%s.*-ga*' % (base_branch[0], base_branch[2]))
elif base_branch in ee_branches:
base_tag = git.describe('HEAD', '--tags', '--abbrev=0',
'--match=fix-pack-base-6%s*' % base_branch[5])
if base_tag is None or len(base_tag) == 0:
if base_branch.find('ee-') == 0:
base_tag = git.describe(
'HEAD', '--tags', '--abbrev=0',
'--match=%s.%s.*-ga*' % (base_branch[3], base_branch[5]))
else:
base_tag = git.describe(
'HEAD', '--tags', '--abbrev=0',
'--match=%s.%s.*-ga*' % (base_branch[0], base_branch[2]))
if base_tag is None:
return base_branch
if base_tag.find('fix-pack-base-') == 0 or base_tag.find(
'fix-pack-de-') == 0 or base_tag.find(
'fix-pack-dxp-') == 0 or base_tag.find('-ga') > -1:
return base_tag
return base_branch
