def process_local_test_settings(args, fileutils=FileUtils()):
def process_local_test_settings_closure(logger):
logger.info('Preparing local test settings for your new instance...')
template = os.sep.join([
'jira-func-tests', 'src', 'main', 'resources', 'localtest.template'
])
template_renderings = {
'jira-func-tests':
'jira-func-tests',
'jira-webdriver-tests':
'jira-webdriver-tests',
os.sep.join(['jira-distribution', 'jira-integration-tests']):
'jira-func-tests'
}
for project, xml_location in template_renderings.items():
dir = fileutils.existing_dir(
os.sep.join(['.', project, 'src', 'main', 'resources']))
dest = os.sep.join([dir, 'localtest.properties'])
# just for unit tests this settings dict is not reused:
settings = {
'${jira.port}': str(args.port),
'${jira.context}': args.jira_context,
'${test.xml.location}': PathUtils.abspath(xml_location)
}
logger.debug('Processing ' + template + ' to ' + dest)
fileutils.filter_file(template, dest, settings)
return Callable.success
return process_local_test_settings_closure
def __init__(self,
args=object,
path=None,
process_utils=ProcessUtils(),
file_utils=FileUtils()):
super().__init__(args, cwd=path)
self.maven_version = None
self.projects = []
self.profiles = []
self.phases = []
self.properties = dict()
self.options = []
self.parallel = False
self.process_utils = process_utils
self.project_prefix = ""
options_from_args = {
'mvn_offline': '-o',
'mvn_update': '-U',
'mvn_debug': '-X',
'mvn_errors': '-e'
}
for key in options_from_args.keys():
if getattr(args, key, False):
self.option(options_from_args[key])
if getattr(args, 'mvn_clean', False):
self.phase('clean')
def __init__(self, fs: FileUtils = FileUtils(),
xml: XmlUtils = XmlUtils()):
super().__init__()
self.roots = []
self.fs = fs
self.xml = xml
self.report_header_logged = False
def __init__(self, fileutils=FileUtils()):
super().__init__(fileutils)
self.command = 'cleanall'
self.description = 'Most thorough clean provided by jmake. '
self.prevent_post_commands = True
self.fileutils = fileutils
self.check_branch = False
def create_jmake_credentials_file(args, fs: FileUtils = FileUtils()):
def jmake_user_credentials_closure(logger):
if fs.file_exists(PGPASS_FILE):
with open(PGPASS_FILE, 'r') as f:
for line in f:
if PGPASS_CREDENTIALS in line:
logger.info(
'Password for user "%s" already exist in %s' %
(DB_USER, PGPASS_FILE))
return Callable.success
logger.info('Adding default password for db user "%s"... ' %
(DB_USER, ))
try:
with open(PGPASS_FILE, 'a') as f:
f.write(PGPASS_CREDENTIALS)
f.write('\n')
# postgres requires this file is with mode 0600 otherwise it won't trust it:
rc = SystemCallable(args,
'chmod 0600 %s' % PGPASS_FILE)(logger)
return rc
except Exception as e:
logger.error(
'Could not add default jmkae credentials to % file.' %
PGPASS_FILE)
return Callable.do_not_proceed
return jmake_user_credentials_closure
def process_webdriver(args, executor, fileutils=FileUtils()):
maven = MavenCallable()
maven.project('jira-distribution/jira-webdriver-tests-runner').option(
'-am')
maven.phase('verify').profile('hallelujahClient').profile(
'distribution')
maven.property('maven.test.unit.skip', 'true')
if fileutils.file_exists(args.replay_file):
replay_file = os.path.abspath(args.replay_file)
else:
replay_file = os.path.abspath(
os.sep.join(['target', 'replays', args.replay_file]))
maven.property('jira.user.extra.jvmargs',
'"-Dhallelujah.local.test.list=%s"' % replay_file)
maven.property('func.mode.plugins').property('reference.plugins')
maven.property(
'system.bamboo.agent.home',
os.path.abspath(
os.sep.join([
'jira-distribution', 'jira-webdriver-tests-runner',
'target'
])))
executor.append(maven.can_run_in_parallel())
def __call__(self, logger, fileutils=FileUtils()):
if not getattr (self.inspector, 'inconsistent', False):
target_dir = fileutils.existing_dir('target')
file = os.sep.join([target_dir, '.jmake-branch'])
with open(file, 'w', encoding='utf-8') as file:
file.write(self.inspector.branch)
return Callable.success
def __hack_maven_settings(fs: FileUtils = FileUtils()):
# attempt to locate maven by running atlas-version
oneliner = OneLiner(None, 'atlas-version | grep "ATLAS Maven"')
oneliner(LOG)
if oneliner.returncode != Callable.success:
return None
maven_home = oneliner.line[oneliner.line.find(':') + 2:]
target_dir = fs.existing_dir(os.sep.join(['.', 'jmake_src', 'target']))
if not fs.file_exists(os.sep.join([target_dir, 'Decipher.class'])):
oneliner = OneLiner(
None,
'$JAVA_HOME/bin/javac -cp %s:. -d %s Decipher.java' %
(os.sep.join([maven_home, 'lib', '*']), target_dir),
cwd=os.sep.join(['.', 'jmake_src', 'jbac', 'java']))
oneliner(LOG)
oneliner = OneLiner(
None, '$JAVA_HOME/bin/java -cp %s:%s Decipher' %
(os.sep.join([maven_home, 'lib', '*']), target_dir))
oneliner(LOG)
if oneliner.returncode != Callable.success:
return None
credentials = oneliner.line.split(':')
return JbacAuthentication(credentials[0], credentials[1])
def __init__(self):
super().__init__()
logging.info('@ Start Auto Refresh Service ... ')
fu = FileUtils()
self.config = fu.load_yaml('od_config.yaml')['order']
self.app = atomacos.getAppRefByBundleId(self.config['appId'])
self.window = self.app.windows()[0]
self.button = self.window.buttons(self.config['defaultCheckBoxTitle'])[0]
def __init__(self, fileutils=FileUtils()):
JmakeModule.__init__(self)
self.command = 'replay'
self.description = 'Replay JBAC failure locally. The most common use case is to replay a file prepared by ' \
'"ci investigate". The autocomplete should provide these filenames for your convenience. ' \
'You can use this to create a "local hallelujah client" to which you can supply a file ' \
'containing tests to run and it will run them as if they were read from the JMS queue.'
self.fileutils = fileutils
def ensure_project_was_opened(logger, fileutils=FileUtils()):
idea_dir = '.%s.idea' % os.sep
if fileutils.dir_exists(idea_dir):
return Callable.success
else:
logger.error(
'Your IDEA directory (%s) is missing. Open the JIRA project by pointing the pom.xml '
'and rerun jmake idea.' % idea_dir)
return Callable.do_not_proceed
def idea_templates(executor, args, fileutils=FileUtils()):
executor.append(
lambda logger: logger.info('Setting up code style templates...') or 0)
executor.append(
MavenCallable(args).project('jira-ide-support').profile('ide-setup').
profile('idea-templates').option('-am').phase('generate-sources'))
executor.append(process_project_local_settings(args))
def __init__(self, fileutils=FileUtils(), surefire: Surefire=None, process_utils=ProcessUtils()):
super().__init__()
self.command = 'unit-tests'
self.description = 'Runs JIRA Unit Tests and verifies dependencies. This can also be used to run the findbugs '\
'analysis on code covered with Unit Tests. This runs unit tests from the JIRA core, '\
'unit tests of the func test module and from bundled plugins modules.'
self.fileutils = fileutils
self.surefire = surefire if surefire else Surefire()
self.process_utils = process_utils
def run(self):
## 1.构建所有url
urlList = self.getAllUrl()
## 2.获取页面数据
fileUtils = FileUtils()
for url in urlList:
htmlContent = self.getHtmlInfo(url)
content = self.parseHtmlContent(htmlContent)
## 3.保存文件
fileUtils.saveFile("qiushibaike.txt", content, "a")
def __init__(self, fileutils=FileUtils()):
super().__init__()
self.command = 'clean'
self.description = 'Cleans built artifacts in workspace. Provides options for cleaning downloaded tomcat '\
'binaries, JIRA home directory or modules that are not enabled. Unless you use the "deep '\
'clean" option, this will generate sources you need for proper compilation after cleaning '\
'target directories.'
self.prevent_post_commands = True
self.fileutils = fileutils
self.check_branch = False
def __call__(self, logger, file_utils: FileUtils = FileUtils()):
svndir = self.args.layout.studio_svn_dir()
svnlink = self.args.layout.studio_svn_link()
if not file_utils.file_exists(svnlink):
# create symlink to svndir
logger.info("Creating symlink: %s -> %s" % (svnlink, svndir))
file_utils.symlink(svndir, svnlink)
return Callable.success
def get_workspace_projects(self, file_utils=FileUtils()):
#get parent directory of jira-project (where jira-project dir is in)
jira_workspace_dir = self.get_jira_workspace_dir(file_utils)
#look for poms in directories siblings to jira-project;
#maybe in the future it should parse workspace pom.xml for that...
return [
dir for dir in file_utils.listdir(jira_workspace_dir)
if file_utils.dir_exists(os.sep.join([jira_workspace_dir, dir]))
and file_utils.file_exists(
os.sep.join([jira_workspace_dir, dir, "pom.xml"]))
]
def process_clean(logger, fileutils=FileUtils()):
logger.info('Removing old run configurations...')
run_cfg_dir = os.sep.join(['.', '.idea', 'runConfigurations'])
legacy_run_configs = [
'Func_Tests', 'JIRA_OnDemand', 'Tomcat_6', 'Tomcat_7',
'Unit_Tests__Fast_', 'Unit_Tests__Legacy_', 'WebDriver_Tests',
'Selenium_Tests'
]
for file in (run_cfg_dir + os.sep + item + '.xml'
for item in legacy_run_configs):
try:
if fileutils.file_exists(file):
fileutils.remove(file)
logger.debug('Removed file: ' + file)
except IOError:
logger.error('Could not remove file:' + file)
return Callable.success - 4
logger.info('Removing old artifacts...')
old_artifact = os.sep.join(['.', '.idea', 'artifacts', 'JIRA.xml'])
try:
if fileutils.file_exists(old_artifact):
fileutils.remove(old_artifact)
logger.debug('Deleted: ' + old_artifact)
except IOError:
logger.error('Could not remove file: ' + old_artifact)
return Callable.success - 4
logger.info('Removing target dirs for old artifacts...')
for target_dir in (os.sep.join(['.', item])
for item in ['classes', 'tomcatBase']):
try:
if fileutils.dir_exists(target_dir):
fileutils.remove_dir(target_dir)
logger.debug('Removed directory: ' + target_dir)
except IOError:
logger.error('Could not remove directory:' + target_dir)
return Callable.success - 4
logger.info('Removing jira.idea.properties...')
legacy_idea_properties = os.sep.join([
'jira-ide-support', 'src', 'main', 'resources', 'jira.idea.properties'
])
try:
if fileutils.file_exists(legacy_idea_properties):
fileutils.remove(legacy_idea_properties)
logger.debug('Deleted: ' + legacy_idea_properties)
except IOError:
logger.error('Could not remove file: ' + legacy_idea_properties)
return Callable.success - 4
return Callable.success
def __init__(self,
metrics_name: str,
file_filter,
description: str = None,
file_utils: FileUtils = FileUtils(),
metrics_logger: MetricsLogger = None):
super().__init__(metrics_name, description, metrics_logger)
self.file_filter = file_filter
self.count = 0
self.size = 0
self.file_utils = file_utils
self.metric_key_count = '.'.join([self.key, 'count'])
self.metric_key_cumulativesize = '.'.join([self.key, 'cumulativesize'])
def __call__(self, logger, fileutils=FileUtils()):
file_name = os.sep.join(['.', 'target', '.jmake-branch'])
try:
with fileutils.open(file_name, 'r', encoding='utf-8') as file:
previous_branch = file.readline().rstrip('\n')
if not previous_branch == self.inspector.branch:
logger.error('jmake detected that it was previously run on a different branch (%s) current branch: (%s).' % (previous_branch, self.inspector.branch))
logger.error('It is advised to clean the old targets.')
logger.info('To override: rm ' + file_name)
self.inspector.inconsistent = True
return Callable.do_not_proceed
except IOError:
pass
return Callable.success
def __init__(self,
git: GitUtils = GitUtils(),
fs: FileUtils = FileUtils(),
metrics_processor=MetricsProcessor(),
json_writer=JsonWriter()):
super().__init__()
self.command = 'eh-metrics'
self.prevent_post_diagnostics = True
self.description = 'Gathers engineering health metrics from the JIRA code. If any violations occur, you will be ' \
'able to add exclusion information interactively. You should pull all latest changes ' \
'before you run this.'
self.git = git
self.fs = fs
self.metrics_processor = metrics_processor
self.json_writer = json_writer
def __init__(self,
args=object,
file_utils: FileUtils = FileUtils(),
process_utils=ProcessUtils(),
horde_status_checker=None,
background=True):
super().__init__(args, args.horde_layout.horde_home_dir(False),
process_utils, file_utils)
self.horde_layout = args.horde_layout
if background:
self.background()
self.redirect_output(self.horde_layout.horde_log_file())
self.file_utils = file_utils
self.horde_status_checker = horde_status_checker if horde_status_checker is not None else HordeStatusChecker(
self.horde_layout)
def process_jmake_module(args, fileutils=FileUtils(), xml=XmlUtils()):
def process_jmake_module_closure(logger):
logger.info('Installing jmake dev module...')
jmake_module_file = os.sep.join(['.', 'jmake_src', 'jmake_src.iml'])
if not fileutils.file_exists(jmake_module_file) or args.force:
# module file does not exist: create it and add it to the module list.
jmake_module_template = os.sep.join([
'jira-ide-support', 'src', 'main', 'resources',
'ideaTemplates', 'jmake', 'jmake_src.iml.template'
])
logger.debug('Copying %s to %s' %
(jmake_module_template, jmake_module_file))
fileutils.copy_file(jmake_module_template, jmake_module_file)
workspace_modules_file = os.sep.join(['.', '.idea', 'modules.xml'])
try:
workspace_tree = xml.parse(workspace_modules_file)
except IOError:
logger.error('Could not open ' + workspace_modules_file)
return Callable.success - 1
module = xml.produce(
workspace_tree.getroot(), ('component', {}), ('modules', {}),
('module', {
'filepath':
os.sep.join(
['$PROJECT_DIR$', 'jmake_src', 'jmake_src.iml'])
}))
if not 'fileurl' in module.attrib:
logger.debug('Adding module entry to %s' %
workspace_modules_file)
module.attrib[
'fileurl'] = 'file://$PROJECT_DIR$/jmake_src/jmake_src.iml'
try:
#os.rename(workspace_file, workspace_file + '.jmake_backup')
workspace_tree.write(workspace_modules_file)
logger.debug('Saved successfully.')
except IOError:
logger.error('Could not save ' + workspace_modules_file)
return Callable.success - 2
return Callable.success
return process_jmake_module_closure
def __init__(self, fast: bool, file_utils: FileUtils=FileUtils()):
collectors = [JavaPackageImports('usageof.easymock',
'Get rid of easymock and use Mockito',
'org.easymock'),
JavaPackageImports('usageof.mockobjects',
'Get rid of mockobjects and use Mockito',
'com.mockobjects'),
JunitAssertWithoutMessage('usageof.assert.true.false',
'Do not use assertTrue or assertFalse without specifying a message')
.unchecked(),
ModulesDescription.commonJUnitFinder,
WebTestFinder().unchecked().neutral(),
InvalidTimedQueries()]
super().__init__(collectors, 'Scan JIRA test files')
self.file_utils = file_utils
def __init__(self,
urlutils=UrlUtils(),
xml=XmlUtils(),
fileutils=FileUtils()):
JmakeModule.__init__(self)
self.command = 'investigate'
self.description = 'Investigates JBAC failures and prepares to reproduce them locally. This command will ' \
'produce a file, that will contain information what tests failed and what should be done ' \
'to replay what happened locally. By default, the failed tests will be accompanied with ' \
'5 tests that preceded it on the particular run in an attempt to repliacate the way the ' \
'instance might have been corrupted by previous tests (which is random on regular rerun ' \
'due to how hallelujah works).'
self.urlutils = urlutils
self.xml = xml
self.fileutils = fileutils
self.replays_dir = self.fileutils.existing_dir(
os.sep.join(['target', 'replays']))
def _change_params_for_workspace(self,
args=None,
path=None,
process_utils=ProcessUtils(),
file_utils=FileUtils()):
if args is None:
return f(self, args, path, process_utils, file_utils)
else:
if getattr(args, 'with_workspace', False) and path is None:
path = file_utils.get_parent_dir_path()
result = f(self, args, path, process_utils, file_utils)
if getattr(args, 'with_workspace', False):
MavenCallableWorkspaceDelegator.after_init(
self, path, file_utils)
return result
def __init__(self,
fs: FileUtils = FileUtils(),
url: UrlUtils = UrlUtils(),
tar_utils=TarUtils(),
bamboo_utils: BambooUtils = BambooUtils()):
super().__init__()
self.command = 'eh-metrics-investigate'
self.prevent_post_diagnostics = True
self.description = wrap_text(
'Finds the reason for failing engineering health metrics. Should work '
'automatically, comparing the latest result of ./jmake eh-metrics with the '
'latest tagged CI build. You should run this command immediately after failing '
'./jmake eh-metrics, and with no parameters. The parameters available to this '
'command are mostly for debug or tricky usages.')
self.fs = fs
self.url = url
self.tar_utils = tar_utils
self.bamboo_utils = bamboo_utils
def __init__(self,
tomcat,
version,
tomcat_directory,
jira_home_dir,
instance_name='work',
ondemand=False,
file_utils=FileUtils()):
self.tomcat_directory = tomcat_directory
self.tomcat_family = tomcat
self.tomcat_version = version
self.file_utils = file_utils
self.instance_name = instance_name
self.ondemand = ondemand
if jira_home_dir == WorkspaceLayout.JIRA_HOME_DEFAULT_MARKER:
self.jira_home_dir = WorkspaceLayout.JIRA_OD_HOME if ondemand else WorkspaceLayout.JIRA_HOME
else:
self.jira_home_dir = jira_home_dir
def __assert_workspace_build_params(self, executor):
self.assertEqual(len(executor), 11, "Should have added tasks to execute")
self.assertIsInstance(executor[0], WorkspaceValidator)
self.assertIsInstance(executor[1], LocalPortInspector)
self.assertIsInstance(executor[2], LocalPortInspector)
self.assertIsInstance(executor[3], types.FunctionType)
self.assertIsInstance(executor[4], MavenCallable)
maven_callable = executor[5]
self.assertIsInstance(maven_callable, MavenCallable)
self.assertIsInstance(executor[6], types.LambdaType)
self.assertIsInstance(executor[7], TomcatDownloader)
self.assertIsInstance(executor[8], ContextPreparer)
self.assertIsInstance(executor[9], types.FunctionType)
self.assertIsInstance(executor[10], TomcatStarter)
self.__assert_property(maven_callable, 'skipTests', 'true')
self.assertListEqual(maven_callable.options, ['-am'])
self.assertIn('package', maven_callable.phases)
self.assertIn(os.sep.join([FileUtils().get_current_dir_name(),'jira-components/jira-webapp']), maven_callable.projects)
return maven_callable
def plugin_resources(self,
fileutils=FileUtils(),
workspace_utils=WorkspaceUtils()):
if self.args.disable_plugin_resources:
return ''
resources_marker = os.sep.join(['src', 'main', 'resources'])
resources_marker_exclude = '.hg'
plugins_roots = [os.sep.join(['jira-components', 'jira-plugins'])]
if self.args.plugin_resources:
plugins_roots.append(fileutils.abs_path(
self.args.plugin_resources))
plugins_roots.extend(
os.sep.join([workspace_utils.get_jira_workspace_dir(), project])
for project in workspace_utils.get_workspace_projects_without_jira(
self.args))
return '-Dplugin.resource.directories=%s' % ','.join(
PathUtils.abspath(path) for plugins_root in plugins_roots
for (path, dirs, files) in fileutils.walk(plugins_root)
if path.endswith(resources_marker)
and not resources_marker_exclude in path)
