def test(): from mergify_engine import gh_pr from mergify_engine import utils utils.setup_logging() config.log() gh_pr.monkeypatch_github() parts = sys.argv[1].split("/") LOG.info("Getting repo %s ..." % sys.argv[1]) if True: # With access_token got from oauth token = sys.argv[2] g = github.Github(token) user = g.get_user(parts[3]) repo = user.get_repo(parts[4]) pull = repo.get_pull(int(parts[6])) update_branch(pull, token) else: # With access_token got from integration integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installation_id = utils.get_installation_id(integration, parts[3]) token = integration.get_access_token(installation_id).token update_branch(pull, "x-access-token:%s" % token)
def run(): parser = argparse.ArgumentParser() parser.add_argument("--clean", action="store_true") parser.add_argument("--dest", default="http://localhost:5000/event") args = parser.parse_args() utils.setup_logging() config.log() session = requests.Session() session.trust_env = False payload_data = os.urandom(250) payload_hmac = utils.compute_hmac(payload_data) if args.clean: r = session.delete( "https://gh.mergify.io/events-testing", data=payload_data, headers={"X-Hub-Signature": "sha1=" + payload_hmac}, ) r.raise_for_status() while True: try: resp = session.get( "https://gh.mergify.io/events-testing", data=payload_data, headers={"X-Hub-Signature": "sha1=" + payload_hmac}, ) events = resp.json() for event in reversed(events): LOG.info("") LOG.info("==================================================") LOG.info( ">>> GOT EVENT: %s %s/%s", event["id"], event["type"], event["payload"].get("state", event["payload"].get("action")), ) data = json.dumps(event["payload"]) hmac = utils.compute_hmac(data.encode("utf8")) session.post( args.dest, headers={ "X-GitHub-Event": event["type"], "X-GitHub-Delivery": event["id"], "X-Hub-Signature": "sha1=%s" % hmac, "Content-type": "application/json", }, data=data, verify=False, ) except Exception: LOG.error("event handling failure", exc_info=True) time.sleep(1)
def main(): utils.setup_logging() config.log() gh_pr.monkeypatch_github() if config.FLUSH_REDIS_ON_STARTUP: utils.get_redis().flushall() with rq.Connection(utils.get_redis()): worker = rq.Worker(['default']) worker.work()
def logger_checker(request, caplog): # daiquiri removes all handlers during setup, as we want to sexy output and the pytest # capability at the same, we must add back the pytest handler utils.setup_logging() logging.getLogger(None).addHandler(caplog.handler) yield for when in ("setup", "call", "teardown"): assert [] == [ rec.getMessage() for rec in caplog.get_records(when) if rec.levelname in ("CRITICAL", "ERROR") ]
def main(): # pragma: no cover utils.setup_logging() config.log() gh_pr.monkeypatch_github() r = utils.get_redis_for_rq() if config.FLUSH_REDIS_ON_STARTUP: r.flushall() with rq.Connection(r): worker = rq.Worker(['default']) if config.SENTRY_URL: client = raven.Client(config.SENTRY_URL, transport=HTTPTransport) register_sentry(client, worker) worker.work()
def main(): # pragma: no cover parser = argparse.ArgumentParser(description='Mergify RQ Worker.') parser.add_argument('--fqdn', help='FQDN of the node', default=utils.get_fqdn()) parser.add_argument("worker_id", type=int, help='Worker ID') args = parser.parse_args() utils.setup_logging() config.log() gh_pr.monkeypatch_github() MergifyWorker(args.fqdn, args.worker_id).work()
def main(): # pragma: no cover utils.setup_logging() LOG.info("Starting") prometheus_client.start_http_server(8889) LOG.info("Started") while True: try: collect_metrics() except Exception: # pragma: no cover LOG.error("Unexpected error during metrics gathering", exc_info=True) # Only generate metrics once per hour time.sleep(60 * 60)
def main(): # pragma: no cover utils.setup_logging() LOG.info("Starting") prometheus_client.start_http_server(8889) LOG.info("Started") while True: try: collect_metrics() except Exception as e: # pragma: no cover if exceptions.need_retry(e): LOG.warning("fail to gather metrics: %s", str(e)) time.sleep(10 * 60) continue else: LOG.error("Unexpected error during metrics gathering", exc_info=True) # Only generate metrics once per hour time.sleep(60 * 60)
# -*- encoding: utf-8 -*- # # Copyright © 2017 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from mergify_engine import utils from mergify_engine.web import app as application # noqa utils.setup_logging()
def setUp(self): super(FunctionalTestBase, self).setUp() self.pr_counter = 0 self.git_counter = 0 self.cassette_library_dir = os.path.join(CASSETTE_LIBRARY_DIR_BASE, self._testMethodName) # Recording stuffs if RECORD: if os.path.exists(self.cassette_library_dir): shutil.rmtree(self.cassette_library_dir) os.makedirs(self.cassette_library_dir) self.recorder = vcr.VCR( cassette_library_dir=self.cassette_library_dir, record_mode="all" if RECORD else "none", match_on=["method", "uri"], filter_headers=[ ("Authorization", "<TOKEN>"), ("X-Hub-Signature", "<SIGNATURE>"), ("User-Agent", None), ("Accept-Encoding", None), ("Connection", None), ], before_record_response=self.response_filter, custom_patches=((github.MainClass, "HTTPSConnection", vcr.stubs.VCRHTTPSConnection), ), ) self.useFixture( fixtures.MockPatchObject(branch_updater.utils, "Gitter", lambda: self.get_gitter())) self.useFixture( fixtures.MockPatchObject(duplicate_pull.utils, "Gitter", lambda: self.get_gitter())) # Web authentification always pass self.useFixture( fixtures.MockPatch("hmac.compare_digest", return_value=True)) reponame_path = os.path.join(self.cassette_library_dir, "reponame") if RECORD: REPO_UUID = str(uuid.uuid4()) with open(reponame_path, "w") as f: f.write(REPO_UUID) else: with open(reponame_path, "r") as f: REPO_UUID = f.read() self.name = "repo-%s-%s" % (REPO_UUID, self._testMethodName) utils.setup_logging() config.log() self.git = self.get_gitter() self.addCleanup(self.git.cleanup) web.app.testing = True self.app = web.app.test_client() # NOTE(sileht): Prepare a fresh redis self.redis = utils.get_redis_for_cache() self.redis.flushall() self.subscription = { "tokens": { "mergifyio-testing": config.MAIN_TOKEN }, "subscription_active": False, "subscription_cost": 100, "subscription_reason": "You're not nice", } self.redis.set( "subscription-cache-%s" % config.INSTALLATION_ID, sub_utils._encrypt(self.subscription), ) # Let's start recording cassette = self.recorder.use_cassette("http.json") cassette.__enter__() self.addCleanup(cassette.__exit__) integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) self.installation_token = integration.get_access_token( config.INSTALLATION_ID).token self.g_integration = github.Github(self.installation_token, base_url="https://api.%s" % config.GITHUB_DOMAIN) self.g_admin = github.Github(config.MAIN_TOKEN, base_url="https://api.%s" % config.GITHUB_DOMAIN) self.g_fork = github.Github(config.FORK_TOKEN, base_url="https://api.%s" % config.GITHUB_DOMAIN) self.o_admin = self.g_admin.get_organization( config.TESTING_ORGANIZATION) self.o_integration = self.g_integration.get_organization( config.TESTING_ORGANIZATION) self.u_fork = self.g_fork.get_user() assert self.o_admin.login == "mergifyio-testing" assert self.o_integration.login == "mergifyio-testing" assert self.u_fork.login == "mergify-test2" self.r_o_admin = self.o_admin.create_repo(self.name) self.r_o_integration = self.o_integration.get_repo(self.name) self.url_main = "https://%s/%s" % ( config.GITHUB_DOMAIN, self.r_o_integration.full_name, ) self.url_fork = "https://%s/%s/%s" % ( config.GITHUB_DOMAIN, self.u_fork.login, self.r_o_integration.name, ) # Limit installations/subscription API to the test account install = { "id": config.INSTALLATION_ID, "target_type": "Org", "account": { "login": "******" }, } self.useFixture( fixtures.MockPatch("mergify_engine.utils.get_installations", lambda integration: [install])) real_get_subscription = sub_utils.get_subscription def fake_retrieve_subscription_from_db(install_id): if int(install_id) == config.INSTALLATION_ID: return self.subscription else: return { "tokens": {}, "subscription_active": False, "subscription_reason": "We're just testing", } def fake_subscription(r, install_id): if int(install_id) == config.INSTALLATION_ID: return real_get_subscription(r, install_id) else: return { "tokens": {}, "subscription_active": False, "subscription_reason": "We're just testing", } self.useFixture( fixtures.MockPatch( "mergify_engine.branch_updater.sub_utils.get_subscription", side_effect=fake_subscription, )) self.useFixture( fixtures.MockPatch( "mergify_engine.branch_updater.sub_utils._retrieve_subscription_from_db", side_effect=fake_retrieve_subscription_from_db, )) self.useFixture( fixtures.MockPatch( "mergify_engine.sub_utils.get_subscription", side_effect=fake_subscription, )) self.useFixture( fixtures.MockPatch( "github.MainClass.Installation.Installation.get_repos", return_value=[self.r_o_integration], )) self._event_reader = EventReader(self.app) self._event_reader.drain()
def setUp(self): super(TestEngineScenario, self).setUp() self.cassette_library_dir = os.path.join(CASSETTE_LIBRARY_DIR_BASE, self._testMethodName) if RECORD_MODE != "none": if os.path.exists(self.cassette_library_dir): shutil.rmtree(self.cassette_library_dir) os.makedirs(self.cassette_library_dir) self.recorder = vcr.VCR( cassette_library_dir=self.cassette_library_dir, record_mode=RECORD_MODE, match_on=['method', 'uri'], filter_headers=[ ('Authorization', '<TOKEN>'), ('X-Hub-Signature', '<SIGNATURE>'), ('User-Agent', None), ('Accept-Encoding', None), ('Connection', None), ], before_record_response=self.response_filter, custom_patches=((github.MainClass, 'HTTPSConnection', vcr.stubs.VCRHTTPSConnection), )) self.useFixture( fixtures.MockPatchObject( branch_updater.utils, 'Gitter', lambda: GitterRecorder(self.cassette_library_dir))) self.useFixture( fixtures.MockPatchObject( backports.utils, 'Gitter', lambda: GitterRecorder(self.cassette_library_dir))) # Web authentification always pass self.useFixture( fixtures.MockPatch('hmac.compare_digest', return_value=True)) reponame_path = os.path.join(self.cassette_library_dir, "reponame") gen_new_uuid = (RECORD_MODE == 'all' or (RECORD_MODE == 'once' and not os.path.exists(reponame_path))) if gen_new_uuid: REPO_UUID = str(uuid.uuid4()) with open(reponame_path, "w") as f: f.write(REPO_UUID) else: with open(reponame_path, "r") as f: REPO_UUID = f.read() self.name = "repo-%s-%s" % (REPO_UUID, self._testMethodName) self.pr_counter = 0 self.remaining_events = [] utils.setup_logging() config.log() self.git = GitterRecorder(self.cassette_library_dir, "tests") self.addCleanup(self.git.cleanup) web.app.testing = True self.app = web.app.test_client() # NOTE(sileht): Prepare a fresh redis self.redis = utils.get_redis_for_cache() self.redis.flushall() subscription = {"token": config.MAIN_TOKEN, "subscribed": False} self.redis.set("subscription-cache-%s" % config.INSTALLATION_ID, json.dumps(subscription)) # Let's start recording cassette = self.recorder.use_cassette("http.json") cassette.__enter__() self.addCleanup(cassette.__exit__) self.session = requests.Session() self.session.trust_env = False # Cleanup the remote testing redis r = self.session.delete( "https://gh.mergify.io/events-testing", data=FAKE_DATA, headers={"X-Hub-Signature": "sha1=" + FAKE_HMAC}) r.raise_for_status() self.g_main = github.Github(config.MAIN_TOKEN) self.g_fork = github.Github(config.FORK_TOKEN) self.u_main = self.g_main.get_user() self.u_fork = self.g_fork.get_user() assert self.u_main.login == "mergify-test1" assert self.u_fork.login == "mergify-test2" self.r_main = self.u_main.create_repo(self.name) self.url_main = "https://github.com/%s" % self.r_main.full_name self.url_fork = "https://github.com/%s/%s" % (self.u_fork.login, self.r_main.name) integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) access_token = integration.get_access_token( config.INSTALLATION_ID).token g = github.Github(access_token) user = g.get_user("mergify-test1") repo = user.get_repo(self.name) # Used to access the cache with its helper self.engine = engine.MergifyEngine(g, config.INSTALLATION_ID, access_token, subscription, user, repo) self.processor = self.engine.get_processor() self.rq_worker = rq.SimpleWorker([ "incoming-events", "localhost-000-high", "localhost-001-high", "localhost-000-low", "localhost-001-low" ], connection=utils.get_redis_for_rq()) if self._testMethodName != "test_creation_pull_of_initial_config": self.git("init") self.git.configure() self.git.add_cred(config.MAIN_TOKEN, "", self.r_main.full_name) self.git.add_cred(config.FORK_TOKEN, "", "%s/%s" % (self.u_fork.login, self.r_main.name)) self.git("config", "user.name", "%s-tester" % config.CONTEXT) self.git("remote", "add", "main", self.url_main) self.git("remote", "add", "fork", self.url_fork) with open(self.git.tmp + "/.mergify.yml", "w") as f: f.write(CONFIG) self.git("add", ".mergify.yml") self.git("commit", "--no-edit", "-m", "initial commit") self.git("push", "--quiet", "main", "master") self.git("checkout", "-b", "stable", "--quiet") self.git("push", "--quiet", "main", "stable") self.git("checkout", "-b", "nostrict", "--quiet") self.git("push", "--quiet", "main", "nostrict") self.git("checkout", "-b", "disabled", "--quiet") self.git("push", "--quiet", "main", "disabled") self.git("checkout", "-b", "enabling_label", "--quiet") self.git("push", "--quiet", "main", "enabling_label") self.r_fork = self.u_fork.create_fork(self.r_main) self.git("fetch", "--quiet", "fork") # NOTE(sileht): Github looks buggy here: # We receive for the new repo the expected events: # * installation_repositories # * integration_installation_repositories # but we receive them 6 times with the same sha1... self.push_events([(None, {"action": "added"})] * 12)
def celery_logging(**kwargs): # pragma: no cover utils.setup_logging() config.log()
def setUp(self): super(TestEngineScenario, self).setUp() self.session = requests.Session() self.session.trust_env = False self.recorder = betamax.Betamax(self.session) self.useFixture( fixtures.MockPatchObject(requests, 'Session', return_value=self.session)) self.useFixture( fixtures.MockPatchObject( gh_update_branch.utils, 'Gitter', lambda: GitterRecorder(self._testMethodName))) # Web authentification always pass self.useFixture( fixtures.MockPatch('hmac.compare_digest', return_value=True)) reponame_path = ( "mergify_engine/tests/fixtures/cassettes/reponame_%s" % self._testMethodName) gen_new_uuid = (RECORD_MODE == 'all' or (RECORD_MODE == 'once' and not os.path.exists(reponame_path))) if gen_new_uuid: REPO_UUID = str(uuid.uuid4()) with open(reponame_path, "w") as f: f.write(REPO_UUID) else: with open(reponame_path, "r") as f: REPO_UUID = f.read() self.name = "repo-%s-%s" % (REPO_UUID, self._testMethodName) self.cassette_counter = 0 self.events_handler_counter = 0 self.events_getter_counter = 0 self.status_counter = 0 self.reviews_counter = 0 self.pr_counter = 0 self.last_event_id = None utils.setup_logging() config.log() self.git = GitterRecorder('%s-tests' % self._testMethodName) self.addCleanup(self.git.cleanup) web.app.testing = True self.app = web.app.test_client() # NOTE(sileht): Prepare a fresh redis self.redis = utils.get_redis() self.redis.flushall() self.redis.set("installation-token-%s" % INSTALLATION_ID, MAIN_TOKEN) with self.cassette("setUp-prepare-repo", allow_playback_repeats=True): # Cleanup the remote testing redis r = self.session.delete("https://gh.mergify.io/events-testing") r.raise_for_status() self.g_main = github.Github(MAIN_TOKEN) self.g_fork = github.Github(FORK_TOKEN) self.u_main = self.g_main.get_user() self.u_fork = self.g_fork.get_user() assert self.u_main.login == "mergify-test1" assert self.u_fork.login == "mergify-test2" self.r_main = self.u_main.create_repo(self.name) self.url_main = "https://%s:@github.com/%s" % ( MAIN_TOKEN, self.r_main.full_name) self.git("init") self.git("config", "user.name", "%s-bot" % config.CONTEXT) self.git("config", "user.email", "*****@*****.**") self.git("remote", "add", "main", self.url_main) with open(self.git.tmp + "/.mergify.yml", "w") as f: f.write(CONFIG) self.git("add", ".mergify.yml") self.git("commit", "--no-edit", "-m", "initial commit") self.git("push", "main", "master") self.git("checkout", "-b", "stable") self.git("push", "main", "stable") self.git("checkout", "-b", "nostrict") self.git("push", "main", "nostrict") self.r_fork = self.u_fork.create_fork(self.r_main) self.url_fork = "https://%s:@github.com/%s" % ( FORK_TOKEN, self.r_fork.full_name) self.git("remote", "add", "fork", self.url_fork) self.git("fetch", "fork") with self.cassette("setUp-login-engine", allow_playback_repeats=True): g = github.Github(ACCESS_TOKEN) user = g.get_user("mergify-test1") repo = user.get_repo(self.name) with self.cassette("setUp-create-engine", allow_playback_repeats=True): self.engine = engine.MergifyEngine(g, INSTALLATION_ID, user, repo) self.useFixture( fixtures.MockPatchObject(worker, 'real_event_handler', self.engine.handle)) queue = rq.Queue(connection=self.redis) self.rq_worker = rq.SimpleWorker([queue], connection=queue.connection) # NOTE(sileht): Github looks buggy here: # We receive for the new repo the expected events: # * installation_repositories # * integration_installation_repositories # but we receive them 6 times with the same sha1... self.push_events(12)