def test(): import github from mergify_engine import gh_pr from mergify_engine import utils utils.setup_logging() config.log() gh_pr.monkeypatch_github() parts = sys.argv[1].split("/") LOG.info("Getting repo %s ..." % sys.argv[1]) integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installation_id = utils.get_installation_id(integration, parts[3]) token = integration.get_access_token(installation_id).token g = github.Github(token) user = g.get_user(parts[3]) repo = user.get_repo(parts[4]) LOG.info("Protecting repo %s branch %s ..." % (sys.argv[1], sys.argv[2])) rule = rules.get_branch_rule(repo, sys.argv[2]) configure_protection_if_needed(repo, sys.argv[2], rule)
def setup_logging(): outputs = [] if config.LOG_STDOUT: outputs.append( daiquiri.output.Stream( sys.stdout, formatter=CustomFormatter(fmt=CELERY_EXTRAS_FORMAT) ) ) if config.LOG_JSON_FILE: outputs.append( daiquiri.output.TimedRotatingFile( filename=config.LOG_JSON_FILE, level=logging.DEBUG, interval=datetime.timedelta(seconds=5), backup_count=48, formatter=daiquiri.formatter.JSON_FORMATTER, ) ) daiquiri.setup( outputs=outputs, level=(logging.DEBUG if config.DEBUG else logging.INFO), ) daiquiri.set_default_log_levels( [ ("celery", "INFO"), ("kombu", "WARN"), ("github.Requester", "WARN"), ("urllib3.connectionpool", "WARN"), ("vcr", "WARN"), ] ) config.log()
def test(): from mergify_engine import gh_pr from mergify_engine import utils utils.setup_logging() config.log() gh_pr.monkeypatch_github() parts = sys.argv[1].split("/") LOG.info("Getting repo %s ..." % sys.argv[1]) if True: # With access_token got from oauth token = sys.argv[2] g = github.Github(token) user = g.get_user(parts[3]) repo = user.get_repo(parts[4]) pull = repo.get_pull(int(parts[6])) update_branch(pull, token) else: # With access_token got from integration integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) installation_id = utils.get_installation_id(integration, parts[3]) token = integration.get_access_token(installation_id).token update_branch(pull, "x-access-token:%s" % token)
def setup_logging(): outputs = [] if config.LOG_STDOUT: outputs.append( daiquiri.output.Stream( sys.stdout, formatter=CustomFormatter(fmt=CELERY_EXTRAS_FORMAT), level=config.LOG_STDOUT_LEVEL, ) ) if config.LOG_DATADOG: outputs.append(daiquiri.output.Datadog()) daiquiri.setup( outputs=outputs, level=(logging.DEBUG if config.DEBUG else logging.INFO), ) daiquiri.set_default_log_levels( [ ("celery", "INFO"), ("kombu", "WARN"), ("github.Requester", "WARN"), ("urllib3.connectionpool", "WARN"), ("urllib3.util.retry", "WARN"), ("vcr", "WARN"), ("httpx", "WARN"), ("cachecontrol", "WARN"), ] ) config.log()
def run(): parser = argparse.ArgumentParser() parser.add_argument("--clean", action="store_true") parser.add_argument("--dest", default="http://localhost:5000/event") args = parser.parse_args() utils.setup_logging() config.log() session = requests.Session() session.trust_env = False payload_data = os.urandom(250) payload_hmac = utils.compute_hmac(payload_data) if args.clean: r = session.delete( "https://gh.mergify.io/events-testing", data=payload_data, headers={"X-Hub-Signature": "sha1=" + payload_hmac}, ) r.raise_for_status() while True: try: resp = session.get( "https://gh.mergify.io/events-testing", data=payload_data, headers={"X-Hub-Signature": "sha1=" + payload_hmac}, ) events = resp.json() for event in reversed(events): LOG.info("") LOG.info("==================================================") LOG.info( ">>> GOT EVENT: %s %s/%s", event["id"], event["type"], event["payload"].get("state", event["payload"].get("action")), ) data = json.dumps(event["payload"]) hmac = utils.compute_hmac(data.encode("utf8")) session.post( args.dest, headers={ "X-GitHub-Event": event["type"], "X-GitHub-Delivery": event["id"], "X-Hub-Signature": "sha1=%s" % hmac, "Content-type": "application/json", }, data=data, verify=False, ) except Exception: LOG.error("event handling failure", exc_info=True) time.sleep(1)
def main(): utils.setup_logging() config.log() gh_pr.monkeypatch_github() if config.FLUSH_REDIS_ON_STARTUP: utils.get_redis().flushall() with rq.Connection(utils.get_redis()): worker = rq.Worker(['default']) worker.work()
def prepare_service(): # pragma: no cover setup_logging() config.log() if config.SENTRY_URL: sentry_client = raven.Client(config.SENTRY_URL, transport=HTTPTransport) handler = raven_logging.SentryHandler(client=sentry_client) handler.setLevel(logging.ERROR) logging.getLogger(None).addHandler(handler) return sentry_client
def main(): # pragma: no cover parser = argparse.ArgumentParser(description='Mergify RQ Worker.') parser.add_argument('--fqdn', help='FQDN of the node', default=utils.get_fqdn()) parser.add_argument("worker_id", type=int, help='Worker ID') args = parser.parse_args() utils.setup_logging() config.log() gh_pr.monkeypatch_github() MergifyWorker(args.fqdn, args.worker_id).work()
def main(): # pragma: no cover utils.setup_logging() config.log() gh_pr.monkeypatch_github() r = utils.get_redis_for_rq() if config.FLUSH_REDIS_ON_STARTUP: r.flushall() with rq.Connection(r): worker = rq.Worker(['default']) if config.SENTRY_URL: client = raven.Client(config.SENTRY_URL, transport=HTTPTransport) register_sentry(client, worker) worker.work()
def setup_logging(): daiquiri.setup( outputs=[ daiquiri.output.Stream( sys.stdout, formatter=CustomFormatter(fmt=CELERY_EXTRAS_FORMAT)) ], level=(logging.DEBUG if config.DEBUG else logging.INFO), ) daiquiri.set_default_log_levels([ ("celery", "INFO"), ("kombu", "WARN"), ("github.Requester", "WARN"), ("urllib3.connectionpool", "WARN"), ("vcr", "WARN"), ]) config.log()
def prepare_service(): # pragma: no cover setup_logging() config.log() return get_sentry_client()
def setUp(self): super(FunctionalTestBase, self).setUp() self.pr_counter = 0 self.git_counter = 0 self.cassette_library_dir = os.path.join(CASSETTE_LIBRARY_DIR_BASE, self._testMethodName) # Recording stuffs if RECORD: if os.path.exists(self.cassette_library_dir): shutil.rmtree(self.cassette_library_dir) os.makedirs(self.cassette_library_dir) self.recorder = vcr.VCR( cassette_library_dir=self.cassette_library_dir, record_mode="all" if RECORD else "none", match_on=["method", "uri"], filter_headers=[ ("Authorization", "<TOKEN>"), ("X-Hub-Signature", "<SIGNATURE>"), ("User-Agent", None), ("Accept-Encoding", None), ("Connection", None), ], before_record_response=self.response_filter, custom_patches=((github.MainClass, "HTTPSConnection", vcr.stubs.VCRHTTPSConnection), ), ) self.useFixture( fixtures.MockPatchObject(branch_updater.utils, "Gitter", lambda: self.get_gitter())) self.useFixture( fixtures.MockPatchObject(duplicate_pull.utils, "Gitter", lambda: self.get_gitter())) # Web authentification always pass self.useFixture( fixtures.MockPatch("hmac.compare_digest", return_value=True)) reponame_path = os.path.join(self.cassette_library_dir, "reponame") if RECORD: REPO_UUID = str(uuid.uuid4()) with open(reponame_path, "w") as f: f.write(REPO_UUID) else: with open(reponame_path, "r") as f: REPO_UUID = f.read() self.name = "repo-%s-%s" % (REPO_UUID, self._testMethodName) utils.setup_logging() config.log() self.git = self.get_gitter() self.addCleanup(self.git.cleanup) web.app.testing = True self.app = web.app.test_client() # NOTE(sileht): Prepare a fresh redis self.redis = utils.get_redis_for_cache() self.redis.flushall() self.subscription = { "tokens": { "mergifyio-testing": config.MAIN_TOKEN }, "subscription_active": False, "subscription_cost": 100, "subscription_reason": "You're not nice", } self.redis.set( "subscription-cache-%s" % config.INSTALLATION_ID, sub_utils._encrypt(self.subscription), ) # Let's start recording cassette = self.recorder.use_cassette("http.json") cassette.__enter__() self.addCleanup(cassette.__exit__) integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) self.installation_token = integration.get_access_token( config.INSTALLATION_ID).token self.g_integration = github.Github(self.installation_token, base_url="https://api.%s" % config.GITHUB_DOMAIN) self.g_admin = github.Github(config.MAIN_TOKEN, base_url="https://api.%s" % config.GITHUB_DOMAIN) self.g_fork = github.Github(config.FORK_TOKEN, base_url="https://api.%s" % config.GITHUB_DOMAIN) self.o_admin = self.g_admin.get_organization( config.TESTING_ORGANIZATION) self.o_integration = self.g_integration.get_organization( config.TESTING_ORGANIZATION) self.u_fork = self.g_fork.get_user() assert self.o_admin.login == "mergifyio-testing" assert self.o_integration.login == "mergifyio-testing" assert self.u_fork.login == "mergify-test2" self.r_o_admin = self.o_admin.create_repo(self.name) self.r_o_integration = self.o_integration.get_repo(self.name) self.url_main = "https://%s/%s" % ( config.GITHUB_DOMAIN, self.r_o_integration.full_name, ) self.url_fork = "https://%s/%s/%s" % ( config.GITHUB_DOMAIN, self.u_fork.login, self.r_o_integration.name, ) # Limit installations/subscription API to the test account install = { "id": config.INSTALLATION_ID, "target_type": "Org", "account": { "login": "******" }, } self.useFixture( fixtures.MockPatch("mergify_engine.utils.get_installations", lambda integration: [install])) real_get_subscription = sub_utils.get_subscription def fake_retrieve_subscription_from_db(install_id): if int(install_id) == config.INSTALLATION_ID: return self.subscription else: return { "tokens": {}, "subscription_active": False, "subscription_reason": "We're just testing", } def fake_subscription(r, install_id): if int(install_id) == config.INSTALLATION_ID: return real_get_subscription(r, install_id) else: return { "tokens": {}, "subscription_active": False, "subscription_reason": "We're just testing", } self.useFixture( fixtures.MockPatch( "mergify_engine.branch_updater.sub_utils.get_subscription", side_effect=fake_subscription, )) self.useFixture( fixtures.MockPatch( "mergify_engine.branch_updater.sub_utils._retrieve_subscription_from_db", side_effect=fake_retrieve_subscription_from_db, )) self.useFixture( fixtures.MockPatch( "mergify_engine.sub_utils.get_subscription", side_effect=fake_subscription, )) self.useFixture( fixtures.MockPatch( "github.MainClass.Installation.Installation.get_repos", return_value=[self.r_o_integration], )) self._event_reader = EventReader(self.app) self._event_reader.drain()
def celery_logging(**kwargs): # pragma: no cover utils.setup_logging() config.log()
def setUp(self): super(TestEngineScenario, self).setUp() self.session = requests.Session() self.session.trust_env = False self.recorder = betamax.Betamax(self.session) self.useFixture( fixtures.MockPatchObject(requests, 'Session', return_value=self.session)) self.useFixture( fixtures.MockPatchObject( gh_update_branch.utils, 'Gitter', lambda: GitterRecorder(self._testMethodName))) # Web authentification always pass self.useFixture( fixtures.MockPatch('hmac.compare_digest', return_value=True)) reponame_path = ( "mergify_engine/tests/fixtures/cassettes/reponame_%s" % self._testMethodName) gen_new_uuid = (RECORD_MODE == 'all' or (RECORD_MODE == 'once' and not os.path.exists(reponame_path))) if gen_new_uuid: REPO_UUID = str(uuid.uuid4()) with open(reponame_path, "w") as f: f.write(REPO_UUID) else: with open(reponame_path, "r") as f: REPO_UUID = f.read() self.name = "repo-%s-%s" % (REPO_UUID, self._testMethodName) self.cassette_counter = 0 self.events_handler_counter = 0 self.events_getter_counter = 0 self.status_counter = 0 self.reviews_counter = 0 self.pr_counter = 0 self.last_event_id = None utils.setup_logging() config.log() self.git = GitterRecorder('%s-tests' % self._testMethodName) self.addCleanup(self.git.cleanup) web.app.testing = True self.app = web.app.test_client() # NOTE(sileht): Prepare a fresh redis self.redis = utils.get_redis() self.redis.flushall() self.redis.set("installation-token-%s" % INSTALLATION_ID, MAIN_TOKEN) with self.cassette("setUp-prepare-repo", allow_playback_repeats=True): # Cleanup the remote testing redis r = self.session.delete("https://gh.mergify.io/events-testing") r.raise_for_status() self.g_main = github.Github(MAIN_TOKEN) self.g_fork = github.Github(FORK_TOKEN) self.u_main = self.g_main.get_user() self.u_fork = self.g_fork.get_user() assert self.u_main.login == "mergify-test1" assert self.u_fork.login == "mergify-test2" self.r_main = self.u_main.create_repo(self.name) self.url_main = "https://%s:@github.com/%s" % ( MAIN_TOKEN, self.r_main.full_name) self.git("init") self.git("config", "user.name", "%s-bot" % config.CONTEXT) self.git("config", "user.email", "*****@*****.**") self.git("remote", "add", "main", self.url_main) with open(self.git.tmp + "/.mergify.yml", "w") as f: f.write(CONFIG) self.git("add", ".mergify.yml") self.git("commit", "--no-edit", "-m", "initial commit") self.git("push", "main", "master") self.git("checkout", "-b", "stable") self.git("push", "main", "stable") self.git("checkout", "-b", "nostrict") self.git("push", "main", "nostrict") self.r_fork = self.u_fork.create_fork(self.r_main) self.url_fork = "https://%s:@github.com/%s" % ( FORK_TOKEN, self.r_fork.full_name) self.git("remote", "add", "fork", self.url_fork) self.git("fetch", "fork") with self.cassette("setUp-login-engine", allow_playback_repeats=True): g = github.Github(ACCESS_TOKEN) user = g.get_user("mergify-test1") repo = user.get_repo(self.name) with self.cassette("setUp-create-engine", allow_playback_repeats=True): self.engine = engine.MergifyEngine(g, INSTALLATION_ID, user, repo) self.useFixture( fixtures.MockPatchObject(worker, 'real_event_handler', self.engine.handle)) queue = rq.Queue(connection=self.redis) self.rq_worker = rq.SimpleWorker([queue], connection=queue.connection) # NOTE(sileht): Github looks buggy here: # We receive for the new repo the expected events: # * installation_repositories # * integration_installation_repositories # but we receive them 6 times with the same sha1... self.push_events(12)
def setUp(self): super(TestEngineScenario, self).setUp() self.cassette_library_dir = os.path.join(CASSETTE_LIBRARY_DIR_BASE, self._testMethodName) if RECORD_MODE != "none": if os.path.exists(self.cassette_library_dir): shutil.rmtree(self.cassette_library_dir) os.makedirs(self.cassette_library_dir) self.recorder = vcr.VCR( cassette_library_dir=self.cassette_library_dir, record_mode=RECORD_MODE, match_on=['method', 'uri'], filter_headers=[ ('Authorization', '<TOKEN>'), ('X-Hub-Signature', '<SIGNATURE>'), ('User-Agent', None), ('Accept-Encoding', None), ('Connection', None), ], before_record_response=self.response_filter, custom_patches=((github.MainClass, 'HTTPSConnection', vcr.stubs.VCRHTTPSConnection), )) self.useFixture( fixtures.MockPatchObject( branch_updater.utils, 'Gitter', lambda: GitterRecorder(self.cassette_library_dir))) self.useFixture( fixtures.MockPatchObject( backports.utils, 'Gitter', lambda: GitterRecorder(self.cassette_library_dir))) # Web authentification always pass self.useFixture( fixtures.MockPatch('hmac.compare_digest', return_value=True)) reponame_path = os.path.join(self.cassette_library_dir, "reponame") gen_new_uuid = (RECORD_MODE == 'all' or (RECORD_MODE == 'once' and not os.path.exists(reponame_path))) if gen_new_uuid: REPO_UUID = str(uuid.uuid4()) with open(reponame_path, "w") as f: f.write(REPO_UUID) else: with open(reponame_path, "r") as f: REPO_UUID = f.read() self.name = "repo-%s-%s" % (REPO_UUID, self._testMethodName) self.pr_counter = 0 self.remaining_events = [] utils.setup_logging() config.log() self.git = GitterRecorder(self.cassette_library_dir, "tests") self.addCleanup(self.git.cleanup) web.app.testing = True self.app = web.app.test_client() # NOTE(sileht): Prepare a fresh redis self.redis = utils.get_redis_for_cache() self.redis.flushall() subscription = {"token": config.MAIN_TOKEN, "subscribed": False} self.redis.set("subscription-cache-%s" % config.INSTALLATION_ID, json.dumps(subscription)) # Let's start recording cassette = self.recorder.use_cassette("http.json") cassette.__enter__() self.addCleanup(cassette.__exit__) self.session = requests.Session() self.session.trust_env = False # Cleanup the remote testing redis r = self.session.delete( "https://gh.mergify.io/events-testing", data=FAKE_DATA, headers={"X-Hub-Signature": "sha1=" + FAKE_HMAC}) r.raise_for_status() self.g_main = github.Github(config.MAIN_TOKEN) self.g_fork = github.Github(config.FORK_TOKEN) self.u_main = self.g_main.get_user() self.u_fork = self.g_fork.get_user() assert self.u_main.login == "mergify-test1" assert self.u_fork.login == "mergify-test2" self.r_main = self.u_main.create_repo(self.name) self.url_main = "https://github.com/%s" % self.r_main.full_name self.url_fork = "https://github.com/%s/%s" % (self.u_fork.login, self.r_main.name) integration = github.GithubIntegration(config.INTEGRATION_ID, config.PRIVATE_KEY) access_token = integration.get_access_token( config.INSTALLATION_ID).token g = github.Github(access_token) user = g.get_user("mergify-test1") repo = user.get_repo(self.name) # Used to access the cache with its helper self.engine = engine.MergifyEngine(g, config.INSTALLATION_ID, access_token, subscription, user, repo) self.processor = self.engine.get_processor() self.rq_worker = rq.SimpleWorker([ "incoming-events", "localhost-000-high", "localhost-001-high", "localhost-000-low", "localhost-001-low" ], connection=utils.get_redis_for_rq()) if self._testMethodName != "test_creation_pull_of_initial_config": self.git("init") self.git.configure() self.git.add_cred(config.MAIN_TOKEN, "", self.r_main.full_name) self.git.add_cred(config.FORK_TOKEN, "", "%s/%s" % (self.u_fork.login, self.r_main.name)) self.git("config", "user.name", "%s-tester" % config.CONTEXT) self.git("remote", "add", "main", self.url_main) self.git("remote", "add", "fork", self.url_fork) with open(self.git.tmp + "/.mergify.yml", "w") as f: f.write(CONFIG) self.git("add", ".mergify.yml") self.git("commit", "--no-edit", "-m", "initial commit") self.git("push", "--quiet", "main", "master") self.git("checkout", "-b", "stable", "--quiet") self.git("push", "--quiet", "main", "stable") self.git("checkout", "-b", "nostrict", "--quiet") self.git("push", "--quiet", "main", "nostrict") self.git("checkout", "-b", "disabled", "--quiet") self.git("push", "--quiet", "main", "disabled") self.git("checkout", "-b", "enabling_label", "--quiet") self.git("push", "--quiet", "main", "enabling_label") self.r_fork = self.u_fork.create_fork(self.r_main) self.git("fetch", "--quiet", "fork") # NOTE(sileht): Github looks buggy here: # We receive for the new repo the expected events: # * installation_repositories # * integration_installation_repositories # but we receive them 6 times with the same sha1... self.push_events([(None, {"action": "added"})] * 12)
def setUp(self): super(FunctionalTestBase, self).setUp() self.pr_counter = 0 self.git_counter = 0 self.cassette_library_dir = os.path.join(CASSETTE_LIBRARY_DIR_BASE, self._testMethodName) # Recording stuffs if RECORD: if os.path.exists(self.cassette_library_dir): shutil.rmtree(self.cassette_library_dir) os.makedirs(self.cassette_library_dir) self.recorder = vcr.VCR( cassette_library_dir=self.cassette_library_dir, record_mode="all" if RECORD else "none", match_on=['method', 'uri'], filter_headers=[ ('Authorization', '<TOKEN>'), ('X-Hub-Signature', '<SIGNATURE>'), ('User-Agent', None), ('Accept-Encoding', None), ('Connection', None), ], before_record_response=self.response_filter, custom_patches=((github.MainClass, 'HTTPSConnection', vcr.stubs.VCRHTTPSConnection), )) self.useFixture( fixtures.MockPatchObject(branch_updater.utils, 'Gitter', lambda: self.get_gitter())) self.useFixture( fixtures.MockPatchObject(backports.utils, 'Gitter', lambda: self.get_gitter())) # Web authentification always pass self.useFixture( fixtures.MockPatch('hmac.compare_digest', return_value=True)) reponame_path = os.path.join(self.cassette_library_dir, "reponame") if RECORD: REPO_UUID = str(uuid.uuid4()) with open(reponame_path, "w") as f: f.write(REPO_UUID) else: with open(reponame_path, "r") as f: REPO_UUID = f.read() self.name = "repo-%s-%s" % (REPO_UUID, self._testMethodName) utils.setup_logging() config.log() self.git = self.get_gitter() self.addCleanup(self.git.cleanup) web.app.testing = True self.app = web.app.test_client() # NOTE(sileht): Prepare a fresh redis self.redis = utils.get_redis_for_cache() self.redis.flushall() self.subscription = {"token": config.MAIN_TOKEN, "subscribed": False} self.redis.set("subscription-cache-%s" % config.INSTALLATION_ID, json.dumps(self.subscription)) # Let's start recording cassette = self.recorder.use_cassette("http.json") cassette.__enter__() self.addCleanup(cassette.__exit__) self.session = requests.Session() self.session.trust_env = False # Cleanup the remote testing redis r = self.session.delete( "https://gh.mergify.io/events-testing", data=FAKE_DATA, headers={"X-Hub-Signature": "sha1=" + FAKE_HMAC}) r.raise_for_status() self.g_main = github.Github(config.MAIN_TOKEN) self.g_fork = github.Github(config.FORK_TOKEN) self.u_main = self.g_main.get_user() self.u_fork = self.g_fork.get_user() assert self.u_main.login == "mergify-test1" assert self.u_fork.login == "mergify-test2" self.r_main = self.u_main.create_repo(self.name) self.url_main = "https://github.com/%s" % self.r_main.full_name self.url_fork = "https://github.com/%s/%s" % (self.u_fork.login, self.r_main.name) # Limit installations/subscription API to the test account install = { "id": config.INSTALLATION_ID, "target_type": "User", "account": { "login": "******" } } self.useFixture( fixtures.MockPatch('mergify_engine.utils.get_installations', lambda integration: [install])) real_get_subscription = utils.get_subscription def fake_subscription(r, install_id): if int(install_id) == config.INSTALLATION_ID: return real_get_subscription(r, install_id) else: return {"token": None, "subscribed": False} self.useFixture( fixtures.MockPatch( "mergify_engine.actions.merge.utils.get_subscription", side_effect=fake_subscription)) self.useFixture( fixtures.MockPatch("mergify_engine.web.utils.get_subscription", side_effect=fake_subscription)) self.useFixture( fixtures.MockPatch( "github.MainClass.Installation.Installation.get_repos", return_value=[self.r_main]))
# -*- encoding: utf-8 -*- # # Copyright © 2017 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from mergify_engine import config from mergify_engine import gh_pr from mergify_engine import utils from mergify_engine import web config.log() utils.setup_logging() gh_pr.monkeypatch_github() application = web.app