def test_choose_runs_to_deploy(): deploy_delay = timedelta(minutes=1) deployer = Deployer(duration=deploy_delay, deploy_policy=DeployPolicy.EveryPassing) runs = [StubPipelineRun(now, now, [StageRun(StageStatus.ok, end_time=now)])] deployer.add_deployments(runs) assert runs[0].deploy_time == now + deploy_delay
def test_deployer_deploy(mock_connect, mock_exec_command): mock_connect.return_value = True mock_exec_command.return_value = True deployer = Deployer() result = deployer.deploy() assert result is True
def test_deployer_make_backup(mock_connect, mock_exec_command): mock_connect.return_value = True mock_exec_command.return_value = True deployer = Deployer() result = deployer.make_backup() assert "backup_" in result
def test_deployer_make_backup_when_command_execution_failed( mock_connect, mock_exec_command): mock_connect.return_value = True mock_exec_command.return_value = False deployer = Deployer() with pytest.raises(SystemExit, match=r".* Can not create a backup .*"): deployer.make_backup()
def test_deployer_revert_deploy(mock_connect, mock_exec_command): mock_connect.return_value = True mock_exec_command.return_value = True deployer = Deployer() backup_name = deployer.make_backup() result = deployer.revert_deploy(backup_name) assert result is True
def test_deployer_revert_deploy_when_command_execution_failed( mock_connect, mock_exec_command): mock_connect.return_value = True mock_exec_command.return_value = False deployer = Deployer() with pytest.raises(SystemExit, match=r".* Can not revert a deploy .*"): deployer.revert_deploy("test_backup_name")
def test_choose_runs_to_deploy_once_a_week(): deploy_delay = timedelta(hours=1) ten = timedelta(minutes=10) deployer = Deployer(duration=deploy_delay, deploy_policy=DeployPolicy.OnceAWeek, deploy_hour=9, deploy_day=6) runs = [StubPipelineRun(now, now, [StageRun(StageStatus.ok, end_time=now)]), StubPipelineRun(now + ten, now + ten, [StageRun(StageStatus.ok, end_time=(now + ten))])] deployer.add_deployments(runs) assert runs[0].deploy_time == "" # deploy on Saturday assert runs[1].deploy_time == datetime(year=2018, month=6, day=2, hour=9) + deploy_delay
def test_choose_runs_to_deploy_same_day(): now = datetime(year=2018,month=6,day=1,hour=15) deploy_delay = timedelta(hours=1) ten = timedelta(minutes=10) deployer = Deployer(duration=deploy_delay, deploy_policy=DeployPolicy.OnceADay, deploy_hour=17) runs = [StubPipelineRun(now, now, [StageRun(StageStatus.ok, end_time=now)]), StubPipelineRun(now + ten, now + ten, [StageRun(StageStatus.ok, end_time=(now + ten))])] deployer.add_deployments(runs) assert runs[0].deploy_time == "" # deploy same day since deploy time is after run is ready assert runs[1].deploy_time == datetime(year=2018, month=6, day=1, hour=17) + deploy_delay
def task(self, logger, nodeName, task): nodes = self.getNodes() if not nodeName in nodes: logger.log("Node {0} does not exist.".format(nodeName), "error") return taskFn = getattr(self.tasksModule, task, None) if not taskFn: logger.log("Task {0} does not exist.".format(task), "error") return; node = nodes[nodeName] deployer = Deployer(node.externalIp, username=node.user, keyFile=node.keyFile) deployer.runTask(taskFn)
def test_deployer_deploy_when_command_execution_failed(mock_connect, mock_exec_command, mock_make_backup, mock_revert_deploy): mock_connect.return_value = True mock_exec_command.return_value = False mock_make_backup.return_value = "test_backup_name" mock_revert_deploy.return_value = True deployer = Deployer() with pytest.raises(SystemExit, match=r".* Error! Deployment failed .*"): deployer.deploy()
def provision(self, logger): status = {} if isfile(self.statusFile): status = yaml.load(open(self.statusFile).read()) self.driver.provision(logger) if self.DNSDriver: self.DNSDriver.provision(self.getNodes(), logger) # Run tasks if self.tasksModule: logger.log("Running tasks.") nodes = self.getNodes().items() # Create a host list that can be used by fabric scripts. hostList = {} for name, node in nodes: hostList[name] = node.externalIp # Always take the management server first, if it is available. # This is necessary since the other nodes could depend on the management server being in place. if "managementServer" in self.data: mgmt = self.data["managementServer"] def sortNodes(item1, item2): if item1[0] == mgmt: return -1 return 0 nodes = sorted(nodes, sortNodes) for name, node in nodes: if node.user: logger.log("Running tasks for {0}".format(name)) deployer = Deployer(node.externalIp, username=node.user, keyFile=node.keyFile, hostList = hostList) meisterFile = "/home/{0}/.meister".format(node.user) status = self.getTaskStatus(deployer, logger, meisterFile) # Filter out tasks that has already been run. tasks = filter(lambda task: task not in status["tasks"], node.tasks) for task in tasks: if isinstance(task, dict): taskFnName = task["name"] args = task["arguments"] else: taskFnName = task args = [] taskFn = getattr(self.tasksModule, taskFnName, None) if taskFn: deployer.runTask(taskFn, args) status["tasks"].append(task) self.putTaskStatus(status, deployer, logger, meisterFile)
def deploy(): if request.method == 'POST': request_data = request.json job_id = request_data["job_id"] assert (job_id is not None), "Missing job id" print("job id:") print(job_id) global _my_deployer try: _my_deployer = Deployer(job_id) except: print("Error init") return "Error" return "Success"
class Pipeline: stages: list = field(default_factory=list) trigger: str = "commits" deployer: Deployer = Deployer(deploy_policy=DeployPolicy.NoDeploys) runs: list = field(default_factory=list) def simulation(self, start_time, commits, duration): now = start_time future_commits = commits while future_commits \ and (now < start_time + duration): commits_this_run, future_commits = commits_in_next_run( future_commits, now) stage_results = self.simulate_stage_results(now) end_time = stage_results[-1].end_time run = PipelineRun( start_time=now, end_time=end_time, changes_included=commits_this_run, stage_results=stage_results, ) now = now + self.stages[0].duration if future_commits and now < future_commits[0].time: now = future_commits[0].time self.runs.append(run) self.deployer.add_deployments(self.runs) return self.runs def simulate_stage_results(self, now): results = [] previous_result = None run_start_time = now for stage in self.stages: result = stage.add_result(now, run_start_time, previous_result) results.append(result) previous_result = result now = previous_result.end_time return results
def test_simulation(): stages = [ Stage("Build & Unit Test", duration=timedelta(minutes=10), failure_rate=0.02), Stage("System Test", duration=timedelta(minutes=20), failure_rate=0.2), Stage("Performance Test", duration=timedelta(minutes=120), failure_rate=0.1, single_threaded=True), ] start_time = datetime(year=2017, month=12, day=11, hour=8) commits = generate_commits(100, start_time, offset=2000, max_interval=100) deployer = Deployer(duration=timedelta(minutes=2), deploy_policy=DeployPolicy.EveryPassing) runs = run_simulation(start_time, stages, commits=commits, deployer=deployer) assert len(runs) <= 100
def ssh(self, logger, nodeName): node = self.getNodes()[nodeName] deployer = Deployer(node.externalIp, username=node.user, keyFile=node.keyFile) deployer.ssh()
from metrics import pipeline_metrics from simulation import run_simulation, print_runs from stages import Stage stages = [ Stage("Build & Unit Test", duration=timedelta(minutes=10), failure_rate=0.01), Stage("Acceptance Test", duration=timedelta(minutes=20), failure_rate=0.02), Stage("Manual Test", duration=timedelta(minutes=120), failure_rate=0.05, manual_stage=True), ] start_time = datetime(year=2017, month=6, day=19, hour=8) commits = generate_commits(100, start_time, offset=1000, max_interval=122) deployer = Deployer(duration=timedelta(minutes=20), deploy_policy=DeployPolicy.EveryPassing, deploy_hour=8) runs = run_simulation(start_time, stages, commits=commits, deployer=deployer) print_runs("simulation2", stages, runs) metrics = pipeline_metrics(runs) print_metrics("simulation2", metrics) print(metrics.pretty_print())
stages = [ Stage("Commit Stage", duration=timedelta(minutes=3), failure_rate=0.005), Stage("Automated Acceptance Test", duration=timedelta(minutes=20), failure_rate=0.01), Stage("Performance Test", duration=timedelta(minutes=20), failure_rate=0.01), Stage("Internal Release", duration=timedelta(minutes=4), failure_rate=0.01, single_threaded=True), ] start_time = datetime(year=2017, month=12, day=11, hour=8) commits = generate_commits(100, start_time, offset=2000, max_interval=100) deployer = Deployer(duration=timedelta(minutes=4), deploy_policy=DeployPolicy.OnceADay, deploy_hour=17, deploy_day=6) runs = run_simulation(start_time, stages, commits=commits, deployer=deployer) print_runs("simulation_farley", stages, runs) metrics_calc = MetricsCalculator(runs) metrics = metrics_calc.metrics() print_metrics("simulation_farley", metrics) print(metrics.pretty_print())
@deploy_app.route('/', methods=['GET']) def index(): return "Running" @deploy_app.route('/predict', methods=['POST']) def predict(): if request.method == 'POST': if _my_deployer is None: return "Error" data = request.json #should be in json string format try: result = _my_deployer.get_prediction_(data['data']) except: print("Error occurred when getting prediction") return "" result_json = json.dumps(result.detach().numpy().tolist()) print("returns json string to client:") print(result_json) return result_json if __name__ == '__main__': job_id = os.environ.get("SLAKING_JOB_ID") try: _my_deployer = Deployer(job_id) except: print("Error init") deploy_app.run(host='0.0.0.0', port=5000)
"--org", dest="buildkite_org", help=( "Lock down to this buildkite org" ) ) args = parser.parse_args() arg_extract_path = args.extract arg_symlink = args.symlink arg_webbook_token = args.webhook_token arg_api_token = args.api_token arg_buildkite_org = args.buildkite_org if not os.path.isdir(arg_extract_path): os.mkdir(arg_extract_path) deployer = Deployer() deployer.bundles_path = args.bundles_dir deployer.should_clean = args.clean for include in args.include: deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(include) }) if args.tarball_uri is not None: build_dir = os.path.join(arg_extract_path, "test-%i" % (time.time())) deploy_tarball(args.tarball_uri, build_dir) else: print( "Listening on port %s. Extracting to %s%s. Symlinking to %s. Include files: %s" % (args.port, arg_extract_path, " (clean after)" if deployer.should_clean else "",
("Don't start an HTTP listener. Instead download a build from Jenkins \ immediately."), ) args = parser.parse_args() if args.jenkins.endswith("/"): # important for urljoin arg_jenkins_url = args.jenkins else: arg_jenkins_url = args.jenkins + "/" arg_extract_path = args.extract arg_symlink = args.symlink if not os.path.isdir(arg_extract_path): os.mkdir(arg_extract_path) deployer = Deployer() deployer.bundles_path = args.bundles_dir deployer.should_clean = args.clean deployer.config_locations = dict(args.configs) if args.configs else {} # we don't pgp-sign jenkins artifacts; instead we rely on HTTPS access to # the jenkins server (and the jenkins server not being compromised and/or # github not serving it compromised source). If that's not good enough for # you, don't use riot.im/develop. deployer.verify_signature = False if args.tarball_uri is not None: build_dir = os.path.join(arg_extract_path, "test-%i" % (time.time())) deploy_tarball(args.tarball_uri, build_dir) else: print(
("Don't start an HTTP listener. Instead download a build from Jenkins \ immediately."), ) args = parser.parse_args() if args.jenkins.endswith("/"): # important for urljoin arg_jenkins_url = args.jenkins else: arg_jenkins_url = args.jenkins + "/" arg_extract_path = args.extract arg_symlink = args.symlink if not os.path.isdir(arg_extract_path): os.mkdir(arg_extract_path) deployer = Deployer() deployer.bundles_path = args.bundles_dir deployer.should_clean = args.clean for include in args.include: deployer.symlink_paths.update( {os.path.basename(pth): pth for pth in glob.iglob(include)}) # we don't pgp-sign jenkins artifacts; instead we rely on HTTPS access to # the jenkins server (and the jenkins server not being compromised and/or # github not serving it compromised source). If that's not good enough for # you, don't use riot.im/develop. deployer.verify_signature = False if args.tarball_uri is not None:
from export import print_metrics from metrics import pipeline_metrics from simulation import run_simulation, print_runs from stages import Stage stages = [ Stage("Build & Unit Test", duration=timedelta(minutes=15), failure_rate=0), Stage("GUI Test", duration=timedelta(minutes=50), failure_rate=0), Stage("Manual Test", duration=timedelta(minutes=120), failure_rate=0, manual_stage=True), ] start_time = datetime(year=2017, month=1, day=2, hour=8) deployer = Deployer(duration=timedelta(minutes=20), deploy_policy=DeployPolicy.OnceAWeek, deploy_hour=8, deploy_day=3) commits = generate_commits_to_weekly_deadline(100, start_time, 3) runs = run_simulation(start_time, stages, commits=commits, deployer=deployer) print_runs("simulation3", stages, runs) metrics = pipeline_metrics(runs) print_metrics("simulation3", metrics) print(metrics.pretty_print())