def test_coercion(self): Sub = Algebraic.Alternative('Sub', I={}, S={}) with self.assertRaises(Exception): Sub.I(Sub.S) X = Algebraic.Alternative('X', A={'val': Sub}) X.A(val=Sub.S()) with self.assertRaises(Exception): X.A(val=Sub.S)
def test_field_lookup(self): X = Algebraic.Alternative('X', A={'a': int}, B={'b': float}) self.assertEqual(X.A(10).a, 10) with self.assertRaises(AttributeError): X.A(10).b self.assertEqual(X.B(11.0).b, 11.0) with self.assertRaises(AttributeError): X.B(11.0).a
def test_basic(self): X = Algebraic.Alternative('X', A={}, B={}) xa = X.A() xb = X.B() self.assertTrue(xa.matches.A) self.assertFalse(xa.matches.B) self.assertTrue(xb.matches.B) self.assertFalse(xb.matches.A)
def test_lists(self): X = Algebraic.Alternative('X') X.A = {'val': int} X.B = {'val': Algebraic.List(X)} xa = X.A(10) xb = X.B([xa, X.A(11)]) self.assertTrue(xa.matches.A) self.assertTrue(xb.matches.B) self.assertTrue(isinstance(xb.val, tuple)) self.assertTrue(len(xb.val) == 2)
def test_stable_sha_hashing(self): #adding default values to a type shouldn't disrupt its hashes leaf = Algebraic.Alternative("Leaf") leaf.A = {'a': int} leaf.B = {'b': int} leaf.setCreateDefault(lambda: leaf.A(0)) not_leaf = Algebraic.Alternative("NotLeaf") not_leaf.A = {'z': float, 'leaf': leaf} not_leaf2 = Algebraic.Alternative("NotLeaf") not_leaf2.A = {'z': float, 'leaf': leaf, 'int': int} a_simple_notleaf = not_leaf.A(z=10.0, _fill_in_missing=True) a_simple_notleaf2 = not_leaf2.A(z=10.0, _fill_in_missing=True) a_different_notleaf = not_leaf.A(z=10.0, leaf=leaf.B(b=10), _fill_in_missing=True) a_different_notleaf2 = not_leaf2.A(z=10.0, leaf=leaf.B(b=10), _fill_in_missing=True) a_final_different_notleaf = not_leaf2.A(z=10.0, leaf=leaf.B(b=10), int=123, _fill_in_missing=True) self.assertEqual(sha_hash(a_simple_notleaf), sha_hash(a_simple_notleaf2)) self.assertNotEqual(sha_hash(a_simple_notleaf), sha_hash(a_different_notleaf)) self.assertEqual(sha_hash(a_different_notleaf), sha_hash(a_different_notleaf2)) self.assertNotEqual(sha_hash(a_simple_notleaf), sha_hash(a_final_different_notleaf)) self.assertNotEqual(sha_hash(a_different_notleaf), sha_hash(a_final_different_notleaf))
def test_coercion_null(self): Sub = Algebraic.Alternative('Sub', I={}, S={}) X = Algebraic.Alternative('X', I={'val': Algebraic.Nullable(Sub)}) self.assertTrue(X(Sub.I()).val.matches.Value)
# you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import test_looper.core.algebraic as Algebraic from test_looper.core.hash import sha_hash import unittest expr = Algebraic.Alternative("Expr") expr.Constant = {'value': int} expr.Add = {'l': expr, 'r': expr} expr.Sub = {'l': expr, 'r': expr} expr.Mul = {'l': expr, 'r': expr} class AlgebraicTests(unittest.TestCase): def test_basic(self): X = Algebraic.Alternative('X', A={}, B={}) xa = X.A() xb = X.B() self.assertTrue(xa.matches.A) self.assertFalse(xa.matches.B)
def __len__(self): return len(self._jsonDict) def __iter__(self): return self._jsonDict.__iter__() def iteritems(self): for k, v in self._jsonDict.iteritems(): if isinstance(v, dict): yield k, DictWrapper(v) else: yield k, v ImportError = algebraic.Alternative("ImportError") ImportError.UnknownRepo = makeDict(repo=str) ImportError.UnknownBranch = makeDict(repo=str, name=str) ImportError.UnknownCommit = makeDict(repo=str, hash=str) ImportError.UnknownTest = makeDict(repo=str, hash=str, test=str) ImportError.TestAlreadyExists = makeDict(identity=str) class ImportExport(object): """ Convert the state of the database to a form that just represents the state but none of the computed information. """ def __init__(self, testManager): self.testManager = testManager self.database = self.testManager.database
import test_looper.core.algebraic as algebraic import test_looper.worker.TestLooperWorker as TestLooperWorker import test_looper.worker.WorkerState as WorkerState import test_looper.core.Config as Config import test_looper.core.machine_management.AwsCloudAPI as AwsCloudAPI import uuid import docker import logging import threading import traceback import os OsConfig = algebraic.Alternative("OsConfig") OsConfig.LinuxWithDocker = {} OsConfig.WindowsWithDocker = {} OsConfig.WindowsVM = {"ami": str, "setupHash": str} OsConfig.LinuxVM = {"ami": str, "setupHash": str} class UnbootableWorkerCombination(Exception): """An exception indicating that we can't meet this request for hardware/software, either because the AMI doesn't exist, or because we don't support some feature yet. """ def __init__(self, hardwareConfig, osConfig): Exception.__init__(self, "Can't boot configuration %s/%s" % (hardwareConfig, osConfig)) self.hardwareConfig = hardwareConfig self.osConfig = osConfig class MachineManagement(object): """Base class for 'machine management' which is responsible for booting workers to work on tests."""
import test_looper.core.algebraic as algebraic HardwareConfig = algebraic.Alternative("HardwareConfig") HardwareConfig.Config = { "cores": int, "ram_gb": int } MachineManagementConfig = algebraic.Alternative("MachineManagementConfig") #boot workers in AWS MachineManagementConfig.Aws = { "region": str, #region to boot into "vpc_id": str, #id of vpc to boot into "subnet": str, #id of subnet to boot into "security_group": str, #id of security group to boot into "keypair": str, #security keypair name to use "bootstrap_bucket": str, #bucket to put windows bootstrap scripts into. "bootstrap_key_prefix": str, #key prefix for windows bootstrap scripts. "windows_password": str, #password to use for booted-up windows boxes "worker_name": str, #name of workers. This should be unique to this install. "worker_iam_role_name": str, #AIM role to boot workers into "linux_ami": str, #default linux AMI to use when booting linux workers "windows_ami": str, #default AMI to use when booting windows workers. Can be overridden for one-shot workers. "path_to_keys": str, #path to ssh keys to place on workers to access source control. "instance_types": algebraic.Dict(HardwareConfig, str), #dict from hardware configuration to instance types we're willing to boot "host_ips": algebraic.Dict(str, str), #dict from hostname to ip address to make available to workers #this is primarily useful when workers don't have access to dns #but we still want certs to all be valid
def setup_types(database): database.BackgroundTask = algebraic.Alternative("BackgroundTask") database.BackgroundTask.RefreshRepos = {} database.BackgroundTask.BootMachineCheck = {} database.BackgroundTask.RefreshBranches = {"repo": database.Repo} database.BackgroundTask.UpdateBranchPins = {"branch": database.Branch} database.BackgroundTask.UpdateBranchTopCommit = {"branch": database.Branch} database.BackgroundTask.UpdateCommitData = {"commit": database.Commit} database.BackgroundTask.CommitTestParse = {"commit": database.Commit} database.BackgroundTask.UpdateTestPriority = {"test": database.Test} database.BackgroundTask.UpdateCommitPriority = {'commit': database.Commit} database.BackgroundTask.CheckBranchAutocreate = {"branch": database.Branch} database.TestPriority = algebraic.Alternative("TestPriority") database.TestPriority.WaitingToRetry = {} database.TestPriority.DependencyFailed = {} database.TestPriority.WaitingOnBuilds = {} database.TestPriority.UnresolvedDependencies = {} database.TestPriority.HardwareComboUnbootable = {} database.TestPriority.NoMoreTests = {} database.TestPriority.FirstBuild = {"priority": int} database.TestPriority.FirstTest = {"priority": int} database.TestPriority.WantsMoreTests = {"priority": int} database.FullyResolvedTestEnvironment = algebraic.Alternative( "FullyResolvedTestEnvironment") database.FullyResolvedTestEnvironment.Unresolved = {} database.FullyResolvedTestEnvironment.Error = {"Error": str} database.FullyResolvedTestEnvironment.Resolved = { "Environment": TestDefinition.TestEnvironment } database.DataTask.define(task=database.BackgroundTask, status=BackgroundTaskStatus, prior=database.DataTask, prior_ct=int, isHead=bool) database.Commit.define(hash=str, repo=database.Repo, data=database.CommitData, userPriority=int, calculatedPriority=int, anyBranch=database.Branch) database.CommitData.define(commit=database.Commit, parents=algebraic.List(database.Commit), subject=str, timestamp=int, commitMessage=str, author=str, authorEmail=str, tests=algebraic.Dict(str, database.Test), repos=algebraic.Dict( str, TestDefinition.RepoReference), testDefinitionsError=str, testsParsed=bool, noTestsFound=bool) database.CommitTestDependency.define(commit=database.Commit, test=database.Test) database.CommitRelationship.define(child=database.Commit, parent=database.Commit) database.TestDefinitionSummary = algebraic.Alternative( "TestDefinitionSummary") database.TestDefinitionSummary.Summary = { "name": str, "machineOs": MachineManagement.OsConfig, "type": str, #Build, Deployment, or Test "configuration": str, "artifacts": algebraic.List(str), "project": str, "disabled": bool, #disabled by default? "timeout": int, #max time, in seconds, for the test "min_cores": int, #minimum number of cores we should be run on, or zero if we don't care "max_cores": int, #maximum number of cores we can take advantage of, or zero "min_ram_gb": int, #minimum GB of ram we need to run, or zero if we don't care "min_disk_gb": int, #minimum GB of disk space we need to run, or zero if we don't care "max_retries": int, #maximum number of times to retry the build "retry_wait_seconds": int, #minimum number of seconds to wait before retrying a build } database.Test.define( hash=str, testDefinitionSummary=database.TestDefinitionSummary, machineCategory=database.MachineCategory, successes=int, totalRuns=int, activeRuns=int, lastTestEndTimestamp=float, totalTestCount=float, totalFailedTestCount=float, calculatedPriority=int, priority=database.TestPriority, targetMachineBoot= int, #the number of machines we want to boot to achieve this runsDesired=int, #the number of runs the _user_ indicated they wanted ) database.UnresolvedTestDependency.define(test=database.Test, dependsOnHash=str, artifact=str) database.UnresolvedCommitSourceDependency.define(commit=database.Commit, repo=database.Repo, commitHash=str) database.UnresolvedCommitRepoDependency.define(commit=database.Commit, reponame=str) database.TestDependency.define(test=database.Test, dependsOn=database.Test, artifact=str) database.TestRun.define( test=database.Test, startedTimestamp=float, lastHeartbeat=float, endTimestamp=float, success=bool, artifactsCompleted=algebraic.List(str), machine=database.Machine, canceled=bool, testNames=database.IndividualTestNameSet, testFailures=Bitstring. Bitstring, #encoded as an 8-bit bitstring, True if successful, False if failed testHasLogs=Bitstring. Bitstring, #True if there are individual test logs for this test totalTestCount=int, totalFailedTestCount=int) database.IndividualTestNameSet.define(shaHash=str, test_names=algebraic.List(str)) database.Repo.define(name=str, isActive=bool, commits=int, commitsWithTests=int, branchCreateTemplates=algebraic.List( database.BranchCreateTemplate), branchCreateLogs=database.LogMessage) database.BranchCreateTemplate.define(globsToInclude=algebraic.List(str), globsToExclude=algebraic.List(str), suffix=str, branchToCopyFrom=str, def_to_replace=str, disableOtherAutos=bool, autoprioritizeBranch=bool, deleteOnUnderlyingRemoval=bool) database.LogMessage.define(msg=str, timestamp=float, prior=database.LogMessage) database.Branch.define(branchname=str, repo=database.Repo, head=database.Commit, isUnderTest=bool, autocreateTrackingBranchName=str) database.BranchPin.define(branch=database.Branch, repo_def=str, pinned_to_repo=str, pinned_to_branch=str, auto=bool, prioritize=bool) database.MachineCategory.define(hardware=Config.HardwareConfig, os=MachineManagement.OsConfig, booted=int, desired=int, hardwareComboUnbootable=bool, hardwareComboUnbootableReason=str) database.Machine.define(machineId=str, hardware=Config.HardwareConfig, os=MachineManagement.OsConfig, bootTime=float, firstHeartbeat=float, lastHeartbeat=float, lastTestCompleted=float, isAlive=bool, lastHeartbeatMsg=str) database.Deployment.define(deploymentId=str, createdTimestamp=float, machine=database.Machine, test=database.Test, isAlive=bool) database.AllocatedGitRepoLocks.define(requestUniqueId=str, testOrDeployId=str, testHash=str) database.addIndex(database.IndividualTestNameSet, 'shaHash') database.addIndex(database.DataTask, 'status', lambda d: d.status if d.isHead else None) database.addIndex( database.DataTask, 'pending_boot_machine_check', lambda d: True if d. status.matches.Pending and d.task.matches.BootMachineCheck else None) database.addIndex( database.DataTask, 'update_commit_priority', lambda d: d.task.commit if d.task.matches.UpdateCommitPriority else None) database.addIndex( database.DataTask, 'update_test_priority', lambda d: d.task.test if d.task.matches.UpdateTestPriority else None) database.addIndex(database.CommitTestDependency, 'test') database.addIndex(database.CommitTestDependency, 'commit') database.addIndex(database.Machine, 'machineId') #don't index the dead ones database.addIndex(database.Machine, 'isAlive', lambda m: True if m.isAlive else None) database.addIndex(database.Machine, 'hardware_and_os', lambda m: (m.hardware, m.os) if m.isAlive else None) database.addIndex(database.MachineCategory, 'hardware_and_os', lambda m: (m.hardware, m.os)) database.addIndex(database.MachineCategory, 'want_more', lambda m: True if (m.desired > m.booted) else None) database.addIndex(database.MachineCategory, 'want_less', lambda m: True if (m.desired < m.booted) else None) database.addIndex(database.UnresolvedTestDependency, 'dependsOnHash') database.addIndex(database.UnresolvedTestDependency, 'test') database.addIndex(database.UnresolvedTestDependency, 'test_and_depends', lambda o: (o.test, o.dependsOnHash, o.artifact)) database.addIndex(database.AllocatedGitRepoLocks, "alive", lambda o: True) database.addIndex(database.AllocatedGitRepoLocks, "requestUniqueId") database.addIndex(database.AllocatedGitRepoLocks, "testOrDeployId") database.addIndex(database.AllocatedGitRepoLocks, "testHash") database.addIndex(database.UnresolvedCommitRepoDependency, 'commit') database.addIndex(database.UnresolvedCommitRepoDependency, 'reponame') database.addIndex(database.UnresolvedCommitRepoDependency, 'commit_and_reponame', lambda o: (o.commit, o.reponame)) database.addIndex(database.UnresolvedCommitSourceDependency, 'commit') database.addIndex(database.UnresolvedCommitSourceDependency, 'repo_and_hash', lambda o: (o.repo, o.commitHash)) database.addIndex(database.UnresolvedCommitSourceDependency, 'commit_and_repo_and_hash', lambda o: (o.commit, o.repo, o.commitHash)) database.addIndex(database.TestDependency, 'test') database.addIndex(database.TestDependency, 'dependsOn') database.addIndex(database.TestDependency, 'test_and_depends', lambda o: (o.test, o.dependsOn, o.artifact)) database.addIndex(database.Repo, 'name') database.addIndex(database.Repo, 'isActive') database.addIndex(database.Branch, 'repo') database.addIndex(database.Branch, 'head') database.addIndex(database.Branch, 'reponame_and_branchname', lambda o: (o.repo.name, o.branchname)) database.addIndex(database.Branch, 'autocreateTrackingBranchName') database.addIndex(database.BranchPin, 'branch') database.addIndex(database.BranchPin, 'pinned_to', lambda o: (o.pinned_to_repo, o.pinned_to_branch)) database.addIndex(database.Commit, 'repo_and_hash', lambda o: (o.repo, o.hash)) database.addIndex(database.CommitRelationship, 'parent') database.addIndex(database.CommitRelationship, 'child') database.addIndex(database.Deployment, 'isAlive', lambda d: d.isAlive or None) database.addIndex(database.Deployment, 'isAliveAndPending', lambda d: d.isAlive and not d.machine or None) database.addIndex(database.Deployment, 'runningOnMachine', lambda d: d.machine if d.isAlive else None) database.addIndex(database.Test, 'hash') database.addIndex( database.Test, 'machineCategoryAndPrioritized', lambda o: o.machineCategory if (not o.priority.matches.NoMoreTests and not o.priority.matches. WaitingToRetry and not o.priority.matches.DependencyFailed and not o.priority.matches.WaitingOnBuilds and not o.priority.matches. UnresolvedDependencies and not o.priority.matches. HardwareComboUnbootable and o.machineCategory) else None) database.addIndex(database.TestRun, 'test') database.addIndex( database.TestRun, 'isRunning', lambda t: True if not t.canceled and t.endTimestamp <= 0.0 else None) database.addIndex( database.TestRun, 'runningOnMachine', lambda t: t.machine if not t.canceled and t.endTimestamp <= 0.0 else None) database.addIndex( database.Test, 'waiting_to_retry', lambda o: True if o.priority.matches.WaitingToRetry else None) database.addIndex( database.Test, 'priority', lambda o: o.priority if (not o.priority.matches.NoMoreTests and not o.priority.matches. WaitingToRetry and not o.priority.matches.DependencyFailed and not o.priority.matches.WaitingOnBuilds and not o.priority.matches. UnresolvedDependencies and not o.priority.matches. HardwareComboUnbootable) else None)
import test_looper.core.Bitstring as Bitstring import test_looper.core.algebraic as algebraic import test_looper.data_model.TestDefinition as TestDefinition import test_looper.core.Config as Config import test_looper.core.machine_management.MachineManagement as MachineManagement BackgroundTaskStatus = algebraic.Alternative("BackgroundTaskStatus") BackgroundTaskStatus.PendingVeryHigh = {} BackgroundTaskStatus.PendingHigh = {} BackgroundTaskStatus.PendingMedium = {} BackgroundTaskStatus.PendingLow = {} BackgroundTaskStatus.PendingVeryLow = {} BackgroundTaskStatus.Running = {} def setup_types(database): database.BackgroundTask = algebraic.Alternative("BackgroundTask") database.BackgroundTask.RefreshRepos = {} database.BackgroundTask.BootMachineCheck = {} database.BackgroundTask.RefreshBranches = {"repo": database.Repo} database.BackgroundTask.UpdateBranchPins = {"branch": database.Branch} database.BackgroundTask.UpdateBranchTopCommit = {"branch": database.Branch} database.BackgroundTask.UpdateCommitData = {"commit": database.Commit} database.BackgroundTask.CommitTestParse = {"commit": database.Commit} database.BackgroundTask.UpdateTestPriority = {"test": database.Test} database.BackgroundTask.UpdateCommitPriority = {'commit': database.Commit} database.BackgroundTask.CheckBranchAutocreate = {"branch": database.Branch} database.TestPriority = algebraic.Alternative("TestPriority") database.TestPriority.WaitingToRetry = {}
import test_looper.core.algebraic as algebraic import test_looper.core.algebraic_to_json as algebraic_to_json import test_looper.data_model.TestDefinition as TestDefinition import yaml import json import simplejson import logging VALID_VERSIONS = [4] Platform = TestDefinition.Platform VariableDict = algebraic.Dict(str, str) Image = algebraic.Alternative("Image") Image.DockerfileInline = {"dockerfile_contents": str} Image.Dockerfile = {"dockerfile": str} Image.AMI = {"base_ami": str, "setup_script_contents": str} DefineEnvironment = algebraic.Alternative("DefineEnvironment") DefineEnvironment.Import = { 'base': algebraic.List(str), 'setup_script_contents': str, "variables": VariableDict, "dependencies": algebraic.Dict(str, str), "test_configuration": str, "test_stages": algebraic.List(TestDefinition.Stage), "test_timeout": int, "test_min_cores": int, "test_max_cores": int,
import os import time import logging import traceback import tempfile import tarfile import shutil import gzip import re import Queue import test_looper.core.algebraic as algebraic import test_looper.core.TimerQueue as TimerQueue timerQueue = TimerQueue.TimerQueue(16) FileContents = algebraic.Alternative("FileContents") FileContents.Inline = { "content_type": str, "content_encoding": str, "content_disposition": str, "content": str } FileContents.Redirect = {"url": str} Encoding = algebraic.Alternative("Encoding") TEST_LOG_NAME_PREFIX = "individual_test_logs/" class ArtifactStorage(object): @staticmethod
""" TestDefinition Objects modeling our tests. """ import test_looper.core.algebraic as algebraic import test_looper.core.GraphUtil as GraphUtil import test_looper.core.algebraic_to_json as algebraic_to_json import logging import re import test_looper.data_model.VariableSubstitution as VariableSubstitution Platform = algebraic.Alternative("Platform") Platform.windows = {} Platform.linux = {} Image = algebraic.Alternative("Image") Image.DockerfileInline = {"dockerfile_contents": str} Image.Dockerfile = {"repo": str, "commitHash": str, "dockerfile": str} Image.AMI = {"base_ami": str, "setup_script_contents": str} TestDependency = algebraic.Alternative("TestDependency") TestDependency.Build = {"name": str, "buildHash": str, "artifact": str} TestDependency.Source = {"repo": str, "commitHash": str, "path": str} #these are intermediate parse states. Resolved builds won't have them. TestDependency.InternalBuild = {"name": str} TestDependency.ExternalBuild = {"repo": str, "commitHash": str, "name": str} #'repo_name' refers to the name of the repo variable the test definitions (not the git repo). This #is an intermediate state that's only used mid-parsing while we're resolving references in external
artifacts = [] for artifact in [ a for stage in testDef.stages for a in stage.artifacts ]: self.manager.recordTestArtifactUploaded( testId, artifact.name, self.timestamp, False) artifacts.append(artifact.name) self.manager.recordTestResults(True, testId, { "ATest": (True, False), "AnotherTest": (False, False) }, artifacts, self.timestamp) self.timestamp += .1 FakeConfig = algebraic.Alternative("FakeConfig") FakeConfig.Config = {"machine_management": Config.MachineManagementConfig} def getHarness(max_workers=1000): return TestManagerTestHarness( TestManager.TestManager( None, MockSourceControl(), MachineManagement.DummyMachineManagement( FakeConfig(machine_management=Config.MachineManagementConfig. Dummy(max_cores=1000, max_ram_gb=1000, max_workers=max_workers)), None, None), InMemoryJsonStore.InMemoryJsonStore(), initialTimestamp=-1000.0))
import threading import traceback import base64 import time import random import socket import test_looper.data_model.TestDefinition as TestDefinition import test_looper.core.SimpleServer as SimpleServer import test_looper.core.socket_util as socket_util import test_looper.core.algebraic as algebraic import test_looper.core.algebraic_to_json as algebraic_to_json CLEANUP_TASK_FREQUENCY = 30 TerminalInputMsg = algebraic.Alternative("TerminalInputMsg") TerminalInputMsg.KeyboardInput = {"bytes": str} TerminalInputMsg.Resize = {"cols": int, "rows": int} ServerToClientMsg = algebraic.Alternative("ServerToClientMsg") ServerToClientMsg.IdentifyCurrentState = {} ServerToClientMsg.TerminalInput = { 'deploymentId': str, 'msg': TerminalInputMsg } ServerToClientMsg.TestAssignment = { 'testId': str, 'testDefinition': TestDefinition.TestDefinition } ServerToClientMsg.CancelTest = {'testId': str} ServerToClientMsg.AcknowledgeFinishedTest = {'testId': str}