def createTextFileAndUpload(fileName, fileContents, fileDir, s3Folder, logger): """ Creates file, add the Contents in the file and upload that file to S3. :param fileName: Name of the file to be created :param fileContents: Contents of the file :param fileDir: The path of the file to upload :param s3Folder: The S3 folder where the file is to be uploaded :param logger: A valid Numenta logger """ with open(fileName, "w") as fHandle: fHandle.write(fileContents) filePath = os.path.join(fileDir, fileName) s3.uploadToS3(g_config, filePath, s3Folder, logger)
def cacheNuPICCore(env, buildWorkspace, nupicCoreSHA, uploadToS3, logger): """ Caches nupic.core to /var/build/NuPIC.core/<SHA> and uploads to S3 :param env: The environment dict :param buildWorkspace: The buildWorkspace were nupic.core is built :param nupicSha: A `string` representing SHA :param uploadToS3: `boolean` defining whether to upload to S3 or not :raises: CommandFailedError if the tar process fails before upload. """ cachedPath = "/var/build/nupic.core/%s" % nupicCoreSHA if not os.path.isdir(cachedPath): logger.info("Caching nupic.core to %s", cachedPath) with changeToWorkingDir(buildWorkspace): shutil.copytree( "nupic.core", ("/var/build/nupic.core/%s/nupic.core" % nupicCoreSHA)) if uploadToS3: nupicCoreZip = "nupic.core-%s.zip" % nupicCoreSHA logger.info("Archiving nupic.core to %s", nupicCoreZip) command = "tar czf %s nupic.core" % nupicCoreZip nupicCoreZipPath = "%s/%s" % (buildWorkspace, nupicCoreZip) try: runWithOutput(command, env, logger=logger) logger.debug("Uploading %s to S3.", nupicCoreZip) s3.uploadToS3(g_config, nupicCoreZipPath, "builds_nupic_core", logger) except: logger.exception("Archiving nupic.core failed.") raise CommandFailedError("Archiving nupic.core failed.") else: logger.info("nupic.core cached locally and to S3.") else: logger.debug("Cached nupic.core already exists.")
def cacheNuPIC(env, nupicSha, uploadToS3, logger): """ Caches a green build of NuPIC to /var/build/nupic/<SHA> :param env: The environment dict :param nupicSha: A `string` representing SHA. """ cachedPath = "/var/build/nupic/%s" % nupicSha if not os.path.isdir(cachedPath): try: logger.info("Caching NuPIC to %s", cachedPath) shutil.copytree(env["NUPIC"], cachedPath) wheelDir = env["NUPIC"] + "/dist" wheelFile = glob.glob("%s/*.whl" % wheelDir)[0] logger.debug("Uploading %s to S3.", wheelFile) s3.uploadToS3(g_config, wheelFile, "builds_nupic_wheel", logger) wheelFileName = os.path.basename(wheelFile) fileDir = os.getcwd() s3Folder = "stable_nupic_version" contents = nupicSha + ":" + wheelFileName createTextFileAndUpload("nupic-package-version.txt", contents, fileDir, s3Folder, logger) createTextFileAndUpload(nupicSha, wheelFileName, fileDir, s3Folder, logger) shutil.move("nupic-package-version.txt", ARTIFACTS_DIR) with open("nupicSHA.txt", "w") as fHandle: fHandle.write(nupicSha) shutil.move("nupicSHA.txt", ARTIFACTS_DIR) except: logger.exception("Caching NuPIC failed.") raise else: logger.info("NuPIC cached locally and to S3.") else: logger.debug("Cached NuPIC already exists.")
def cacheNuPICCore(env, buildWorkspace, nupicCoreSHA, uploadToS3, logger): """ Caches nupic.core to /var/build/NuPIC.core/<SHA> and uploads to S3 :param env: The environment dict :param buildWorkspace: The buildWorkspace were nupic.core is built :param nupicSha: A `string` representing SHA :param uploadToS3: `boolean` defining whether to upload to S3 or not :raises: CommandFailedError if the tar process fails before upload. """ cachedPath = "/var/build/nupic.core/%s" % nupicCoreSHA if not os.path.isdir(cachedPath): logger.info("Caching nupic.core to %s", cachedPath) with changeToWorkingDir(buildWorkspace): shutil.copytree("nupic.core", ("/var/build/nupic.core/%s/nupic.core" % nupicCoreSHA)) if uploadToS3: nupicCoreZip = "nupic.core-%s.zip" % nupicCoreSHA logger.info("Archiving nupic.core to %s", nupicCoreZip) command = "tar czf %s nupic.core" % nupicCoreZip nupicCoreZipPath = "%s/%s" % (buildWorkspace, nupicCoreZip) try: runWithOutput(command, env, logger=logger) logger.debug("Uploading %s to S3.", nupicCoreZip) s3.uploadToS3(g_config, nupicCoreZipPath, "builds_nupic_core", logger) except: logger.exception("Archiving nupic.core failed.") raise CommandFailedError("Archiving nupic.core failed.") else: logger.info("nupic.core cached locally and to S3.") else: logger.debug("Cached nupic.core already exists.")
def main(jsonArgs=None): """ Creates an AMI using a HTM-IT RPM for a given SHA. 1) Downloads the HTM-IT RPM corresponding to a given SHA to local disk 2) Calls bake_ami.sh with the name of the HTM-IT RPM and the AMI name. to launch an instance with Packer, install the HTM-IT RPM from 1 products, runs integration tests, if green then stamps AMI """ try: jsonArgs = jsonArgs or {} parsedArgs = addAndParseArgs(jsonArgs) amiName = parsedArgs["amiName"] if not (os.environ.get("AWS_ACCESS_KEY_ID") and os.environ.get("AWS_SECRET_ACCESS_KEY")): g_logger.error("Please set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY") raise MissingAWSKeysInEnvironment("AWS keys are not set") else: g_config["AWS_ACCESS_KEY_ID"] = os.environ["AWS_ACCESS_KEY_ID"] g_config["AWS_SECRET_ACCESS_KEY"] = os.environ["AWS_SECRET_ACCESS_KEY"] artifactsDir = createOrReplaceArtifactsDir(logger=g_logger) g_logger.info("Creating the Ami") pipeLineSrc = os.path.join(os.environ["PRODUCTS"], "htm-it", "htm-it", "pipeline", "src") with changeToWorkingDir(pipeLineSrc): g_logger.info("\n\n########## Baking AMI ##########") g_logger.debug("########## AMI Name: %s ##########", amiName) # Baking AMI takes around 15 mins, so print as it runs so we see # progress in the jenkins console during the run runWithOutput("./bake_ami %s" % amiName, env=os.environ, logger=g_logger) amiIDPath = os.path.join(os.getcwd(), "ami.txt") with open(amiIDPath, "r") as amiFileHandler: readAmiId = (amiFileHandler.readline()).split(":") amiID = readAmiId[1].strip() g_logger.info("AMI ID generated is: %s", amiID) buildNumber = getBuildNumber(logger=g_logger) artifactAmiIdPath = os.path.join(artifactsDir, "ami_%s.txt" % buildNumber) shutil.copy(amiIDPath, artifactAmiIdPath) print "#############################################################" print "Running the Integration Tests" runIntegrationTestScriptPath = os.path.join(os.environ["PRODUCTS"], "htm-it", "htm-it", "pipeline", "src") runIntegrationTestCommand = ("python " + "%s/run_htm-it_integration_tests.py" % runIntegrationTestScriptPath + " --ami " + amiID) if parsedArgs["pipelineJson"]: runIntegrationTestCommand += (" --pipeline-json %s" % parsedArgs["pipelineJson"]) g_logger.info(runIntegrationTestCommand) runWithOutput(runIntegrationTestCommand, env=os.environ, logger=g_logger) #Load the json file again and check the status of test with open(parsedArgs["pipelineJson"]) as jsonFile: params = json.load(jsonFile) integrationTestStatus = params.get("integration_test").get("testStatus") # Upload the ami-id to S3 if the pipeline was triggred with production # forks. if integrationTestStatus: g_logger.info("Uploading %s to S3 which contains the generated AMI: %s", os.path.basename(artifactAmiIdPath), amiID) uploadToS3(config=g_config, filePath=artifactAmiIdPath, s3Folder="stable_ami", logger=g_logger) except TestsFailed: g_logger.error("There was a failure executing the HTM-IT integration tests") raise except PipelineError: g_logger.exception("External process failed while baking the AMI") raise except Exception: g_logger.exception("Unknown error occurred while baking the AMI") raise
def main(jsonArgs=None): """ Creates an AMI using a YOMP RPM for a given SHA. 1) Downloads the YOMP RPM corresponding to a given SHA to local disk 2) Calls bake_ami.sh with the name of the YOMP RPM and the AMI name. to launch an instance with Packer, install the YOMP RPM from 1 products, runs integration tests, if green then stamps AMI """ try: jsonArgs = jsonArgs or {} parsedArgs = addAndParseArgs(jsonArgs) amiName = parsedArgs["amiName"] if not (os.environ.get("AWS_ACCESS_KEY_ID") and os.environ.get("AWS_SECRET_ACCESS_KEY")): g_logger.error( "Please set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY") raise MissingAWSKeysInEnvironment("AWS keys are not set") else: g_config["AWS_ACCESS_KEY_ID"] = os.environ["AWS_ACCESS_KEY_ID"] g_config["AWS_SECRET_ACCESS_KEY"] = os.environ[ "AWS_SECRET_ACCESS_KEY"] artifactsDir = createOrReplaceArtifactsDir() g_logger.info("Creating the Ami") pipeLineSrc = os.path.join(os.environ["PRODUCTS"], "YOMP", "YOMP", "pipeline", "src") with changeToWorkingDir(pipeLineSrc): g_logger.info("\n\n########## Baking AMI ##########") g_logger.debug("########## AMI Name: %s ##########", amiName) # Baking AMI takes around 15 mins, so print as it runs so we see # progress in the jenkins console during the run runWithOutput("./bake_ami %s" % amiName, env=os.environ, logger=g_logger) amiIDPath = os.path.join(os.getcwd(), "ami.txt") with open(amiIDPath, "r") as amiFileHandler: readAmiId = (amiFileHandler.readline()).split(":") amiID = readAmiId[1].strip() g_logger.info("AMI ID generated is: %s", amiID) buildNumber = getBuildNumber() artifactAmiIdPath = os.path.join(artifactsDir, "ami_%s.txt" % buildNumber) shutil.copy(amiIDPath, artifactAmiIdPath) print "#############################################################" print "Running the Integration Tests" runIntegrationTestScriptPath = os.path.join(os.environ["PRODUCTS"], "YOMP", "YOMP", "pipeline", "src") runIntegrationTestCommand = ( "python " + "%s/run_YOMP_integration_tests.py" % runIntegrationTestScriptPath + " --ami " + amiID) if parsedArgs["pipelineJson"]: runIntegrationTestCommand += (" --pipeline-json %s" % parsedArgs["pipelineJson"]) g_logger.info(runIntegrationTestCommand) runWithOutput(runIntegrationTestCommand, env=os.environ, logger=g_logger) #Load the json file again and check the status of test with open(parsedArgs["pipelineJson"]) as jsonFile: params = json.load(jsonFile) integrationTestStatus = params.get("integration_test").get( "testStatus") # Upload the ami-id to S3 if the pipeline was triggred with production # forks. if integrationTestStatus: g_logger.info( "Uploading %s to S3 which contains the generated AMI: %s", os.path.basename(artifactAmiIdPath), amiID) uploadToS3(config=g_config, filePath=artifactAmiIdPath, s3Folder="stable_ami", logger=g_logger) except TestsFailed: g_logger.error( "There was a failure executing the YOMP integration tests") raise except PipelineError: g_logger.exception("External process failed while baking the AMI") raise except Exception: g_logger.exception("Unknown error occurred while baking the AMI") raise