def update_local_bokchoy_db_from_s3(): """ Update the MYSQL database for bokchoy testing: * Determine if your current cache files are up to date with all the migrations * If not then check if there is a copy up at s3 * If so then download then extract it * Otherwise apply migrations as usual """ fingerprint = fingerprint_bokchoy_db_files(MIGRATION_OUTPUT_FILES, ALL_DB_FILES) if does_fingerprint_on_disk_match(fingerprint): print("DB cache files match the current migrations.") # TODO: we don't really need to apply migrations, just to # load the db cache files into the database. apply_migrations(BOKCHOY_DB_FILES, update_cache_files=False) elif is_fingerprint_in_bucket(fingerprint, CACHE_BUCKET_NAME): print("Found updated bokchoy db files at S3.") refresh_bokchoy_db_cache_from_s3(fingerprint=fingerprint) # TODO: we don't really need to apply migrations, just to # load the db cache files into the database. apply_migrations(BOKCHOY_DB_FILES, update_cache_files=False) # Write the new fingerprint to disk so that it reflects the # current state of the system. compute_fingerprint_and_write_to_disk(MIGRATION_OUTPUT_FILES, ALL_DB_FILES) else: msg = "{} {} {}".format("Did not find updated bokchoy db files at S3.", "Loading the bokchoy db files from disk", "and running migrations.") print(msg) apply_migrations(BOKCHOY_DB_FILES, update_cache_files=True) # Write the new fingerprint to disk so that it reflects the # current state of the system. # E.g. you could have added a new migration in your PR. compute_fingerprint_and_write_to_disk(MIGRATION_OUTPUT_FILES, ALL_DB_FILES)
def update_local_bokchoy_db_from_s3(options): """ Prepare the local MYSQL test database for running bokchoy tests. Since most pull requests do not introduce migrations, this task provides an optimization for caching the state of the db when migrations are added into a bucket in s3. Subsequent commits can avoid rerunning migrations by using the cache files from s3, until the local cache files are updated by running the `update_bokchoy_db_cache` Paver task, and committing the updated cache files to github. Steps: 1. Determine which migrations, if any, need to be applied to your current db cache files to make them up to date 2. Compute the sha1 fingerprint of the local db cache files and the output of the migration 3a. If the fingerprint computed in step 2 is equal to the local fingerprint file, load the cache files into the MYSQL test database 3b. If the fingerprints are not equal, but there is bucket matching the fingerprint computed in step 2, download and extract the contents of bucket (db cache files) and load them into the MYSQL test database 3c. If the fingerprints are not equal AND there is no bucket matching the fingerprint computed in step 2, load the local db cache files into the MYSQL test database and apply any needed migrations. Create a bucket in s3 named the fingerprint computed in step 2 and push the newly updated db cache files to the bucket. NOTE: the computed fingerprints referenced in this and related functions represent the state of the db cache files and migration output PRIOR to running migrations. The corresponding s3 bucket named for a given fingerprint contains the db cache files AFTER applying migrations """ fingerprint = fingerprint_bokchoy_db_files(MIGRATION_OUTPUT_FILES, ALL_DB_FILES) fingerprints_match = does_fingerprint_on_disk_match(fingerprint) # Calculating the fingerprint already reset the DB, so we don't need to # do it again (hence use_existing_db=True below) if fingerprints_match: print("DB cache files match the current migrations.") reset_test_db(BOKCHOY_DB_FILES, update_cache_files=False, use_existing_db=True) elif is_fingerprint_in_bucket(fingerprint, CACHE_BUCKET_NAME): print("Found updated bokchoy db files at S3.") refresh_bokchoy_db_cache_from_s3(fingerprint, CACHE_BUCKET_NAME, BOKCHOY_DB_FILES) reset_test_db(BOKCHOY_DB_FILES, update_cache_files=False, use_existing_db=True) else: msg = u"{} {} {}".format( "Did not find updated bokchoy db files at S3.", "Loading the bokchoy db files from disk", "and running migrations." ) print(msg) reset_test_db(BOKCHOY_DB_FILES, update_cache_files=True, use_existing_db=True) # Check one last time to see if the fingerprint is present in # the s3 bucket. This could occur because the bokchoy job is # sharded and running the same task in parallel if not is_fingerprint_in_bucket(fingerprint, CACHE_BUCKET_NAME): upload_db_cache_to_s3(fingerprint, BOKCHOY_DB_FILES, CACHE_BUCKET_NAME) else: msg = u"{} {}. {}".format( "Found a matching fingerprint in bucket ", CACHE_BUCKET_NAME, "Not pushing to s3" ) print(msg) rewrite_fingerprint = getattr(options, 'rewrite_fingerprint', False) # If the rewrite_fingerprint flag is set, and the fingerpint has changed, # write it to disk. if not fingerprints_match and rewrite_fingerprint: print("Updating fingerprint and writing to disk.") compute_fingerprint_and_write_to_disk(MIGRATION_OUTPUT_FILES, ALL_DB_FILES)
def update_local_bokchoy_db_from_s3(options): """ Prepare the local MYSQL test database for running bokchoy tests. Since most pull requests do not introduce migrations, this task provides an optimization for caching the state of the db when migrations are added into a bucket in s3. Subsequent commits can avoid rerunning migrations by using the cache files from s3, until the local cache files are updated by running the `update_bokchoy_db_cache` Paver task, and committing the updated cache files to github. Steps: 1. Determine which migrations, if any, need to be applied to your current db cache files to make them up to date 2. Compute the sha1 fingerprint of the local db cache files and the output of the migration 3a. If the fingerprint computed in step 2 is equal to the local fingerprint file, load the cache files into the MYSQL test database 3b. If the fingerprints are not equal, but there is bucket matching the fingerprint computed in step 2, download and extract the contents of bucket (db cache files) and load them into the MYSQL test database 3c. If the fingerprints are not equal AND there is no bucket matching the fingerprint computed in step 2, load the local db cache files into the MYSQL test database and apply any needed migrations. Create a bucket in s3 named the fingerprint computed in step 2 and push the newly updated db cache files to the bucket. NOTE: the computed fingerprints referenced in this and related functions represent the state of the db cache files and migration output PRIOR to running migrations. The corresponding s3 bucket named for a given fingerprint contains the db cache files AFTER applying migrations """ fingerprint = fingerprint_bokchoy_db_files(MIGRATION_OUTPUT_FILES, ALL_DB_FILES) fingerprints_match = does_fingerprint_on_disk_match(fingerprint) # Calculating the fingerprint already reset the DB, so we don't need to # do it again (hence use_existing_db=True below) if fingerprints_match: print("DB cache files match the current migrations.") reset_test_db(BOKCHOY_DB_FILES, update_cache_files=False, use_existing_db=True) elif is_fingerprint_in_bucket(fingerprint, CACHE_BUCKET_NAME): print("Found updated bokchoy db files at S3.") refresh_bokchoy_db_cache_from_s3(fingerprint, CACHE_BUCKET_NAME, BOKCHOY_DB_FILES) reset_test_db(BOKCHOY_DB_FILES, update_cache_files=False, use_existing_db=True) else: msg = "{} {} {}".format("Did not find updated bokchoy db files at S3.", "Loading the bokchoy db files from disk", "and running migrations.") print(msg) reset_test_db(BOKCHOY_DB_FILES, update_cache_files=True, use_existing_db=True) # Check one last time to see if the fingerprint is present in # the s3 bucket. This could occur because the bokchoy job is # sharded and running the same task in parallel if not is_fingerprint_in_bucket(fingerprint, CACHE_BUCKET_NAME): upload_db_cache_to_s3(fingerprint, BOKCHOY_DB_FILES, CACHE_BUCKET_NAME) else: msg = "{} {}. {}".format("Found a matching fingerprint in bucket ", CACHE_BUCKET_NAME, "Not pushing to s3") print(msg) rewrite_fingerprint = getattr(options, 'rewrite_fingerprint', False) # If the rewrite_fingerprint flag is set, and the fingerpint has changed, # write it to disk. if not fingerprints_match and rewrite_fingerprint: print("Updating fingerprint and writing to disk.") compute_fingerprint_and_write_to_disk(MIGRATION_OUTPUT_FILES, ALL_DB_FILES)