Example #1
0
def create_patch(oldDir, newDir, outDir):
    patchDir = os.path.join(outDir, "patch_temp")

    if not os.path.exists(oldDir):
        print "Directory to the old version is invalid! Aborting."
        return None

    if not os.path.exists(newDir):
        print "Directory to the new version is invalid! Aborting."
        return None

    if not validate_environment():
        return None

    if not os.path.exists(patchDir):
        os.mkdir(patchDir)
    elif not is_empty_directory(patchDir):
        print "patch_temp directory is not empty! Aborting."
        return None

    walk_old_dir(oldDir, newDir, patchDir)
    walk_new_dir(oldDir, newDir, patchDir)
    merge_index(patchDir)
    zip_directory(patchDir, patchDir + ".7z")
    return patchDir + ".7z"
Example #2
0
def create_patch(oldDir, newDir, outDir):
    patchDir = os.path.join(outDir, 'patch_temp')
    
    if not os.path.exists(oldDir):
        print 'Directory to the old version is invalid! Aborting.'
        return None
    
    if not os.path.exists(newDir):
        print 'Directory to the new version is invalid! Aborting.'
        return None
    
    if not validate_environment():
        return None
    
    if not os.path.exists(patchDir):
        os.mkdir(patchDir)
    elif not is_empty_directory(patchDir):
        print 'patch_temp directory is not empty! Aborting.'
        return None
    
    walk_old_dir(oldDir, newDir, patchDir)
    walk_new_dir(oldDir, newDir, patchDir)
    merge_index(patchDir)
    zip_directory(patchDir, patchDir + '.7z')
    return patchDir + '.7z'
Example #3
0
def zip_full_game():
    print 'zipping game'
    binDir = os.path.join(TEMP_DIR, 'bin')
    if os.path.exists(binDir):
        print 'WARNING: Replacing directory ', binDir
        shutil.rmtree(binDir)
    outFile = os.path.join(OUT_DIR, 'latest.7z')
    if os.path.exists(outFile):
        print 'WARNING: Replacing archive ', outFile
        os.remove(outFile)
    os.rename(NEW_DIR, binDir)
    zip_directory(binDir, outFile)
    os.rename(binDir, NEW_DIR)
Example #4
0
def zip_full_game():
    print 'zipping game'
    binDir = os.path.join(TEMP_DIR, 'bin')
    if os.path.exists(binDir):
        print 'WARNING: Replacing directory ', binDir
        shutil.rmtree(binDir)
    outFile = os.path.join(OUT_DIR, 'latest.7z')
    if os.path.exists(outFile):
        print 'WARNING: Replacing archive ', outFile
        os.remove(outFile)
    os.rename(NEW_DIR, binDir)
    zip_directory(binDir, outFile)
    os.rename(binDir, NEW_DIR)
Example #5
0
def run_simulation(mongo_host, binary_id, executable, arguments):
    client = pymongo.MongoClient(mongo_host)
    gfs = gridfs.GridFS(client.opp)

    binary_zip = gfs.get(binary_id).read()

    # housekeeping
    shutil.rmtree(MODEL_DIR, ignore_errors=True)
    os.makedirs(MODEL_DIR)
    os.chdir(MODEL_DIR)

    # unzip binary_zip
    utils.unzip_bytes(binary_zip)

    subprocess.call(["chmod", "+x", executable])  # eh

    # execute binary with args
    subprocess.call([executable] + arguments)

    # zip results
    results_zip = utils.zip_directory("results")

    # cleaning up
    shutil.rmtree(MODEL_DIR, ignore_errors=True)

    job_id = rq.get_current_job().get_id()

    gfs.put(results_zip, _id=job_id)
Example #6
0
def build_model(mongo_host, source_id):
    client = pymongo.MongoClient(mongo_host)
    gfs = gridfs.GridFS(client.opp)

    source_zip = gfs.get(source_id).read()

    # housekeeping
    shutil.rmtree(MODEL_DIR, ignore_errors=True)
    os.makedirs(MODEL_DIR)
    os.chdir(MODEL_DIR)

    # unzip source_zip
    utils.unzip_bytes(source_zip)

    # run make, first clean just to be sure
    subprocess.call(["make", "clean"])
    subprocess.call(["make", "MODE=release"])

    # zip built model
    binary_zip = utils.zip_directory(".")

    # cleaning up
    shutil.rmtree(MODEL_DIR, ignore_errors=True)

    job_id = rq.get_current_job().get_id()

    gfs.put(binary_zip, _id=job_id)
Example #7
0
def run_simulation(source_zip, executable, arguments):

    # housekeeping
    shutil.rmtree(MODEL_DIR, ignore_errors=True)
    os.makedirs(MODEL_DIR)
    os.chdir(MODEL_DIR)

    # unzip source_zip
    utils.unzip_bytes(source_zip)

    # run make, first clean just to be sure
    subprocess.call(["make", "clean"])
    subprocess.call(["make", "MODE=release"])

    # execute binary with args
    subprocess.call([executable] + arguments)

    # zip results
    results_zip = utils.zip_directory("results")

    # cleaning up
    shutil.rmtree(MODEL_DIR, ignore_errors=True)

    # return zip
    return results_zip
Example #8
0
def run(target, outpath, output_directory=False):
    print "Marking the directory as built by the build process"
    utils.mark_as_built(target)

    print "Generating pyo files"
    utils.generate_pyo_files(target)

    print "disabling i18n"
    utils.append_to_local_settings(target, "USE_I18N = False")
    utils.append_to_local_settings(target, "USE_L10N = False")

    print "Disabling DEBUG mode"
    utils.append_to_local_settings(target, "DEBUG = False")

    print "Deleting some blacklisted files"
    utils.delete_blacklisted_files(target)

    print "Collecting static files"
    utils.collectstatic(target)

    print "Deleting extra static files"
    utils.delete_blacklisted_files(target, removejuststatic=True)

    # Django doesn't like deleted py files. Have to investigate more
    # print "Deleting the .py files"
    # utils.delete_py_files(target)

    print "Pregenerating the database"
    utils.generate_db(target)

    print "Adding in a default facility"
    utils.create_default_facility(target)

    # zip up the tmp directory by getting kalite's grandparent
    top_build_directory = target.parent.parent

    outpath = outpath / "ka-lite"
    if output_directory:
        print "Copying built KA Lite to {}".format(outpath)
        shutil.copytree(str(top_build_directory), str(outpath))
    else:
        print "Zipping up everything"
        utils.zip_directory(top_build_directory, out=outpath)
Example #9
0
                    default="localhost",
                    help="""the address of the Redis server to use
                            (default: localhost)""")

args = parser.parse_args()

print("Connecting to Redis at '" + args.redis_host + "'...")

redis_conn = Redis(
    host=args.redis_host)  # Tell RQ what Redis connection to use
q = Queue(connection=redis_conn)  # no args implies the default queue

runs = get_runs_from_filter(args.configuration, args.runfilter)
print("Matched runs: " + ", ".join(runs))

model_source_zip = utils.zip_directory(
    ".", exclude_dirs=["results", "frames", "out"])
print("Size of sources: " + str(len(model_source_zip)) + "B")

jobs = []

print("Enqueueing " + str(len(runs)) + " jobs...")
for r in runs:
    j = q.enqueue(worker.run_simulation, model_source_zip, args.executable,
                  ["-c", args.configuration, "-r", r])
    j.meta['runnumber'] = r
    j.save_meta()
    jobs.append(j)

print("Waiting for results...")

while jobs: