Пример #1
0
def run():
    """
        1. Read in image file
        2. Pull summary statistics from file
        3. Compress image
        4. Pad image as appropriate
        5. Add date onto image
        6. Save images locally
        7. Push images to S3 Bucket
        8. Remove local copies of images
        9. Create appropriate objects
        10. Push new object references to database
    """

    today = datetime.now().strftime("%Y-%m-%d")
    log_file = os.path.abspath("logs/{}.log".format(today))
    logger = RsmasLogger("pipeline", log_file)

    images = get_list_of_images()
    # LOG: list of images to process
    logger.log(loglevel.INFO, [img.key for img in images])

    for im in images:

        logger.log(loglevel.INFO, "Processing image: {}".format(im.key))

        file_path = "{}/{}".format(im.bucket_name, im.key)
        full_path = "{}_full.jpg"
        mod_path = "{}_mod.jpg"
        aws_path = "{}/{}/{}/{}"
        try:
            haz_id, haz_name, sat_name, sat_dir, img_type, img_date, center = summary.pull_summary_data(
                "/vsis3/{}".format(file_path))
            sat_id = Satellite.from_params(sat_name, bool(sat_dir))
        except:
            # LOG: error in image metadata format
            logger.log(
                loglevel.ERROR,
                '\tThere was an error in the metadata format of the image. Skipping.'
            )
            continue

        aws_path = aws_path.format(haz_id, sat_id, img_type, img_date)
        full_path = full_path.format(img_date)
        mod_path = mod_path.format(img_date)

        # 1. Read in image file
        with rasterio.open("s3://{}".format(file_path)) as data:
            band = data.read(1)
            img = plot.show(band)
            img.get_figure().savefig(full_path, dpi=300)

        # 3. Compress image
        compressed = immanip.compress_image(full_path, compression_amount=0.3)

        # 4 - 5. Pad image and add date on image
        text_image = immanip.add_text_to_image(compressed, img_date)

        # 6. Save image locally
        text_image.save(mod_path.format(img_date))
        mod_path_aws = save.get_s3_url("{}/{}".format(aws_path, mod_path))
        full_path_aws = save.get_s3_url("{}/{}".format(aws_path, full_path))

        tif_path_aws = save.get_s3_url("{}/{}".format(aws_path, im.key))

        # LOG: images successfully moved to S3 bucket
        # LOG: mod_path_aws, full_path_aws, tif_path_aws

        hazard = Hazard(haz_id, haz_name, HazardType.VOLCANO,
                        Location(center[0], center[1]), Date(img_date), 0)
        satellite = Satellite.from_params(sat_name, bool(sat_dir))
        image = Image(str(randint(1, 10000000)), haz_id, satellite,
                      ImageType.from_string(img_type), Date(img_date),
                      ImageURL(full_path_aws), ImageURL(tif_path_aws),
                      ImageURL(mod_path_aws))

        try:
            db = Database()
        except ConnectionError:
            logger.log(
                loglevel.ERROR,
                "\tThere was an error while connecting to the database. Skipping this image."
            )
            continue

        db.create_new_hazard(hazard)
        db.create_new_satellite(satellite)
        db.create_new_image(image)

        db.close()

        # LOG: database successfully updated
        logger.log(loglevel.INFO, "\tDatabase succesfully updated.")

        save.save_image_s3(mod_path, "{}/{}".format(aws_path, mod_path))
        save.save_image_s3(full_path, "{}/{}".format(aws_path, full_path))
        save.move_tif(im.key, "{}/{}".format(aws_path, im.key))

        logger.log(loglevel.INFO,
                   "\tImages were successfully uploaded to the S3 bucket")
        logger.log(loglevel.INFO, "\t\tmod_path_aws: {}".format(mod_path_aws))
        logger.log(loglevel.INFO,
                   "\t\tfull_path_aws: {}".format(full_path_aws))
        logger.log(loglevel.INFO, "\t\ttif_path_aws: {}".format(tif_path_aws))

        # LOG: image completed
        logger.log(loglevel.INFO,
                   "\tProcessing of {} completed.".format(im.key))

    # LOG: finished processing images
    logger.log(loglevel.INFO, "Processing complete.")
Пример #2
0
        # 6. Save image locally
        text_image.save(mod_path.format(img_date))
        mod_path_aws = save.save_image_s3(mod_path, "{}/{}".format(aws_path, mod_path))
        full_path_aws = save.save_image_s3(full_path, "{}/{}".format(aws_path, full_path))

        tif_path_aws = save.move_tif(im.key, "{}/{}".format(aws_path, im.key))

        print(mod_path_aws)
        print(full_path_aws)
        print(tif_path_aws)

        hazard = Hazard(haz_id, haz_name, HazardType.VOLCANO, Location(LatLong(center[0], center[1])), Date(img_date))
        sat_id = SatelliteEnum.from_string(sat_name)
        satellite = Satellite(sat_id, sat_dir)
        image = Image(str(randint(1, 10000000)),
                      haz_id,
                      satellite,
                      ImageType.from_string(img_type),
                      Date(img_date),
                      ImageURL(full_path_aws),
                      ImageURL(tif_path_aws),
                      ImageURL(mod_path_aws))

        db = Database()
        db.create_new_hazard(hazard)
        db.create_new_satellite(satellite)
        db.create_new_image(image)
        db.close()