def run(): """ 1. Read in image file 2. Pull summary statistics from file 3. Compress image 4. Pad image as appropriate 5. Add date onto image 6. Save images locally 7. Push images to S3 Bucket 8. Remove local copies of images 9. Create appropriate objects 10. Push new object references to database """ today = datetime.now().strftime("%Y-%m-%d") log_file = os.path.abspath("logs/{}.log".format(today)) logger = RsmasLogger("pipeline", log_file) images = get_list_of_images() # LOG: list of images to process logger.log(loglevel.INFO, [img.key for img in images]) for im in images: logger.log(loglevel.INFO, "Processing image: {}".format(im.key)) file_path = "{}/{}".format(im.bucket_name, im.key) full_path = "{}_full.jpg" mod_path = "{}_mod.jpg" aws_path = "{}/{}/{}/{}" try: haz_id, haz_name, sat_name, sat_dir, img_type, img_date, center = summary.pull_summary_data( "/vsis3/{}".format(file_path)) sat_id = Satellite.from_params(sat_name, bool(sat_dir)) except: # LOG: error in image metadata format logger.log( loglevel.ERROR, '\tThere was an error in the metadata format of the image. Skipping.' ) continue aws_path = aws_path.format(haz_id, sat_id, img_type, img_date) full_path = full_path.format(img_date) mod_path = mod_path.format(img_date) # 1. Read in image file with rasterio.open("s3://{}".format(file_path)) as data: band = data.read(1) img = plot.show(band) img.get_figure().savefig(full_path, dpi=300) # 3. Compress image compressed = immanip.compress_image(full_path, compression_amount=0.3) # 4 - 5. Pad image and add date on image text_image = immanip.add_text_to_image(compressed, img_date) # 6. Save image locally text_image.save(mod_path.format(img_date)) mod_path_aws = save.get_s3_url("{}/{}".format(aws_path, mod_path)) full_path_aws = save.get_s3_url("{}/{}".format(aws_path, full_path)) tif_path_aws = save.get_s3_url("{}/{}".format(aws_path, im.key)) # LOG: images successfully moved to S3 bucket # LOG: mod_path_aws, full_path_aws, tif_path_aws hazard = Hazard(haz_id, haz_name, HazardType.VOLCANO, Location(center[0], center[1]), Date(img_date), 0) satellite = Satellite.from_params(sat_name, bool(sat_dir)) image = Image(str(randint(1, 10000000)), haz_id, satellite, ImageType.from_string(img_type), Date(img_date), ImageURL(full_path_aws), ImageURL(tif_path_aws), ImageURL(mod_path_aws)) try: db = Database() except ConnectionError: logger.log( loglevel.ERROR, "\tThere was an error while connecting to the database. Skipping this image." ) continue db.create_new_hazard(hazard) db.create_new_satellite(satellite) db.create_new_image(image) db.close() # LOG: database successfully updated logger.log(loglevel.INFO, "\tDatabase succesfully updated.") save.save_image_s3(mod_path, "{}/{}".format(aws_path, mod_path)) save.save_image_s3(full_path, "{}/{}".format(aws_path, full_path)) save.move_tif(im.key, "{}/{}".format(aws_path, im.key)) logger.log(loglevel.INFO, "\tImages were successfully uploaded to the S3 bucket") logger.log(loglevel.INFO, "\t\tmod_path_aws: {}".format(mod_path_aws)) logger.log(loglevel.INFO, "\t\tfull_path_aws: {}".format(full_path_aws)) logger.log(loglevel.INFO, "\t\ttif_path_aws: {}".format(tif_path_aws)) # LOG: image completed logger.log(loglevel.INFO, "\tProcessing of {} completed.".format(im.key)) # LOG: finished processing images logger.log(loglevel.INFO, "Processing complete.")
# 6. Save image locally text_image.save(mod_path.format(img_date)) mod_path_aws = save.save_image_s3(mod_path, "{}/{}".format(aws_path, mod_path)) full_path_aws = save.save_image_s3(full_path, "{}/{}".format(aws_path, full_path)) tif_path_aws = save.move_tif(im.key, "{}/{}".format(aws_path, im.key)) print(mod_path_aws) print(full_path_aws) print(tif_path_aws) hazard = Hazard(haz_id, haz_name, HazardType.VOLCANO, Location(LatLong(center[0], center[1])), Date(img_date)) sat_id = SatelliteEnum.from_string(sat_name) satellite = Satellite(sat_id, sat_dir) image = Image(str(randint(1, 10000000)), haz_id, satellite, ImageType.from_string(img_type), Date(img_date), ImageURL(full_path_aws), ImageURL(tif_path_aws), ImageURL(mod_path_aws)) db = Database() db.create_new_hazard(hazard) db.create_new_satellite(satellite) db.create_new_image(image) db.close()