Beispiel #1
0
 def reset_files_extension(self, storage_name, prefix):
     bucket = self.__s3.get_bucket(storage_name)
     for key in bucket.list(prefix=prefix):
         if key.name.endswith('-done'):
             new_key_name = key.name.replace('-done','')
             bucket.copy_key(new_key_name=new_key_name, src_bucket_name=storage_name, src_key_name=key.name)
             bucket.delete_key(key.name)
Beispiel #2
0
 def __get_file_contents_list_from_bucket(bucket, prefix, bucket_name):
     json_files_list = []
     for key in bucket.list(prefix=prefix):
         if key.name.endswith('/') or key.name.endswith('-done'):
             continue
         try:
             new_key_name = "{}-done".format(key.name)
             bucket.copy_key(new_key_name=new_key_name, src_bucket_name=bucket_name, src_key_name=key.name)
             bucket.delete_key(key.name)
             new_key = bucket.get_key(new_key_name)
             new_key.get_contents_to_filename(filename="tmp.json.gz")
             f = gzip.open('tmp.json.gz', 'rb')
             json_files_list.append(f.read())
             f.close()
         except Exception as ex:
             Logger.log("warning", "{} FAILED: {}".format(key.name, ex.message))
     return json_files_list