BUCKET_NAME='sa-data-engineering-exercise'
AWS_KEY='AKIAWJ4POKDXO3PJGMOV'
AWS_SECRET_KEY='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'
s3 = boto3.resource(
service_name='s3',
region_name='us-east-2',
aws_access_key_id=AWS_KEY,
aws_secret_access_key=AWS_SECRET_KEY
)
#spark = SparkSession.builder.appName("TTD").getOrCreate()
sc=SparkContext()
hadoopConf = sc.hadoopConfiguration()
spark._jsc.hadoopConfiguration().set("fs.s3a.access.key", "mykey")
spark._jsc.hadoopConfiguration().set("fs.s3a.secret.key", "mysecret")
dataframe = spark.createDataFrame(data, columns)
dataframe.show()
'''
for obj in s3.Bucket(BUCKET_NAME).objects.all():
print(obj)
try:
