def init_spark_with_s3(self, s3):
"""
:param s3:
:return:
"""
success = False
message = ""
for i in range(10):
try:
self.spark_context = SparkContext().getOrCreate()
self.hadoop_conf = self.spark_context._jsc.hadoopConfiguration(
)
self.hadoop_conf.set(
"fs.s3n.impl",
"org.apache.hadoop.fs.s3native.NativeS3FileSystem")
self.hadoop_conf.set("fs.s3n.awsAccessKeyId", s3["ID"])
self.hadoop_conf.set("fs.s3n.awsSecretAccessKey", s3["key"])
self.spark_session = SparkSession(self.spark_context)
success = True
continue
except Exception as e:
logger.error(e)
message = e
pass
return success, message
def read(self, address="", file_format="csv", s3={}):
"""
:param address:
:param file_format:
:param s3:
:return:
"""
try:
status, message = self.init_spark_with_s3(s3)
if status is False:
return {"success": status, "message": message}
self.s3 = s3
self.address = address
if file_format == "csv":
self.dataframe = self.read_csv(address)
elif file_format == "excel":
self.dataframe = self.read_excel(address)
elif file_format == "parquet":
self.dataframe = self.read_parquet(address)
else:
print(
"File format " + file_format +
" is currently not supported. Please create a feature request on Github"
)
return self.dataframe
except Exception as e:
logger.error(e)
def read_parquet(self, path):
try:
self.dataframe = self.spark_session.read.load('s3n://' +
self.s3["bucket"] +
'/' + path)
return self.dataframe
except Exception as e:
logger.error(e)
return {"success": False, "message": e}
def date_formatting(self, x):
"""
dateutill library is used to convert the different format of dates into standard format
:param x: row wise date values
:return: standard format of date
"""
try:
return str(parser.parse(x))
except Exception as e:
logger.error(e)
return str(x)
def read_parquet(self, path):
"""
:param path:
:return:
"""
try:
self.dataframe = self.spark_session.read.load(path)
return self.dataframe
except Exception as e:
logger.error(e)
return {"success": False, "message": e}
def read_excel(self, path):
"""
:param path:
:return:
"""
try:
self.dataframe = self.spark_session.read.csv(path, inferSchema=True, header=True)
return self.dataframe
except Exception as e:
logger.error(e)
return {"success": False, "message": e}