def generate_preview_table(self,
number_of_rows_in_preview=None,
path_file=None):
if (number_of_rows_in_preview is None):
number_of_rows_in_preview = self.get_number_of_rows_in_preview()
if (self.get_number_of_rows_in_preview() is None):
raise exceptions.CustomError(
"Error in default values. Cannot find default number of rows in preview"
)
if (path_file is None):
path_file = self.get_path_file()
if (self.get_path_file() is None):
raise exceptions.CustomError("Cannot find file path.")
self.set_path_file(path_file)
with open(self.get_path_file(), "r", newline="") as csvfile:
list_lines = csvfile.readlines()
list_lines = list_lines[:self.get_number_of_rows_in_preview()]
min_number_of_columns = min(
[len(line.split(self.get_delimiter())) for line in list_lines])
list_rows = []
for line in list_lines:
row = []
counter = 0
for cell in line.split(self.get_delimiter()):
if (counter == min_number_of_columns):
row.append(str(line.split(self.get_delimiter())[counter:]))
else:
row.append(cell)
list_rows.append(row)
return (list_rows)
def generate_delimiter(self, path_file=None):
"""
Returns a list of probable delimiters in decreasing order of probability
"""
if (path_file is None):
path_file = self.get_path_file()
if (self.get_path_file() is None):
raise exceptions.CustomError("Cannot find source file")
self.set_path_file(path_file)
with open(self.get_path_file(), "r", newline="") as csvfile:
list_lines = csvfile.readlines()
if (len(list_lines) > 0):
dict_delimiter_count = {}
for line_number in range(
0,
min(len(list_lines),
self.get_number_of_rows_in_preview())):
for delimiter in self.list_allowed_delimiters:
if (delimiter not in dict_delimiter_count.keys()):
dict_delimiter_count[delimiter] = 0
else:
dict_delimiter_count[delimiter] += list(
list_lines[line_number]).count(delimiter)
list_tuple_delimiter_count = sorted(dict_delimiter_count.items(),
key=lambda item: item[1],
reverse=True)
self.set_delimiter(list_tuple_delimiter_count[0][0])
self.set_path_file(path_file)
return (list_tuple_delimiter_count)
else:
return (None)
def set_server():
try:
server = smtplib.SMTP_SSL()
server = smtplib.SMTP_SSL(host=config.configs.email_host, port=465)
server.login(config.configs.sender_email, config.configs.sender_pswd)
except Exception:
raise exceptions.CustomError("Error when trying to login to {}".format(
config.configs.sender_email))
return server
def send_message(json_message):
producer = get_kafka_producer()
try:
producer.send(config.configs.kafka_topic, json_message)
producer.flush()
except Exception:
raise exceptions.CustomError(
"Error when trying to connect to send message to Kafka Server",
500)
def change_columns_names_dataset(self, list_columns, dataset=None):
if (dataset is None):
dataset = self.get_dataset()
if (self.get_dataset() is None):
raise exceptions.CustomError("Cannot find dataset.")
dataset.columns = list_columns
self.set_dataset(dataset)
return
def get_kafka_producer():
global kafka_instance
if kafka_instance is None:
try:
kafka_instance = KafkaProducer(
bootstrap_servers=[config.configs.kafka_bootstrap_server],
value_serializer=lambda v: json.dumps(v).encode('utf-8'))
except Exception:
raise exceptions.CustomError(
"Error when trying to connect to Kafka Server", 500)
return kafka_instance
def get_kafka_producer():
global kafka_instance
if kafka_instance is None:
try:
kafka_instance = KafkaConsumer(
config.configs.kafka_topic,
bootstrap_servers=[config.configs.kafka_bootstrap_server],
auto_offset_reset='earliest',
consumer_timeout_ms=1000)
except Exception:
raise exceptions.CustomError(
"Error when trying to connect to Kafka Server")
return kafka_instance
def generate_dataset(self, path_file=None):
if (path_file is None):
path_file = self.get_path_file()
if (self.get_path_file() is None):
raise exceptions.CustomError("Cannot find file path.")
try:
self.set_dataset(
pd.read_csv(self.get_path_file(),
sep=self.get_delimiter(),
index_col=False))
except Exception as e:
print(e)
return
def send_email(server, subject, body):
msg = MIMEMultipart()
msg['From'] = config.configs.sender_email
msg['To'] = config.configs.manager_email
msg['Subject'] = "ShoppingCartAPI notification: " + subject
msg.attach(MIMEText(body, 'plain'))
try:
server.send_message(msg)
except Exception:
raise exceptions.CustomError("Error when trying to send email")
del msg
def set_delimiter(self, delimiter):
if (delimiter not in self.list_allowed_delimiters):
raise exceptions.CustomError("Invalid delimiter")
self.delimiter = delimiter
return