def test_generator(self, keep_data=False):
B, W, H = self.flags.batch_size, self.flags.width, self.flags.height
path = self.flags.input_path
loaded = {}
if "cv" not in self.flags.task:
# test images to submit
imgs = ["%s/%s" % (path, i) for i in os.listdir(path)]
else:
imgs = split(self.flags)[self.flags.fold]
self.test_imgs = imgs
xs, ns = [], []
for img in imgs:
if len(xs) == B:
yield np.array(xs), ns
del xs, ns
xs, ns = [], []
if img in loaded:
x = loaded[img]
else:
x = np.array(Image.open(img).resize([W, H]))
if keep_data:
loaded[img] = x
xs.append(x)
ns.append(img.split('/')[-1])
if len(xs):
yield np.array(xs), ns
def va_generator(self, keep_data=True, first=False):
fold = self.flags.fold
dic = split(self.flags)
imgs = dic[fold]
del dic
if first:
print("Valid using fold {} images {}".format(fold, len(imgs)))
return self._gen_random_batch(1, imgs, keep_data)
def tr_generator(self, keep_data=True):
fold = self.flags.fold
epochs = self.flags.epochs
dic = split(self.flags)
imgs = sum([dic[i] for i in dic if i != fold], [])
folds = [i for i in dic if i != fold]
print("Train using folds {} images {}".format(folds, len(imgs)))
del dic
return self._gen_random_batch(epochs, imgs, keep_data)
def analysis_timeout_order(self):
print("===> analysis timeout order")
item = self.by_xpath(
'//*[@id="container"]/div/div[2]/div[2]/div/div[2]/div[2]/div[4]/div[2]/div[7]/div[2]'
)
item.click()
try:
orders = []
time.sleep(5)
curr_orders, count = [], 0
while len(curr_orders) < 8 and count < 30:
time.sleep(2)
timeout_orders = self.by_xpath(
'/html/body/div[2]/div/div[2]/div[2]/div[2]/div/div[2]/table/tbody'
)
curr_orders = timeout_orders.text.split("\n")
count += 1
page_orders = split(curr_orders, 8)
orders += page_orders
next_button = self.by_xpath(
'/html/body/div[2]/div/div[2]/div[2]/div[3]/div/button[2]')
while next_button.is_enabled():
next_button.click()
curr_orders = [], 0
while len(curr_orders) < 8 and count < 30:
time.sleep(2)
timeout_orders = self.by_xpath(
'/html/body/div[2]/div/div[2]/div[2]/div[2]/div/div[2]/table/tbody'
)
curr_orders = timeout_orders.text.split("\n")
count += 1
page_odrers = split(curr_orders, 8)
orders += page_orders
larger_than_35 = len(
[item for item in orders if item != '-' and int(item) >= 35])
except:
print("===> can't find such orders!")
larger_than_35 = 0
close_button = self.by_xpath_with_sleep(
'/html/body/div[2]/div/div[2]/a/i', 1)
close_button.click()
return larger_than_35
def parser(self, content: str):
lines = [l.strip() for l in content.split('\n') if l.strip()]
lines = [l for l in lines if not l.startswith(';')]
section_list = utils.split(
lines, lambda l: l.startswith('[') and l.endswith(']'))
ini = INI()
for s in section_list:
self.parser_section(ini, s)
logger.info(ini)
return ini
def write_cv_tfrecords(self):
folds = self.flags.folds
dic = split(self.flags)
for i in range(folds):
record_path = self.flags.record_path.replace(
".tfrecords", "_%d.tfrecords" % i)
imgs = dic[i]
labels = [
img.replace(".jpg",
"_mask.gif").replace("train", "train_masks")
for img in imgs
]
self.write_tfrecord(imgs, labels, record_path)
def get_bboxes(img, gt_path):
h, w = img.shape[0:2]
# lines = read_lines(gt_path)
bboxes = []
tags = []
with open(gt_path, encoding='utf-8', mode='r') as f:
for line in f.readlines():
line = remove_all(line, '\xef\xbb\xbf')
gt = split(line, ',')
if gt[-1][0] == '#':
tags.append(False)
else:
tags.append(True)
box = [int(gt[i]) for i in range(8)]
box = np.asarray(box) / ([w * 1.0, h * 1.0] * 4)
bboxes.append(box)
return np.array(bboxes), tags
def get_bboxes(img, gt_path):
h, w = img.shape[0:2]
lines = read_lines(gt_path)
bboxes = []
tags = []
for line in lines:
line = remove_all(line, '\xef\xbb\xbf')
gt = split(line, ',')
x1 = np.int(gt[0])
y1 = np.int(gt[1])
bbox = [np.int(gt[i]) for i in range(4, 32)]
bbox = np.asarray(bbox) + ([x1 * 1.0, y1 * 1.0] * 14)
bbox = np.asarray(bbox) / ([w * 1.0, h * 1.0] * 14)
bboxes.append(bbox)
tags.append(True)
return np.array(bboxes), tags
# For the supervised learning, I used two common technics on categorical data: # * Counting: Which consist of creating new features by counting the frequences of unique values of each feature. # * Target encoding: Which consist of encoding the data using the target variable. # %% df_count_encode = utils.count_encoding(df.copy()) df_targ_encoded = utils.target_encode(df_count_encode, target_2_classes) # %% df_targ_encoded.head() # %% [markdown] # ## Splitting the data into train and validation sets # %% x_train, x_val, y_train, y_val = utils.split(df_targ_encoded, target_2_classes) # %% models_common_params_GBM = dict(random_state=constants.RANDOM_STATE, verbose=0) # %% [markdown] # ## Defining models # %% [markdown] # We will be using 6 models: # * LightGBM # * XGBoost # * CatBoost # * 2 layers Neural Nets # * Explainable Boosting Classifier # * Hist Gradient Boosting Classifier
)
while True:
option: str = input(f"{fore}Please select the option you want to use: ")
while option not in ["1", "2", "3"]:
delete_lines(1)
option = input(f"{fore}Please select a correct option: ")
if option == "1":
text: str = input(
f"{fore}\nPlease write something for the webhook to send: ")
wb.send(content=split(text),
username=data.username,
avatar_url=data.avatar_url)
print(f"\n{fore}Message sent successfully.")
time.sleep(1)
delete_lines(5)
elif option == "2":
title, description, image, thumbnail, footer = handle_embed_input()
embed: Embed = Embed(
title=split(title),
description=split(description),
)