def instance(every=None):
global _instance
if not _instance:
if every:
_instance = Timer(every)
_instance.start()
print("Timers started to loop every", every, "s")
else:
raise ValueError("No timer set up.")
return _instance
def __init__(self,
corpus,
fonts,
bgs,
cfg,
width=256,
height=32,
clip_max_chars=False,
debug=False,
gpu=False,
strict=False):
self.corpus = corpus
self.fonts = fonts
self.bgs = bgs
self.out_width = width
self.out_height = height
self.clip_max_chars = clip_max_chars
self.max_chars = math.floor(width / 4) - 1
self.debug = debug
self.gpu = gpu
self.strict = strict
self.cfg = cfg
self.timer = Timer()
self.liner = Liner(cfg)
self.noiser = Noiser(cfg)
self.remaper = Remaper(cfg)
self.create_kernals()
if self.strict:
self.font_chars = get_fonts_chars(self.fonts, corpus.chars_file)
def __init__(self, corpus, fonts, bgs, cfg, width=256, height=32,
clip_max_chars=False, debug=False, gpu=False, strict=False, fonts_by_image=False):
self.corpus = corpus
self.fonts = fonts
self.bgs = bgs
self.out_width = width
self.out_height = height
self.clip_max_chars = clip_max_chars
self.max_chars = math.floor(width / 4) - 1
self.debug = debug
self.gpu = gpu
self.strict = strict
self.cfg = cfg
self.fonts_by_image = fonts_by_image
self.timer = Timer()
self.liner = Liner(cfg)
self.noiser = Noiser(cfg)
self.texture = Texture(cfg)
self.remaper = Remaper(cfg)
self.start = time.time()
self.create_kernals()
if self.strict:
self.font_unsupport_chars = font_utils.get_unsupported_chars(
self.fonts, corpus.chars_file, self.fonts_by_image)
def fit(self, embeddings):
X = embeddings
if self.is_full_matrix:
X = X[self.data.indices]
with Timer(disable=not self.verbose):
clu = self.algo(**self.params).fit(X)
return self.set_clustering(clu)
def __init__(self,
corpus,
fonts,
bgs,
cfg,
width=256,
height=32,
debug=False,
gpu=False,
strict=False):
self.corpus = corpus
self.fonts = fonts
self.bgs = bgs
self.out_width = width
self.out_height = height
self.debug = debug
self.gpu = gpu
self.strict = strict
self.cfg = cfg
self.timer = Timer()
self.liner = Liner(cfg)
self.noiser = Noiser(cfg)
if self.strict:
self.font_chars = get_fonts_chars(self.fonts, corpus.chars_file)
def train_and_validate_xgboost(params, train_features, train_labels,
validation_features, num_boost_round):
n_classes = train_labels.shape[1]
y_val_pred = np.zeros((validation_features.shape[0], n_classes))
time_results = defaultdict(list)
for class_i in tqdm(range(n_classes)):
dtrain = xgb.DMatrix(data=train_features,
label=train_labels[:, class_i],
nthread=-1)
dtest = xgb.DMatrix(data=validation_features, nthread=-1)
with Timer() as t:
model = xgb.train(params, dtrain, num_boost_round=num_boost_round)
time_results['train_time'].append(t.interval)
with Timer() as t:
y_val_pred[:, class_i] = model.predict(dtest)
time_results['test_time'].append(t.interval)
return y_val_pred, time_results
def train_and_validate_lightgbm(params, train_features, train_labels,
validation_features, num_boost_round):
n_classes = train_labels.shape[1]
y_val_pred = np.zeros((validation_features.shape[0], n_classes))
time_results = defaultdict(list)
for class_i in tqdm(range(n_classes)):
lgb_train = lgb.Dataset(train_features,
train_labels[:, class_i],
free_raw_data=False)
with Timer() as t:
model = lgb.train(params,
lgb_train,
num_boost_round=num_boost_round)
time_results['train_time'].append(t.interval)
with Timer() as t:
y_val_pred[:, class_i] = model.predict(validation_features)
time_results['test_time'].append(t.interval)
return y_val_pred, time_results
def _compute_mapping(self, clustering, allow_root=False, **kwargs):
F = get_F_matrix(clustering) if "F" not in kwargs else kwargs["F"]
if not allow_root:
# Prevent root cluster from being associated with a type
F.iloc[clustering.root_id] = 0.0
# Compute F-matrix : Fij = F1(clu_i, cls_j)
#F = get_F_matrix(clustering)
# Compute a mapping m: cls -> clu that maximizes sum_j F1(m(cls_j), cls_j)
with Timer():
rows, cols = linear_sum_assignment(-F.values)
int_to_cls = {
i: cls
for i, cls in enumerate(clustering.data.name2cls.keys())
}
cls_to_clu = {int_to_cls[i]: clust for clust, i in zip(rows, cols)}
return cls_to_clu
def __init__(self,
corpus,
fonts,
bgs,
texteffect,
width=256,
height=32,
debug=False,
gpu=False):
self.corpus = corpus
self.fonts = fonts
self.bgs = bgs
self.out_width = width
self.out_height = height
self.debug = debug
self.gpu = gpu
self.timer = Timer()
self.textstate = TextState()
self.texteffect = texteffect
self.liner = Liner()
self.noiser = Noiser()
def __init__(self,
corpus,
fonts,
bgs,
cfg,
width=256,
height=32,
clip_max_chars=False,
debug=True,
gpu=False,
strict=False):
self.cnt = 0
self.corpus = corpus
self.fonts = fonts
self.bgs = bgs
self.out_width = width
self.out_height = height
self.clip_max_chars = clip_max_chars
self.max_chars = math.floor(width / 4) - 1
self.debug = debug
self.gpu = gpu
self.strict = strict
self.cfg = cfg
self.timer = Timer()
self.liner = Liner(cfg)
self.noiser = Noiser(cfg)
self.remaper = Remaper(cfg)
self.create_kernals()
if not self.is_bgr():
for i, bg in enumerate(self.bgs):
self.bgs[i] = cv2.cvtColor(bg, cv2.COLOR_BGR2GRAY)
if self.strict:
self.font_unsupport_chars = font_utils.get_unsupported_chars(
self.fonts, corpus.chars_file)
def __init__(self, corpus, fonts, bgs, cfg, width=256, height=32, space_ratio=0.0,
clip_max_chars=False, max_chars=None, debug=False, gpu=False, strict=False):
self.corpus = corpus
self.fonts = fonts
self.bgs = bgs
self.out_width = width
self.out_height = height
self.space_ratio = space_ratio
self.clip_max_chars = clip_max_chars
self.max_chars = math.floor(width / 4) - 1 if max_chars is None else max_chars
self.debug = debug
self.gpu = gpu
self.strict = strict
self.cfg = cfg
self.timer = Timer()
self.liner = Liner(cfg)
self.noiser = Noiser(cfg)
self.remaper = Remaper(cfg)
self.create_kernals()
if self.strict:
self.font_unsupport_chars = font_utils.get_unsupported_chars(self.fonts, corpus.chars_file)
params = {'max_depth':2,
'objective':'binary:logistic',
'min_child_weight':1,
'learning_rate':0.1,
'scale_pos_weight':2,
'gamma':0.1,
'reg_lamda':1,
'subsample':1,
'tree_method':'gpu_hist'
}
# In[13]:
with Timer() as t_train:
xgb_hist_clf_pipeline = xgb.train(params, dtrain, num_boost_round=num_rounds)
with Timer() as t_test:
y_prob_xgb_hist = xgb_hist_clf_pipeline.predict(dtest)
# In[14]:
y_pred_xgb_hist = binarize_prediction(y_prob_xgb_hist)
# In[15]:
def test_lightgbm(clf, X):
with Timer() as t:
y_pred = clf.predict(X.values)
return y_pred, t.interval
def train_lightgbm(parameters, X, y, num_rounds=50):
ddata = lgb.Dataset(X.values, y.values, free_raw_data=False)
with Timer() as t:
clf = lgb.train(parameters, ddata, num_boost_round=num_rounds)
return clf, t.interval
def train_xgboost(parameters, X, y, num_rounds=50):
ddata = xgb.DMatrix(data=X, label=y, nthread=-1)
with Timer() as t:
clf = xgb.train(parameters, ddata, num_boost_round=num_rounds)
return clf, t.interval
logging.basicConfig(level=logging.INFO)
# Setup Flask
app = Flask(__name__)
app.secret_key = urandom(75)
# Setup Spark
spark = Spark(secrets['spark']['server'], 'Project-IV-Demo-app',
secrets['elastic']['version'], secrets['elastic']['internal'],
secrets['spark']['port'])
# Setup Elasticsearch
elastic = Elastic(secrets['elastic']['server'], secrets['elastic']['port'])
# Create timer
timer = Timer()
# Create threader
threader = Threader()
def load_dataset(idx):
global LOADED_INDICES, ACTIVE_INDICES
if idx not in LOADED_INDICES:
logging.info(f'Loading index {idx} into Spark')
spark.load_index(idx)
LOADED_INDICES.add(idx)
ACTIVE_INDICES.add(idx)
def reload_datasets_into_memory():
# https://github.com/pytorch/pytorch/issues/3492#issuecomment-382660636
if utils.get_platform() == "OS X":
mp.set_start_method('spawn', force=True)
if flags.viz == 1:
flags.num_processes = 1
tmp_label_path = os.path.join(flags.save_dir, 'tmp_labels.txt')
label_path = os.path.join(flags.save_dir, 'labels.txt')
manager = mp.Manager()
q = manager.Queue()
start_index = restore_exist_labels(label_path)
timer = Timer(Timer.SECOND)
timer.start()
with mp.Pool(processes=get_num_processes(flags)) as pool:
if not flags.viz:
pool.apply_async(start_listen, (q, tmp_label_path))
pool.starmap(
generate_img,
zip(range(start_index, start_index + flags.num_img), repeat(q)))
q.put(STOP_TOKEN)
pool.close()
pool.join()
timer.end("Finish generate data")
if not flags.viz:
def test_xgboost(clf, X, y):
ddata = xgb.DMatrix(data=X, label=y, nthread=-1)
with Timer() as t:
y_pred = clf.predict(ddata)
return y_pred, t.interval
