def __init__(self):
# Zeno task manager
self.tasks = zeno.TasksManager("tasks",
host=config.DB_HOST,
user=config.DB_USER,
passwd=config.DB_PASSWD)
# Database connection
self.db = MyMySQL(db=config.DB_NAME,
host=config.DB_HOST,
user=config.DB_USER,
passwd=config.DB_PASSWD)
# Logging configuration
self.log = utils.config_logging(
'tokenizer',
stream=sys.stdout,
level=logging.DEBUG,
format='%(asctime)s (%(name)s) [%(levelname)6s]: %(message)s',
datefmt="%Y-%m-%d %H:%M:%S")
self.MIN_TOKENS = 10
# Create folders with non existing
utils.ensure_folder(os.path.dirname(config.TOKENS_PATH))
utils.ensure_folder(os.path.dirname(config.TOKENS_PATH_PARTS))
def upload_file():
if request.method == 'POST':
start = time.time()
ensure_folder('static')
file1 = request.files['file1']
filename_1 = secure_filename(file1.filename)
full_path_1 = os.path.join('static', filename_1)
file1.save(full_path_1)
resize(full_path_1)
file2 = request.files['file2']
filename_2 = secure_filename(file2.filename)
full_path_2 = os.path.join('static', filename_2)
file2.save(full_path_2)
resize(full_path_2)
try:
prob, is_same = compare(full_path_1, full_path_2)
elapsed = time.time() - start
message = '是否同一个人: {}, 置信度为 {:.4f}, 耗时: {:.4f} 秒。'.format(
is_same, prob, elapsed)
except FaceNotFoundError as err:
message = '对不起，[{}] 图片中没有检测到人类的脸。'.format(err)
return render_template('show.html',
message=message,
filename_1=filename_1,
filename_2=filename_2)
def __init__(self, **params):
self.params = params
if not os.path.exists(config.MENG_GRAPH_PATH):
log.debug("Meng graph file not found. Building one at '%s'" %
config.MENG_GRAPH_PATH)
mb = MengModelBuilder()
self.graph = mb.build()
del mb
log.debug(
"Meng graph built. %d nodes and %d edges." %
(self.graph.number_of_nodes(), self.graph.number_of_edges()))
utils.ensure_folder(os.path.dirname(config.MENG_GRAPH_PATH))
nx.write_gexf(self.graph, config.MENG_GRAPH_PATH)
log.debug("Meng graph saved.")
else:
log.debug("Reading Meng graph file at '%s'" %
config.MENG_GRAPH_PATH)
self.graph = nx.read_gexf(config.MENG_GRAPH_PATH, node_type=int)
def playlist_meta(playlist_url):
# try parsing the playlist url
try:
parsed = urlparse(playlist_url)
except ValueError:
click.echo('Failed parsing playlist url, stopping.')
return
# parse the query string
query_dict = parse_qs(parsed.query)
# try to get the playlist id out from it
try:
playlist_id = query_dict['list'][0]
except KeyError:
click.echo('Invalid playlist url, stopping.')
return
# create/get playlists folder
playlists_folder = os.path.abspath(os.path.join(config.BASE_FOLDER, 'playlists'))
ensure_folder(playlists_folder)
# create/get playlist output folder
output_folder = os.path.join(playlists_folder, playlist_id)
ensure_folder(output_folder)
runnable = create_runnable(
query=[playlist_url],
output=os.path.join(output_folder, config.PLAYLIST_FORMAT),
archive=os.path.join(output_folder, 'archive'))
subprocess.call(runnable)
def process_data():
with open(tran_file, 'r', encoding='utf-8') as file:
lines = file.readlines()
samples = []
for i, line in enumerate(lines):
tokens = line.strip().split()
audiopath = 'data/km_kh_male/wavs/{}.wav'.format(tokens[0])
text = ''.join(tokens[1:])
for token in text:
build_vocab(token)
samples.append('{}|{}\n'.format(audiopath, text))
valid_ids = random.sample(range(len(samples)), 100)
train = []
valid = []
for id in range(len(samples)):
sample = samples[id]
if id in valid_ids:
valid.append(sample)
else:
train.append(sample)
ensure_folder('filelists')
# print(samples)
with open(training_files, 'w', encoding='utf-8') as file:
file.writelines(train)
with open(validation_files, 'w', encoding='utf-8') as file:
file.writelines(valid)
print('num_train: ' + str(len(train)))
print('num_valid: ' + str(len(valid)))
def main(args):
out_dir = args.out_dir
basename = os.path.basename(args.input)
basename = basename.strip('.json.gz')
create_time_field_mapping = {
'reddit': 'created_utc', # timestamp
'twitter': 'created_at' # date_str
}
date_transform_func = {
'reddit': get_day_from_timestamp,
'twitter': get_day_from_date
}
keywords_count = defaultdict(dict)
for line in gzip.open(args.input, 'r'):
data = json.loads(line.decode('utf8'))
kw_list = data['annotations']['keywords']
for kw in kw_list:
create_time = data[create_time_field_mapping[args.platform]]
date_str = date_transform_func[args.platform](create_time)
if kw not in keywords_count[date_str]:
keywords_count[date_str][kw] = 1
else:
keywords_count[date_str][kw] += 1
out_fn = os.path.join(out_dir, 'keywords_count')
out_fn = os.path.join(out_fn, '{}_counts_date_keywords.csv'.format(basename))
ensure_folder(out_fn)
with open(out_fn, 'w') as outf:
outf.write('date,keyword,count\n')
for date_str, sub_dict in keywords_count.items():
for kw, count in sub_dict.items():
outf.write('{},{},{}\n'.format(date_str, kw, count))
def ensure_vgg16_weights():
import os
if not os.path.isfile('models/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5'):
ensure_folder('models')
import urllib.request
urllib.request.urlretrieve(
"https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5",
filename="models/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5")
def ensure_yolo_weights():
import os
if not os.path.isfile('models/yolo.weights'):
ensure_folder('models')
import urllib.request
urllib.request.urlretrieve(
"https://pjreddie.com/media/files/yolo.weights",
filename="models/yolo.weights")
def detail_trimap(img_folder, trimap_folder, output_folder):
checkpoint = 'preModel/BEST_checkpoint.tar'
checkpoint = torch.load(checkpoint)
model = checkpoint['model'].module
model = model.to(device)
model.eval()
transformer = data_transforms['valid']
ensure_folder(output_folder)
files = [
f for f in os.listdir(img_folder)
if f.endswith('.png') or f.endswith('.jpg')
]
for file in tqdm(files):
filename = os.path.join(img_folder, file)
img = cv.imread(filename)
print(img.shape)
h, w = img.shape[:2]
x = torch.zeros((1, 4, h, w), dtype=torch.float)
image = img[..., ::-1] # RGB
image = transforms.ToPILImage()(image)
image = transformer(image)
x[0:, 0:3, :, :] = image
file = os.path.splitext(file)[0] + ".png"
filename = os.path.join(trimap_folder, file)
print('reading {}...'.format(filename))
trimap = cv.imread(filename, 0)
x[0:, 3, :, :] = torch.from_numpy(trimap.copy() / 255.)
# print(torch.max(x[0:, 3, :, :]))
# print(torch.min(x[0:, 3, :, :]))
# print(torch.median(x[0:, 3, :, :]))
# Move to GPU, if available
x = x.type(torch.FloatTensor).to(device)
with torch.no_grad():
pred = model(x)
pred = pred.cpu().numpy()
pred = pred.reshape((h, w))
pred[trimap == 0] = 0.0
pred[trimap == 255] = 1.0
out = (pred.copy() * 255).astype(np.uint8)
filename = os.path.join(output_folder, file)
cv.imwrite(filename, out)
print('wrote {}.'.format(filename))
def fit(self, x_train, y_train):
# TODO: Validation of parameters
# Train/validation split
x_train, x_val, y_train, y_val = train_test_split(
x_train,
y_train,
# TODO: mb make `test_size` tunable?
test_size=0.4,
random_state=42)
# NOTE: we make category matrix from y_train here!
y_train = (y_train > 0).astype(int)
self.model_ = self._create_model(x_train.shape[1:])
weights_folder = os.path.join(os.getcwd(), self.label,
WEIGHTS_DIRECTORY)
# TODO: hardcoded monitor variable. Move it to config file
callbacks = [
SaveCallback(monitor_variable='val_classifier_output_acc',
save_dir=weights_folder,
model=self.model_,
verbose=self.verbose,
save_best_only=self.save_best_only,
debug=self.debug)
]
# Take care of tensorboard
tb_folder = os.path.join(os.getcwd(), self.label,
TENSORBOARD_DIRECTORY)
ensure_folder(tb_folder)
callbacks.append(TensorBoard(log_dir=tb_folder))
if self.load_weights_from_file:
self.model_ = load_model(self.load_weights_from_file)
else:
# Train it
self.model_.fit(
# Train data
x_train,
# Test data. Note that each output has its own data to train on!
{
'decoded_output': x_train,
'classifier_output': y_train
},
epochs=self.num_epochs,
batch_size=self.batch_size,
shuffle=True,
validation_data=(x_val, {
'classifier_output': y_val,
'decoded_output': x_val
}),
callbacks=callbacks)
return self
def save_models(opt, model, optimizer, feeder):
model_options = ['char_hidden_size', 'encoder_hidden_size', 'rnn_type']
model_options = {k: getattr(opt, k) for k in model_options}
utils.ensure_folder(opt.ckpt_path)
torch.save(
{
'model': model.state_dict(),
'optimizer': optimizer.state_dict(),
'feeder': feeder.state(),
'model_options': model_options
}, opt.ckpt_path)
def channel(query): # find and ensure the channels folder channels_folder = os.path.abspath(os.path.join(config.BASE_FOLDER, 'channels')) ensure_folder(channels_folder) runnable = create_runnable( query=query, archive=os.path.join(channels_folder, 'archive'), # global channels archive output=os.path.join(channels_folder, '%(uploader_id)s', config.CHANNEL_FORMAT)) # set full output format # run youtube-dl subprocess.call(runnable)
def get_data(split, n_samples):
print('getting {} data...'.format(split))
global VOCAB
with open(tran_file, 'r', encoding='utf-8') as file:
lines = file.readlines()
tran_dict = dict()
for line in lines:
tokens = line.split()
key = tokens[0]
trn = ''.join(tokens[1:])
tran_dict[key] = trn
samples = []
#n_samples = 5000
rest = n_samples
folder = os.path.join(wav_folder, split)
ensure_folder(folder)
dirs = [os.path.join(folder, d) for d in os.listdir(folder) if os.path.isdir(os.path.join(folder, d))]
for dir in tqdm(dirs):
files = [f for f in os.listdir(dir) if f.endswith('.wav')]
rest = len(files) if n_samples <= 0 else rest
for f in files[:rest]:
wave = os.path.join(dir, f)
key = f.split('.')[0]
if key in tran_dict:
trn = tran_dict[key]
trn = list(trn.strip()) + ['<eos>']
for token in trn:
build_vocab(token)
trn = [VOCAB[token] for token in trn]
samples.append({'trn': trn, 'wave': wave})
rest = rest - len(files) if n_samples > 0 else rest
if rest <= 0 :
break
print('split: {}, num_files: {}'.format(split, len(samples)))
return samples
def visualize(threshold, show=True):
with open(angles_file) as file:
lines = file.readlines()
ones = []
zeros = []
for line in lines:
tokens = line.split()
angle = float(tokens[0])
type = int(tokens[1])
if type == 1:
ones.append(angle)
else:
zeros.append(angle)
bins = np.linspace(0, 180, 181)
plt.hist(zeros, bins, density=True, alpha=0.5, label='0', facecolor='red')
plt.hist(ones, bins, density=True, alpha=0.5, label='1', facecolor='blue')
mu_0 = np.mean(zeros)
sigma_0 = np.std(zeros)
y_0 = scipy.stats.norm.pdf(bins, mu_0, sigma_0)
plt.plot(bins, y_0, 'r--')
mu_1 = np.mean(ones)
sigma_1 = np.std(ones)
y_1 = scipy.stats.norm.pdf(bins, mu_1, sigma_1)
plt.plot(bins, y_1, 'b--')
plt.xlabel('theta')
plt.ylabel('theta j Distribution')
plt.title(
r'Histogram : mu_0={:.4f},sigma_0={:.4f}, mu_1={:.4f},sigma_1={:.4f}'.
format(mu_0, sigma_0, mu_1, sigma_1))
print('threshold: ' + str(threshold))
print('mu_0: ' + str(mu_0))
print('sigma_0: ' + str(sigma_0))
print('mu_1: ' + str(mu_1))
print('sigma_1: ' + str(sigma_1))
plt.legend(loc='upper right')
plt.plot([threshold, threshold], [0, 0.05], 'k-', lw=2)
if show:
plt.show()
ensure_folder('images')
plt.savefig(image_name)
def face_search():
start = time.time()
ensure_folder(STATIC_DIR)
ensure_folder(UPLOAD_DIR)
file = request.files['file']
filename = secure_filename(file.filename)
filename = filename.lower()
if filename in ['jpg', 'jpeg', 'png', 'gif']:
filename = str(random.randint(0, 101)) + '.' + filename
file_upload = os.path.join(UPLOAD_DIR, filename)
file.save(file_upload)
resize(file_upload)
print('file_upload: ' + file_upload)
name, prob, file_star = search(file_upload)
elapsed = time.time() - start
return name, prob, file_star, file_upload, float(elapsed)
def init_logger():
logger = logging.getLogger()
log_formatter = logging.Formatter("%(message)s")
filename = os.path.join(args.output_dir, 'info.log')
utils.ensure_folder(filename)
file_handler = logging.FileHandler(filename, mode='w')
file_handler.setFormatter(log_formatter)
logger.addHandler(file_handler)
console_handler = logging.StreamHandler()
console_handler.setFormatter(log_formatter)
logger.addHandler(console_handler)
logger.setLevel(logging.INFO)
return logger
def main():
ensure_folder(download_folder)
ensure_folder(image_folder)
train = get_data('train')
test = get_data('test')
print('num_train: ' + str(len(train)))
print('num_test: ' + str(len(test)))
print('train[:10]: ' + str(train[:10]))
print('test[:10]: ' + str(test[:10]))
with open('data/train.pkl', 'wb') as f:
pickle.dump(train, f)
with open('data/test.pkl', 'wb') as f:
pickle.dump(test, f)
def face_verify():
start = time.time()
ensure_folder('static')
file1 = request.files['file1']
fn_1 = secure_filename(file1.filename)
full_path_1 = os.path.join('static', fn_1)
file1.save(full_path_1)
resize(full_path_1)
file2 = request.files['file2']
fn_2 = secure_filename(file2.filename)
full_path_2 = os.path.join('static', fn_2)
file2.save(full_path_2)
resize(full_path_2)
prob, is_same = compare(full_path_1, full_path_2)
elapsed = time.time() - start
return is_same, prob, elapsed, fn_1, fn_2
def main():
checkpoint = '{}/BEST_checkpoint.tar'.format(save_folder) # model checkpoint
print('checkpoint: ' + str(checkpoint))
# Load model
checkpoint = torch.load(checkpoint)
model = checkpoint['model']
model = model.to(device)
model.eval()
test_path = 'data/test/'
test_images = [os.path.join(test_path, f) for f in os.listdir(test_path) if f.endswith('.jpg')]
num_test_samples = 10
samples = random.sample(test_images, num_test_samples)
imgs = torch.zeros([num_test_samples, 3, imsize, imsize], dtype=torch.float, device=device)
ensure_folder('images')
for i, path in enumerate(samples):
# Read images
img = imread(path)
img = imresize(img, (imsize, imsize))
imsave('images/{}_image.png'.format(i), img)
img = img.transpose(2, 0, 1)
assert img.shape == (3, imsize, imsize)
assert np.max(img) <= 255
img = torch.FloatTensor(img / 255.)
imgs[i] = img
imgs = torch.tensor(imgs)
with torch.no_grad():
preds = model(imgs)
for i in range(num_test_samples):
out = preds[i]
out = out.cpu().numpy()
out = np.transpose(out, (1, 2, 0))
out = out * 255.
out = np.clip(out, 0, 255)
out = out.astype(np.uint8)
out = cv.cvtColor(out, cv.COLOR_RGB2BGR)
cv.imwrite('images/{}_out.png'.format(i), out)
def get_data(split):
print('getting {} data...'.format(split))
global VOCAB
with open(transcript_file, 'r', encoding='utf-8') as file:
lines = file.readlines()
tran_dict = dict()
for line in lines:
tokens = line.split()
key = tokens[0]
trn = ''.join(tokens[1:])
tran_dict[key] = trn # tran_dict: {'BAC0009123': wav1.wav, ...}
samples = []
folder = os.path.join(wav_folder, split) # data/data_aishell/wav/train
ensure_folder(folder) # 确保floder是一个目录
dirs = [
os.path.join(folder, d) for d in os.listdir(folder)
if os.path.isdir(os.path.join(folder, d))
] # data/data_aishell/wav/train/S0003
for dir in tqdm(dirs):
files = [f for f in os.listdir(dir)
if f.endswith('.wav')] # [wav1, wav2, .....]
for f in files:
wave = os.path.join(
dir, f) # data/data_aishell/wav/train/S0003/wav1.wav
key = f.split('.')[0]
if key in tran_dict:
trn = tran_dict[key]
trn = list(trn.strip()) + ['<eos>']
for token in trn:
build_vocab(token)
trn = [VOCAB[token] for token in trn]
samples.append({'trn': trn, 'wave': wave})
print('split: {}, num_files: {}'.format(split, len(samples)))
return samples
def match_image():
start = time.time()
ensure_folder(STATIC_DIR)
ensure_folder(UPLOAD_DIR)
file1 = request.files['file1']
fn_1 = secure_filename(file1.filename)
full_path_1 = os.path.join(UPLOAD_DIR, fn_1)
file1.save(full_path_1)
resize(full_path_1)
file2 = request.files['file2']
fn_2 = secure_filename(file2.filename)
full_path_2 = os.path.join(UPLOAD_DIR, fn_2)
file2.save(full_path_2)
resize(full_path_2)
is_match = compare(full_path_1, full_path_2)
elapsed = time.time() - start
return is_match, elapsed, fn_1, fn_2
def get_data(mode):
print('getting {} data...'.format(mode))
global VOCAB
with open(tran_file, 'r', encoding='utf-8') as file:
lines = file.readlines()
tran_dict = dict()
for line in lines:
tokens = line.split()
key = tokens[0]
trn = ''.join(tokens[1:])
tran_dict[key] = trn
samples = []
folder = os.path.join(wav_folder, mode)
ensure_folder(folder)
dirs = [
os.path.join(folder, d) for d in os.listdir(folder)
if os.path.isdir(os.path.join(folder, d))
]
for dir in tqdm(dirs):
files = [f for f in os.listdir(dir) if f.endswith('.wav')]
for f in files:
wave = os.path.join(dir, f)
key = f.split('.')[0]
if key in tran_dict:
trn = tran_dict[key]
trn = list(trn.strip()) + ['<EOS>']
for token in trn:
build_vocab(token)
trn = [VOCAB[token] for token in trn]
samples.append({'trn': trn, 'wave': wave})
return samples
def run_filter(fn, reddit_type):
cmd_filter = "bash filter_reddit_posts.sh {input} {output} {command}"
if fn.endswith('.xz'):
command = 'xzgrep'
elif fn.endswith('.bz2'):
command = 'bzgrep'
elif fn.endswith('.zst'):
# .1 for exception in 2019-09
command = 'zstdgrep'
elif fn.endswith('.gz'):
command = 'zgrep'
else:
raise RuntimeError("ERROR: unknown input format @ {}".format(fn))
fn_basename = os.path.basename(fn).split('.')[0]
output = os.path.join(REDDIT_PROCESSED_DATA_DIR, 'reddit_{}_filtered'.format(reddit_type), fn_basename+'.json.gz')
ensure_folder(output)
os.system(cmd_filter.format(input=fn, output=output, command=command))
return output, fn_basename
def match_video():
start = time.time()
ensure_folder(STATIC_DIR)
ensure_folder(UPLOAD_DIR)
file = request.files['file']
upload_file = secure_filename(file.filename)
full_path = os.path.join(UPLOAD_DIR, upload_file)
file.save(full_path)
resize(full_path)
print('full_path: ' + full_path)
with torch.no_grad():
x = gen_feature(full_path)
cosine = np.dot(features, x)
cosine = np.clip(cosine, -1, 1)
print('cosine.shape: ' + str(cosine.shape))
max_index = int(np.argmax(cosine))
max_value = cosine[max_index]
name = name_list[max_index]
fps = fps_list[max_index]
idx = idx_list[max_index]
image_fn = image_fn_list[max_index]
print('max_index: ' + str(max_index))
print('max_value: ' + str(max_value))
print('name: ' + name)
print('fps: ' + str(fps))
print('idx: ' + str(idx))
theta = math.acos(max_value)
theta = theta * 180 / math.pi
print('theta: ' + str(theta))
prob = get_prob(theta)
print('prob: ' + str(prob))
time_in_video = idx / fps
print('time_in_video: ' + str(time_in_video))
prob = get_prob(theta)
elapsed = time.time() - start
return name, prob, idx, float(time_in_video), float(elapsed), str(
upload_file), image_fn
def visual_img(model):
with open(pickle_file, 'rb') as file:
data = pickle.load(file)
samples = [item for item in data]
samples = random.sample(samples, img_num)
imgs = torch.zeros([img_num, 3, im_size, im_size], dtype=torch.float)
ensure_folder('images')
origin_pts = []
for i in range(img_num):
sample = samples[i]
fullpath = sample['fullpath']
raw = cv.imread(fullpath)
raw = cv.resize(raw, (im_size, im_size))
img = raw[..., ::-1] # RGB
img = transforms.ToPILImage()(img)
img = transformer(img)
imgs[i] = img
cv.imwrite('images/{}_img.jpg'.format(i), raw)
# print(sample['pts'])
raw = draw_bboxes2(raw, sample['pts'], thick=3)
origin_pts.append(sample['pts'])
cv.imwrite('images/{}_true.jpg'.format(i), raw)
with torch.no_grad():
outputs = model(imgs.to(device))
iou_sum = 0
for i in range(img_num):
output = outputs[i].cpu().numpy()
output = output * im_size
# print('output: ' + str(output))
# print('output.shape: ' + str(output.shape))
img = cv.imread('images/{}_img.jpg'.format(i))
# print(output)
img = draw_bboxes2(img, output, thick=3)
iou_sum += get_iou(origin_pts[i], output)
cv.imwrite('images/{}_out.jpg'.format(i), img)
return iou_sum / img_num
def group(group_name, query):
# create/get groups folder(s)
group_folder = os.path.abspath(os.path.join(config.BASE_FOLDER, 'groups'))
ensure_folder(group_folder)
group_folder = os.path.join(group_folder, group_name)
ensure_folder(group_folder)
# if no query is specified, just create the folder and exit
if not query:
click.echo('Group folder created.')
return
archive = os.path.join(group_folder, 'archive')
runnable = create_runnable(
query=query,
archive=archive,
output=os.path.join(group_folder, config.GROUP_FORMAT))
subprocess.call(runnable)
def __init__(self, model, monitor_variable, verbose=0,
save_best_only=True,
save_dir='unlabeled/trained',
mode='auto', period=1, debug=False):
super(SaveCallback, self).__init__()
self.model_to_save = model
self.monitor_variable = monitor_variable
self.verbose = verbose
self.debug = debug
self.save_dir = save_dir
# Check if directory exists
ensure_folder(save_dir)
_datetime = datetime.now().strftime("%d.%m.%Y-%H:%M:%S")
self.filepath = os.path.join(save_dir, '%s-model_trained.epoch={epoch:02d}-%s={%s:.2f}.h5'
% (_datetime, monitor_variable, monitor_variable))
self.save_best_only = save_best_only
self.period = period
self.epochs_since_last_save = 0
if mode not in ['auto', 'min', 'max']:
warnings.warn('ModelCheckpoint mode %s is unknown, '
'fallback to auto mode.' % mode,
RuntimeWarning)
mode = 'auto'
if mode == 'min':
self.monitor_op = np.less
self.best = np.Inf
elif mode == 'max':
self.monitor_op = np.greater
self.best = -np.Inf
else:
if 'acc' in self.monitor_variable or self.monitor_variable.startswith('fmeasure'):
self.monitor_op = np.greater
self.best = -np.Inf
else:
self.monitor_op = np.less
self.best = np.Inf
def face_detect():
start = time.time()
ensure_folder(STATIC_DIR)
ensure_folder(UPLOAD_DIR)
file = request.files['file']
fn = secure_filename(file.filename)
full_path = os.path.join(UPLOAD_DIR, fn)
file.save(full_path)
# resize(full_path)
print('full_path: ' + full_path)
img = Image.open(full_path).convert('RGB')
bboxes, landmarks = detect_faces(img)
num_faces = len(bboxes)
if num_faces > 0:
img = cv.imread(full_path)
draw_bboxes(img, bboxes, landmarks)
cv.imwrite(full_path, img)
elapsed = time.time() - start
return num_faces, float(elapsed), str(fn), bboxes, landmarks
extract('/content/msceleb.zip')
#pre_process.py
import os
import pickle
import cv2 as cv
import mxnet as mx
from mxnet import recordio
from tqdm import tqdm
from config import path_imgidx, path_imgrec, IMG_DIR, pickle_file
from utils import ensure_folder
if __name__ == "__main__":
ensure_folder(IMG_DIR)
imgrec = recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, 'r')
print(len(imgrec))
samples = []
class_ids = set()
# %% 1 ~ 5179510
for i in tqdm(range(5179510)):
print(i)
try:
header, s = recordio.unpack(imgrec.read_idx(i + 1))
img = mx.image.imdecode(s).asnumpy()
print(img.shape)
img = cv.cvtColor(img, cv.COLOR_RGB2BGR)
print(header.label)
def save_model(model):
filename = os.path.join(args.output_dir, 'model.pt')
utils.ensure_folder(filename)
torch.save(model, filename)