def copyOut(url):
min_YCrCb = np.array([0, 133, 77], np.uint8)
max_YCrCb = np.array([255, 173, 127], np.uint8)
folder_name, filename = url.split("/")
# Grab video frame, decode it and return next video frame
sourceImage = cv2.imread(root_path + "/" + url)
if os.path.exists(ldmk_path + "/" + folder_name + "/" + filename[:-4] +
".txt"):
return
#sourceImage = sourceImage[:,:,[2,1,0]]
# Convert image to YCrCb
imageYCrCb = cv2.cvtColor(sourceImage, cv2.COLOR_BGR2YCR_CB)
# Find region with skin tone in YCrCb image
skinRegion = cv2.inRange(imageYCrCb, min_YCrCb, max_YCrCb)
cv2.imwrite("z.png", skinRegion * 100)
percent = np.count_nonzero(skinRegion) / (sourceImage.shape[0] *
sourceImage.shape[1] * 1.0)
if percent > 0.35:
print url
utils.mkdirP(out_path + "/" + folder_name)
shutil.copy(root_path + "/" + url, out_path + "/" + url)
def gen_class_centers(root_path):
path_feat = root_path + "_features"
path_outerliers = root_path + "_outliers"
path_class_center = root_path + "_cls_centers"
print "Loading features..."
files = loader.load_prefix(path_feat, "../tmp/" + path_feat.split("/")[-1],
"npy", partial(is_outlier, path_outerliers))
utils.mkdirP(path_class_center)
func = partial(computer_center, path_feat, path_outerliers,
path_class_center)
#for feat in feature_ms20k_files'
pool = Pool()
pool.map(func, files)
def make(self):
print "Loading data..."
saved_data_name = "../tmp/" + self.root_path.split(
"/")[-1] + "_img_label"
img_labels = []
if not os.path.exists(saved_data_name + ".npy"):
img_labels = loader.load_img_labels(self.root_path,
saved_data_name)
else:
img_labels = np.load(saved_data_name + ".npy")
data = self._getUrls(img_labels)
print "Reading out {0} images.".format(len(data))
utils.mkdirP(self.feat_path)
chunk_size = 15000 # or whatever
chunks = [
data[i:i + chunk_size] for i in xrange(0, len(data), chunk_size)
]
N_chunks = len(chunks)
pool = Pool()
iidx = 0
for chunk in chunks[iidx:]:
print 'Working chunks: {0}/{1}...'.format(iidx, N_chunks)
new_chunk = filter(self._filteringOut, chunk)
print "Filtering out: {0}".format(len(chunk) - len(new_chunk))
_getDatum(new_chunk[0])
results = pool.map(_getDatum, new_chunk)
print "Reading out: {0}".format(len(results))
bs = 256
batches = [results[i:i + bs] for i in xrange(0, len(results), bs)]
N_bs = len(batches)
iidx_bs = 0
for batch in batches:
print '--->Working batches: {0}/{1}...'.format(iidx_bs, N_bs)
feats = self.net.extract(batch)
pool.map(_saveFeature, feats)
iidx_bs += 1
iidx += 1
def make(self):
self.detector = FaceDetector(minsize=20,
gpuid=self.gpu_id,
fastresize=False)
if self.save_aligned_img:
utils.mkdirP(self.root_path + self.save_path_suffix)
if self.save_landmarks:
utils.mkdirP(self.root_path + "_landmarks")
#f_list = open(list_path, "r")
all_files = self.getLines()
N = len(all_files)
cnt = 0
for line in all_files:
filename = line['filename']
class_num = line['class_num']
if cnt >= 0:
self.align_and_save_face(filename)
print("{0}/{1}".format(cnt + 1, N))
cnt += 1
def getLines(self):
lines = []
root_path = self.root_path
class_label = 0
for parent, dirnames, filenames in os.walk(root_path):
for dirname in dirnames:
for sub_parent, sub_dirnames, sub_filenames in os.walk(
root_path + "/" + dirname):
for sub_filename in sub_filenames:
if self.save_aligned_img:
utils.mkdirP(root_path + self.save_path_suffix +
"/" + dirname)
if self.save_landmarks:
utils.mkdirP(root_path + "_landmarks/" + dirname)
if (sub_filename.endswith("png")
or sub_filename.endswith("jpg")
or sub_filename.endswith("bmp")):
tmp = {}
tmp['filename'] = dirname + "/" + sub_filename
tmp['class_num'] = class_label
lines.append(tmp)
class_label += 1
return lines
Column 3: Physical coordinate (GRCh37 assumed)
Column 4: Allele 1 (row retained only if allele 1 in ALLELE_SET)
Column 5: Allele 2 (if present)
'''
CHROMOSOME_SET = { '24', 'Y' }
ALLELE_SET = set('ACGTDI')
fnEnding2fnTypeDict = {
'.ped': 'ped',
'.23andMe.txt': 'ttam',
'.acom.txt': 'ttam',
}
outDir = 'converted'
utils.mkdirP(outDir)
#----------------------------------------------------------------------
# ped and map
def convertPed(pedFN, fnRoot, fnEnding):
'reads a .ped and a .map and converts to .genos.txt'
mapFN = pedFN.replace(fnEnding, '.map')
outFN = '%s/%s.genos.txt' % (outDir, fnRoot)
outFile = open(outFN, 'w')
indexList = readMap(mapFN, outFile)
processPed(pedFN, indexList, outFile)
return
#sourceImage = sourceImage[:,:,[2,1,0]]
# Convert image to YCrCb
imageYCrCb = cv2.cvtColor(sourceImage, cv2.COLOR_BGR2YCR_CB)
# Find region with skin tone in YCrCb image
skinRegion = cv2.inRange(imageYCrCb, min_YCrCb, max_YCrCb)
cv2.imwrite("z.png", skinRegion * 100)
percent = np.count_nonzero(skinRegion) / (sourceImage.shape[0] *
sourceImage.shape[1] * 1.0)
if percent > 0.35:
print url
utils.mkdirP(out_path + "/" + folder_name)
shutil.copy(root_path + "/" + url, out_path + "/" + url)
#persons = loader.load_img_labels(root_path, "../tmp/MsCelebV1-Faces-Cropped-aligned_182")
persons = np.load("../tmp/MsCelebV1-Faces-Cropped-aligned_182.npy")
urls = []
for p in persons:
for z in p["pathes"]:
urls.append(z)
utils.mkdirP(out_path)
pool = Pool()
pool.map(copyOut, urls)
def makeOutputDirectories(self):
'make output directories if they do not already exist'
if not self.suppressOutputAndLog:
utils.mkdirP(self.outDir)
utils.mkdirP(self.phyloOutDir)