def getMovieCast(dataF, movieID, indexF, keyF, attrIF, attrKF, offsList=[],
charNF=None, doCast=0, doWriters=0):
"""Read the specified files and return a list of Person objects,
one for every people in offsList."""
resList = []
_globoff = []
for offset in offsList:
# One round for person is enough.
if offset not in _globoff: _globoff.append(offset)
else: continue
personID, movies = getRawData(dataF, offset, doCast, doWriters)
# Consider only the current movie.
movielist = [x for x in movies if x.get('movieID') == movieID]
# XXX: a person can be listed more than one time for a single movie:
# think about directors of TV series.
# XXX: here, 'movie' is a dictionary as returned by the getRawData
# function, not a Movie class instance.
for movie in movielist:
name = getLabel(personID, indexF, keyF)
if not name: continue
curRole = movie.get('currentRole', u'')
roleID = None
if curRole and charNF:
curRole, roleID = getCharactersIDs(curRole, charNF)
p = Person(name=name, personID=personID,
currentRole=curRole, roleID=roleID,
accessSystem='local')
if movie.has_key('attributeID'):
attr = getLabel(movie['attributeID'], attrIF, attrKF)
if attr: p.notes = attr
# Used to sort cast.
if movie.has_key('position'):
p.billingPos = movie['position'] or None
resList.append(p)
return resList
def getMovieCast(dataF, movieID, indexF, keyF, attrIF, attrKF, offsList=[],
charNF=None, doCast=0, doWriters=0):
"""Read the specified files and return a list of Person objects,
one for every people in offsList."""
resList = []
_globoff = []
for offset in offsList:
# One round for person is enough.
if offset not in _globoff: _globoff.append(offset)
else: continue
personID, movies = getRawData(dataF, offset, doCast, doWriters)
# Consider only the current movie.
movielist = [x for x in movies if x.get('movieID') == movieID]
# XXX: a person can be listed more than one time for a single movie:
# think about directors of TV series.
# XXX: here, 'movie' is a dictionary as returned by the getRawData
# function, not a Movie class instance.
for movie in movielist:
name = getLabel(personID, indexF, keyF)
if not name: continue
curRole = movie.get('currentRole', u'')
roleID = None
if curRole and charNF:
curRole, roleID = getCharactersIDs(curRole, charNF)
p = Person(name=name, personID=personID,
currentRole=curRole, roleID=roleID,
accessSystem='local')
if movie.has_key('attributeID'):
attr = getLabel(movie['attributeID'], attrIF, attrKF)
if attr: p.notes = attr
# Used to sort cast.
if movie.has_key('position'):
p.billingPos = movie['position'] or None
resList.append(p)
return resList
def getFilmography(dataF,
indexF,
keyF,
attrIF,
attrKF,
offset,
charNF=None,
doCast=0,
doWriters=0):
"""Gather information from the given files about the
person entry found at offset; return a list of Movie objects,
with the relevant attributes."""
name, res = getRawData(dataF, offset, doCast, doWriters)
resList = []
for movie in res:
title = getLabel(movie['movieID'], indexF, keyF)
if not title: continue
curRole = movie.get('currentRole', u'')
roleID = None
if curRole and charNF:
curRole, roleID = getCharactersIDs(curRole, charNF)
m = Movie(title=title,
movieID=movie['movieID'],
currentRole=curRole,
roleID=roleID,
accessSystem='local')
if movie.has_key('attributeID'):
attr = getLabel(movie['attributeID'], attrIF, attrKF)
if attr: m.notes = attr
resList.append(m)
return resList
def getAkaTitles(movieID, akaDF, titlesIF, titlesKF, attrIF , attrKF):
"""Return a list of aka titles."""
entries = getFullIndex(akaDF, movieID, kind='akatdb',
rindex=None, multi=1, default=[])
res = []
for entry in entries:
akaTitle = getLabel(entry[1], titlesIF, titlesKF)
if not akaTitle: continue
attr = getLabel(entry[2], attrIF, attrKF)
if attr: akaTitle += '::%s' % attr
if akaTitle: res.append(akaTitle)
return res
def getAkaTitles(movieID, akaDF, titlesIF, titlesKF, attrIF , attrKF):
"""Return a list of aka titles."""
entries = getFullIndex(akaDF, movieID, kind='akatdb',
rindex=None, multi=1, default=[])
res = []
for entry in entries:
akaTitle = getLabel(entry[1], titlesIF, titlesKF)
if not akaTitle: continue
attr = getLabel(entry[2], attrIF, attrKF)
if attr: akaTitle += '::%s' % attr
if akaTitle: res.append(akaTitle)
return res
def gen(batch_size=8, imglist=[]):
k = 0
imgCount = len(imglist)
shuffle(imglist)
def getLabel(imgname):
vin = imgname.split('_')[0]
label = np.zeros((17, len(vocabulary)))
for i, char in enumerate(vin):
label[i][vocabulary.index(char)] = 1
return label
while True:
x = np.zeros((batch_size, im_heigth, im_width, 1), dtype='float32')
y = np.zeros((batch_size, 17, len(vocabulary)))
for i in range(0, batch_size):
if k >= imgCount:
shuffle(imglist)
k = 0
k = k % imgCount
x[i] = cv2.imread(imglist[k])[:, :, :1] / 127.5 - 1
y[i] = getLabel(imglist[k].split('/')[-1])
k += 1
yield (x, y)
def getAkaNames(personID, akaDF, namesIF, namesKF):
"""Return a list of aka names."""
entries = getFullIndex(akaDF, personID, kind='akandb',
rindex=None, multi=1, default=[])
res = []
for entry in entries:
akaName = getLabel(entry[1], namesIF, namesKF)
if akaName: res.append(akaName)
return res
def encodeRecord(fileName, ignoreDict, fromCell, toCell, minGrade):
recordData = openpyxl.load_workbook(fileName)
sheet = recordData.active
cells = sheet[fromCell:toCell]
sequences = [[]]
studentIDs = []
# Index:
# [Student ID, Year, Semester, N/A, Course's name, Course's grade]
curStudentID = cells[0][1].value
curSemester = cells[0][3].value
sequences[0] = []
sequences[0].append("")
studentIDs.append(curStudentID)
for row in cells:
studentID = row[1].value
# year = row[2].value
semester = row[3].value
courseName = row[5].value
courseGrade = row[6].value
# if courseName in ignoreDict:
# print("Ignore courseName {0}".format(courseName))
# continue
if courseGrade == "NULL" or courseGrade == None: # Special case: user has no course's grade
#print("Ignore unknow course {0} with grade: {1} of student {2}".format(courseName, courseGrade, studentID))
continue
if courseGrade < minGrade:
#print("Ignore course \"{0}\" with grade: {1} of student {2}".format(courseName, courseGrade, studentID))
continue
if curStudentID != studentID:
curStudentID = studentID
sequences.append([])
curSemester = semester
sequences[-1].append("")
studentIDs.append(curStudentID)
elif curSemester != semester:
curSemester = semester
sequences[-1].append("")
encodedKey = str(Util.getLabel(courseName)) # Remove unicode
if len(sequences[-1][-1]) > 0:
sequences[-1][-1] += "."
sequences[-1][-1] += encodedKey
Util.cacheLabel()
sequences = [Helper.sortAdv(seq) for seq in sequences]
return (sequences, studentIDs)
def test_one_batch(X):
sorted_items = X[0].numpy()
groundTrue = X[1]
r = utils.getLabel(groundTrue, sorted_items)
pre, recall, ndcg = [], [], []
for k in world.topks:
ret = utils.RecallPrecision_ATk(groundTrue, r, k)
pre.append(ret['precision'])
recall.append(ret['recall'])
ndcg.append(utils.NDCGatK_r(groundTrue,r,k))
return {'recall':np.array(recall),
'precision':np.array(pre),
'ndcg':np.array(ndcg)}
def getFilmography(dataF, indexF, keyF, attrIF, attrKF, offset,
charNF=None, doCast=0, doWriters=0):
"""Gather information from the given files about the
person entry found at offset; return a list of Movie objects,
with the relevant attributes."""
name, res = getRawData(dataF, offset, doCast, doWriters)
resList = []
for movie in res:
title = getLabel(movie['movieID'], indexF, keyF)
if not title: continue
curRole = movie.get('currentRole', u'')
roleID = None
if curRole and charNF:
curRole, roleID = getCharactersIDs(curRole, charNF)
m = Movie(title=title, movieID=movie['movieID'],
currentRole=curRole, roleID=roleID,
accessSystem='local')
if movie.has_key('attributeID'):
attr = getLabel(movie['attributeID'], attrIF, attrKF)
if attr: m.notes = attr
resList.append(m)
return resList
def getAkaNames(personID, akaDF, namesIF, namesKF):
"""Return a list of aka names."""
entries = getFullIndex(akaDF,
personID,
kind='akandb',
rindex=None,
multi=1,
default=[])
res = []
for entry in entries:
akaName = getLabel(entry[1], namesIF, namesKF)
if akaName: res.append(akaName)
return res
def getMovieLinks(movieID, dataF, movieTitlIF, movieTitlKF):
"""Return a dictionary with movie connections."""
entries = getFullIndex(dataF, movieID, kind='mlinks',
rindex=None, multi=1, default=[])
res = {}
for entry in entries:
title = getLabel(entry[2], movieTitlIF, movieTitlKF)
if not title: continue
m = Movie(title=title, movieID=entry[2],
accessSystem='local')
sect = _links_sect.get(entry[1])
if not sect: continue
res.setdefault(sect, []).append(m)
return res
def getMovieLinks(movieID, dataF, movieTitlIF, movieTitlKF):
"""Return a dictionary with movie connections."""
entries = getFullIndex(dataF, movieID, kind='mlinks',
rindex=None, multi=1, default=[])
res = {}
for entry in entries:
title = getLabel(entry[2], movieTitlIF, movieTitlKF)
if not title: continue
m = Movie(title=title, movieID=entry[2],
accessSystem='local')
sect = _links_sect.get(entry[1])
if not sect: continue
res.setdefault(sect, []).append(m)
return res
def joinVectors():
allData = []
allLabels = []
for imgFeatures in featureVectorList:
if imgFeatures.endswith('.swp'):
continue
featureVectors = np.load('./{0}/{1}'.format(folderPath, imgFeatures))
data = featureVectors[:, 1:] # Get rid of None in first column
currLabel = utils.getLabel(imgFeatures)
labelArr = np.tile(currLabel, len(data))
if allData == []:
allData = data
allLabels = labelArr
else:
allData = np.concatenate((allData, data), 0)
allLabels = np.concatenate((allLabels, labelArr), 0)
print(allLabels.shape)
print(allData.shape)
return allData, allLabels
def process(z):
rot_coords = np.zeros((4, 2), dtype='float32')
flips = [
Image.FLIP_LEFT_RIGHT, Image.FLIP_TOP_BOTTOM, Image.ROTATE_90,
Image.ROTATE_180, Image.ROTATE_270, Image.TRANSPOSE
]
def getLabel(vin):
label = np.zeros((17, len(vocabulary)))
for i, char in enumerate(vin):
label[i][vocabulary.index(char)] = 1
return label
while True:
vin = generateVin(vin_length, max_spaces)
if len(vin) < maxLength:
for i in range(maxLength - len(vin)):
vin += ' '
ff = random.choice(fonts)
font = ImageFont.truetype(ff, random.randint(32, 256))
t_width, t_height_orig = font.getsize(vin)
t_height = t_height_orig #int(t_height_orig * (1 + height_padding))
# t_width = int(t_width * (1 + width_padding))
# text image is a square
t_img_size = max(t_height, t_width)
# rot_deg = random.uniform(0, 360)
rot_deg = random.uniform(-max_angle, max_angle)
# text image will have original text located horizontally in the middle
# original coordinates will be (0, (s-h)/2) - top left and (s, (s+h)/2) - bottom right
# here s denotes text image size (image width), h - text height
# we rotate all four corners of text around the center of the image (s/2, s/2) on the chosen angle
# and check if rotated text can be fitted in background image
rot_coords[0] = rotate_coord(0, (t_img_size - t_height) / 2.,
t_img_size / 2., t_img_size / 2., rot_deg)
rot_coords[1] = rotate_coord(0, (t_img_size + t_height) / 2.,
t_img_size / 2., t_img_size / 2., rot_deg)
rot_coords[2] = rotate_coord(t_img_size, (t_img_size + t_height) / 2.,
t_img_size / 2., t_img_size / 2., rot_deg)
rot_coords[3] = rotate_coord(t_img_size, (t_img_size - t_height) / 2.,
t_img_size / 2., t_img_size / 2., rot_deg)
min_tx = round(np.min(rot_coords[:, 0]))
max_tx = round(np.max(rot_coords[:, 0]))
min_ty = round(np.min(rot_coords[:, 1]))
max_ty = round(np.max(rot_coords[:, 1]))
dx = max_tx - min_tx
dy = max_ty - min_ty
if dx < im_width and dy < im_heigth:
try:
bcg_img = Image.open(random.choice(bcgs)).convert('L')
except:
continue
text_image = Image.new('L', (t_img_size, t_img_size))
draw = ImageDraw.Draw(text_image)
draw.text((t_width * width_padding / 2,
round(t_img_size / 2. - t_height_orig / 2.)),
vin,
font=font,
fill=255)
text_image = text_image.rotate(rot_deg)
x = random.randint(-min_tx, im_width -
max_tx) # int((-min_tx + im_width-max_tx)/2)
y = random.randint(-min_ty, im_heigth -
max_ty) # int((-min_ty + im_heigth-max_ty)/2)
color = random.randint(0, 255)
if random.uniform(0, 1) > 0.5:
bcg_img = bcg_img.transpose(random.choice(flips))
bcg_img = bcg_img.resize((im_width, im_heigth))
colorization = ImageOps.colorize(text_image, (0, 0, 0),
(color, color, color))
bcg_img.paste(colorization, (x, y), text_image)
sigma = random.uniform(0, 2)
bcg_img = gaussian_filter(bcg_img, sigma)
totalmask = Image.new('L', (im_width, im_heigth))
colorization = ImageOps.colorize(text_image, (0, 0, 0),
(255, 255, 255))
totalmask.paste(colorization, (x, y), text_image)
totalmask = np.array(totalmask, dtype=np.float)
totalmask[totalmask > 0] = 1
totalmask = np.expand_dims(totalmask, axis=-1)
# from scipy.misc import imsave
# imsave('tmp/%s_%s_%s.jpg' % (vin, x, y), bcg_img)
bcg_img = np.array(bcg_img, dtype='uint8')
rot_coords[:] += [x, y]
mask = Image.new('L', (t_width, t_height_orig))
draw = ImageDraw.Draw(mask)
draw.text((0, 0), vin, font=font, fill=255)
mask = cv2.resize(np.array(mask, dtype=np.float), (32 * 16, 32))
# from scipy.misc import imsave
# imsave('res/%s_%s_%s.jpg' % (vin, x, y), mask.astype(np.uint8))
mask[mask > 0] = 1
mask = np.expand_dims(mask, axis=-1)
# output endpoints and height to define rectangle
c = [
2 * (rot_coords[0, 0] / im_width) - 1,
2 * (rot_coords[0, 1] / im_heigth) - 1,
2 * (rot_coords[2, 0] / im_width) - 1,
2 * (rot_coords[2, 1] / im_heigth) - 1,
# m_point_a[0]/im_width,
# m_point_a[1]/im_heigth,
# m_point_b[0]/im_width,
# m_point_b[1]/im_heigth,
rot_deg / max_angle
# t_height#/t_width
]
# crop = crop_rotate(bcg_img, -rot_deg, rot_coords[0],rot_coords[2])
#
# from scipy.misc import imsave
# try:
# imsave('tmp/%s_%s_%s.jpg' % (vin, x, y), crop)
# except:
# print('trololo')
label = getLabel(vin)
matrix = getMatrix(rot_coords)
# crop = cv2.warpAffine(bcg_img, matrix, (1000, 1000))
# cv2.imwrite('temp_crop.png',crop)
return np.reshape(bcg_img,
(im_heigth, im_width,
1)), label, mask, matrix.flatten(), totalmask
def patch_generator(folder,
all_patch_list,
det_patch_list,
batch_size=64,
detection_ratio=0.5,
levels=[0, 1, 2],
dims=(512, 512)):
'''
Returns (via yields) the sample image patch and corresponding ground truth mask, in given batch_size, using
one level in levels list per patch with equal probability
'''
true_batch = int(detection_ratio * batch_size) + 1
all_batch_size = batch_size - true_batch
#print('true_batch_size: {} \t all_batch_size: {}'.format(true_batch, all_batch_size))
while 1:
all_patch_list = shuffle(all_patch_list)
det_patch_list = shuffle(det_patch_list)
det_patch_list_cycle = cycle(det_patch_list)
for offset in range(0, len(all_patch_list), all_batch_size):
## Get file and coords list from each patch list and combine them
all_samples = all_patch_list[offset:offset + all_batch_size]
true_sample = []
count = 0
for sample in det_patch_list_cycle:
true_sample.append(sample)
count += 1
if count >= true_batch:
break
combined_sample_list = all_samples
combined_sample_list.extend(true_sample)
combined_sample_list = shuffle(combined_sample_list)
patch = []
ground_truth = []
for sample in combined_sample_list:
filename = folder + sample[0]
coords = sample[1]
level = levels[np.random.randint(0, len(levels),
dtype=np.int8)]
patch.append(
getRegionFromSlide(getWSI(filename),
level=level,
start_coord=coords,
dims=dims))
ground_truth.append(getLabel(filename, level, coords, dims))
#print('Level used: {}'.format(level))
X_train = np.array(patch)
y_train = np.array(ground_truth)
yield shuffle(X_train, y_train)
def getMovieMisc(movieID, dataF, indexF, attrIF, attrKF):
"""Return information from files like production-companies.data,
keywords.data and so on."""
index = getFullIndex(indexF, movieID, kind='idx2idx')
if index is None: return []
result = []
try:
fdata = open(dataF, 'rb')
except IOError, e:
raise IMDbDataAccessError, str(e)
fdata.seek(index)
# Eat the first offset.
if len(fdata.read(3)) != 3:
fdata.close()
return []
while 1:
length = convBin(fdata.read(1), 'length')
strval = latin2utf(fdata.read(length))
attrid = convBin(fdata.read(3), 'attrID')
if attrid != 0xffffff:
attr = getLabel(attrid, attrIF, attrKF)
if attr: strval += ' %s' % attr
result.append(strval)
nextBin = fdata.read(3)
# There can be multiple values.
if not (len(nextBin) == 3 and \
convBin(nextBin, 'movieID') == movieID):
break
fdata.close()
return result
length = convBin(dfptr.read(2), 'longlength')
# Skip company name.
latin2utf(dfptr.read(length))
nrItems = convBin(dfptr.read(3), 'nrCompanyItems')
filmography = {}
# Yes: kindID values are hard-coded in the companies4local.py script.
_kinds = {0: 'distributors', 1: 'production companies',
2: 'special effect companies', 3: 'miscellaneous companies'}
for i in xrange(nrItems):
kind = _kinds.get(ord(dfptr.read(1)))
if kind is None:
import warnings
warnings.warn('Unidentified kindID for a company.')
break
movieID = convBin(dfptr.read(3), 'movieID')
title = getLabel(movieID, movieIF, movieKF)
m = Movie(title=title, movieID=movieID, accessSystem='local')
filmography.setdefault(kind, []).append(m)
dfptr.close()
return filmography
def _convChID(companyID):
"""Return a numeric value for the given string, or None."""
if companyID is None:
return None
return convBin(companyID, 'companyID')
def getCompanyID(name, compNF):
"""Return a companyID for a name."""
keywords.data and so on."""
index = getFullIndex(indexF, movieID, kind='idx2idx')
if index is None: return []
result = []
try:
fdata = open(dataF, 'rb')
except IOError, e:
raise IMDbDataAccessError, str(e)
fdata.seek(index)
# Eat the first offset.
if len(fdata.read(3)) != 3:
fdata.close()
return []
while 1:
length = convBin(fdata.read(1), 'length')
strval = latin2utf(fdata.read(length))
attrid = convBin(fdata.read(3), 'attrID')
if attrid != 0xffffff:
attr = getLabel(attrid, attrIF, attrKF)
if attr: strval += ' %s' % attr
result.append(strval)
nextBin = fdata.read(3)
# There can be multiple values.
if not (len(nextBin) == 3 and \
convBin(nextBin, 'movieID') == movieID):
break
fdata.close()
return result
def process(z):
rot_coords = np.zeros((4, 2), dtype='float32')
flips = [
Image.FLIP_LEFT_RIGHT,
Image.FLIP_TOP_BOTTOM,
Image.ROTATE_90,
Image.ROTATE_180,
Image.ROTATE_270,
Image.TRANSPOSE
]
while True:
vin = generateVin(vin_length, max_spaces)
if len(vin) < maxLength:
for i in range(maxLength - len(vin)):
vin +=' '
ff = random.choice(fonts)
font = ImageFont.truetype(ff, random.randint(32, 256))
t_width, t_height = font.getsize(vin)
# text image is a square
t_img_size = max(t_height, t_width)
# rot_deg = random.uniform(0, 360)
rot_deg = random.uniform(-max_angle, max_angle)
# text image will have original text located horizontally in the middle
# original coordinates will be (0, (s-h)/2) - top left and (s, (s+h)/2) - bottom right
# here s denotes text image size (image width), h - text height
# we rotate all four corners of text around the center of the image (s/2, s/2) on the chosen angle
# and check if rotated text can be fitted in background image
rot_coords[0] = rotate_coord(0, (t_img_size - t_height)/2., t_img_size/2., t_img_size/2., rot_deg)
rot_coords[1] = rotate_coord(0, (t_img_size + t_height)/2., t_img_size/2., t_img_size/2., rot_deg)
rot_coords[2] = rotate_coord(t_img_size, (t_img_size + t_height)/2., t_img_size/2., t_img_size/2., rot_deg)
rot_coords[3] = rotate_coord(t_img_size, (t_img_size - t_height)/2., t_img_size/2., t_img_size/2., rot_deg)
min_tx = round(np.min(rot_coords[:, 0]))
max_tx = round(np.max(rot_coords[:, 0]))
min_ty = round(np.min(rot_coords[:, 1]))
max_ty = round(np.max(rot_coords[:, 1]))
dx = max_tx - min_tx
dy = max_ty - min_ty
if dx < im_width and dy < im_heigth:
try:
bcg_img = Image.open(random.choice(bcgs)).convert('L')
except:
continue
text_image = Image.new('L', (t_img_size, t_img_size))
draw = ImageDraw.Draw(text_image)
draw.text((0, round(t_img_size/2. - t_height/2.)), vin, font=font, fill=255)
text_image = text_image.rotate(rot_deg)
x = random.randint(-min_tx, im_width - max_tx) #int((-min_tx + im_width-max_tx)/2)
y = random.randint(-min_ty, im_heigth - max_ty) #int((-min_ty + im_heigth-max_ty)/2)
color = random.randint(0, 255)
if random.uniform(0, 1) > 0.5:
bcg_img = bcg_img.transpose(random.choice(flips))
bcg_img = bcg_img.resize((im_width, im_heigth))
colorization = ImageOps.colorize(text_image, (0, 0, 0), (color, color, color))
bcg_img.paste(colorization, (x, y), text_image)
sigma = random.uniform(0, 2)
bcg_img = gaussian_filter(bcg_img, sigma)
from scipy.misc import imsave
imsave('tmp/%s_%s_%s.jpg' % (vin, x, y), bcg_img)
bcg_img = np.array(bcg_img, dtype='uint8')
rot_coords[:] += [x, y]
# bcg_img = cv2.polylines(bcg_img, np.int32([rot_coords]), 1, (255,255,255))
# end points of center line going through text
# m_point_a = (rot_coords[0] + rot_coords[1])/2
# m_point_b = (rot_coords[2] + rot_coords[3])/2
# output endpoints and height to define rectangle
c = [
2 * (rot_coords[0, 0]/im_width)-1,
2 * (rot_coords[0, 1]/im_heigth)-1,
2 * (rot_coords[2, 0]/im_width)-1,
2 * (rot_coords[2, 1]/im_heigth)-1,
# m_point_a[0]/im_width,
# m_point_a[1]/im_heigth,
# m_point_b[0]/im_width,
# m_point_b[1]/im_heigth,
rot_deg / max_angle
#t_height#/t_width
]
# crop = crop_rotate(bcg_img, -rot_deg, rot_coords[0],rot_coords[2])
#
# from scipy.misc import imsave
# try:
# imsave('tmp/%s_%s_%s.jpg' % (vin, x, y), crop)
# except:
# print('trololo')
indexes, vinLen = getLabel(vin)
filmography = {}
# Yes: kindID values are hard-coded in the companies4local.py script.
_kinds = {
0: 'distributors',
1: 'production companies',
2: 'special effect companies',
3: 'miscellaneous companies'
}
for i in xrange(nrItems):
kind = _kinds.get(ord(dfptr.read(1)))
if kind is None:
import warnings
warnings.warn('Unidentified kindID for a company.')
break
movieID = convBin(dfptr.read(3), 'movieID')
title = getLabel(movieID, movieIF, movieKF)
m = Movie(title=title, movieID=movieID, accessSystem='local')
filmography.setdefault(kind, []).append(m)
dfptr.close()
return filmography
def _convChID(companyID):
"""Return a numeric value for the given string, or None."""
if companyID is None:
return None
return convBin(companyID, 'companyID')
def getCompanyID(name, compNF):
"""Return a companyID for a name."""
#############################################################################################################
#############################################################################################################
if __name__ == '__main__':
# FILENAME = args.file
FOLDERPATH = args.folderPath
#FILETYPE = args.fileType
# img = cv2.imread('testImages/' + FILENAME)
categoryFileList = os.listdir(FOLDERPATH)
# The 'ground truths' for getting rid of background blobs
rgbBackgroundFeatures = background.handleBackgroundBlobs()
for folderName in categoryFileList:
label = utils.getLabel(folderName)
imgFileList = os.listdir("{0}{1}".format(FOLDERPATH, folderName))
if folderName == 'assorted':
continue
for imgName in imgFileList:
img = cv2.imread("{0}{1}/{2}".format(FOLDERPATH,folderName,imgName))
# We need to iterate over all the files in the dir (we will split it 80/20 for training vs test)
#segment the image
blobs, markers,labels = utils.extractBlobs(img)
rgbFeatureList = []
#multiprocess the feature extraction
#!/usr/bin/python3
import utils
import numpy
import colors
imageWidth = 224
if __name__ == "__main__":
# Load the model
model = utils.loadModel("model/keras_model.h5")
# Get an image from the webcam
image = utils.getWebcam("/dev/video0")
normalImage = utils.prepareImage(image, imageWidth)
# Run the prediction
prediction = model.predict(normalImage)
# get the largest value
largest = max(prediction[0])
index = int(numpy.where(prediction[0] == largest)[0])
# Get the labels
labels = utils.openLabels("model/labels.txt")
label = utils.getLabel(labels, index)
# Run the user defined function
colors.runColor(label, index, prediction)
def patch_generator(folder,
all_patch_list,
det_patch_list,
batch_size=64,
sample_factor=1,
levels=[0, 1, 2],
dims=(512, 512),
save_labels=False,
labels_list=None):
'''
Returns (via yields) the sample image patch and corresponding ground truth mask, in given batch_size, using
one level in levels list per patch with equal probability
if save_labels is True then the function appends the labels generated to labels_list and yields
only the image patches - used for inference purposes (metrics)
folder: location of the image slides
sample_factor = det_list_size / mix_list_size
if sample_factor = 1 then all patch sample size is same as detections patch sample size
if sample_factor = 2 then all patch sample size is *half* as detections patch sample size
if sample_factor = 0.5 then all patch sample size is *double* as detections patch sample size
#detection_ratio = det_list_size / combined_list_size
#cls = dls + mls = dls + dls/sf = dls(1+1/sf) = dls (sf+1)/sf
#detection_ratio = dls/cls = dls/(dls * (sf+1)/sf) = sf/(sf+1)
'''
#detection_ratio = sample_factor / (sample_factor + 1)
#true_batch = math.ceil(detection_ratio * batch_size)
#all_batch_size = batch_size - true_batch
#print('true_batch_size: {} \t all_batch_size: {}'.format(true_batch, all_batch_size))
IDG = ImageDataGenerator()
while 1:
all_patch_list = shuffle(all_patch_list)
#det_patch_list = shuffle(det_patch_list)
all_patch_list_size = math.ceil(len(det_patch_list) / sample_factor)
all_patch_list_sub = all_patch_list[:all_patch_list_size]
sampleset_size = len(all_patch_list_sub) + len(det_patch_list)
## Create a combined sample list and shuffle
all_patch_list_sub.extend(det_patch_list)
combined_sample_list = shuffle(all_patch_list_sub)
for offset in range(0, sampleset_size, batch_size):
sample_batch = combined_sample_list[offset:offset + batch_size]
patch = []
ground_truth = []
for sample in sample_batch:
filename = folder + sample[0]
coords = sample[1]
level = levels[np.random.randint(0, len(levels),
dtype=np.int8)]
brightness = (0.95 + np.random.rand() *
(1.05 - 0.95)) if np.random.rand() > 0.5 else 1
dims_factor = (0.9 + np.random.rand() *
(1. - 0.9)) if np.random.rand() > 0.5 else 1
zoom_dims = int(dims[0] / dims_factor)
flip_v = np.random.rand() > 0.5
flip_h = np.random.rand() > 0.5
transformation = {
'brightness': brightness,
'flip_horizontal': flip_h,
'flip_vertical': flip_v
}
patch_img = getRegionFromSlide(
getWSI(filename),
level=level,
start_coord=coords,
dims=(zoom_dims, zoom_dims)).astype(np.float)
if zoom_dims != dims[0]:
patch_img = cv2.resize(patch_img, dims)
patch_img = IDG.apply_transform(patch_img, transformation)
patch_img = (patch_img - 128) / 128
patch.append(patch_img)
if len(sample) == 3:
ground_truth.append(np.array(sample[2]))
else:
ground_truth.append(
getLabel(filename, level, coords,
(zoom_dims, zoom_dims)))
#print('Level used: {}'.format(level))
X_train = np.array(patch)
if save_labels:
labels_list.extend(ground_truth)
#print('||----------------------------------------------||')
#print('len(patch): {}'.format(len(patch)))
#print('len(ground_truth): {}'.format(len(ground_truth)))
#print('len(labels_list): {}'.format(len(labels_list)))
yield X_train
else:
y_train = np.array(ground_truth)
yield X_train, y_train
dfptr.seek(idx)
# Check characterID.
chID = dfptr.read(3)
if characterID != convBin(chID, 'characterID'):
dfptr.close()
return None
length = convBin(dfptr.read(2), 'longlength')
# Skip character name.
latin2utf(dfptr.read(length))
nrItems = convBin(dfptr.read(3), 'nrCharacterItems')
if limit is not None and nrItems/2 > limit:
nrItems = limit*2
filmography = []
for i in xrange(nrItems/2):
personID = convBin(dfptr.read(3), 'personID')
name = getLabel(personID, personIF, personKF)
movieID = convBin(dfptr.read(3), 'movieID')
title = getLabel(movieID, movieIF, movieKF)
# XXX: notes are not retrieved: they can be found scanning
# actors.list and acresses.list, but it will slow down everything.
m = Movie(title=title, movieID=movieID, currentRole=name,
roleID=personID, roleIsPerson=True, accessSystem='local')
filmography.append(m)
dfptr.close()
return filmography
def _convChID(characterID):
"""Return a numeric value for the given string, or None."""
if characterID is None:
return None