def main():
patches = {}
merged_champions = {}
with codecs.open(os.path.join(data_dir, "patches"), "r", "utf-8") as file:
patches = json.load(file)
summaries = ""
for patch in patches.values():
summaries += patch['summary']
for champion in patch['champions']:
if champion['name'] in merged_champions:
merged_champions[champion['name']].short_summary += ' ' + champion['short_summary']
merged_champions[champion['name']].summary += ' ' + champion['summary']
else:
merged_champions[champion['name']] = Champion(champion['name'])
merged_champions[champion['name']].short_summary = champion['short_summary']
merged_champions[champion['name']].summary = champion['summary']
summary = generate_patch_summary(summaries)
for champion in merged_champions.values():
champion.short_summary = generate_sentence(champion.short_summary, 20, 1)
champion.summary = generate_sentence(champion.summary, 15, 2)
generated_patch = Patch('generated', summary)
generated_patch.champions = merged_champions
with codecs.open(os.path.join(out_dir, "patch"), "w", "utf-8") as file:
json.dump(generated_patch, file, default=patch_serialize, indent=4)
def processFile(self, github_file, committed_by):
patch = Patch(patch_text=github_file['patch'],
filepath=github_file['filename'],
repo=self.repo,
account=self.user,
committed_by=committed_by)
patch.updateTodos()
def main(argv):
print(sys.argv)
patchfile = sys.argv[-2]
clogname = sys.argv[-1]
p = Patch(patchfile)
payload = p.parse_payload()
for subdir, text in payload.clogs.iteritems():
print('subdir: %r' % subdir)
print('text:\n%s' % text)
clogfile = os.path.join(subdir, clogname)
if os.path.exists(clogfile):
with open(clogfile) as f:
content = f.read()
else:
content = """Copyright (C) 2014 Free Software Foundation, Inc.
Copying and distribution of this file, with or without modification,
are permitted in any medium without royalty provided the copyright
notice and this notice are preserved.
"""
today = date.today()
header = '%s %s <%s>\n\n' % (today.isoformat(),
AUTHOR.name,
AUTHOR.email)
content = (header + ('\t%s\n\n' % p.summary) + text.rstrip() + '\n\n' + content)
with open(clogfile, 'w') as f:
f.write(content)
def processFile(self, github_file, committed_by):
patch = Patch(patch_text=github_file['patch'],
filepath=github_file['filename'],
repo=self.repo,
account=self.user,
committed_by=committed_by)
patch.updateTodos()
def load(self, filename):
"""
Loads a surface from a text file.
The file must have the following structure:
Each line is a control point. It contains all 3 coordinates, separated by spaces only.
The file must contain n * 16 control points, where n is a positive integer.
"""
self.patches = []
control_points = np.loadtxt(filename)
nb_of_patches = int(len(control_points) / 16)
patches_control_points = np.split(control_points, nb_of_patches)
for patch_nb in range(len(patches_control_points)):
patch = Patch()
patch_control_points = patches_control_points[patch_nb]
for i in range(4):
for j in range(4):
z = patch_control_points[4 * i + j, 2]
y = patch_control_points[4 * i + j, 1]
x = patch_control_points[4 * i + j, 0]
patch.control_points[i, j] = np.array([x, y, z])
self.patches.append(patch)
def add_patch(self, patch=None):
'''
Add a patch to the environment.
Set the location and radius of the patch in
the environment randomly.
'''
if patch is None:
# create a new empty patch
patch = Patch(parent=self)
if patch.parent is None:
patch.parent = self
# set the size of the patch
# TODO: Allow for instance parent max ratio
if patch.radius is None or patch.radius == 0.0:
patch.radius = random() * (
self.width * Environment.MAX_PATCH_RADIUS_RATIO)
# set the location of the patch
if (
patch.x_pos is None or patch.y_pos is None or
patch.x_pos == 0.0 or patch.y_pos == 0.0
):
patch.x_pos, patch.y_pos = self.patch_location(patch.radius)
self.children.append(patch)
def ResetToVanilla(self) -> None:
self._ReadOverworldData()
self.level_1_to_6_rooms = self._ReadDataForLevelGrid(
self.level_1_to_6_raw_data)
self.level_7_to_9_rooms = self._ReadDataForLevelGrid(
self.level_7_to_9_raw_data)
self.level_metadata = list(
open("data/level-metadata.bin", 'rb').read(0x9D8))
self.sprite_set_patch = Patch()
def create_bigM_without_mask(cnn, volume, volume_prob, volume_segmentation, oversampling=False, verbose=False, max=10000):
bigM = []
global_patches = []
if type(volume) is list:
z_s = len(volume)
else:
z_s = volume.shape[0]
t0 = time.time()
for slice in range(z_s):
image = volume[slice]
prob = volume_prob[slice]
segmentation = volume_segmentation[slice]
patches = Patch.patchify(image, prob, segmentation, oversampling=oversampling, max=max)
if verbose:
print len(patches), 'generated in', time.time()-t0, 'seconds.'
t0 = time.time()
grouped_patches = Patch.group(patches)
if verbose:
print 'Grouped into', len(grouped_patches.keys()), 'patches in', time.time()-t0, 'seconds.'
global_patches.append(patches)
hist = Util.get_histogram(segmentation.astype(np.float))
labels = len(hist)
# create Matrix
M = np.zeros((labels, labels), dtype=np.float)
# .. and initialize with -1
M[:,:] = -1
for l_n in grouped_patches.keys():
l = int(l_n.split('-')[0])
n = int(l_n.split('-')[1])
# test this patch group for l and n
prediction = Patch.test_and_unify(grouped_patches[l_n], cnn)
# fill value into matrix
M[l,n] = prediction
M[n,l] = prediction
# now the matrix for this slice is filled
bigM.append(M)
return bigM
def _GetPatchForLevelGrid(self, start_address: int,
rooms: List[Room]) -> Patch:
patch = Patch()
for room_num in Range.VALID_ROOM_NUMBERS:
room_data = rooms[room_num].GetRomData()
assert len(room_data) == self.NUM_BYTES_OF_DATA_PER_ROOM
for table_num in range(0, self.NUM_BYTES_OF_DATA_PER_ROOM):
patch.AddData(
start_address + table_num * self.LEVEL_TABLE_SIZE +
room_num, [room_data[table_num]])
return patch
def __init__(self) -> None:
self.overworld_raw_data = list(
open("data/overworld-data.bin", 'rb').read(0x300))
self.level_1_to_6_raw_data = list(
open("data/level-1-6-data.bin", 'rb').read(0x300))
self.level_7_to_9_raw_data = list(
open("data/level-7-9-data.bin", 'rb').read(0x300))
self.level_metadata = list(
open("data/level-metadata.bin", 'rb').read(0x9D8))
self.overworld_caves: List[Cave] = []
self.level_1_to_6_rooms: List[Room] = []
self.level_7_to_9_rooms: List[Room] = []
self.sprite_set_patch = Patch()
self.misc_data_patch = Patch()
def update_tags(index, entries):
"""
Update the Git-repo tag (possibly by removing it) of patches.
"""
for entry in entries:
with Patch(open(entry.name, mode="r+b")) as patch:
message = "Failed to update tag \"%s\" in patch \"%s\". This " \
"tag is not found."
if entry.dest_head == git_sort.remotes[0]:
tag_name = "Patch-mainline"
try:
patch.change(tag_name, index.describe(entry.dest.index))
except KeyError:
raise exc.KSNotFound(message % (
tag_name,
entry.name,
))
except git_sort.GSError as err:
raise exc.KSError("Failed to update tag \"%s\" in patch "
"\"%s\". %s" % (
tag_name,
entry.name,
str(err),
))
patch.remove("Git-repo")
else:
tag_name = "Git-repo"
try:
patch.change(tag_name, repr(entry.new_url))
except KeyError:
raise exc.KSNotFound(message % (
tag_name,
entry.name,
))
def computePatch(feature, potentialFeature, referenceImage) :
sensedImage = potentialFeature.getImage()
opticalCentre1 = referenceImage.getOpticalCentre()
projectionMatrix1 = referenceImage.getProjectionMatrix()
projectionMatrix2 = sensedImage.getProjectionMatrix()
centre = triangulate(feature, potentialFeature, projectionMatrix1, projectionMatrix2)
centre = np.array([centre[0][0], centre[0][1], centre[0][2], centre[0][3]])
normal = opticalCentre1 - centre
patch = Patch(centre, normal, referenceImage)
# gridCoordinate1 = projectPatch(referenceImage.getProjectionMatrix(), patch)
# gridCoordinate2 = projectPatch(sensedImage.getProjectionMatrix(), patch)
# ref = cv.imread(referenceImage.getImageName())
# fundamentalMatrix = referenceImage.getFundamentalMatrix(potentialFeature.getImage().getImageID())
# coordinate = np.array([
# feature.getX(),
# feature.getY(),
# 1
# ])
# epiline = fundamentalMatrix @ coordinate
# img = sensedImage.computeFeatureMapSatisfyingEpiline(epiline)
# epiline_x = (int(-epiline[2] / epiline[0]), 0)
# epiline_y = (int((-epiline[2] - (epiline[1]*480)) / epiline[0]), 480)
# cv.line(img, epiline_x, epiline_y, (255, 0, 0), 1)
# cv.circle(ref, (int(feature.getX()), int(feature.getY())), 4, (0, 0, 255), -1)
# cv.rectangle(ref, (int(gridCoordinate1[0][0][0]), int(gridCoordinate1[0][0][1])), (int(gridCoordinate1[4][4][0]), int(gridCoordinate1[4][4][1])), (0, 255, 0), -1)
# cv.rectangle(img, (int(gridCoordinate2[0][0][0]), int(gridCoordinate2[0][0][1])), (int(gridCoordinate2[4][4][0]), int(gridCoordinate2[4][4][1])), (0, 255, 0), -1)
# cv.imshow(f'Reference Image', ref)
# cv.imshow(f'Sensed Image', img)
# cv.waitKey(0)
# cv.destroyAllWindows()
return patch
def find_commit(commit, series, mode="rb"):
"""
Caller must chdir to where the entries in series can be found.
"""
for name in filter_series(series):
patch = Patch(open(name, mode="rb"))
found = False
if commit in [firstword(value)
for value in patch.get("Git-commit")
if value]:
found = True
yield name, patch
patch.writeback()
if found:
return
raise exc.KSNotFound()
def split(image, prob, segmentation, n=1, min_pixels=100):
'''
'''
ugly_segmentation = np.array(segmentation)
hist = Util.get_histogram(ugly_segmentation.astype(np.uint64))
labels = len(hist)
for l in range(labels):
if l == 0:
continue
for s in range(n):
binary_mask = Util.threshold(ugly_segmentation, l)
splitted_label, border = Patch.split_label(image, binary_mask)
# check if splitted_label is large enough
if len(splitted_label[splitted_label == 1]) < min_pixels:
continue
ugly_segmentation[splitted_label ==
1] = ugly_segmentation.max() + 1
return ugly_segmentation
def generate_patches(x, y, split, balanced=False):
for i in tqdm(range(20)):
if split == 'train':
I = np.array(
Image.open('DRIVE/training/images/' + format(i + 21, '02d') +
'_training.tif'))
J = np.array(
Image.open('DRIVE/training/1st_manual/' +
format(i + 21, '02d') + '_manual1.tif'))
mask = np.array(
Image.open('DRIVE/training/mask/' + format(i + 21, '02d') +
'_training_mask.tif'))
elif split == 'test':
I = np.array(
Image.open('DRIVE/test/images/' + format(i + 1, '02d') +
'_test.tif'))
J = np.array(
Image.open('DRIVE/test/1st_manual/' + format(i + 1, '02d') +
'_manual1.tif'))
mask = np.array(
Image.open('DRIVE/test/mask/' + format(i + 1, '02d') +
'_test_mask.tif'))
for j in range(20000):
index = i * 20000 + j
try:
label = None
if balanced:
label = int(np.random.rand() < 0.5)
patch = Patch(I, J, mask, size=x.shape[1], label=label)
x[index] = patch.data
y[index, patch.label] = 1
except ValueError:
j -= 1
def create_patch(self, replace_current = False):
patch = Patch(self.rom, self.patch_range, self.patch_size, 'increment')
if replace_current == True:
self.patches[len(self.patches) - 1] = patch
else:
self.patches.append(patch)
def create_patch(self) -> Patch:
last = self.version
self._increment_version()
current = self.version
patch = Patch(last, current)
self.patches[last] = patch
self.current_patch = patch
return patch
def generate_patches(scaled_imgs, constants):
patch_size = constants.PATCH_SIZE
std_dev_threshold = constants.STD_DEV_THRESHOLD
patches = []
for k, sc in enumerate(scaled_imgs):
img_patches = []
for i in range(sc.shape[0] - patch_size):
for j in range(sc.shape[1] - patch_size):
raw_patch = sc[i:i + patch_size, j:j + patch_size, :]
patch = Patch(
raw_patch=raw_patch,
patch_size=patch_size,
)
if patch.std_dev > std_dev_threshold:
patch.store(sc, [i, j])
img_patches.append(patch)
patches.append(img_patches)
return patches
def generate_patches(scaled_imgs, constants, all_patches):
"""Generate patches for all scaled images."""
patch_size = constants.PATCH_SIZE
step = 1 if all_patches else 2
patches = []
for k, sc in enumerate(scaled_imgs):
img_patches = []
for i in range(0, sc.shape[0] - patch_size, step):
for j in range(0, sc.shape[1] - patch_size, step):
raw_patch = sc[i:i + patch_size, j:j + patch_size, :]
patch = Patch(
raw_patch=raw_patch,
patch_size=patch_size,
)
patch.store(sc, [i, j])
img_patches.append(patch)
patches.append(img_patches)
return patches
def setup():
with open("input.txt") as f:
for line in f:
coord = line.split(',')
x = int(coord[0])
y = int(coord[1].strip())
patches.append(Patch(x, y))
usedPoints.append([x, y])
def __init__(self, match_id):
connect = Connection_to_internet()
self.match_id = match_id
self.PATCH = Patch.getCurrentPatch(Patch)
self.players_signatures = {}
self.players_7day_ago = {}
self.__get_json_from_url = connect.get_json_from_url
# Нельзя вызвать данные в лайве в одном месте, так иногда бывает что не прогрузились данные
# Поэтому необходимо вызывать каждый раз с self.match_id
self.get_live_data_for_match = connect.get_live_data_for_match
def GetPatch(self) -> Patch:
patch = Patch()
patch += self._GetPatchForLevelGrid(
self.LEVEL_1_TO_6_DATA_START_ADDRESS, self.level_1_to_6_rooms)
patch += self._GetPatchForLevelGrid(
self.LEVEL_7_TO_9_DATA_START_ADDRESS, self.level_7_to_9_rooms)
patch += self._GetPatchForLevelMetadata()
patch += self._GetPatchForOverworldCaveData()
patch += self._GetPatchForOverworldData()
patch += self.sprite_set_patch
return patch
def parse_patch(number):
patch = None
url = url_start + number + url_end
print_bullet_point(url, 2)
request = requests.get(url)
if request.status_code == requests.codes.ok:
soup = BeautifulSoup(request.text, "html.parser")
container = soup.find("div", {"id": "patch-notes-container"})
if container is None:
print_bullet_point("ERROR: Could not find patch-notes-container", 4)
else:
patch_summary = parse_summary(container)
patch = Patch(number, patch_summary)
patch.champions = parse_champions(container)
else:
print_bullet_point("ERROR: status_code " + str(request.status_code), 4)
return patch
def computePatch(feat1, feat2, ref):
logging.info(f'IMAGE {ref.id:02d}:Constructing patch.')
img = feat2.image
opticalCentre = ref.opticalCentre
projectionMatrix1 = ref.projectionMatrix
projectionMatrix2 = img.projectionMatrix
centre = triangulate(feat1, feat2, projectionMatrix1, projectionMatrix2)[0]
normal = opticalCentre - centre
normal /= norm(normal)
patch = Patch(centre, normal, ref)
return patch
def generate_correct(image, prob, segmentation):
hist = Util.get_histogram(segmentation.astype(np.uint64))
labels = range(1, len(hist)) # do not include zeros
np.random.shuffle(labels)
for l in labels:
binary_mask = Util.threshold(segmentation, l)
borders = mh.labeled.borders(binary_mask, Bc=mh.disk(2))
labeled_borders = skimage.measure.label(borders)
labeled_borders[borders == 0] = 0
relabeled_borders, no_relabeled_borders = mh.labeled.relabel(
labeled_borders.astype(np.uint16))
for border in range(1, no_relabeled_borders + 1):
isolated_border = Util.threshold(relabeled_borders, border)
patches = Patch.analyze_border(image, prob, binary_mask,
binary_mask.invert(),
isolated_border)
for s in patches:
yield s
yield Patch.fliplr(s)
yield Patch.flipud(s)
yield Patch.rotate(s, 90)
yield Patch.rotate(s, 180)
yield Patch.rotate(s, 270)
def _GetPatchForOverworldCaveData(self) -> Patch:
patch = Patch()
for cave_type in Range.VALID_CAVE_TYPES:
cave_num = int(cave_type) - self.CAVE_TYPE_CAVE_NUM_OFFSET
if cave_type == CaveType.ARMOS_ITEM_VIRTUAL_CAVE:
patch.AddData(
self.ARMOS_ITEM_ADDRESS,
[self.overworld_caves[cave_num].GetItemAtPosition(2)])
continue
if cave_type == CaveType.COAST_ITEM_VIRTUAL_CAVE:
patch.AddData(
self.COAST_ITEM_ADDRESS,
[self.overworld_caves[cave_num].GetItemAtPosition(2)])
continue
# Note that the Cave class is responsible for protecting bits 6 and 7 in its item data
patch.AddData(
self.OVERWORLD_DATA_START_ADDRESS +
self._GetOverworldCaveDataIndex(
cave_type, 0, is_second_byte=False),
self.overworld_caves[cave_num].GetItemData())
patch.AddData(
self.OVERWORLD_DATA_START_ADDRESS +
self._GetOverworldCaveDataIndex(
cave_type, 0, is_second_byte=True),
self.overworld_caves[cave_num].GetPriceData())
return patch
def find_commit(commit, series, mode="rb"):
"""
commit: unabbreviated git commit id
series: list of lines from series.conf
mode: mode to open the patch files in, should be "rb" or "r+b"
Caller must chdir to where the entries in series can be found.
Returns patch.Patch instances
"""
for name in filter_series(series):
patch = Patch(open(name, mode=mode))
found = False
if commit in [firstword(value)
for value in patch.get("Git-commit")
if value]:
found = True
yield name, patch
patch.writeback()
if found:
return
raise exc.KSNotFound()
def randomize(self,
patch_length,
patch_width,
min_height=0,
max_height=0.2):
"""
Creates a random Bézier surface. The resulting surface will be C0 only.
patch_length is the number of patches along the x axis that will be generated.
patch_width is the number of patches along the y axis that will be generated.
"""
self.patches = []
for w, l in product(range(patch_width), range(patch_length)):
# Patches are created and stored in a 1D array.
# The patch at coordinate (x, y) can be retrieved as self.patches[x + y * patch_length]
# However this structure is true ONLY in this generation fuction, as in general, if
# we import another surface, it may not follow this particular pattern.
# It is however useful in this generation, as it allows us to easily correct random
# patches to make them c0, without checking each control points 1 by 1.
patch = Patch()
patch.randomize(min_x=l,
max_x=l + 1,
min_y=w,
max_y=w + 1,
min_z=min_height,
max_z=max_height)
self.patches.append(patch)
# make the connections continuous
for l, w in product(range(1, patch_length), range(patch_width)):
previous_patch = self.patches[(l - 1) + w * patch_length]
current_patch = self.patches[l + w * patch_length]
for i in range(4):
current_patch[0, i] = previous_patch[3, i]
for l, w in product(range(patch_length), range(1, patch_width)):
previous_patch = self.patches[l + (w - 1) * patch_length]
current_patch = self.patches[l + w * patch_length]
for i in range(4):
current_patch[i, 0] = previous_patch[i, 3]
def find_commit(commit, series, mode="rb"):
"""
commit: unabbreviated git commit id
series: list of lines from series.conf
mode: mode to open the patch files in, should be "rb" or "r+b"
Caller must chdir to where the entries in series can be found.
Returns patch.Patch instances
"""
for name in filter_series(series):
patch = Patch(open(name, mode=mode))
found = False
if commit in [
firstword(value) for value in patch.get("Git-commit") if value
]:
found = True
yield name, patch
patch.writeback()
if found:
return
raise exc.KSNotFound()
def evaluate_grid_masked(self, x0, y0, mask, fx, fy, derivs=False):
'''
mask: np array of booleans (NOT Patch object!)
'''
from mix import c_gauss_2d_masked
h, w = mask.shape
result = np.zeros((h, w), np.float32)
xderiv = yderiv = None
if derivs:
xderiv = np.zeros_like(result)
yderiv = np.zeros_like(result)
# print 'gauss_2d_masked:', int(x0), int(y0), int(w), int (h), float(fx), float(fy),
# print ' ', self.amp.astype(np.float32),
# print ' ', self.mean.astype(np.float32),
# print ' ', self.var.astype(np.float32),
# print ' ', 'res', (result.shape, result.dtype),
# if xderiv is not None:
# print ' ', 'xd', (xderiv.shape, xderiv.dtype),
# if yderiv is not None:
# print ' ', 'yd', (yderiv.shape, yderiv.dtype),
# print ' ', 'mask', mask.shape, mask.dtype
rtn = c_gauss_2d_masked(int(x0), int(y0), int(w), int(h), float(fx),
float(fy), self.amp.astype(np.float32),
self.mean.astype(np.float32),
self.var.astype(np.float32), result, xderiv,
yderiv, mask)
# print 'gauss_2d_masked returned.'
assert (rtn == 0)
if derivs:
return (Patch(x0, y0,
result), Patch(x0, y0,
xderiv), Patch(x0, y0, yderiv))
return Patch(x0, y0, result)
def evaluate_grid_dstn(self, x0, x1, y0, y1, cx, cy):
'''
[x0,x1): (int) X values to evaluate
[y0,y1): (int) Y values to evaluate
(cx,cy): (float) pixel center of the MoG
'''
from mix import c_gauss_2d_grid
assert(self.D == 2)
result = np.zeros((y1-y0, x1-x0))
rtn = c_gauss_2d_grid(x0, x1, y0, y1, cx, cy,
self.amp, self.mean,self.var, result)
if rtn == -1:
raise RuntimeError('c_gauss_2d_grid failed')
return Patch(x0, y0, result)
def add_new_label_to_M(cnn, m, input_image, input_prob, input_rhoana, label1):
# calculate neighbors of the two new labels
label1_neighbors = Util.grab_neighbors(input_rhoana, label1)
for l_neighbor in label1_neighbors:
# recalculate new neighbors of l
if l_neighbor == 0:
# ignore neighbor zero
continue
prediction = Patch.grab_group_test_and_unify(cnn, input_image, input_prob, input_rhoana, label1, l_neighbor, oversampling=False)
m[label1,l_neighbor] = prediction
m[l_neighbor,label1] = prediction
return m
def rank(self, image, prob, segmentation, label1, label2):
'''
Rank the intersection between label1 and label2 for split errors.
This is the same as an affinity score.
Returns
1 If label1 and label2 should be merged.
..
0 If label1 and label2 should NOT be merged.
or
-1 If there was a problem.
'''
return Patch.grab_group_test_and_unify(self._cnn, image, prob,
segmentation, label1, label2)
def generate_split_error(image, prob, segmentation, n=3):
hist = Util.get_histogram(segmentation.astype(np.uint64))
labels = range(1, len(hist)) # do not include zeros
np.random.shuffle(labels)
for l in labels:
binary_mask = Util.threshold(segmentation, l)
labeled_parts = skimage.measure.label(binary_mask)
labeled_parts += 1 # avoid the 0
labeled_parts[binary_mask == 0] = 0
labeled_parts, no_labeled_parts = mh.labeled.relabel(labeled_parts)
for i in range(n):
for part in range(1, no_labeled_parts + 1):
binary_part = Util.threshold(labeled_parts, part)
split_binary_mask, split_isolated_border = Patch.split_label(
image, binary_part)
if split_binary_mask.max() == 0:
# the strange err (label too small or sth)
print 'Caught empty..'
continue
patches = Patch.analyze_border(image, prob,
split_binary_mask,
binary_mask.invert(),
split_isolated_border)
for s in patches:
yield s
yield Patch.fliplr(s)
yield Patch.flipud(s)
yield Patch.rotate(s, 90)
yield Patch.rotate(s, 180)
yield Patch.rotate(s, 270)
def get_patches_from_slide(slide, tile_size=1024, overlap=0, limit_bounds=False):
"""
Splits an OpenSlide object into nonoverlapping patches
Args:
slide: OpenSlide object
tile_size: Width and height of a single tile
overlap: Number of extra pixels to add to each interior edge of a tile
limit_bounds: If True, renders only non-empty slide region
Returns:
Array of patches
"""
tiles = DeepZoomGenerator(slide, tile_size=tile_size, overlap=overlap,
limit_bounds=limit_bounds)
level = len(tiles.level_tiles) - 1
x_tiles, y_tiles = tiles.level_tiles[level] #Note: Highest level == Highest resolution
x, y = 0, 0
count, batch_count = 0, 0
patches = []
coordinate_list = []
tiled_dims = (y_tiles, x_tiles)
while y < y_tiles:
while x < x_tiles:
new_patch_img = np.array(tiles.get_tile(level, (x,y)), dtype=np.uint8)
new_patch_coords = tiles.get_tile_coordinates(level, (x,y))
if np.shape(new_patch_img) == (tile_size, tile_size, 3):
new_patch = Patch(new_patch_img, new_patch_coords)
patches.append(new_patch)
patch_coordinates = (y,x)
coordinate_list.append(patch_coordinates)
count += 1
x += 1
y += 1
x = 0
return (patches, coordinate_list, tiled_dims)
def expand(self):
it = 1
queue = self.patches.copy()
while len(queue) > 0:
p = queue.pop(0)
for cell in p.cells:
C = []
for i in range(-1, 2):
for j in range(-1, 2):
if i == 0 and j == 0:
continue
idx = (cell.y + i * 2, cell.x + j * 2)
if idx in self.cells[cell.ref_idx]:
C.append(self.cells[cell.ref_idx][idx])
# cp = p.cam_center + p.k * p.d
# p_depth = (self.extrinsics[p.ref_idx] @ np.array([*cp, 1]))[-1]
C_filtered = []
for c in C:
neccessary = True
if len(c.q) > 0:
for pp in c.q:
# cpp = pp.cam_center + pp.k * pp.d
# pp_depth = (self.extrinsics[pp.ref_idx] @ np.array([*cpp, 1]))[-1]
# rho1 = 2 * self.intrinsic_inv[0, 0] * (p_depth + pp_depth) / 2
# if np.abs(np.dot(cp - cpp, sph2norm(p.theta, p.phi))) + np.abs(np.dot(cp - cpp, sph2norm(pp.theta, pp.phi))) < 2 * rho1:
if self.is_neighbor(p, pp):
neccessary = False
break
if neccessary and len(c.q_star) > 0:
for pp in c.q_star:
if self.gp(pp, pp.ref_idx, pp.v_star) < 0.3:
neccessary = False
break
if neccessary:
C_filtered.append(c)
C = C_filtered
# print(len(C))
for c in C:
pprime = Patch(patch=p)
# set new pc
n = sph2norm(p.theta, p.phi)
p_center = p.cam_center + p.k * p.d
plane = -np.dot(n, p_center)
pt = np.array([c.x + 1, c.y + 1, 1])
pt_world = np.linalg.pinv(self.extrinsics[pprime.ref_idx]) @ self.intrinsic_inv @ pt
pt_world = pt_world[:3] / pt_world[-1]
cam2pt = pt_world - p.cam_center
pprime_center = -(plane + np.dot(n, p.cam_center)) / np.dot(n, cam2pt) * cam2pt + p.cam_center
# print(p_center)
# print(pprime_center)
pprime.d = pprime_center - p.cam_center
pprime.k = np.linalg.norm(pprime.d)
pprime.d /= pprime.k
# import pdb; pdb.set_trace()
proj_ref = pprime.project_onto_img(self.intrinsic, self.extrinsics[pprime.ref_idx])
v_star = []
for cand in pprime.v:
proj_cand = pprime.project_onto_img(self.intrinsic, self.extrinsics[cand])
zncc = calc_zncc(self.grays[pprime.ref_idx], self.grays[cand], proj_ref, proj_cand)
if 1. - zncc < 0.6:
v_star.append(cand)
pprime.v_star = v_star
if len(pprime.v_star) < 2:
continue
def gp(x):
backup = pprime.k, pprime.theta, pprime.phi
pprime.k, pprime.theta, pprime.phi = x
res = self.gp(pprime, pprime.ref_idx, pprime.v_star)
pprime.k, pprime.theta, pprime.phi = backup
return res
def gp_grad(x):
backup = pprime.k, pprime.theta, pprime.phi
pprime.k, pprime.theta, pprime.phi = x
res = self.gp_grad(pprime, pprime.ref_idx, pprime.v_star)
pprime.k, pprime.theta, pprime.phi = backup
return res
# print(patch.k, patch.theta, patch.phi)
best_x = optimize.fmin_cg(gp, np.array([pprime.k, pprime.theta, pprime.phi]), fprime=gp_grad, disp=False)
pprime.k, pprime.theta, pprime.phi = best_x
# determine depth test threshold
n = sph2norm(pprime.theta, pprime.phi)
pprime_center = pprime.cam_center + pprime.k * pprime.d
plane = -np.dot(n, pprime_center)
pt = np.array([c.x + 3, c.y + 1, 1]) # offset two pixels
pt_world = np.linalg.pinv(self.extrinsics[pprime.ref_idx]) @ self.intrinsic_inv @ pt
pt_world = pt_world[:3] / pt_world[-1]
cam2pt = pt_world - pprime.cam_center
delta = np.linalg.norm(-(plane + np.dot(n, pprime.cam_center)) / np.dot(n, cam2pt) * cam2pt + pprime.cam_center - pprime_center)
pprime_depth = (self.extrinsics[pprime.ref_idx] @ np.array([*pprime_center, 1]))[-1]
for cand in range(len(self.grays)):
if cand == pprime.ref_idx:
continue
proj = self.intrinsic @ self.extrinsics[cand] @ np.array([*(pprime.cam_center + pprime.k * pprime.d), 1])
proj = (proj[:2] / proj[-1]).astype(np.int)
if (proj[1], proj[0]) in self.cells[cand]:
if len(self.cells[cand][(proj[1], proj[0])].q) > 0:
valid = True
for pp in self.cells[cand][(proj[1], proj[0])].q:
pp = self.cells[cand][(proj[1], proj[0])].q[0]
pp_proj = self.extrinsics[pp.ref_idx] @ np.array([*(pp.cam_center + pp.k * pp.d), 1])
depth = pp_proj[-1]
if pprime_depth > depth + delta:
valid = False
break
if valid:
pprime.v.append(cand)
pprime.v = list(set(pprime.v))
v_star = []
for cand in pprime.v:
proj_cand = pprime.project_onto_img(self.intrinsic, self.extrinsics[cand])
zncc = calc_zncc(self.grays[pprime.ref_idx], self.grays[cand], proj_ref, proj_cand)
if 1. - zncc < 0.3:
v_star.append(cand)
pprime.v_star = list(set(v_star))
if len(pprime.v_star) < 2:
continue
queue.append(pprime)
self.patches.append(pprime)
for idx in pprime.v:
proj = self.intrinsic @ self.extrinsics[idx] @ np.array([*pprime_center, 1])
proj = (proj[:2] / proj[-1]).astype(np.int)
if (proj[1], proj[0]) in self.cells[idx]:
self.cells[idx][(proj[1], proj[0])].q.append(pprime)
for idx in pprime.v_star:
proj = self.intrinsic @ self.extrinsics[idx] @ np.array([*pprime_center, 1])
proj = (proj[:2] / proj[-1]).astype(np.int)
if (proj[1], proj[0]) in self.cells[idx]:
self.cells[idx][(proj[1], proj[0])].q_star.append(pprime)
pprime.cells.append(self.cells[idx][(proj[1], proj[0])])
print('processed one: {}, current queue length: {}'.format(len(self.patches), len(queue)))
colors = np.zeros((len(self.patches), 3))
vis = self.rgbs[0].copy()
for i, patch in enumerate(self.patches):
pts = patch.project_onto_img(self.intrinsic, self.extrinsics[0])
pt = pts[pts.shape[0] // 2].astype(np.int)
if pt[0] >= 0 and pt[1] >= 0 and pt[0] < self.grays[0].shape[1] and pt[1] < self.grays[0].shape[0]:
vis = cv2.circle(vis, (int(pt[0]), int(pt[1])), 1, color=(255, 0, 0), thickness=-1)
colors[i] = self.rgbs[0][pt[1], pt[0]]
cv2.imshow('patch', cv2.resize(vis[..., ::-1], (0, 0), fx=4, fy=4))
cv2.waitKey(10)
head = repository.repository_head(repository.full_path, branch)
logging.info("branch/revision: " + branch + '/' + head + ' (' + config.get('build-revision') + ')')
log_data['revision'] = head
else:
logging.info("branch/revision: " + branch + '/' + config.get('build-revision'))
log_data['revision'] = config.get('build-revision')
# use the current timestamp and the branch name as build dir name
build_dir_name = str(current_time) + '_' + branch
if (config.get('build-revision') != 'HEAD'):
# add the revision name, if it's not HEAD
build_dir_name += '_' + config.get('build-revision')
if (config.get('run-configure') is True):
# create "Patch" instance before creating the repository
# https://github.com/andreasscherbaum/buildfarm-client/issues/1
patch = Patch(config.get('patch'), config, repository, None, config.get('cache-dir'))
if (patch.have_patches() is True):
# retrieve all patches
result_retrieve_patches = patch.retrieve_patches()
log_data['patches'] = '|'.join(patch.patches)
if (result_retrieve_patches is False):
# retrieving patches failed, don't bother with the rest of the job
# continue with next branch in list
# don't care about logging, this is manual mode
continue
build_dir = repository.copy_repository(build_dir_name, branch, config.get('build-revision'))
# the "Patch" instance is initialized without the build_dir information
patch.set_build_dir(build_dir)
log_data['build_dir'] = build_dir
def main():
args = parse_command_line()
root_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
config.config_init(args.version, args.revision, args.version_name,
args.sdk_root_path, args.sdk_version,
root_dir)
sdk = Sdk(config.ARTIFACTS_ROOT, config.SDK_VERSION)
# Assumes that source root dir is .. since build.py is ran in directory build_scripts
patch_dir = os.path.join(root_dir, 'patches')
try:
if args.clean:
clean.clean_all(sdk)
if args.create_patch_filename:
sdk.prepare_sdk()
old_path = sdk.path
tempdir_path = None
try:
tempdir_path = tempfile.mkdtemp(prefix='nordicsemi')
sdk.path = os.path.join(tempdir_path, "orig_sdk")
sdk.prepare_sdk()
p = Patch(patch_dir=patch_dir,
apply_patch_root_dir=sdk.path,
strip=0 # The old patches has 4 more components
)
p.create_patch(tempdir_path, old_path, os.path.join(patch_dir, args.create_patch_filename))
finally:
sdk.path = old_path
shutil.rmtree(tempdir_path)
if args.dependencies:
sdk.prepare_sdk()
patch_tag_file = posixpath.join(sdk.path, ".patched")
if os.path.exists(patch_tag_file):
logger.info("Patches are already applied to this SDK, skipping patching")
else:
open(patch_tag_file, 'w').close()
p = Patch(patch_dir=patch_dir,
apply_patch_root_dir=sdk.path,
strip=0 # The old patches has 4 more components
)
p.apply_patches(dry_run=False)
if args.build:
serialization_dll.build()
if args.test:
error_code = tests.do_testing()
if error_code != 0:
return error_code
if args.package:
package_release()
if args.examples:
examples.build_examples()
except Exception, ex:
logger.exception(ex)
return -1
def calibrate(cls, band, ra, dec, rng, calibid, maxmag=22, limit=None):
"""
Given a band and coordinates, calibrate a patch and return the patch.
:param band:
The band to use.
:param ra:
The R.A. of the center of the patch.
:param dec:
The Dec. of the center of the patch.
:param rng:
A tuple giving the range of the patch in degrees. This should
have the form ``(ra_range, dec_range)``.
:param calibid:
The ``id`` of the :class:`CalibRun`.
:param limit: (optional)
Passed to :func:`Measurement.find()`. This should probably always
be ``None`` except for testing.
:param maxmag: (optional)
The limiting magnitude to use for calibration.
:param limit: (optional)
Passed to :func:`Measurement.find()`. This should probably always
be ``None`` except for testing.
"""
# Create a new empty `CalibPatch` object.
doc = dict([(k, None) for k in cls.columns])
doc["band"] = band
doc["ramin"], doc["ramax"] = ra - rng[0], ra + rng[0]
doc["decmin"], doc["decmax"] = dec - rng[1], dec + rng[1]
doc["calibid"] = calibid
self = cls(**doc)
# Get the photometry.
tai, runs, stars, flux, ivar, fp, ivp = self.get_photometry(maxmag)
self.doc["runs"] = runs
self.doc["stars"] = stars
self.save()
_id = self["id"]
patch = Patch(flux, ivar)
patch.optimize(fp, ivp)
with DBConnection() as cursor:
# Build the photometrically calibrated models too.
zero = patch.f0
for i, run in enumerate(runs):
for j, star in enumerate(stars):
if ivar[i, j] > 0 and zero[i] > 0.0:
doc = {"calibid": calibid, "patchid": _id,
"band": band, "runid": run, "starid": star}
doc["tai"] = tai[i]
doc["flux"] = flux[i, j] / zero[i]
doc["fluxivar"] = ivar[i, j] * zero[i] ** 2
p = Photometry(**doc)
cursor.execute(*p._save_cmd)
# Save the zero points.
beta2 = patch.b2
delta2 = patch.d2
for i, run in enumerate(runs):
doc = {"calibid": calibid, "patchid": _id, "band": band,
"runid": run, "ramin": self.ramin, "ramax": self.ramax,
"decmin": self.decmin, "decmax": self.decmax}
doc["zero"] = zero[i]
doc["beta2"] = beta2[i]
doc["delta2"] = delta2[i]
doc["zeroivar"] = 1.0
doc["nstars"] = np.sum(ivar[i, :] > 0)
z = Zero(**doc)
cursor.execute(*z._save_cmd)
# Save the mean fluxes.
fs = patch.fs
eta2 = patch.e2
for j, star in enumerate(stars):
doc = {"calibid": calibid, "patchid": _id, "band": band,
"starid": star, "mean_flux": fs[j], "eta2": eta2[j]}
f = Flux(**doc)
cursor.execute(*f._save_cmd)
return self
