def test(self):
self.model.eval()
avg_psnr = 0
name = 0
pae = 6
with torch.no_grad():
for batch_num, (data, target) in enumerate(self.testing_loader):
data, target = data.to(self.device), target.to(self.device)
prediction = self.model(data)
loss = self.criterion(prediction, target)
prediction = np.around(prediction.cpu() * 128 * pae * 0.7 +
128)
target = np.around(target.cpu() * 128 * pae * 0.7 + 128)
mse = self.criterion(prediction, target)
psnr = 10 * log10(65025 / mse.item())
avg_psnr += psnr
progress_bar(batch_num, len(self.testing_loader),
'PSNR: %.4f' % (avg_psnr / (batch_num + 1)))
prediction = prediction.cpu().numpy()
prediction = np.reshape(prediction,
(prediction.shape[2], prediction.shape[3]))
cv2.imwrite('./result/test.png', prediction)
print(" Average PSNR: {:.4f} dB".format(avg_psnr /
len(self.testing_loader)))
return avg_psnr / len(self.testing_loader)
def valid(self):
"""
data: [torch.cuda.FloatTensor], 10 batches: [10, 10, 10, 10, 10, 10, 10, 10, 10, 10]
"""
self.model.eval()
avg_psnr = 0
avg_niqe = 0
with torch.no_grad():
for batch_num, (data, target) in enumerate(self.valid_loader):
data = self.img_preprocess(data) # resize input image size
data, target = data.to(self.device), target.to(self.device)
_, prediction = self.model(data)
#calculate psnr
mse = torch.mean(((prediction - target)**2), dim=[1, 2, 3])
psnr = -10 * mse.log10().mean().item()
avg_psnr += psnr
#calculate niqe
niqe = np.mean(
niqe_metric.niqe(
prediction.permute(0, 2, 3, 1).cpu().numpy() * 255,
RGB=True,
video_params=False))
avg_niqe += niqe
progress_bar(
batch_num, len(self.valid_loader),
'PSNR: %.3f || NIQE: %.3f' %
(avg_psnr / (batch_num + 1), avg_niqe / (batch_num + 1)))
print(" Average PSNR: {:.4f} dB".format(avg_psnr /
len(self.valid_loader)))
return avg_psnr
def progress_info(self):
# Show the progress
self.amount = len(self.link_list_set)
scraped = len(self.scraped_list_set)
print(f"=====\nOverall Progress for {self.domain}")
progress_bar(scraped, self.amount)
old = 0
for link in self.link_list_set:
if str(link) in self.scraped_list_set:
old += 1
else:
pass
diff = self.amount - old
print("=====")
print(f"{diff} new links found and ready for scraping")
if go():
print("Let's go!")
pass
else:
y.graph() # not that intereseting at the moment
df = self.create_df()
self.create_csv(df, self.seo_file)
sys.exit()
def recurs(self):
self.progress_info()
count = 0
for link in self.link_list_set.copy():
print(f"Link : {link}")
count += 1
progress_bar(count, self.amount)
try:
if link in self.scraped_list_set.copy():
pass
elif self.load(
link
): # load(link) grabs robots to see if no-index in it if so, skip link
pass
else:
self.load(link)
self.find(seo_tags)
# df = self.create_df()
# self.create_csv(df, self.seo_file, "a", False)
self.next_link()
except:
print(f"❌ Status Code für Link {link}")
# Starting again
self.recurs()
def train(self):
self.model.train()
train_loss = 0
for batch_num, (data) in enumerate(self.training_loader):
data, target = data[:, :2, ::3, ::3].to(
self.device), data[:, :2].to(self.device)
self.optimizer.zero_grad()
loss = self.criterion(self.model(data), target)
train_loss += loss.item()
loss.backward()
self.optimizer.step()
progress_bar(
batch_num, len(self.training_loader), 'Loss: %.4f, LR: %.4f' %
(train_loss /
(batch_num + 1), self.optimizer.param_groups[0]['lr']))
self.writer.add_scalar(
'Train/TrainIterLoss_%s' % self.config.comment, loss.item(),
self.epoch * len(self.training_loader) + batch_num)
self.writer.add_scalar(
'Train/TrainAvgLoss_%s' % self.config.comment,
train_loss / (batch_num + 1),
self.epoch * len(self.training_loader) + batch_num)
print(" Average Loss: {:.4f}".format(train_loss /
len(self.training_loader)))
def test(self):
self.model.eval()
avg_psnr = 0
with torch.no_grad():
for batch_num, (data) in enumerate(self.testing_loader):
data, target = data[:, :2, ::3, ::3].to(
self.device), data[:, :2].to(self.device)
data = F.interpolate(data,
size=(target.shape[-2], target.shape[-1]))
_, prediction = self.model(data)
mse = self.criterion(prediction, target)
avg_psnr += mse.item()
progress_bar(batch_num, len(self.testing_loader),
'PSNR: %.4f' % (avg_psnr / (batch_num + 1)))
self.writer.add_scalar(
'Valid/ValidIterLoss_%s' % self.config.comment, mse.item(),
self.epoch * len(self.testing_loader) + batch_num)
self.writer.add_scalar(
'Valid/ValidAvgLoss_%s' % self.config.comment,
avg_psnr / (batch_num + 1),
self.epoch * len(self.testing_loader) + batch_num)
print(" Average Loss: {:.4f}".format(avg_psnr /
len(self.testing_loader)))
return avg_psnr / len(self.testing_loader)
def train(self):
self.model.train()
train_loss = 0
for batch_num, (data, target) in enumerate(self.training_loader):
data = data.to(self.device)
target = [
target[i].to(self.device) for i in range(self.num_targets)
]
self.optimizer.zero_grad()
out = self.model(data)
for i in range(self.num_targets):
if i == 0:
loss = self.criterion(out[i], target[i])
else:
loss += self.criterion(out[i], target[i])
train_loss += loss.item()
loss.backward()
self.optimizer.step()
progress_bar(batch_num, len(self.training_loader),
'Loss: {:.4f}'.format((train_loss / (batch_num + 1))))
avg_train_loss = train_loss / len(self.training_loader)
print(" Average Loss: {:.4f}".format(avg_train_loss))
return avg_train_loss
def main():
# ===========================================================
# Set train dataset & valid dataset
# ===========================================================
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print('===> Loading datasets')
if args.dataset == 'test':
test_set = get_test_set(args.upscale_factor)
elif args.dataset == 'valid':
test_set = get_valid_set(args.upscale_factor)
else:
raise NotImplementedError
test_data_loader = DataLoader(dataset=test_set,
batch_size=args.batchSize,
shuffle=False)
file_name = args.model + "_generator.pth" if "gan" in args.model else "model.pth"
model_name = args.model + ("_diff" if args.diff else "")
model_path = "/home/teven/canvas/python/super-resolution/results/models/{}/{}".format(
model_name, file_name)
model = torch.load(model_path, map_location=lambda storage, loc: storage)
model = model.to(device)
model.eval()
avg_psnr = 0
avg_baseline_psnr = 0
criterion = nn.MSELoss()
with torch.no_grad():
for batch_num, (data, target) in enumerate(test_data_loader):
data, target = data.to(device), target.to(device)
prediction = model(data)
mse = criterion(prediction, target)
psnr = 10 * log10(1 / mse.item())
avg_psnr += psnr
progress_bar(batch_num, len(test_data_loader),
'PSNR: %.3f' % (avg_psnr / (batch_num + 1)))
baseline = F.interpolate(data,
scale_factor=args.upscale_factor,
mode='bilinear',
align_corners=False)
baseline_mse = criterion(baseline, target)
baseline_psnr = 10 * log10(1 / baseline_mse.item())
avg_baseline_psnr += baseline_psnr
progress_bar(batch_num, len(test_data_loader),
'PSNR: %.3f' % (avg_baseline_psnr / (batch_num + 1)))
print(" Average PSNR: {:.3f} dB".format(avg_psnr /
len(test_data_loader)))
print(" Average Baseline PSNR: {:.3f} dB".format(avg_baseline_psnr /
len(test_data_loader)))
def train(self):
self.model.train()
train_loss = 0
for batch_num, (data, target) in enumerate(self.training_loader):
data, target = data.to(self.device), target.to(self.device)
self.optimizer.zero_grad()
loss = self.criterion(self.model(data), target)
train_loss += loss.item()
loss.backward()
self.optimizer.step()
progress_bar(batch_num, len(self.training_loader), 'Loss: %.4f' % (train_loss / (batch_num + 1)))
print(" Average Loss: {:.4f}".format(train_loss / len(self.training_loader)))
def load_county(county_state_country,progress_bar=False):
"""
Given a country name (str)/list, return a dictionary with time history data in a dataframe and rest in other fields
"""
if isinstance(county_state_country,list):
pass
else:
county_state_country = [county_state_country]
for n, csc in enumerate(county_state_country):
if not csc.endswith(', US'):
county_state_country[n] = csc+', US'
dfd = load_Counties()
Combined_Keys = dfd['positive']['Combined_Key'].tolist()
# Find location and population data
aux = dfd['death'].T.loc[['Lat','Long_','Population','Combined_Key']]
aux = aux.rename({'Lat':'lat','Long_':'lon','Population':'pop'},axis='index')
aux.columns = aux.loc['Combined_Key'] # assign the combined key row as the columns
aux = aux.drop('Combined_Key') # now the combined key row is redundant since its embedded in defining the column names
# Drop unused columns
for key in dfd:
if key is 'positive':
dfd[key] = dfd[key].drop(columns=['UID','iso2','iso3','code3','FIPS','Province_State','Country_Region','Lat','Long_']).groupby('Combined_Key').sum()
else:
dfd[key] = dfd[key].drop(columns=['UID','iso2','iso3','code3','FIPS','Province_State','Country_Region','Lat','Long_','Population']).groupby('Combined_Key').sum()
out = {}
global mp_dic
mp_dic = {
'dfd': dfd,
'aux': aux,
'Combined_Keys':Combined_Keys
}
Nitems = len(county_state_country)
Ncpu = min([mp.cpu_count(),Nitems]) # use maximal number of local CPUs
chunksize = 1
pool = mp.Pool(processes=Ncpu)
for n,d in enumerate(pool.imap_unordered(mp_load_county,county_state_country,chunksize=chunksize)):
# for n,c in enumerate(county_state_country):
# d = mp_load_county(c)
if progress_bar:
pb.progress_bar(n,-(-Nitems/chunksize)-1)
out[d['key']]=d['data'];
pool.terminate()
return out
def test(self):
self.model.eval()
avg_psnr = 0
with torch.no_grad():
for batch_num, (data, target) in enumerate(self.testing_loader):
data, target = data.to(self.device), target.to(self.device)
prediction = self.model(data)
mse = self.criterion(prediction, target)
psnr = 10 * log10(1 / mse.item())
avg_psnr += psnr
progress_bar(batch_num, len(self.testing_loader), 'PSNR: %.4f' % (avg_psnr / (batch_num + 1)))
print(" Average PSNR: {:.4f} dB".format(avg_psnr / len(self.testing_loader)))
def load_pre_built(self):
querier = SQL_querier()
numLines = querier.count_entries(self.graph_table_name)
records = querier.select_table(self.graph_table_name)
print("\nReading from as_graph table")
progress = progress_bar(numLines)
for record in records:
current_as = AS(record.asn)
current_as.customers = record.customers
current_as.peers = record.peers
current_as.providers = record.providers
current_as.SCC_id = record.asn
current_as.rank = record.rank
if (current_as.rank not in self.ases_by_rank):
self.ases_by_rank[current_as.rank] = list()
self.ases_by_rank[current_as.rank].append(current_as.asn)
self.strongly_connected_components[
current_as.SCC_id] = record.members
for asn in record.members:
current_as.asn = asn
self.ases[asn] = current_as
progress.update()
progress.finish()
return
def combine_components(self):
"""Takes the SCCs of this graph and exchanges them for "super nodes".
These super nodes have the providers, peers and customers than all
nodes in the SCC would have. These providers, peers, and customers
point to the new super node.
"""
print("\nCombining Components")
large_components = list()
for component in self.strongly_connected_components:
comp = self.strongly_connected_components[component]
if(len(comp)>1):
large_components.append(comp)
progress = progress_bar(0,len(large_components))
for component in large_components:
progress.next_job(len(component))
#Create an AS using an "inner" AS to avoid collision
#TODO maybe change it to some known unique value, ideally integer
#grab ASN of first AS in component
new_asn = self.ases[component[0]].SCC_id
combined_AS = AS(new_asn)
combined_AS.SCC_id = new_asn
combined_cust_anns = list()
combined_peer_prov_anns = list()
#get providers, peers, customers from "inner" ASes
#only if they aren't also in "inner" ASes
for asn in component:
for provider in self.ases[asn].providers:
if(provider not in component):
combined_AS.append_no_dup(combined_AS.providers,provider)
# combined_AS.add_neighbor(provider,0)
#replace old customer reference from provider
prov_AS = self.ases[provider]
prov_AS.customers.remove(asn)
prov_AS.append_no_dup(prov_AS.customers,new_asn)
for peer in self.ases[asn].peers:
if(peer not in component):
combined_AS.append_no_dup(combined_AS.peers,peer)
# combined_AS.add_neighbor(peer,1)
peer_AS = self.ases[peer]
peer_AS.peers.remove(asn)
peer_AS.append_no_dup(peer_AS.peers,new_asn)
for customer in self.ases[asn].customers:
if(customer not in component):
combined_AS.append_no_dup(combined_AS.customers,customer)
# combined_AS.add_neighbor(customer,2)
cust_AS = self.ases[customer]
cust_AS.providers.remove(asn)
cust_AS.append_no_dup(cust_AS.providers,new_asn)
self.ases[asn] = combined_AS
progress.update()
self.ases[combined_AS.asn] = combined_AS
progress.finish()
return
def mosaic(tiles, pad=False, scatter=False, margin=0, scaled_margin=False,
background=(255, 255, 255)):
"""Return the mosaic image."""
# Infer dimensions so they don't have to be passed in the function call.
dimensions = map(max, zip(*[(1 + tile.x, 1 + tile.y) for tile in tiles]))
mosaic_size = map(lambda (x, y): x*y,
zip(*[tiles[0].ancestor_size, dimensions]))
mos = Image.new('RGB', mosaic_size, background)
pbar = progress_bar(len(tiles), "Scaling and placing tiles")
random.shuffle(tiles)
for tile in tiles:
if tile.blank:
pbar.next()
continue
if pad:
size = shrink_by_lightness(pad, tile.size, tile.match['dL'])
if margin == 0:
margin = min(tile.size[0] - size[0], tile.size[1] - size[1])
else:
size = tile.size
if scaled_margin:
pos = tile_position(tile, size, scatter, margin//(1 + tile.depth))
else:
pos = tile_position(tile, size, scatter, margin)
mos.paste(crop_to_fit(tile.match_img, size), pos)
pbar.next()
return mos
def analyze(tiles):
"""Determine dominant colors of target tiles, and save that information
in the Tile object."""
pbar = progress_bar(len(tiles), "Analyzing images")
for tile in tiles:
analyze_one(tile)
pbar.next()
def assemble(self, pad=False, scatter=False, margin=0, scaled_margin=False,
background=(255, 255, 255), new_width=None):
"""Create the mosaic image."""
# Infer dimensions so they don't have to be passed in the function call.
mosaic_size = map(max, zip(*[(tile.x+tile.w, tile.y+tile.h) for tile in self.tiles]))
if new_width is None:
scale = 1.0
else:
scale = new_width / mosaic_size[0]
mosaic_size = (new_width, int(scale * mosaic_size[1]))
mos = Image.new('RGB', mosaic_size, background)
pbar = progress_bar(len(self.tiles), "Scaling and placing tiles")
random.shuffle(self.tiles)
for tile in self.tiles:
if False: #pad: TODO!
size = shrink_by_lightness(pad, tile.size, tile.match['dL'])
if margin == 0:
margin = min(tile.size[0] - size[0], tile.size[1] - size[1])
else:
size = int(tile.w*scale), int(tile.h*scale)
if scaled_margin:
pos = tile.get_position(size, scatter, margin//(1 + tile.depth)) #TODO
else:
pos = int(tile.x*scale), int(tile.y*scale) #tile.get_position(size, scatter, margin)
fn = self.matches[tile][4]
mi = Image.open(fn)
mos.paste(crop_to_fit(mi, size), pos)
pbar.next()
self.mos = mos
def test(self):
for i in range(self.numModels):
self.models[i].eval()
avg_psnr = 0
with torch.no_grad():
for batch_num, (datas, targets) in enumerate(self.testing_loader):
for i in range(len(datas)):
data_unbaselined = datas[i if not self.
predictColors else 0]
data = self.prepArr(data_unbaselined).to(self.device)
target_unbaselined = targets[i]
target = self.prepArr(target_unbaselined).to(self.device)
prediction = self.models[i if self.allLayers else 0](data)
prediction_squeezed = self.prepArr(prediction)
prediction_unbaselined = self.unprepArr(
prediction_squeezed, data_unbaselined)
mse = self.criterion(prediction_squeezed, target)
psnr = 10 * log10(1 / mse.item())
avg_psnr += psnr
global print1
if print1 and batch_num == 0 and theEpoch == self.nEpochs:
print("data shape ", data.shape)
plt.imshow(
self.unprepArr(data, data_unbaselined)[
0, 0, :, :].numpy()), plt.title('data')
plt.xticks([]), plt.yticks([])
plt.show()
print("prediction shape ", prediction.shape)
plt.imshow(prediction[
0, 0].detach().numpy()), plt.title('prediction')
plt.xticks([]), plt.yticks([])
plt.show()
if i == 2:
print1 = False
progress_bar(batch_num, len(self.testing_loader),
'PSNR: %.4f' % (avg_psnr / (batch_num + 1)))
print(" Average PSNR: {:.4f} dB".format(avg_psnr /
len(self.testing_loader)))
def writeDataset(self,
data_dir: dir,
dataset_file: dir,
train_on_tiles=False):
"""
Creates a dataset out of the images and labels found in data_dir and writes it to the dataset_file.
Warning! It will overwrite the dataset_file!
Inputs:
data_dir : dir - folder which contains the pictures and labels
dataset_file : dir - location where the dataset is to be written
Outputs:
None
"""
pics = self.get_image_list(data_dir)
images, labels = (0, 0)
#random.seed(RANDOM_SEED)
training_set_size = int(TRAINING_SET_RATIO * len(pics))
training_pics = pics[:training_set_size]
#training_pics = pics
with tf.python_io.TFRecordWriter(dataset_file) as output_file:
for i in range(len(training_pics)):
picloc = training_pics[i]
progress_bar(i + 1, len(training_pics), "Writing dataset")
fullpicloc = join(data_dir, picloc)
pic = scipy.ndimage.imread(fullpicloc, mode="L")
label = get_bounding_box(fullpicloc)
pic, label = self.preprocess(pic, label, train_on_tiles)
pic_feature = _createBytesFeature(pic)
label_feature = _createBytesFeature(np.asarray(label))
feature = {
'train/image': pic_feature,
'train/label': label_feature
}
example = tf.train.Example(features=tf.train.Features(
feature=feature))
output_file.write(example.SerializeToString())
def train(self):
self.model.train()
train_loss = 0
for batch_num, (data) in enumerate(self.training_loader):
data, target = data[:, :2, ::3, ::3].to(
self.device), data[:, :2].to(self.device)
data = F.interpolate(data,
size=(target.shape[-2], target.shape[-1]))
target_d, output = self.model(data)
# loss1
loss_1 = 0
for d in range(self.num_recursions):
loss_1 += (self.criterion(target_d[d], target) /
self.num_recursions)
# loss2
loss_2 = self.criterion(output, target)
# regularization
reg_term = 0
for theta in self.model.parameters():
reg_term += torch.mean(torch.sum(theta**2))
# total loss
loss = self.loss_alpha * loss_1 + (
1 - self.loss_alpha) * loss_2 + self.loss_beta * reg_term
loss.backward()
train_loss += loss.item()
self.optimizer.step()
progress_bar(
batch_num, len(self.training_loader), 'Loss: %.4f, LR: %.4f' %
(train_loss /
(batch_num + 1), self.optimizer.param_groups[0]['lr']))
self.writer.add_scalar(
'Train/TrainIterLoss_%s' % self.config.comment, loss.item(),
self.epoch * len(self.training_loader) + batch_num)
self.writer.add_scalar(
'Train/TrainAvgLoss_%s' % self.config.comment,
train_loss / (batch_num + 1),
self.epoch * len(self.training_loader) + batch_num)
print(" Average Loss: {:.4f}".format(train_loss /
len(self.training_loader)))
def add_directory(self, image_dir, skip_errors=True):
walker = DirectoryWalker(image_dir)
file_count = len(list(walker)) # stupid but needed but progress bar
pbar = progress_bar(file_count, "Analyzing images and building db")
for filename in walker:
self.add_image(filename, skip_errors)
pbar.next()
logger.info('Collection %s built with %d images'%(self.db_name, len(self)))
def backpropagation(self):
grad_dim1 = np.zeros((785, 50))
grad_dim2 = np.zeros((51, 50))
grad_dim3 = np.zeros((51, 10))
for i in range(self.m):
ex = self.images[i]
# First layer forward prop
z2 = np.matmul(self.THETA1.T, ex)
a2 = self.sigmoid(z2)
a2 = np.insert(a2, 0, 1, axis=0)
# Second layer forward prop
z3 = np.matmul(self.THETA2.T, a2)
a3 = self.sigmoid(z3)
a3 = np.insert(a3, 0, 1, axis=0)
# Output layer forward prop
z4 = np.matmul(self.THETA3.T, a3)
a4 = self.sigmoid(z4)
# Backprop first layer
delta4 = a4 - self.labels[i]
# Backprop second layer
delta3 = np.multiply(np.matmul(self.THETA3, delta4),
np.multiply(a3, 1 - a3))
# Backprop third layer
delta2 = np.multiply(np.matmul(self.THETA2, delta3[1:]),
np.multiply(a2, 1 - a2))
grad1 = np.matmul(ex.reshape(785, 1), delta2[1:].reshape(1, 50))
grad2 = np.matmul(a2.reshape(51, 1), delta3[1:].reshape(1, 50))
grad3 = np.matmul(a3.reshape(51, 1), delta4.reshape(1, 10))
grad_dim1 += grad1
grad_dim2 += grad2
grad_dim3 += grad3
progress_bar(100, int(i / (self.m / 100)), "Backpropagation")
print("")
grad_dim1 = (1 / self.m) * grad_dim1
grad_dim2 = (1 / self.m) * grad_dim2
grad_dim3 = (1 / self.m) * grad_dim3
grad_dim1[1:] = grad_dim1[1:] + (self.regul_lambda /
self.m) * self.THETA1[1:]
grad_dim2[1:] = grad_dim2[1:] + (self.regul_lambda /
self.m) * self.THETA2[1:]
grad_dim3[1:] = grad_dim3[1:] + (self.regul_lambda /
self.m) * self.THETA3[1:]
return np.array([grad_dim1, grad_dim2, grad_dim3], dtype=object)
def load_country(country, progress_bar=False, chunksize=1):
"""
Given a country name/s (str/list), return a dictionary with the data for that country
"""
if isinstance(country, list):
pass
else:
country = [country]
out = {}
dfd = load_Global()
pop = get_country_population(country)
lonlat = get_country_location(country)
Combined_Keys = dfd['positive']['Country/Region'].tolist()
for key in dfd:
dfd[key] = dfd[key].drop(columns=['Province/State', 'Lat', 'Long'
]).groupby('Country/Region').sum()
global mp_dic
mp_dic = {
'dfd': dfd,
'pop': pop,
'lonlat': lonlat,
'Combined_Keys': Combined_Keys,
}
Nitems = len(country)
Ncpu = min([mp.cpu_count(), Nitems]) # use maximal number of local CPUs
chunksize = 1
pool = mp.Pool(processes=Ncpu)
#for n,d in enumerate(pool.imap_unordered(mp_load_country,country,chunksize=chunksize)):
for n, do in enumerate(country):
d = mp_load_country(do)
if progress_bar:
pb.progress_bar(n, -(-Nitems / chunksize) - 1)
out[d['key']] = d['data']
pool.terminate()
return out
def test(self):
"""
data: [torch.cuda.FloatTensor], 10 batches: [10, 10, 10, 10, 10, 10, 10, 10, 10, 10]
"""
self.model.eval()
avg_psnr = 0
with torch.no_grad():
for batch_num, (data, target) in enumerate(self.testing_loader):
data = self.img_preprocess(data) # resize input image size
data, target = data.to(self.device), target.to(self.device)
_, prediction = self.model(data)
mse = self.criterion(prediction, target)
psnr = 10 * log10(1 / mse.item())
avg_psnr += psnr
progress_bar(batch_num, len(self.testing_loader),
'PSNR: %.4f' % (avg_psnr / (batch_num + 1)))
print(" Average PSNR: {:.4f} dB".format(avg_psnr /
len(self.testing_loader)))
def match(self, tolerance=1, usage_penalty=1, usage_impunity=2):
"""Assign each tile a new image, and open that image in the Tile object."""
if len(self.pool)==0:
logger.error('No images in pool to match!')
exit(-1)
self.pool.reset_usage()
pbar = progress_bar(len(self.tiles), "Choosing and loading matching images")
for tile in self.tiles:
self.match_one(tile, tolerance, usage_penalty, usage_impunity)
pbar.next()
def createLabelFiles(path: dir):
samples_with_label = collectSamplesWithLabel(path, 'license plate')
number_of_samples = len(samples_with_label)
i = 0
for file_loc in samples_with_label:
progress_bar(i, number_of_samples, 'Converting labels')
example = Example(file_loc)
bb_label_loc = example.basename + ".txt"
if not os.path.isfile(bb_label_loc):
license_plate_mask = [50, 159]
license_plates = findBoundingBox(example, 'license plate',
license_plate_mask)
license_plates_as_list = map(lambda x: x[1].getAsList(),
license_plates.items())
with open(bb_label_loc, 'w') as f:
for license_plate_coords in license_plates_as_list:
line = str(license_plate_coords)[
1:-1] # get rid of leading '[' and trailing ']'
f.write(line + '\n')
i += 1
progress_bar(number_of_samples, number_of_samples, 'Converting label')
def extract_embeddings_from_directory(unknown_photo_directory,
alpha_named_files=True):
'''
Takes the name of a directory
Creates a list of embeddings of all photos in the directory
Creates a list of the file names in the directory
'''
count = 1
embeddings = []
img_files = []
file_list = os.listdir(unknown_photo_directory)
print('Extracting embeddings from ' + unknown_photo_directory)
for img_file in file_list:
full_filename = unknown_photo_directory + '/' + img_file
progress_bar(count, len(file_list))
#print("Loading file ",count," of ",len(file_list),'\t',img_file)
count += 1
# Load image file
try:
img = face_recognition.load_image_file(full_filename)
except (ValueError, OSError) as e:
print("Failed to open file ", img_file, file=sys.stderr)
continue
# Extract embedding
try:
encoding = face_recognition.face_encodings(img)[0]
except IndexError:
print("Unable to locate face in photo. Check image file",
img_file,
file=sys.stderr)
continue
# Add to list
embeddings.append(encoding)
if alpha_named_files:
alpha = img_file[:6]
img_files.append(alpha)
else:
img_files.append(img_file)
print('\r' + ' ' * 80 + '\r' + '\033[2A')
return embeddings, img_files
def afterburner_train(language,
phase,
release,
model_fn,
new_model_fn,
epochs,
batch_size=32):
C, model, SRC, TRG, device, train_iterator, _ = afterburner_pretrained_model(
language, phase, release, model_fn, batch_size)
LEARNING_RATE = 0.0005
optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)
TRG_PAD_IDX = TRG.vocab.stoi[TRG.pad_token]
criterion = torch.nn.CrossEntropyLoss(ignore_index=TRG_PAD_IDX)
model.train()
print(f'{len(train_iterator)} batches / epoch')
epoch_loss = 9999999999999999
fig, ax = plt.subplots(1, 1)
ax.set_xlabel('epochs')
ax.set_ylabel('Loss')
losses = []
for j in tqdm(range(epochs)):
epoch_loss = 0
for i, batch in enumerate(tqdm(train_iterator)):
src = batch.src.to(device)
trg = batch.trg.to(device)
optimizer.zero_grad()
output, _ = model(src, trg[:, :-1])
output_dim = output.shape[-1]
output = output.contiguous().view(-1, output_dim)
trg = trg[:, 1:].contiguous().view(-1)
loss = criterion(output, trg)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1)
optimizer.step()
epoch_loss += loss.item()
losses.append(epoch_loss)
progress_bar(fig, ax, losses)
torch.save(model.state_dict(), f'{new_model_fn}.{j}')
torch.save(model.state_dict(), new_model_fn)
def train(self):
# models setup
self.netG.train()
self.netD.train()
g_train_loss = 0
d_train_loss = 0
for batch_num, (data, target) in enumerate(self.training_loader):
# setup noise
real_label = torch.ones(data.size(0), data.size(1)).to(self.device)
fake_label = torch.zeros(data.size(0), data.size(1)).to(self.device)
data, target = data.to(self.device), target.to(self.device)
# Train Discriminator
self.optimizerD.zero_grad()
d_real = self.netD(target)
d_real_loss = self.criterionD(d_real, real_label)
d_fake = self.netD(self.netG(data))
d_fake_loss = self.criterionD(d_fake, fake_label)
d_total = d_real_loss + d_fake_loss
d_train_loss += d_total.item()
d_total.backward()
self.optimizerD.step()
# Train generator
self.optimizerG.zero_grad()
g_real = self.netG(data)
g_fake = self.netD(g_real)
gan_loss = self.criterionD(g_fake, real_label)
mse_loss = self.criterionG(g_real, target)
g_total = mse_loss + 1e-3 * gan_loss
g_train_loss += g_total.item()
g_total.backward()
self.optimizerG.step()
progress_bar(batch_num, len(self.training_loader), 'G_Loss: %.4f | D_Loss: %.4f' % (g_train_loss / (batch_num + 1), d_train_loss / (batch_num + 1)))
print(" Average G_Loss: {:.4f}".format(g_train_loss / len(self.training_loader)))
def test(path, testing_loader, pae):
model = torch.load(path)
model.eval()
mse = torch.nn.MSELoss()
avg_ssim = 0
avg_psnr = 0
max_bound = 0
with torch.no_grad():
for batch_num, (data, target) in enumerate(testing_loader):
data, target = data.cuda(), target.cuda()
prediction = model(data)
prediction = (prediction.cpu() * 128 * pae * 0.7 + 128)
target = (target.cpu() * 128 * pae * 0.7 + 128)
mse_value = mse(prediction, target)
psnr = 10 * log10(65025 / mse_value.item())
avg_psnr += psnr
progress_bar(batch_num, len(testing_loader),
'PSNR: %.4f' % (avg_psnr / (batch_num + 1)))
abs_value = np.abs(prediction - target)
abs_value = abs_value.numpy()
if abs_value.max() > max_bound:
max_bound = abs_value.max()
prediction = prediction.numpy()
prediction = np.reshape(prediction,
(prediction.shape[2], prediction.shape[3]))
target = np.reshape(target, (target.shape[2], target.shape[3]))
ssim = compute_ssim(prediction, target)
avg_ssim += ssim
# break
print(" Average PSNR: {:.4f} dB".format(avg_psnr / len(testing_loader)))
print(" Average SSIM: {:.4f} dB".format(avg_ssim / len(testing_loader)))
print("bound: " + str(max_bound))
return avg_psnr / len(testing_loader), avg_ssim / len(
testing_loader), max_bound
def partition_tiles(self, dimensions=10, depth=0, hdr=80, analyze=True):
"Partition the target image into a list of Tile objects."
self.p = Partition(self.img, self.mask)
self.p.simple_partition(dimensions)
self.p.recursive_split(depth, hdr)
self.tiles = self.p.get_tiles()
if not analyze:
return
pbar = progress_bar(len(self.tiles), "Analyzing images")
for tile in self.tiles:
self.analyze_one(tile)
pbar.next()
def train(self):
"""
data: [torch.cuda.FloatTensor], 4 batches: [64, 64, 64, 8]
"""
self.model.train()
train_loss = 0
for batch_num, (data, target) in enumerate(self.training_loader):
data = self.img_preprocess(data) # resize input image size
data, target = data.to(self.device), target.to(self.device)
target_d, output = self.model(data)
# loss1
loss_1 = 0
for d in range(self.num_recursions):
loss_1 += (self.criterion(target_d[d], target) /
self.num_recursions)
# loss2
loss_2 = self.criterion(output, target)
# regularization
reg_term = 0
for theta in self.model.parameters():
reg_term += torch.mean(torch.sum(theta**2))
# total loss
loss = self.loss_alpha * loss_1 + (
1 - self.loss_alpha) * loss_2 + self.loss_beta * reg_term
loss.backward()
train_loss += loss.item()
self.optimizer.step()
progress_bar(batch_num, len(self.training_loader),
'Loss: %.4f' % (train_loss / (batch_num + 1)))
print(" Average Loss: {:.4f}".format(train_loss /
len(self.training_loader)))
def matchmaker(tiles, db_name, tolerance=1, usage_penalty=1, usage_impunity=2):
"""Assign each tile a new image, and open that image in the Tile object."""
db = connect(db_name)
try:
reset_usage(db)
pbar = progress_bar(len(tiles), "Choosing and loading matching images")
for tile in tiles:
if tile.blank:
pbar.next()
continue
tile.match = choose_match(tile.lab, db, tolerance,
usage_penalty if tile.depth < usage_impunity else 0)
pbar.next()
finally:
db.close()
def train(self):
self.model.train()
train_loss = 0
for batch_num, (data, target) in enumerate(self.training_loader):
data, target = data.to(self.device), target.to(self.device)
self.optimizer.zero_grad()
loss = self.criterion(self.model(data), target)
train_loss += loss.item()
loss.backward()
self.optimizer.step()
total_time = progress_bar(
batch_num, len(self.training_loader),
'Loss: %.4f' % (train_loss / (batch_num + 1)))
avg_loss = train_loss / len(self.training_loader)
return [avg_loss, total_time]
def save_anns_to_db(self):
print("Saving Propagation results to DB")
progress = progress_bar(len(self.graph.ases))
start_time = time.time()
for asn in self.graph.ases:
AS = self.graph.ases[asn]
# if asn == AS.SCC_id:
sql_anns_arg = AS.anns_to_sql()
self.querier.insert_results(asn, sql_anns_arg)
progress.update()
progress.finish()
end_time = time.time()
print("Time To Save Announcements: " + str(end_time - start_time) +
"s")
return
def download_video(url, folder):
try:
yt = YouTube(url)
except Exception as e:
print ("Error:", e, "- skipping video with url:",url)
return
#video should be downloaded in 720p
try:
vid = yt.get("mp4","720p")
#else tries to get the highest resolution available
except Exception:
vid = yt.filter("mp4")[-1]
#download video
try:
bar = progress_bar()#init progress_bar
vid.download(folder,on_progress = bar.print_progress, on_finish = bar.print_end)
print("Successfully downloaded", yt.filename, " !")
except OSError:
print(yt.filename, "already exists in the directory. Skipping video...")
return
#converts video to audio
try:
aud = "ffmpeg -i " + folder + "/" + "\"" + str(yt.filename) + "\"" + ".mp4 " + folder + "/" + "\"" + str(yt.filename) + "\"" + ".mp3"
print (aud)
os.system(aud)
if os.path.exists(folder +"\\" + yt.filename + ".mp4"):
os.remove(folder +"\\" + yt.filename + ".mp4")
print("Succesfully converted",yt.filename, "to mp3!")
except OSError:
print("There are some problems with the file name(s), skipping video..")
return
def pool(image_dir, db_name):
"""Analyze all the images in image_dir, and store the results in
a sqlite database at db_name."""
db = connect(db_name)
try:
create_tables(db)
walker = DirectoryWalker(image_dir)
file_count = len(list(walker)) # stupid but needed but progress bar
pbar = progress_bar(file_count, "Analyzing images and building db")
walker = DirectoryWalker(image_dir)
for filename in walker:
if in_db(filename, db):
logger.warning("Image %s is already in the table. Skipping it."%filename)
pbar.next()
continue
try:
img = Image.open(filename)
except IOError:
logger.warning("Cannot open %s as an image. Skipping it.",
filename)
pbar.next()
continue
if img.mode != 'RGB':
logger.warning("RGB images only. Skipping %s.", filename)
pbar.next()
continue
w, h = img.size
try:
regions = split_quadrants(img)
rgb = map(dominant_color, regions)
lab = map(cs.rgb2lab, rgb)
except:
logger.warning("Unknown problem analyzing %s. Skipping it.",
filename)
continue
insert(filename, w, h, rgb, lab, db)
pbar.next()
db.commit()
logger.info('Collection %s built with %d images'%(db_name, get_size(db)))
finally:
db.close()
def fit(
fn,
cor,
tmin,
tmax,
filestub=None,
bootstraps=NBOOTSTRAPS,
return_quality=False,
return_chi=False,
writecor=True,
tstride=1,
options=None,
):
if tmax - tmin < len(fn.parameter_names):
raise InvalidFit("Can not fit to less points than parameters")
if options.random:
logging.info("Setting random seed to %s", options.random)
np.random.seed(options.random)
eval_file = None
results = logging.getLogger("results")
if filestub and not results.handlers:
filename = filestub + ".stats"
filehandler = logging.FileHandler(filename)
filehandler.level = OUTPUT
results.addHandler(filehandler)
logging.info("Writing output to file {}".format(filename))
if filestub:
eval_filename = filestub + ".evals"
eval_file = open(eval_filename, "w")
eval_file.write("# evals seqeuntial\n")
tstride_filename = filestub + ".tstride"
tstride_file = open(tstride_filename, "w")
tstride_file.write("{}\n".format(tstride))
results.info(
"Fitting data to {} from t={} to t={} using {} bootstrap samples".format(fn.description, tmin, tmax, bootstraps)
)
if fn.subtract:
logging.debug("before suctracted correlator is:")
logging.debug(cor.average_sub_vev())
ccor = deepcopy(cor)
cor = ccor
fn.subtract = tmin - 1
cor.subtract(tmin - 1)
logging.debug("subtracted correlator is:")
logging.debug(cor.average_sub_vev())
# tmax = tmax+1 # I use ranges, so this needs to be offset by one
fitrange = range(tmin, tmax + 1, tstride)
fun = lambda v, mx, my: (fn.formula(v, mx) - my)
initial_guess = fn.starting_guess(cor, options.period, tmax, tmin)
logging.info("Starting with initial_guess: {}".format(repr(initial_guess)))
try:
ranges = []
for i in fn.indexes:
ranges.extend(range(i[0], i[1] + 1, tstride))
fitrange = ranges
except AttributeError:
logging.info("no indexes on fit function, using normal fitrange")
x = np.array(fitrange)
dof = len(x) - len(fn.parameter_names)
orig_ave_cor = cor.average_sub_vev()
y = [orig_ave_cor[t] for t in fitrange]
logging.info("x {}".format(x))
logging.info("y {}".format(y))
original_ensamble_params, success = leastsq(fun, initial_guess, args=(x, y), maxfev=10000)
original_cov = covariance_matrix(cor, fitrange)
logging.info("inverting original cov")
igonre_error_original_cov = options.debug_ignoreinverterror or options.debug_uncorrelated
inv_original_cov = bestInverse(original_cov, print_error=True, ignore_error=igonre_error_original_cov)
logging.info("original ensemble full cov")
matrix_stats(original_cov, None, cond=True)
if options.debug_singlecov:
logging.info("original ensemble single cov")
original_cov = covariance_matrix(cor, fitrange)
inv_original_cov = bestInverse(original_cov, print_error=True, ignore_error=options.debug_ignoreinverterror)
matrix_stats(original_cov, eval_file, cond=True)
if options.debug_singleuncorrelated:
logging.debug("Using uncorrlated")
jke = cor.jackknifed_errors()
original_cov = np.diag([jke[t] ** 2 for t in fitrange])
inv_original_cov = bestInverse(original_cov, print_error=True, ignore_error=options.debug_ignoreinverterror)
matrix_stats(original_cov, eval_file, cond=True)
if options.debug_outputcov:
def invert_error(M, i):
return np.max(np.abs((np.dot(M, i) - np.identity(len(i)))))
def invert_error_one(M, i):
return np.sum(np.abs((np.dot(M, i) - np.identity(len(i)))))
def invert_error_two(M, i):
return np.sum(((np.dot(M, i) - np.identity(len(i)))) ** 2)
logging.info("inv=\n{}".format(inv_original_cov))
logging.info("invert errors:")
logging.info("inv error max norm {}".format(invert_error(original_cov, inv_original_cov)))
logging.info("inv error one norm {}".format(invert_error_one(original_cov, inv_original_cov)))
logging.info("inv error two norm {}".format(invert_error_two(original_cov, inv_original_cov)))
exit(0)
if options.debugguess:
# return original_ensamble_params, [0.01, 0.01, 0.01, 0.01] # For testing initila guess in plot
if options.plot:
plot_fit(fn, cor, tmin, tmax, options, initial_guess)
return initial_guess, [0.01, 0.01, 0.01, 0.01] # For testing initila guess in plot
if not success:
raise InvalidFit("original exnamble leastsq failed")
if options.first_pass:
initial_guess = original_ensamble_params
logging.info("initial_guess after first pass: {}".format(repr(initial_guess)))
def cov_fit(correlator, guess):
ave_cor = correlator.average_sub_vev()
y = [ave_cor[t] for t in fitrange]
if options.debug_uncorrelated:
logging.debug("Using uncorrlated")
fcov = covariance_matrix(correlator, fitrange)
cov = np.diag(np.diag(fcov))
# jke = correlator.jackknifed_errors()
# cov = np.diag([jke[t]**2 for t in fitrange])
elif options.debug_singlecov:
cov = original_cov
else:
cov = covariance_matrix(correlator, fitrange)
matrix_stats(cov, eval_file)
if options.debug_singlecov or options.debug_singleuncorrelated:
inv_cov = inv_original_cov
else:
inv_cov = bestInverse(cov, ignore_error=options.debug_ignoreinverterror)
matrix_stats(cov, eval_file)
if options.debug_identcov:
results.log(30, "using identcov debug option")
inv_cov = np.identity(len(cov))
aoc = np.array([ave_cor[t] for t in fitrange])
# logging.debug("guess {}".format(str(guess)))
def cov_fun(g):
""" Function to be minizied. computed using matrix mult"""
vect = aoc - fn.formula(g, x)
return vect.dot(inv_cov).dot(vect)
if options.first_pass:
uncorrelated_fit_values, success = leastsq(fun, guess, args=(x, y), maxfev=100000)
if not success:
raise InvalidFit("leastsq failed")
logging.debug("firstpass guess {}".format(str(uncorrelated_fit_values)))
if guess[0] < 0.0:
logging.warn("first pass found mass to be negative {}, lets not use it".format(guess[0]))
else:
guess = uncorrelated_fit_values
if len(guess) > 2 and guess[2] < 0.0:
logging.warn("first pass found mass2 to be negative {}, lets flip it".format(guess[2]))
logging.info("first pass results are {}".format(repr(guess)))
guess[2] = -guess[2]
def clamp(n, minn, maxn):
return max(min(maxn, n), minn)
bounded_guess = [clamp(g, b[0], b[1]) for g, b in zip(guess, fn.bounds)]
# logging.debug("guess {}, bounded guess {}".format(repr(guess), repr(bounded_guess)))
m = fn.custom_minuit(aoc, inv_cov, x, guess=bounded_guess)
# m.set_strategy(2)
migradinfo = m.migrad()
minuit_results = [m.values[name] for name in fn.parameter_names]
chisqr = migradinfo[0]["fval"]
if m.get_fmin().is_valid:
return minuit_results, chisqr
else:
logging.error("minuit failed!!")
logging.error("was at {}".format(minuit_results))
raise InvalidFit("minuit failed")
# end cov_fit
original_ensamble_correlatedfit, original_ensamble_chisqr = cov_fit(cor, initial_guess)
isvalidfit = fn.valid(original_ensamble_correlatedfit)
if not isvalidfit:
raise InvalidFit("Full ensamble failed")
boot_params = []
boot_chisqr = []
failcount = 0
attempted = 0
for strap in bootstrap_ensamble(cor, N=bootstraps, filelog=filestub, jackknife=options.jackknife):
if options.jackknife:
bootstraps = len(cor.configs)
pb = progress_bar.progress_bar(bootstraps)
attempted += 1
pb.update(attempted)
if options.reguess:
newguess = fn.starting_guess(strap, options.period, tmax, tmin)
else:
newguess = initial_guess
try:
fitted_params, fitted_chisqr = cov_fit(strap, newguess)
except (InversionError, InvalidFit) as e:
if options.debug_ignoreinverterror:
fitted_params = None
else:
raise e
if fitted_params is not None:
boot_params.append(fitted_params)
boot_chisqr.append(fitted_chisqr)
logging.debug("bootstrap converged")
if options.write_each_boot:
write_fitted_cor(
fn, strap, tmin, tmax, options, fitted_params, postfix=".bootstrap{}".format(attempted)
)
if options.debug:
plot_fit(fn, strap, tmin, tmax, options, fitted_params, postfix=".bootstrap{}".format(attempted))
else:
logging.error("bootstrap failed to converge!")
# raise InvalidFit("one bootstrap failed")
# raw_input("test")
failcount += 1
logging.debug("fails:{} attempts:{}, ratio:{}".format(failcount, attempted, failcount / float(attempted)))
# if failcount/float(attempted) > 0.15 and attempted > 40:
# raise InvalidFit("more than 20% of boostraps failed to converge")
del strap
pb.done()
if failcount > 0:
logging.warn("{} bootstraps did not converge!".format(bootstraps - len(boot_params)))
if len(boot_params) < bootstraps * 0.9:
logging.error("More that 10% of the straps failed")
raise InvalidFit("more than 10% of boostraps failed to converge")
if options.histo:
plot_histograms(fn.parameter_names, boot_params, options)
results.info("")
results.info("Uncorelated total fit: %s", {n: p for n, p in zip(fn.parameter_names, original_ensamble_params)})
results.info(
"Correlated total fit: %s", {n: p for n, p in zip(fn.parameter_names, original_ensamble_correlatedfit)}
)
factor = 1
if options.jackknife:
factor = np.sqrt(len(cor.configs) - 1)
boot_averages = np.mean(boot_params, 0)
boot_std = factor * np.std(boot_params, 0)
boota = np.array(boot_params)
upper_quartiles = [stats.scoreatpercentile(boota[:, i], 75) for i in range(len(boot_averages))]
medians = [stats.scoreatpercentile(boota[:, i], 50) for i in range(len(boot_averages))]
lower_quartiles = [stats.scoreatpercentile(boota[:, i], 25) for i in range(len(boot_averages))]
inter_range = [
stats.scoreatpercentile(boota[:, i], 75) - stats.scoreatpercentile(boota[:, i], 25)
for i in range(len(boot_averages))
]
for name, boot, original, err in zip(fn.parameter_names, boot_averages, original_ensamble_correlatedfit, boot_std):
bias = abs(boot - original)
percent_bias = abs(boot - original) / original
results.info("Bootstrap Bias in {:<10}: {:.3%}".format(name, percent_bias))
if bias > err * 2 and (err > 0.0):
results.error("Bootstrap Bias in {:<10}: {:.3%}".format(name, percent_bias))
results.error(
"Bootstrap average does not agree with ensamble average!"
"\nNot enough statistics for this for to be valid!!!\n"
)
if not options.unsafe:
results.critical("Exiting! Run with --unsafe to fit anyway")
raise InvalidFit("Bootstrap average does not agree with ensamble average")
for name, ave, med, std, iqr in zip(fn.parameter_names, boot_averages, medians, boot_std, inter_range):
skew = abs(ave - med) / ave
dist_skew = abs(std - iqr) / iqr
if skew > 1.0:
results.error("{}: diff of bstrap average and bstrap med is {:.3%}".format(name, skew))
results.error("Bootstrap distrubtion is skewed!!")
if not options.unsafe:
results.critical("Exiting! Run with --unsafe to fit anyway")
raise InvalidFit("Bootstrap average does not agree with ensamble average")
else:
results.info("{}: diff of bstrap average and bstrap med is {:.3%}".format(name, skew))
results.info("")
try:
results.log(OUTPUT, "Fit ranges ({}), ({})".format(*fn.ranges))
except Exception as e:
pass
results.log(OUTPUT, "Full ensemble fitted parameters t={}---------------------".format(fitrange))
results.log(OUTPUT, "Name : Average, STD, (1st Quart, Median, 3rd Quart, IQR)")
for name, ave, std, low, med, up, iqr in zip(
fn.parameter_names, boot_averages, boot_std, upper_quartiles, medians, lower_quartiles, inter_range
):
results.log(
OUTPUT,
u"{:<10}: {:<15.10f} \u00b1 {:<10g} ({:<9.6f}, {:<9.6f}, {:<9.6f}, {:<9.6f})".format(
name, ave, std, low, med, up, iqr
),
)
results.log(OUTPUT, "--------------------------------------------------------")
v = original_ensamble_correlatedfit
chi_average = np.mean(boot_chisqr, 0)
chi_median = np.median(boot_chisqr, 0)
chi_min = min(boot_chisqr)
chi_std = np.std(boot_chisqr, 0)
chi_range = stats.scoreatpercentile(boot_chisqr, 84) - stats.scoreatpercentile(boot_chisqr, 16)
dof = len(x) - len(fn.parameter_names)
chi_sqr = original_ensamble_chisqr
results.log(
OUTPUT,
u"\u03c7\u00b2 ={}, \u03c7\u00b2 / dof = {}, Qual {}\n".format(
chi_sqr, chi_sqr / dof, quality_of_fit(dof, chi_sqr)
),
)
logging.debug(
"chiave:{}, chi_med:{}, chi_min:{}, chi_std:{}, chi_range{}".format(
chi_average, chi_median, chi_min, chi_std, chi_range
)
)
if bootstraps > 1 and filestub:
bootfilename = filestub + ".boot"
if options.jackknife:
bootfilename = filestub + ".jack"
results.info("writing each bootstrap parameter to {}".format(bootfilename))
with open(bootfilename, "w") as bootfile:
str_ensamble_params = ", ".join([str(p) for p in original_ensamble_correlatedfit])
bootfile.write(
"#bootstrap, {}, \t ensamble mean: {}\n".format(", ".join(fn.parameter_names), str_ensamble_params)
)
for i, params in enumerate(boot_params):
strparams = ", ".join([str(p) for p in params])
bootfile.write("{}, {}\n".format(i, strparams))
if options.output_stub and writecor:
write_fitted_cor(fn, cor, tmin, tmax, options, boot_averages, errors=boot_std)
if options.plot and bootstraps > 1:
plot_fit(fn, cor, tmin, tmax, options, boot_averages, errors=boot_std)
if return_chi:
return boot_averages, boot_std, chi_sqr / dof
if return_quality:
return boot_averages, boot_std, quality_of_fit(dof, chi_sqr)
else:
return boot_averages, boot_std
