def ShowResults(sampleFile="sample.txt",sample_infoFile="sample_info.txt", levelsFile="levels.txt",posteriorFile="posterior_sample.txt",DataFile='Data/dummy_data.dat',weightFile='weights.txt',plotfilebase=None,clearall=True,showfullsample=True):
import postprocess
import display
if clearall:
for p in range(0,6):
figure(p+1)
clf()
postprocess.postprocess(sampleFile=sampleFile,sample_infoFile=sample_infoFile, levelsFile=levelsFile,posteriorFile=posteriorFile,weightFile=weightFile)
display.Display(posteriorFile=posteriorFile,DataFile=DataFile,plotfilebase=plotfilebase,showfullsample=showfullsample)
def main():
questions = open("qs1.txt", "r")
qs = questions.readlines()
for line in qs:
print(line)
firstWord = line.partition(' ')[0]
wQuestions = ["who", "what", "when", "where", "why", "is"]
query = ""
found = False
if line.lower() == "what is the meaning of life?":
print "42"
elif firstWord.lower() in wQuestions:
#print "in wQuestions"
query = rewriteQuestion(line)
plainQuery = query[1:]
print(query)
if query != "none":
query = "\"" + query + "\""
allTweets = returnTweets(query)
if firstWord.lower() == "who" or firstWord.lower() == "where" or firstWord.lower() == "what":
#print "postprocessWhoWhere"
#print "query is" + plainQuery
found = postprocessWhoWhereWhat(allTweets, plainQuery)
else:
found = postprocess(firstWord.lower(), allTweets, query)
if not found:
#print "hi"
findAnswer(line)
else:
findAnswer(line)
def main():
parser = argparse.ArgumentParser(description='Postprocess word sense induction file for all files in a directory.')
parser.add_argument('ddt_dir', help='Path to a directory with csv files with DDTs: "word<TAB>sense-id<TAB>keyword<TAB>cluster" w/o header by default. Here <cluster> is "word:sim<SPACE><SPACE>word:sim<SPACE><SPACE>..."')
parser.add_argument('-min_size', help='Minimum cluster size. Default -- 5.', default="5")
args = parser.parse_args()
print "Input DDT directory (pattern):", args.ddt_dir
print "Min size:", args.min_size
#postprocess(args.ddt, output_fpath, filtered_fpath, int(args.min_size))
for cluster_fpath in glob.glob(args.ddt_dir):
if splitext(cluster_fpath)[-1] == ".csv":
print "\n>>>", cluster_fpath
postprocess(
cluster_fpath,
cluster_fpath+"-minsize" + args.min_size + ".csv",
cluster_fpath+"-minsize" + args.min_size + "-filtered.csv",
args.min_size)
def main():
# Define a timer
t = Timer()
# Start the timer
t.start()
# Start by getting all the data from twitter
# Note that this should be commented out if there is no API keys
# Note max is 100 tweets per senator due to twitter limits
createCSV.getData(100)
# Preprocess the data and create new CSV files
preprocess.preprocess()
# Analyze the data and generate new CSV files
analyze.analyze()
# Postprocess the data and create new CSV files
postprocess.postprocess()
# Generate the visualizations of the data
visualize.visualize()
# Stop the timer and display runtime
elapsed_time = t.stop()
def collect_images(file_name, input_dir, output_dir, num_views):
"""Collects images from sim."""
if not os.path.exists(output_dir):
os.makedirs(output_dir)
f = h5py.File(os.path.join(input_dir, file_name), 'r')
pregrasp, postgrasp = postprocess(f['pregrasp'], f['postgrasp'])
if pregrasp is None or postgrasp is None:
print('No data in %s' % file_name)
return
num_samples = len(pregrasp[pregrasp.keys()[0]])
out_file = h5py.File(os.path.join(output_dir, file_name), 'w')
pregrasp_group = out_file.create_group('pregrasp')
postgrasp_group = out_file.create_group('postgrasp')
dt = h5py.special_dtype(vlen=unicode)
for key in pregrasp.keys():
num_var = pregrasp[key].shape[1]
pregrasp_group.create_dataset(key, (num_samples * num_views, num_var))
postgrasp_group.create_dataset(key, (num_samples * num_views, num_var))
# Collecting this from the sim
pregrasp_group.create_dataset('grasp_wrt_cam', (num_samples * num_views, 18))
pregrasp_group.create_dataset('image_name', (num_samples * num_views, ), dtype=dt)
pregrasp_group.create_dataset('frame_work2cam', (num_samples * num_views, 12))
# Start collection
object_name = file_name.split('.')[0]
for i in range(num_samples):
print('Querying for image set %d / %d ' % (i, num_samples))
low = i * num_views
high = (i + 1) * num_views
for key in pregrasp.keys():
pregrasp_group[key][low:high] = np.repeat(pregrasp[key][i:i + 1],
num_views, axis=0)
postgrasp_group[key][low:high] = np.repeat(postgrasp[key][i:i + 1],
num_views, axis=0)
# Query the simulator for some images; note that we're only collecting
# this info for the pregrasp here.
grasp_wrt_cam, frame_work2cam, image_names = \
query_minibatch(pregrasp, i, num_views, object_name)
pregrasp_group['grasp_wrt_cam'][low:high] = grasp_wrt_cam
pregrasp_group['image_name'][low:high] = image_names
pregrasp_group['frame_work2cam'][low:high] = frame_work2cam
f.close()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--path', type=str, help='path to the image')
parser.add_argument('--sl', type=str, help='source language')
parser.add_argument('--tl', type=str, default='en', help='target language')
parser.add_argument('--region',
type=str,
default='us-east-2',
help='AWS region name')
args = parser.parse_args()
path = args.path
sl = args.sl
tl = args.tl
region = args.region
mask_generator(path=path, region_name=region)
inpainter()
postprocess()
bucket = ''
document = ''
block_count, text, coord = text_extractor(bucket,
document,
path,
region_name=region)
#Getting the translated text i.e. ttext
ttext = translator(text, sl, tl, region_name=region)
#Opening the inpainted image
image = cv2.imread('./results/image.jpg')
#Writing the text on inpainted image
written_img = writer(image, ttext, coord)
#Output Image
cv2.imwrite('./Output/Output.jpg', written_img)
print("Blocks detected: " + str(block_count))
def diffusive_integrator(sampler, tolerance = 0.01):
# sample from sampler
samples = [] # coordinates
samplesInfo = [] # logX, likelihood, (tieBreaker), livePointID
finfo = open('sample_info.txt', 'w')
finfo.write("# index, logLikelihood, tieBreaker, ID.\n")
#numpy.savetxt(f, self.samplesInfo, fmt='%d %e %f %d')
fsample = open('sample.txt', 'w')
fsample.write("# Samples file. One sample per line.\n")
#numpy.savetxt(f, self.samples)
for i in range(1000):
# keep sampling
sample, sampleInfo = sampler.next()
print 'new sample:', sample, sampleInfo
# each sample file contains one line per live point / particle
# sampleFile :: self.live_pointsu
# sampleInfoFile :: self.indices, self.live_pointsL, (tieBreaker), ID?
# levelFile :: logX, cutoff, (tieBreaker), accepts, tries, exceeds, visits
# adding an extra number because postprocess can't deal with 1d data
fsample.write(' '.join(['%e' % s for s in sample]) + " 1\n")
fsample.flush()
finfo.write("%d %e %f %d\n" % sampleInfo)
finfo.flush()
samples.append(sample)
samplesInfo.append(sampleInfo)
levels = numpy.array([[l.logX, l.cutoff, numpy.random.uniform(), l.accepts, l.tries, l.exceeds, l.visits] for l in sampler.levels])
flevels = open('levels.txt', 'w')
flevels.write("# logX, logLikelihood, tieBreaker, accepts, tries, exceeds, visits.\n")
numpy.savetxt(flevels, levels, fmt='%f %e %f %d %d %d %d')
flevels.close()
if i % 20 == 19:
# check if tolerance achieved already
logZ, H, weights, logZerr = postprocess.postprocess(
loaded=(levels, numpy.asarray(samplesInfo), numpy.asarray(samples)),
save=False, plot=False,
verbose=False, numResampleLogX=10)
print 'logZ = %.3f +- %.3f' % (logZ, logZerr)
if logZerr < tolerance / 3:
break
return dict(logZ=logZ, logZerr=logZerr,
samples=sampler.samples, weights=weights, information=H)
def main(args):
'''Example script to train a model on the sample lyrics dataset'''
c = Configuration()
if args.artist:
c.set_artist(args.artist)
print("Hyperparameters: ", c)
print("Loading data from path: ", c.path)
lyrics_dataset = pp.read_lyrics_files(c.path)
tokenized = pp.tokenize(lyrics_dataset)
x, y, dictionary = pp.preprocess(tokenized, c.window_size)
training_data = DataLoader(list(zip(x, y)),
batch_size=c.train_batch_size,
shuffle=True)
model = LyricPredictor(len(dictionary), c.output_size)
print("Training model...")
model, _, _ = train(model=model,
training_data=training_data,
num_epochs=c.num_epochs,
lr=c.lr,
grad_norm=c.grad_max_norm)
print("Saving model: ", c.model_path)
torch.save(model, c.model_path)
print("Saving dictionary: ", c.dictionary_path)
torch.save(dictionary, c.dictionary_path)
print("Generating lyrics...")
seed_lyrics = generate_seed_lyrics(tokenized, c.window_size, args.censored)
predicted_lyrics = predict(model,
seed_lyrics,
dictionary,
num_words=args.words,
topk=c.predict_topk)
predicted_lyrics = postprocess(predicted_lyrics, args.censored)
print(predicted_lyrics)
def main(config_path, results_dir):
r"""
This function runs the whole analysis pipeline
"""
starttime = time.time()
logger.define_logging(logpath=results_dir + '/optimisation_results')
# Preproccessing
logging.info('Preprocess data')
connect_to_oep(config_path=config_path, results_dir=results_dir)
prepare_timeseries(config_path=config_path, results_dir=results_dir)
preprocess_closed_data(config_path=config_path, results_dir=results_dir)
# Run the optimisation model
logging.info('Run optimisation model')
run_model_dessau(config_path=config_path, results_dir=results_dir)
# Postprocessing
logging.info('Postprocess data')
postprocess(config_path=config_path, results_dir=results_dir)
# Plotting
logging.info('Create plots')
create_plots(config_path=config_path, results_dir=results_dir)
# Build a report
# cmd = ['pdflatex', '-interaction=nonstopmode', '--output-directory={0}/presentation/build'.format(abs_path), '{0}/presentation/report.tex'.format(results_dir)]
# process = subprocess.call(cmd) # , stdout=open(os.devnull, 'wb'))
endtime = time.time()
logging.info(f'Analysis lastet {endtime-starttime} sec.')
return True
def simulation_wrapper_noparams(host, COMSOL_model, paramfile):
#CST
#Preprocessing - calculate current distribution
current_density_file, paramfile, frequency = preprocess(paramfile)
if not within_frequency_bounds(frequency, 7.03e09, 100e6):
return (-dist_from_clock(frequency, 7.03e09), 0
) # Negative as want to optimize against this
else:
#COMSOL simulation
remote_interface(host, COMSOL_model, paramfile, current_density_file)
#Postprocess - generate g_ens and pi_fidelity
file_gens2_number = os.getcwd(
) + '/downloads/exports/g_ens2_number.csv'
g_ens, FWHM = postprocess(file_gens2_number)
return (g_ens, FWHM)
def detect_card_details(card_img,
net,
save=False,
image_path=".jpg",
prepro=False):
# Preprocess image
if prepro:
card_img = preprocess(card_img)
# Predict classes from image
card_img, outs = detection(card_img, net)
# Postprocess to extract card details
image_name = os.path.split(image_path)[1]
_card_details = postprocess(card_img, outs, image_name, save)
return _card_details
def main(args):
global NUMBER_FILES_TO_GENERATE
if args and len(args) > 0:
try:
NUMBER_FILES_TO_GENERATE = int(args[0])
except:
raise ValueError("The first argument to main.py must be an integer!")
util.init()
global heads
global pcfg
global objects
global primitives
# Call the main rule-parsing/generation module first to get our pcfg
# and other data ready to go
(heads, pcfg, objects, raw_primitives) = generate_rules.process_all();
# Prepare primitives dictionaries by doing some pre-processing on them
# so that future stuff is accessible much quicker:
primitives = prepare_primitives(raw_primitives)
print "\nStarting generation process. Don't be alarmed if it takes awhile."
numGenerated = 0
while True:
try:
print "Attempting to generate artificial Python file {}/{}".format(numGenerated+1, NUMBER_FILES_TO_GENERATE)
tree = makeNode("Module", 0, [])
outcode = "generated/code{}.py".format(numGenerated+1)
outast = "generated/AST{}.txt".format(numGenerated+1)
with open(outcode, "w") as out:
Unparser.Unparser(tree, out)
with open(outcode, "r") as in_f:
text = in_f.read()
with open(outcode, 'w') as out:
out.write(postprocess.postprocess(text))
with open(outast, "w") as out:
out.write(ast.dump(tree))
print "Successfully created {} and {}".format(outcode, outast)
numGenerated += 1
if numGenerated >= NUMBER_FILES_TO_GENERATE:
break
except Exception as e:
pass # The Fifth Amendment allows me to do this. Shhh...
def preview(api: sly.Api, task_id, context, state, app_logger):
bg_images = update_bg_images(api, state)
if len(bg_images) == 0:
sly.logger.warn("There are no background images")
else:
cache_dir = os.path.join(app.data_dir, "cache_images_preview")
sly.fs.mkdir(cache_dir)
sly.fs.clean_dir(cache_dir)
img, ann, res_meta = synthesize(api, task_id, state, meta, images_info,
labels, bg_images, cache_dir)
res_meta, ann = postprocess(state, ann, res_meta, sly.ProjectMeta())
if state["taskType"] == "inst-seg" and state[
"highlightInstances"] is True:
res_meta, ann = highlight_instances(res_meta, ann)
src_img_path = os.path.join(cache_dir, "res.png")
dst_img_path = os.path.join(f"/flying_object/{task_id}", "res.png")
sly.image.write(src_img_path, img)
file_info = None
if api.file.exists(team_id, dst_img_path):
api.file.remove(team_id, dst_img_path)
file_info = api.file.upload(team_id, src_img_path, dst_img_path)
gallery = dict(empty_gallery)
gallery["content"]["projectMeta"] = res_meta.to_json()
gallery["content"]["annotations"] = {
"preview": {
"url": file_info.full_storage_url,
"figures": [label.to_json() for label in ann.labels]
}
}
gallery["content"]["layout"] = [["preview"]]
fields = [
{
"field": "data.gallery",
"payload": gallery
},
{
"field": "state.previewLoading",
"payload": False
},
]
api.task.set_fields(task_id, fields)
def app_estimate():
if request.headers.getlist("X-Forwarded-For"):
ip = request.headers.getlist("X-Forwarded-For")[0]
else:
ip = request.remote_addr
slack.notify(text=f"[ESTIMATE] : \n from {ip}")
test_csv = request.files.get('test_csv')
if test_csv is None:
return abort(404)
if test_csv.content_type == 'text/csv' or test_csv.content_type == 'application/vnd.ms-excel':
test_df = pd.read_csv(test_csv)
else:
return abort(
404, {
'code': 'Wrong file',
'message': 'the file is not csv please send me csv'
})
test_encoded = preprocess(test_df)
test_preds_df = estimate(test_encoded)
submit_df = postprocess(test_preds_df, test_df)
temp_time = time.time()
test_df.to_csv(osp.join(DATA_DIR, f'dip_input_{temp_time}.csv'),
index=False,
header=True)
submit_df.to_csv(osp.join(OUTPUT_DIR, f'dip_estimete_{temp_time}.csv'),
index=False,
header=True)
output_name = f"estimate_{test_csv.filename}"
if output_name[-4:] != ".csv":
output_name += ".csv"
textStream = StringIO()
submit_df.to_csv(textStream, index=False, header=True)
return Response(
textStream.getvalue(),
mimetype="text/csv",
headers={"Content-disposition": f"attachment; filename={output_name}"})
def quantile_regression():
df = init()
raw_input, processing_info = preprocess(df)
raw_predictions = train_and_predict(raw_input)
postprocess(raw_input, raw_predictions, processing_info)
def put_out(input):
# this code sounds weird :-)
return sep.join(postprocess.postprocess(input.split(sep), sep))
def generate(api: sly.Api, task_id, context, state, app_logger):
bg_images = update_bg_images(api, state)
if len(bg_images) == 0:
sly.logger.warn("There are no background images")
else:
cache_dir = os.path.join(app.data_dir, "cache_images_generate")
sly.fs.mkdir(cache_dir)
sly.fs.clean_dir(cache_dir)
if state["destProject"] == "newProject":
res_project_name = state["resProjectName"]
if res_project_name == "":
res_project_name = "synthetic"
res_project = api.project.create(workspace_id,
res_project_name,
change_name_if_conflict=True)
elif state["destProject"] == "existingProject":
res_project = api.project.get_info_by_id(state["destProjectId"])
res_dataset = api.dataset.get_or_create(res_project.id,
state["resDatasetName"])
res_meta = sly.ProjectMeta.from_json(
api.project.get_meta(res_project.id))
progress = sly.Progress("Generating images", state["imagesCount"])
refresh_progress_images(api, task_id, progress)
for i in range(state["imagesCount"]):
img, ann, cur_meta = synthesize(api,
task_id,
state,
meta,
images_info,
labels,
bg_images,
cache_dir,
preview=False)
merged_meta, new_ann = postprocess(state, ann, cur_meta, res_meta)
if res_meta != merged_meta:
api.project.update_meta(res_project.id, merged_meta.to_json())
res_meta = merged_meta
image_info = api.image.upload_np(
res_dataset.id, f"{i + res_dataset.items_count}.png", img)
api.annotation.upload_ann(image_info.id, new_ann)
progress.iter_done_report()
if progress.need_report():
refresh_progress_images(api, task_id, progress)
res_project = api.project.get_info_by_id(res_project.id)
fields = [
{
"field": "data.started",
"payload": False
},
{
"field": "data.resProjectId",
"payload": res_project.id
},
{
"field": "data.resProjectName",
"payload": res_project.name
},
{
"field":
"data.resProjectPreviewUrl",
"payload":
api.image.preview_url(res_project.reference_image_url, 100, 100)
},
]
api.task.set_fields(task_id, fields)
f = open('OPTIONS_TEMPLATE', 'r')
o = f.read()
f.close()
o = o.replace('XXX', '500')
o = o.replace('YYY', '400')
f = open('OPTIONS', 'w')
f.write(o)
f.close()
os.system('./main -t 8 -s 0')
l = loadtxt('levels.txt')
l = l[l[:,1] <= -30., :]
f = open('OPTIONS_TEMPLATE', 'r')
o = f.read()
f.close()
o = o.replace('XXX', str(l.shape[0]))
o = o.replace('YYY', '3000')
f = open('OPTIONS', 'w')
f.write(o)
f.close()
os.system('./main -t 8 -s 0 -l levels.txt')
postprocess.postprocess(cut=0.25, plot=False)
os.system('cp posterior_sample.txt Results/' + str(i) + '.txt')
def main():
# constants
num_channels = cfg.config['num_channels']
classes = cfg.config['classes']
num_classes = len(classes)
height = cfg.config['height']
width = cfg.config['width']
anchors = cfg.config['anchors']
num_anchors = len(anchors[0])
confidency = cfg.config['confidency']
tp_iou = cfg.config['TP_IoU']
nms_iou = cfg.config['NMS_IoU']
cuda = cfg.config['CUDA']
target_dir = cfg.config['path']['test']
image_dir = cfg.config['path']['image']
weight_dir = cfg.config['path']['weight_test']
image_paths = pre.load_image_paths(image_dir, target_dir)
target_paths = pre.load_dir_paths(target_dir)
weight_paths = pre.load_dir_paths(weight_dir)
num_images = len(image_paths)
# network
net = model.YOLOv3(num_channels, num_classes, num_anchors)
if cuda:
net = net.cuda()
# calculate loss or mAP for each weights
for weight_path in weight_paths:
net.load_state_dict(torch.load(weight_path))
net.eval()
loss_giou = 0.0
loss_obj = 0.0
loss_cls = 0.0
loss_blc = 0.0
predictions = []
targets = []
t0 = time.time()
for i in range(num_images):
# load image and target
image = pre.load_image(image_paths[i], height, width, cuda)
image = image.unsqueeze(0)
target = pre.load_targets(target_paths[i:i + 1], num_classes,
height, width, cuda)
# predict bbox
prediction = [pred.detach() for pred in net(image)]
# calculate loss
loss = post.calculate_loss(prediction, target, anchors, height,
width, cuda).detach()
loss_giou += float(loss[0])
loss_obj += float(loss[1])
loss_cls += float(loss[2])
loss_blc += float(loss[3])
# save prediction and target as numpy array
prediction = post.postprocess(prediction, anchors, height, width,
confidency, nms_iou, cuda)
predictions.extend([pred.detach() for pred in prediction])
targets.extend([tagt.detach() for tagt in target])
# normalize loss
loss_giou /= num_images
loss_obj /= num_images
loss_cls /= num_images
loss_blc /= num_images
# calculate AP
AP = post.calculate_AP(predictions, targets, tp_iou, cuda)
elapsed_time = time.time() - t0
print((
'Weight: {}, Elapsed Time: {:.2f}s, ' +
# 'GIoU Loss: {:.2f}, ' +
# 'Objectness Loss: {:.2f}, ' +
# 'Class Loss: {:.2f}, ' +
# 'Balance Loss: {:.2f}, ' +
'Loss: {:.2f}, ' + 'AP: ' +
', '.join(['{:.2f}'.format(ap * 100) for ap in AP]) + ', '
'mAP: {:.2f}'.format(100 * sum(AP) / len(AP))).format(
weight_path, elapsed_time, loss_giou, loss_obj, loss_cls,
loss_blc, loss_giou + loss_obj + loss_cls + loss_blc))
# Copyright (c) 2009, 2010, 2011, 2012 Brendon J. Brewer. # # This file is part of DNest3. # # DNest3 is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # DNest3 is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with DNest3. If not, see <http://www.gnu.org/licenses/>. # Import postprocess from two directories up # http://stackoverflow.com/a/9806045 import os parentdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) parentdir = os.path.dirname(parentdir) os.sys.path.insert(0, parentdir) import postprocess postprocess.postprocess() import display
import postprocess postprocess.postprocess(cut=0., plot=False, moreSamples=1.) # import display import make_plots import histograms
import postprocess postprocess.postprocess(cut=0.) import display
def convert(self, text):
text = '\n'+text+'\n'# add embed in newlines (makes regex replaces work better)
# first we search for [[code]] statements as we don't want any replacement to happen inside those code blocks!
code_blocks = dict()
code_blocks_found = re.findall(re.compile(r'(\[\[code( type="([\S]+)")?\]\]([\s\S ]*?)\[\[/code\]\])',re.MULTILINE), text)
for code_block_found in code_blocks_found:
tmp_hash = str(uuid.uuid4())
text = text.replace(code_block_found[0],tmp_hash,1) # replace code block with a hash - to fill it in later
code_blocks[tmp_hash] = "\n"+string.join([" " + l for l in code_block_found[-1].strip().split("\n") ],"\n")+"\n"
for search, replacement in self.static_replacements.items():
text = text.replace(search,replacement,1)
# search for any of the simpler replacements in the dictionary regex_replacements
for s_reg, r_reg in self.regex_replacements.items():
text = re.sub(re.compile(s_reg,re.MULTILINE),r_reg,text)
# TITLES -- replace '+++ X' with '=== X ==='
for titles in re.finditer(r"^(\++)([^\n]*)$", text, re.MULTILINE):
header = ("=" * len(titles.group(1)))
text = text.replace(titles.group(0), header + (titles.group(2) + " ") + header)
# LISTS(*) -- replace ' *' with '***' and so on
for stars in re.finditer(r"^([ \t]+)\*", text, re.MULTILINE):
text = text[:stars.start(1)] + ("*" * len(stars.group(1))) + text[stars.end(1):]
# LISTS(#) -- replace ' #' with '###' and so on
for hashes in re.finditer(r"^([ \t]+)\*", text, re.MULTILINE):
text = text[:hashes.start(1)] + ("#" * len(hashes.group(1))) + text[hashes.end(1):]
# INTERNAL LINKS -- replace [[[bink]]] with [[bink]]
for inlink in re.finditer(r"\[\[\[([\s\S ]*?)\]\]\]", text):
text = text.replace(inlink.group(0), "[["+inlink.group(1)+"]]")
# IMAGES
for image in re.finditer(r"\[\[image([\s\S ]*?)\]\]", text):
text = text.replace(image.group(0), "[[File:" + image.group(1) + "]]")
# START TABLE
for table in re.finditer(r"\[\[table([\s\S ]*?)\]\]", text):
#text = text.replace(table.group(0), "{|" + table.group(1))
text = text.replace(table.group(0), "{|")
# START ROW
for row in re.finditer(r"\[\[row([\s\S ]*?)\]\]", text):
#text = text.replace(row.group(0), "|-" + row.group(1))
text = text.replace(row.group(0), "|-")
# START CELL
for cell in re.finditer(r"\[\[cell([\s\S ]*?)\]\]", text):
#text = text.replace(cell.group(0), "|" + cell.group(1))
text = text.replace(cell.group(0), "|")
# ENDS
for end in re.finditer(r"\[\[/([\s\S ]*?)\]\]", text):
token = end.group(1)
if token == "table":
text = text.replace(end.group(0), "|}")
elif token == "row":
# end row tabs are not necessary in mediawiki
text = text.replace(end.group(0), "")
elif token == "cell":
# end cell tabs are not necessary in mediawiki
text = text.replace(end.group(0), "")
# now we substitute back our code blocks
for tmp_hash, code in code_blocks.items():
text = text.replace(tmp_hash, code, 1)
# Process color corrections
startpos = 0
while -1 != startpos:
startpos = text.find("##", startpos)
pipepos = text.find("|", startpos + 2)
endpos = text.find("##", startpos + 2)
if startpos != -1 and pipepos != -1 and endpos != -1 and endpos > pipepos:
color = text[startpos + 2 : pipepos].strip()
colored = text[pipepos + 1 : endpos].strip()
text = text[: startpos] + "<span style=\"color:" + color + "\">" + colored + \
"</span>" + text[endpos + 2 :]
startpos = endpos
# Process math corrections
startpos = 0
while -1 != startpos:
startpos = text.find("[[$", startpos)
endpos = text.find("$]]", startpos)
if startpos != -1 and endpos != -1:
mathtext = text[startpos + 3 : endpos].strip()
text = text[: startpos] + "<math>" + mathtext + "</math>" + text[endpos + 3 :]
startpos = endpos
# Process table corrections
startpos = 0
while -1 != startpos:
startpos = text.find("\n||", startpos)
if startpos == -1:
break
# Find end of table
endpos = text.find("\n", startpos + 3)
while endpos < len(text) - 3 and "||" == text[endpos + 1: endpos + 3]:
endpos = text.find("\n", endpos + 3)
# Found bounds of text chunk: reformat table
fixup = text[startpos + 1 : endpos].replace("||~", "!!")
fixup = fixup.split("\n")
fixout = ["", "{| class=\"wikitable\""]
for i in xrange(len(fixup)):
if fixup[i][0 : 2] == "||" or fixup[i][0 : 2] == "!!":
out = fixup[i].strip()[1 : ]
fixout.append(out[ : -2 if out[-2 : ] in ["||", "!!"] else 0])
else:
print("Failed to parse item %d/%d: '%s'" % (i, len(fixup), fixup[i]))
sys.exit(-1)
fixout.append("|}" if i == len(fixup) - 1 else "|-")
# Construct output table text
fullout = "\n".join(fixout)
text = text[ : startpos] + fullout + text[endpos : ]
startpos = startpos + len(fullout)
# Repair multi-newlines
text = re.sub(r"\n\n+", "\n\n", text, re.M)
# Repair starting newlines
text = text.strip()
# Optional postprocessing stage
text = postprocess.postprocess(text)
return text
def main():
parser = argparse_ArgumentParser("Input parameters")
parser.add_argument("--input_file_name",
default="input_toy.yaml",
help="Input parameters file name")
parser.add_argument("--out_dir_name",
default="/results",
help="Output directory name")
parser.add_argument("--train_test_files_dir",
default="",
help="Train test file path")
parser.add_argument("--n_pts",
default=1,
help="number of partitions (computers)")
parser.add_argument("--over_t", help="Overlap threshold")
parser.add_argument("--model_dir", help="Directory containing model")
parser.add_argument("--sample_dir", help="Sample files dir + /res")
parser.add_argument("--sample_folders_prefix",
help="Input parameters file name /results..")
parser.add_argument(
"--sample_folders_prefix_final",
help="Input file name to use final merged results Use as /results..")
parser.add_argument(
"--sample_folders_list",
nargs='+',
help="Input parameters file name /results.. separated by commas")
parser.add_argument("--graph_files_dir",
default="",
help="Graph files' folder path")
parser.add_argument("--overlap_method",
default=1,
help="Overlap method option: qi, default: jaccard")
parser.add_argument("--infer_overlap_threshold",
default='n',
help="y or n")
args = parser.parse_args()
with open(args.input_file_name, 'r') as f:
inputs = yaml_load(f, yaml_Loader)
if args.overlap_method:
inputs['overlap_method'] = args.overlap_method
if args.over_t:
inputs['over_t'] = float(args.over_t)
if args.sample_dir:
inputs['sample_dir'] = args.sample_dir
if args.model_dir:
inputs['model_dir'] = args.model_dir
if args.infer_overlap_threshold:
inputs['infer_overlap_threshold'] = args.infer_overlap_threshold
# Override output directory name if same as gen
if args.out_dir_name or inputs['out_comp_nm'] == "/results/res":
if not os_path.exists(inputs['dir_nm'] + args.out_dir_name):
os_mkdir(inputs['dir_nm'] + args.out_dir_name)
inputs['out_comp_nm'] = args.out_dir_name + "/res"
inputs['train_test_files_dir'] = ''
if args.train_test_files_dir:
if not os_path.exists(inputs['dir_nm'] + args.train_test_files_dir):
os_mkdir(inputs['dir_nm'] + args.train_test_files_dir)
inputs['train_test_files_dir'] = args.train_test_files_dir
inputs['graph_files_dir'] = ''
if args.graph_files_dir:
if not os_path.exists(inputs['dir_nm'] + args.graph_files_dir):
os_mkdir(inputs['dir_nm'] + args.graph_files_dir)
inputs['graph_files_dir'] = args.graph_files_dir
logging_basicConfig(filename=inputs['dir_nm'] + inputs['out_comp_nm'] +
"_logs.yaml",
level=logging_INFO)
# fin_list_graphs = control(myGraph,inputs,n=50)
if "sample_dir" not in inputs:
inputs['sample_dir'] = inputs['out_comp_nm']
myGraphName = inputs['dir_nm'] + inputs['graph_files_dir'] + "/res_myGraph"
with open(myGraphName, 'rb') as f:
myGraph = pickle_load(f)
if 'infer_overlap_threshold' in inputs:
if inputs['infer_overlap_threshold'] == 'y':
pp_flag = 0
if inputs['dir_nm'] == 'yeast':
pp_flag = 1
if 'overlap_method' in inputs:
if inputs['overlap_method'] == 'qi':
inputs['over_t'] = get_overlap_threshold_qi(
inputs, pp_flag, myGraph)
else:
inputs['over_t'] = get_overlap_threshold(
inputs, pp_flag, myGraph)
with open(inputs['dir_nm'] + inputs['out_comp_nm'] + "_input_pp.yaml",
'w') as outfile:
yaml_dump(inputs, outfile, default_flow_style=False)
out_comp_nm = inputs['dir_nm'] + inputs['out_comp_nm']
out_comp_nm_sample = inputs['dir_nm'] + inputs['sample_dir']
out_comp_nm_model = inputs['dir_nm'] + inputs['model_dir']
modelfname = out_comp_nm_model + "_model"
scalerfname = out_comp_nm_model + "_scaler"
with open(scalerfname, 'rb') as f:
scaler = pickle_load(f)
pred_comp_list = []
sdndap = pred_comp_list.append
if args.sample_folders_list:
for folder in args.sample_folders_list:
allfiles = './' + inputs['dir_nm'] + folder + '/res_pred_comp_list*'
for fname in glob(allfiles, recursive=True):
with open(fname, 'rb') as f:
pred_comp_tmp = pickle_load(f)
for snode in pred_comp_tmp:
sdndap(snode)
elif args.sample_folders_prefix_final:
allsubd = './' + inputs[
'dir_nm'] + args.sample_folders_prefix_final + '*/res_pred.out'
for fname in glob(allsubd, recursive=True):
with open(fname) as f:
complexes_score = [
line.rstrip().split() for line in f.readlines()
]
pred_comp_tmp = [(frozenset(comp[:-1]), float(comp[-1]))
for comp in complexes_score]
for snode in pred_comp_tmp:
sdndap(snode)
elif args.sample_folders_prefix:
allsubd = './' + inputs[
'dir_nm'] + args.sample_folders_prefix + '*/res_pred_comp_list*'
for fname in glob(allsubd, recursive=True):
with open(fname, 'rb') as f:
pred_comp_tmp = pickle_load(f)
for snode in pred_comp_tmp:
sdndap(snode)
else:
for i in range(int(args.n_pts)):
with open(out_comp_nm_sample + "_pred_comp_list" + str(i),
'rb') as f:
pred_comp_tmp = pickle_load(f)
for snode in pred_comp_tmp:
sdndap(snode)
len_pred_comp_list = 'No. of complexes before pp = ' + str(
len(pred_comp_list))
logging_info(len_pred_comp_list)
test_complex_path = inputs['dir_nm'] + inputs[
'train_test_files_dir'] + "/res_test_known_complex_nodes_list"
test_prot_list = get_prot_list(test_complex_path)
train_complex_path = inputs['dir_nm'] + inputs[
'train_test_files_dir'] + "/res_train_known_complex_nodes_list"
train_prot_list = get_prot_list(train_complex_path)
protlistfname = inputs['dir_nm'] + inputs[
'train_test_files_dir'] + "/res_protlist"
with open(protlistfname, 'rb') as f:
prot_list = pickle_load(f)
start_time_pp = time_time()
fin_list_graphs = postprocess(pred_comp_list, modelfname, scaler, inputs,
myGraph, prot_list, train_prot_list,
test_prot_list)
pp_time = time_time() - start_time_pp
tot_time = time_time() - start_time
# Write to yaml file instead
with open(out_comp_nm + '_runtime_performance.out', "a") as fid:
print("--- Runtime performance ---", file=fid)
print("Post processing complex time (s) = ",
pp_time,
"[",
round(100 * float(pp_time) / tot_time, 2),
"%]",
file=fid)
print("Total time (s) = ", tot_time, file=fid)
# # Copyright (C) 2011 Brendon J. Brewer # This file is part of DNest, the Diffusive Nested Sampler. # # DNest is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # DNest is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with DNest. If not, see <http://www.gnu.org/licenses/>. import postprocess postprocess.postprocess(temperature=10.) import display
def hanso(func, x0=None, grad=None, nvar=None, nstart=None, sampgrad=False,
funcrtol=1e-20, gradnormtol=1e-6, verbose=2, fvalquit=-np.inf,
cpumax=np.inf, maxit=100, callback=None, **kwargs):
"""
HANSO: Hybrid Algorithm for Nonsmooth Optimization
The algorithm is two-fold. Viz,
BFGS phase: BFGS is run from multiple starting points, taken from
the columns of x0 parameter, if provided, and otherwise 10 points
generated randomly. If the termination test was satisfied at the
best point found by BFGS, or if nvar > 100, HANSO terminates;
otherwise, it continues to:
Gradient sampling phases: 3 gradient sampling phases are run from
lowest point found, using sampling radii:
10*evaldist, evaldist, evaldist/10
Termination takes place immediately during any phase if
cpumax CPU time is exceeded.
References
----------
A.S. Lewis and M.L. Overton, Nonsmooth Optimization via Quasi-Newton
Methods, Math Programming, 2012
J.V. Burke, A.S. Lewis and M.L. Overton, A Robust Gradient Sampling
Algorithm for Nonsmooth, Nonconvex Optimization
SIAM J. Optimization 15 (2005), pp. 751-779
Parameters
----------
func: callable function on 1D arrays of length nvar
function being optimized
grad: callable function
gradient of func
fvalquit: float, optional (default -inf)
param passed to bfgs1run function
gradnormtol: float, optional (default 1e-4)
termination tolerance for smallest vector in convex hull of saved
gradients
verbose: int, optional (default 1)
param passed to bfgs1run function
cpumax: float, optional (default inf)
quit if cpu time in secs exceeds this (applies to total running time)
sampgrad: boolean, optional (default False)
if set, the gradient-sampling will be used to continue the algorithm
in case the BFGS fails
**kwargs: param-value dict
optional parameters passed to bfgs backend. Possible key/values are:
x0: 2D array of shape (nvar, nstart), optional (default None)
intial points, one per column
nvar: int, optional (default None)
number of dimensions in the problem (exclusive x0)
nstart: int, optional (default None)
number of starting points for BFGS algorithm (exclusive x0)
maxit: int, optional (default 100)
param passed to bfgs1run function
wolfe1: float, optional (default 0)
param passed to bfgs1run function
wolfe2: float, optional (default .5)
param passed to bfgs1run function
Returns
-------
x: D array of same length nvar = len(x0)
final iterate
f: list of nstart floats
final function values, one per run of bfgs1run
d: list of nstart 1D arrays, each of same length as input nvar
final smallest vectors in convex hull of saved gradients,
one array per run of bfgs1run
H: list of nstarts 2D arrays, each of shape (nvar, nvar)
final inverse Hessian approximations, one array per run of bfgs1run
itrecs: list of nstart int
numbers of iterations, one per run of bfgs1run; see bfgs1run
for details
inforecs: list of int
reason for termination; see bfgs1run for details
pobj: list of tuples of the form (duration of iteration, final func value)
trajectory for best starting point (i.e of the starting point that
led to the greatest overall decrease in the cost function.
Note that the O(1) time consumed by the gradient-sampling stage is not
counted.
Optional Outputs (in case output_records is True):
Xrecs: list of nstart 2D arrays, each of shape (iter, nvar)
iterates where saved gradients were evaluated; one array per run
of bfgs1run; see bfgs1run
for details
Grecs: ist of nstart 2D arrays, each of shape (nvar, nvar)
gradients evaluated at these points, one per run of bfgs1run;
see bfgs1run for details
wrecs: list of nstart 1D arrays, each of length iter
weights defining convex combinations d = G*w; one array per
run of bfgs1run; see bfgs1run for details
fevalrecs: list of nstart 1D arrays, each of length iter
records of all function evaluations in the line searches;
one array per run of bfgs1run; see bfgs1run for details
xrecs: list of nstart 2D arrays, each of length (iter, nvar)
record of x iterates
Hrecs: list of nstart 2D arrays, each of shape (iter, nvar)
record of H (Hessian) iterates; one array per run of bfgs1run;
see bfgs1run for details
Raises
------
RuntimeError
"""
def _log(msg, level=0):
if verbose > level:
print msg
# sanitize x0
if x0 is None:
assert not nvar is None, (
"No value specified for x0, expecting a value for nvar")
assert not nstart is None, (
"No value specified for x0, expecting a value for nstart")
x0 = setx0(nvar, nstart)
else:
assert nvar is None, (
"Value specified for x0, expecting no value for nvar")
assert nstart is None, (
"Value specified for x0, expecting no value for nstart")
x0 = np.array(x0)
if x0.ndim == 1:
x0 = x0.reshape((-1, 1))
nvar, nstart = x0.shape
cpufinish = time.time() + cpumax
# run BFGS step
kwargs['output_records'] = 1
x, f, d, H, _, info, X, G, w, pobj = bfgs(
func, x0=x0, grad=grad, fvalquit=fvalquit, funcrtol=funcrtol,
gradnormtol=gradnormtol, cpumax=cpumax, maxit=maxit,
verbose=verbose, callback=callback, **kwargs)
# throw away all but the best result
assert len(f) == np.array(x).shape[1], np.array(x).shape
indx = np.argmin(f)
f = f[indx]
x = x[..., indx]
d = d[..., indx]
H = H[indx] # bug if do this when only one start point: H already matrix
X = X[indx]
G = G[indx]
w = w[indx]
pobj = pobj[indx]
dnorm = linalg.norm(d, 2)
# the 2nd argument will not be used since x == X(:,1) after bfgs
loc, X, G, w = postprocess(x, np.nan, dnorm, X, G, w, verbose=verbose)
if np.isnan(f) or np.isinf(f):
_log('hanso: f is infinite or nan at all starting points')
return x, f, loc, X, G, w, H, pobj
if time.time() > cpufinish:
_log('hanso: cpu time limit exceeded')
_log('hanso: best point found has f = %g with local optimality '
'measure: dnorm = %5.1e, evaldist = %5.1e' % (
f, loc['dnorm'], loc['evaldist']))
return x, f, loc, X, G, w, H, pobj
if f < fvalquit:
_log('hanso: reached target objective')
_log('hanso: best point found has f = %g with local optimality'
' measure: dnorm = %5.1e, evaldist = %5.1e' % (
f, loc['dnorm'], loc['evaldist']))
return x, f, loc, X, G, w, H, pobj
if dnorm < gradnormtol:
_log('hanso: verified optimality within tolerance in bfgs phase')
_log('hanso: best point found has f = %g with local optimality '
'measure: dnorm = %5.1e, evaldist = %5.1e' % (
f, loc['dnorm'], loc['evaldist']))
return x, f, loc, X, G, w, H, pobj
if sampgrad:
# launch gradient sampling
# time0 = time.time()
f_BFGS = f
# save optimality certificate info in case gradient sampling cannot
# improve the one provided by BFGS
dnorm_BFGS = dnorm
loc_BFGS = loc
d_BFGS = d
X_BFGS = X
G_BFGS = G
w_BFGS = w
x0 = x.reshape((-1, 1))
# otherwise gradient sampling is too expensivea
if maxit > 100:
maxit = 100
# # otherwise grad sampling will augment with random starts
# x0 = x0[..., :1]
# assert 0, x0.shape
cpumax = cpufinish - time.time() # time left
# run gradsamp proper
x, f, g, dnorm, X, G, w = gradsamp(func, x0, grad=grad, maxit=maxit,
cpumax=cpumax)
if f == f_BFGS: # gradient sampling did not reduce f
_log('hanso: gradient sampling did not reduce f below best point'
' found by BFGS\n')
# use the better optimality certificate
if dnorm > dnorm_BFGS:
loc = loc_BFGS
d = d_BFGS
X = X_BFGS
G = G_BFGS
w = w_BFGS
elif f < f_BFGS:
loc, X, G, w = postprocess(x, g, dnorm, X, G, w, verbose=verbose)
_log('hanso: gradient sampling reduced f below best point found'
' by BFGS\n')
else:
raise RuntimeError(
'hanso: f > f_BFGS: this should never happen'
) # this should never happen
x = x[0]
f = f[0]
if callback:
callback(x)
return x, f, loc, X, G, w, H, pobj
else:
if callback:
callback(x)
return x, f, loc, X, G, w, H, pobj
from pylab import *
import postprocess
from postprocess import logsumexp
# Load the levels from the DNest run
levels = loadtxt('levels.txt')
sample_info = loadtxt('sample_info.txt')
# Get logx of samples from the DNest run
result = postprocess.postprocess()
# Relationship between logl and logx based on the DNest run
logx = hstack([levels[:,0], result[2].flatten()])
logl = vstack([levels[:,1:3], sample_info[:,1:3]])
order = argsort(logx)[::-1]
logx = logx[order]
logl = logl[order, :]
# Now treat logx, logl as "levels" and use them as scaffolding to locate
# the points from the directional runs
levels = empty((logx.size, levels.shape[1]))
levels[:, 0] = logx
levels[:, 1:3] = logl
scalars = loadtxt('scalars.txt')
sample = loadtxt('output.txt')
sample_info = empty((scalars.shape[0], sample_info.shape[1]))
sample_info[:,1:3] = scalars[:, 0:2]
# log prior weights
result = postprocess.postprocess(loaded=[levels, sample_info, sample],\
temperature=1E300, plot=False)
def main(): img = cv2.imread(sys.argv[1]) gray= cv2.cvtColor(img,cv2.COLOR_BGR2GRAY) p = postprocess()
def hanso(func,
x0=None,
grad=None,
nvar=None,
nstart=None,
sampgrad=False,
funcrtol=1e-20,
gradnormtol=1e-6,
verbose=2,
fvalquit=-np.inf,
cpumax=np.inf,
maxit=100,
**kwargs):
"""
HANSO: Hybrid Algorithm for Nonsmooth Optimization
The algorithm is two-fold. Viz,
BFGS phase: BFGS is run from multiple starting points, taken from
the columns of x0 parameter, if provided, and otherwise 10 points
generated randomly. If the termination test was satisfied at the
best point found by BFGS, or if nvar > 100, HANSO terminates;
otherwise, it continues to:
Gradient sampling phases: 3 gradient sampling phases are run from
lowest point found, using sampling radii:
10*evaldist, evaldist, evaldist/10
Termination takes place immediately during any phase if
cpumax CPU time is exceeded.
References
----------
A.S. Lewis and M.L. Overton, Nonsmooth Optimization via Quasi-Newton
Methods, Math Programming, 2012
J.V. Burke, A.S. Lewis and M.L. Overton, A Robust Gradient Sampling
Algorithm for Nonsmooth, Nonconvex Optimization
SIAM J. Optimization 15 (2005), pp. 751-779
Parameters
----------
func: callable function on 1D arrays of length nvar
function being optimized
grad: callable function
gradient of func
fvalquit: float, optional (default -inf)
param passed to bfgs1run function
gradnormtol: float, optional (default 1e-4)
termination tolerance for smallest vector in convex hull of saved
gradients
verbose: int, optional (default 1)
param passed to bfgs1run function
cpumax: float, optional (default inf)
quit if cpu time in secs exceeds this (applies to total running time)
sampgrad: boolean, optional (default False)
if set, the gradient-sampling will be used to continue the algorithm
in case the BFGS fails
**kwargs: param-value dict
optional parameters passed to bfgs backend. Possible key/values are:
x0: 2D array of shape (nvar, nstart), optional (default None)
intial points, one per column
nvar: int, optional (default None)
number of dimensions in the problem (exclusive x0)
nstart: int, optional (default None)
number of starting points for BFGS algorithm (exclusive x0)
maxit: int, optional (default 100)
param passed to bfgs1run function
wolfe1: float, optional (default 0)
param passed to bfgs1run function
wolfe2: float, optional (default .5)
param passed to bfgs1run function
Returns
-------
x: D array of same length nvar = len(x0)
final iterate
f: list of nstart floats
final function values, one per run of bfgs1run
d: list of nstart 1D arrays, each of same length as input nvar
final smallest vectors in convex hull of saved gradients,
one array per run of bfgs1run
H: list of nstarts 2D arrays, each of shape (nvar, nvar)
final inverse Hessian approximations, one array per run of bfgs1run
itrecs: list of nstart int
numbers of iterations, one per run of bfgs1run; see bfgs1run
for details
inforecs: list of int
reason for termination; see bfgs1run for details
pobj: list of tuples of the form (duration of iteration, final func value)
trajectory for best starting point (i.e of the starting point that
led to the greatest overall decrease in the cost function.
Note that the O(1) time consumed by the gradient-sampling stage is not
counted.
Optional Outputs (in case output_records is True):
Xrecs: list of nstart 2D arrays, each of shape (iter, nvar)
iterates where saved gradients were evaluated; one array per run
of bfgs1run; see bfgs1run
for details
Grecs: ist of nstart 2D arrays, each of shape (nvar, nvar)
gradients evaluated at these points, one per run of bfgs1run;
see bfgs1run for details
wrecs: list of nstart 1D arrays, each of length iter
weights defining convex combinations d = G*w; one array per
run of bfgs1run; see bfgs1run for details
fevalrecs: list of nstart 1D arrays, each of length iter
records of all function evaluations in the line searches;
one array per run of bfgs1run; see bfgs1run for details
xrecs: list of nstart 2D arrays, each of length (iter, nvar)
record of x iterates
Hrecs: list of nstart 2D arrays, each of shape (iter, nvar)
record of H (Hessian) iterates; one array per run of bfgs1run;
see bfgs1run for details
Raises
------
RuntimeError
"""
def _log(msg, level=0):
if verbose > level:
print msg
# sanitize x0
if x0 is None:
assert not nvar is None, (
"No value specified for x0, expecting a value for nvar")
assert not nstart is None, (
"No value specified for x0, expecting a value for nstart")
x0 = setx0(nvar, nstart)
else:
assert nvar is None, (
"Value specified for x0, expecting no value for nvar")
assert nstart is None, (
"Value specified for x0, expecting no value for nstart")
x0 = np.array(x0)
if x0.ndim == 1:
x0 = x0.reshape((-1, 1))
nvar, nstart = x0.shape
cpufinish = time.time() + cpumax
# run BFGS step
kwargs['output_records'] = 1
x, f, d, H, _, info, X, G, w, pobj = bfgs(func,
x0=x0,
grad=grad,
fvalquit=fvalquit,
funcrtol=funcrtol,
gradnormtol=gradnormtol,
cpumax=cpumax,
maxit=maxit,
verbose=verbose,
**kwargs)
# throw away all but the best result
assert len(f) == np.array(x).shape[1], np.array(x).shape
indx = np.argmin(f)
f = f[indx]
x = x[..., indx]
d = d[..., indx]
H = H[indx] # bug if do this when only one start point: H already matrix
X = X[indx]
G = G[indx]
w = w[indx]
pobj = pobj[indx]
dnorm = linalg.norm(d, 2)
# the 2nd argument will not be used since x == X(:,1) after bfgs
loc, X, G, w = postprocess(x, np.nan, dnorm, X, G, w, verbose=verbose)
if np.isnan(f) or np.isinf(f):
_log('hanso: f is infinite or nan at all starting points')
return x, f, loc, X, G, w, H, pobj
if time.time() > cpufinish:
_log('hanso: cpu time limit exceeded')
_log('hanso: best point found has f = %g with local optimality '
'measure: dnorm = %5.1e, evaldist = %5.1e' %
(f, loc['dnorm'], loc['evaldist']))
return x, f, loc, X, G, w, H, pobj
if f < fvalquit:
_log('hanso: reached target objective')
_log('hanso: best point found has f = %g with local optimality'
' measure: dnorm = %5.1e, evaldist = %5.1e' %
(f, loc['dnorm'], loc['evaldist']))
return x, f, loc, X, G, w, H, pobj
if dnorm < gradnormtol:
_log('hanso: verified optimality within tolerance in bfgs phase')
_log('hanso: best point found has f = %g with local optimality '
'measure: dnorm = %5.1e, evaldist = %5.1e' %
(f, loc['dnorm'], loc['evaldist']))
return x, f, loc, X, G, w, H, pobj
if sampgrad:
# launch gradient sampling
# time0 = time.time()
f_BFGS = f
# save optimality certificate info in case gradient sampling cannot
# improve the one provided by BFGS
dnorm_BFGS = dnorm
loc_BFGS = loc
d_BFGS = d
X_BFGS = X
G_BFGS = G
w_BFGS = w
x0 = x.reshape((-1, 1))
# otherwise gradient sampling is too expensivea
if maxit > 100:
maxit = 100
# # otherwise grad sampling will augment with random starts
# x0 = x0[..., :1]
# assert 0, x0.shape
cpumax = cpufinish - time.time() # time left
# run gradsamp proper
x, f, g, dnorm, X, G, w = gradsamp(func,
x0,
grad=grad,
maxit=maxit,
cpumax=cpumax)
if f == f_BFGS: # gradient sampling did not reduce f
_log('hanso: gradient sampling did not reduce f below best point'
' found by BFGS\n')
# use the better optimality certificate
if dnorm > dnorm_BFGS:
loc = loc_BFGS
d = d_BFGS
X = X_BFGS
G = G_BFGS
w = w_BFGS
elif f < f_BFGS:
loc, X, G, w = postprocess(x, g, dnorm, X, G, w, verbose=verbose)
_log('hanso: gradient sampling reduced f below best point found'
' by BFGS\n')
else:
raise RuntimeError('hanso: f > f_BFGS: this should never happen'
) # this should never happen
x = x[0]
f = f[0]
# pobj.append((time.time() - time0, f))
return x, f, loc, X, G, w, H, pobj
else:
return x, f, loc, X, G, w, H, pobj
# Copyright (c) 2009, 2010, 2011, 2012 Brendon J. Brewer. # # This file is part of DNest3. # # DNest3 is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # DNest3 is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with DNest3. If not, see <http://www.gnu.org/licenses/>. import postprocess postprocess.postprocess(cut=0.5) import display
def interactive_plot():
# import data from post-processing script
run_data = postprocess()
# setup
accuracies = []
losses = []
test_accuracies = []
step = 0
acc = 0
loss = 0
handle = 0
#######################################################
# GRAPHING VALIDATION ACCURACY AS A FUNCTION OF EPOCH #
#######################################################
fig, ax = plt.subplots()
ax.set_title('Validation Accuracy vs Epoch')
for run_num in run_data:
# print "processing run", run_num
step = np.asarray(run_data[run_num]['step'])
acc = 100 * np.asarray(run_data[run_num]['validation_accuracy'])
loss = np.asarray(run_data[run_num]['loss'])
handle, = ax.plot(step, acc, lw=1.0, label=str(run_num))
accuracies.append(handle)
test_accuracies.append(run_data[run_num]['test_accuracy'])
# shrink x axis by 20% so we can fit our huge legend
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.9, box.height])
# create legend, put to right of plot area
leg = ax.legend(fancybox=True,
shadow=True,
ncol=3,
loc='upper right',
bbox_to_anchor=(1.2, 1.0))
leg.get_frame().set_alpha(0.4)
# I have no idea what this does, but I think it links legend entries to the data
lined = dict()
for legline, origline in zip(leg.get_lines(), accuracies):
legline.set_picker(5) # 5 pts tolerance
# origline.set_visible(False)
# legline.set_alpha(0.2)
lined[legline] = origline
# legend click callback
def onpick(event):
legline = event.artist
origline = lined[legline]
vis = not origline.get_visible()
origline.set_visible(vis)
# change the alpha on the line in the legend so we can see what lines have been toggled
if vis:
legline.set_alpha(1.0)
else:
legline.set_alpha(0.2)
fig.canvas.draw()
fig.canvas.mpl_connect('pick_event', onpick)
ax.set_xlim([0, 500])
ax.set_ylim([80, 100])
plt.xlabel('Epoch Number')
plt.ylabel('Validation Accuracy [%]')
plt.show()
########################################
# GRAPHING LOSS AS A FUNCTION OF EPOCH #
########################################
fig, ax = plt.subplots()
ax.set_title('Softmax Loss vs Epoch')
for run_num in run_data:
# print "processing run", run_num
step = np.asarray(run_data[run_num]['step'])
loss = np.asarray(run_data[run_num]['loss'])
handle, = ax.plot(step, loss, lw=1.0, label=str(run_num))
# losses[run_num] = handle
losses.append(handle)
# shrink x axis by 20% so we can fit our huge legend
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.9, box.height])
# create legend, put to right of plot area
leg = ax.legend(fancybox=True,
shadow=True,
ncol=3,
loc='upper right',
bbox_to_anchor=(1.2, 1.0))
leg.get_frame().set_alpha(0.4)
# I have no idea what this does, but I think it links legend entries to the data
lined = dict()
for legline, origline in zip(leg.get_lines(), losses):
legline.set_picker(5) # 5 pts tolerance
# origline.set_visible(False)
# legline.set_alpha(0.2)
lined[legline] = origline
# legend click callback
def onpick2(event):
legline = event.artist
origline = lined[legline]
vis = not origline.get_visible()
origline.set_visible(vis)
# change the alpha on the line in the legend so we can see what lines have been toggled
if vis:
legline.set_alpha(1.0)
else:
legline.set_alpha(0.2)
fig.canvas.draw()
fig.canvas.mpl_connect('pick_event', onpick2)
ax.set_xlim([0, 500])
ax.set_ylim([0.0, 0.5])
plt.xlabel('Epoch Number')
plt.ylabel('Softmax Loss')
plt.show()
#######################################################
# GRAPHING FINAL TEST ACCURACY AS A FUNCTION OF EPOCH #
#######################################################
plt.plot(test_accuracies)
# ax.set_xlim([0, 500])
ax.set_ylim([0.0, 0.5])
plt.xlabel('Epoch Number')
plt.ylabel('Softmax Loss')
plt.show()
plt.show()
# Copyright (c) 2009, 2010, 2011, 2012 Brendon J. Brewer. # # This file is part of DNest3. # # DNest3 is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # DNest3 is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with DNest3. If not, see <http://www.gnu.org/licenses/>. # Import postprocess from two directories up # http://stackoverflow.com/a/9806045 import os parentdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) parentdir = os.path.dirname(parentdir) os.sys.path.insert(0, parentdir) import postprocess postprocess.postprocess(temperature=1) import display
def print_help():
print('''Usage: {0}: [OPTION] scan|text|learn file [file...]\
\nExtract string from image file
\nvalid argument may be:\
\n -h, --help: print this help message\
\n -d, --dir: dataset directory\
\n scan: scan for valid text in file'''.format(sys.argv[0]))
sys.exit(0)
if __name__ == "__main__":
optlist, args = getopt.getopt(sys.argv[1:], "hd:", ["help", "dir"])
direc = "dataset"
p = ps.postprocess()
for o, a in optlist:
if o in ("-h", "--help"):
print_help()
if o in ("-d", "--dir"):
direc = a
if len(args) >= 2 and args[0] in ("s", "scan"):
knn = learnLetter()
for filename in args[1:]:
scan(knn, filename, p)
elif len(args) >= 2 and args[0] in ("t", "text"):
print(direc)
knn = learnLetter(directory=direc)
for filename in args[1:]:
scantext(knn, filename, p)
elif len(args) >= 1 and args[0] in ("l", "learn"):
def main():
st.title("picTranslate")
'''Hii there! We are happy to help you. '''
'''picTranslate is an AI based app that enables you to intuitively
translate the language of the text on your images, so that you can understand your picture completely.'''
st.subheader('Instructions')
'''1. Just select the image you want to change the text of.'''
'''2. Select the source language i.e. the current language on the image.'''
'''3. Select the target language i.e. the language you want the translated text to be in and Press Start.'''
'''That's it!!'''
st.header('Upload the required image')
uploaded_file = st.file_uploader("")
if uploaded_file is not None:
image = Image.open(uploaded_file)
image.save('Uploadedimg.jpg')
st.image(image, caption='Uploaded Image.', use_column_width=True)
st.success('Uploaded Successfully!!')
sl = st.text_input("Source Language", 'en')
tl = st.text_input("Target Language", 'en')
if st.button('More Target Languages'):
webbrowser.open_new_tab(url)
region = st.text_input("AWS Region name", 'us-east-2')
url = 'https://docs.aws.amazon.com/translate/latest/dg/what-is.html'
if st.button('Start Translation'):
path = './Uploadedimg.jpg'
st.write("(1/4) Generating Mask...")
mask_generator(path, region_name=region)
st.write(
"(2/4) Inpainting the image (This might take sometime if running on CPU)..."
)
inpainter()
postprocess()
bucket = ''
document = ''
st.write('(3/4) Detecting text...')
block_count, text, coord = text_extractor(bucket,
document,
path,
region_name=region)
#Getting the translated text i.e. ttext
ttext = translator(text, sl, tl, region_name=region)
#Opening the inpainted image
image = cv2.imread('./results/image.jpg')
#Writing the text on inpainted image
st.write("(4/4) Just there...")
written_img = writer(image, ttext, coord)
#Output Image
cv2.imwrite('./Output/Output.jpg', written_img)
st.success('Finished!!')
image2 = Image.open('./Output/Output.jpg')
st.image(image2, caption='Output image', use_column_width=True)
st.warning('Very rich backgrounds might create problems!!')
st.markdown(
"<h1 style='text-align: center; color: black; font-size:16px;'>Developed by Vaibhav Tiwari</h1>",
unsafe_allow_html=True)
# Copyright (c) 2009, 2010, 2011, 2012 Brendon J. Brewer. # # This file is part of DNest3. # # DNest3 is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # DNest3 is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with DNest3. If not, see <http://www.gnu.org/licenses/>. # Import postprocess from two directories up # http://stackoverflow.com/a/9806045 import os #parentdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) #parentdir = os.path.dirname(parentdir) #os.sys.path.insert(0, parentdir) import postprocess postprocess.postprocess(plot=False)
# [micro m]
SL1 = SL0 * .97
# [micro m]
SLt0 = .05
# [s]
SLt1 = .35
# [s]
SLtau0 = .05
# [s]
SLtau1 = .02
# [s]
SL_base = lambda t: SL0 + (SL1 - SL0) * (np.clip(
1 - np.exp((SLt0 - t) / SLtau0), 0, None) - np.clip(
1 - np.exp((SLt1 - t) / SLtau1), 0, None))
#%% Input definition
inputs = {}
inputs['times'] = np.arange(0, Tmax, 1e-4)
inputs['Ca'] = Ca_base(inputs['times'])
inputs['SL'] = SL_base(inputs['times'])
#%% Simulation
output = model.solve(inputs)
#%% Saving results to csv file
pandas.DataFrame(output).to_csv('output_' + model.model_name + '.csv',
index=False)
#%% Postprocessing
postprocess(output, 'output_' + model.model_name + '.pdf')
print e.content
except TA_error.Preprocess_Error, e:
# preprocess has some problems
n_tests_fail += 1
print_error(self.test_name, t_elapsed, "PREPROCESS")
print e.content
except TA_error.Process_Error, e:
# process has some problems
n_tests_fail += 1
print_error(self.test_name, t_elapsed, "PROCESS")
print e.content
finally:
try:
if test_pass == True:
print_pass(self.test_name, t_elapsed)
postprocess.postprocess(self.parser)
except TA_error.Postprocess_Error, e:
# print "postprocess error"
print_error(self.test_name, t_elapsed, "POSTPROCESS")
print e.content
def main(self):
self._parse_test_cfg() # parse test cfg file
if self.macro == "open":
# print "open"
self.macro_ele_list = extract_macro(self.parser)
macro.new_macro(self.macro_ele_list, self)
else:
"""
no macro, only run once
"""
def _process(video):
"""
This route will perform 3 steps:
1. split video into frames directory and audio file
2. add frames into service bus queue
3. this function will poll storage until the number of
input images matches the number of processed images
4. download processed frames and stitch video back together
"""
# get varaibles from environment
namespace = os.getenv("SB_NAMESPACE")
queue = os.getenv("SB_QUEUE")
sb_key_name = os.getenv("SB_SHARED_ACCESS_KEY_NAME")
sb_key_value = os.getenv("SB_SHARED_ACCESS_KEY_VALUE")
mount_dir = os.getenv("MOUNT_DIR", "data")
terminate = os.getenv("TERMINATE")
# start time
t0 = time.time()
# set video name
video_name = video.split(".")[0]
input_dir = os.path.join(mount_dir, video_name, "input_frames")
output_dir = os.path.join(mount_dir, video_name, "output_frames")
audio_file = os.path.join(mount_dir, video_name, "audio.aac")
# create parent directory in mount_dir
if not os.path.exists(os.path.join(mount_dir, video_name)):
os.makedirs(os.path.join(mount_dir, video_name))
# setup logger
handler_format = get_handler_format()
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setFormatter(handler_format)
log_file = "{}.log".format(video_name)
file_handler = RotatingFileHandler(os.path.join(mount_dir, video_name,
log_file),
maxBytes=20000)
file_handler.setFormatter(handler_format)
logger = logging.getLogger("root")
logger.setLevel(logging.DEBUG)
logger.addHandler(console_handler)
logger.addHandler(file_handler)
logger.propagate = False
# process video and upload output frames and audio file to blob
logger.debug("Preprocessing video {}".format(video))
preprocess(video=video, mount_dir=mount_dir)
t1 = time.time()
# service bus client
bus_service = ServiceBusService(
service_namespace=namespace,
shared_access_key_name=sb_key_name,
shared_access_key_value=sb_key_value,
)
# add all images from frame_dir to the queue
logger.debug("Adding images from {} to queue {}".format(input_dir, queue))
image_count = add_images_to_queue(
mount_dir=mount_dir,
queue=queue,
video_name=video_name,
bus_service=bus_service,
)
t2 = time.time()
# terminate if testing
if terminate:
exit(0)
# poll storage for output
logger.debug(
"Polling for input images {} to equal output images {}".format(
input_dir, output_dir))
while True:
path, dirs, files = next(os.walk(output_dir))
output_frames_length = len(files)
if output_frames_length == image_count:
t3 = time.time()
# postprocess video
logger.debug(
"Stitching video together with processed frames dir '{}' and audio file '{}'."
.format(output_dir, audio_file))
postprocess(video_name=video_name, mount_dir=mount_dir)
t4 = time.time()
break
else:
logger.debug(
"Images are still processing. Retrying in 10 seconds...")
time.sleep(10)
continue
t5 = time.time()
logger.debug(
"Preprocessing video finished.... Time taken in seconds: {:.2f}".
format(t1 - t0))
logger.debug(
"Adding image to queue finished.. Time taken in seconds: {:.2f}".
format(t2 - t1))
logger.debug(
"Detecting vehicles.............. Time taken in seconds: {:.2f}".
format(t3 - t2))
logger.debug(
"Postprocessing video finished... Time taken in seconds: {:.2f}".
format(t4 - t3))
logger.debug(
"Total process................... Time taken in seconds: {:.2f}".
format(t5 - t0))
loadName = loadName[
0::
2] # get the elements in the even position. start at the first element (0) and skip 1 each time
# print(loadName) # debug
# CREATE HEADERS IN RESULTS TABLE
column_names = ["specimen", "timePL", "cyclesPL", "timeYP", "cyclesYP"]
results = pd.DataFrame(columns=column_names)
# CALL POSTPROCESS FUNCTION AND FILL IN RESULTS TABLE
for i in range(len(names)): # range creates iterable object
index = loadName.index(
names[i].split(" ")[0]
) # find the name that is being evaluated and return target load from load files
output = pp.postprocess(os.path.join(entry_file_path, names[i] + '.csv'),
os.path.join(script_dir, 'temporary.csv'),
float(load[index].split(" ")[1]), 0.01, 0.05,
5) #0.01 and 0.05 for all files
print('aqui', output)
preLoadData = list(map(
float, output[0].split(",")))[0:2] # get first and second value
yieldPointData = []
for j in output[1].split(","):
if j == '':
yieldPointData.append('not found')
yieldPointData.append('not found')
else:
yieldPointData.append(float(j))
yieldPointData = yieldPointData[0:2]
row = [names[i].split(" ")[0]] # extract name until the first blank space
row.extend(preLoadData) # extend te alarga la lista
row.extend(yieldPointData)
def diffusive_integrator(sampler,
tolerance=0.01,
maxEvaluations=0,
minEvaluations=0):
# sample from sampler
samples = [] # coordinates
samplesInfo = [] # logX, likelihood, (tieBreaker), livePointID
finfo = open('sample_info.txt', 'w')
finfo.write("# index, logLikelihood, tieBreaker, ID.\n")
#numpy.savetxt(f, self.samplesInfo, fmt='%d %e %f %d')
fsample = open('sample.txt', 'w')
fsample.write("# Samples file. One sample per line.\n")
#numpy.savetxt(f, self.samples)
logZs = []
i = 1
nlevelcomplete = None
while True:
# keep sampling
sample, sampleInfo = next(sampler)
#print 'new sample:', sample, sampleInfo
# each sample file contains one line per live point / particle
# sampleFile :: self.live_pointsu
# sampleInfoFile :: self.indices, self.live_pointsL, (tieBreaker), ID?
# levelFile :: logX, cutoff, (tieBreaker), accepts, tries, exceeds, visits
# adding an extra number because postprocess can't deal with 1d data
fsample.write(' '.join(['%e' % s for s in sample]) + "\n")
fsample.flush()
finfo.write("%d %e %f %d\n" % sampleInfo)
finfo.flush()
samples.append(sample)
samplesInfo.append(sampleInfo)
levels = numpy.array([[
l.logX, l.cutoff,
numpy.random.uniform(), l.accepts, l.tries, l.exceeds, l.visits
] for l in sampler.levels])
flevels = open('levels.txt', 'w')
flevels.write(
"# logX, logLikelihood, tieBreaker, accepts, tries, exceeds, visits.\n"
)
numpy.savetxt(flevels, levels, fmt='%f %e %f %d %d %d %d')
flevels.close()
if i % 20 == 0:
# check if tolerance achieved already
logZ, H, weights, logZerr, ESS = postprocess.postprocess(
loaded=(levels, numpy.asarray(samplesInfo),
numpy.asarray(samples)),
save=False,
plot=False,
verbose=False,
numResampleLogX=10)
logZs.append(logZ)
print(
'logZ = %.3f +- %.3f +- %.3f | %.1f samples | iteration %d[warmup:%s,neval:%d]'
% (logZ, logZerr, numpy.std(
logZs[-10:]), ESS, i, nlevelcomplete, sampler.neval))
if nlevelcomplete is None and ESS > 60 and sampler.maxLevels <= 0:
sampler.maxLevels = len(sampler.levels)
if nlevelcomplete is None and sampler.levelsComplete():
nlevelcomplete = i
print('# levels sufficient after %d iterations' %
nlevelcomplete)
logZerr_total = (logZerr**2 + numpy.var(logZs[-10:]))**0.5
converged = nlevelcomplete is not None and logZerr_total < tolerance and numpy.std(
logZs[-10:]) * 10 < tolerance
if (minEvaluations is not None and sampler.neval > minEvaluations
) and (converged or (maxEvaluations > 0
and sampler.neval >= maxEvaluations)):
# do some plotting
postprocess.postprocess(loaded=(levels,
numpy.asarray(samplesInfo),
numpy.asarray(samples)),
save=True,
plot=True,
numResampleLogX=1)
break
i = i + 1
return dict(logZ=logZ,
logZerr=logZerr,
samples=samples,
points=[(p, numpy.median(w), si[0])
for p, si, w in zip(samples, samplesInfo, weights)],
information=H)
