def parseParameters(self, params):
parameters = []
for para in params:
mtype = Selector.solve(para.type)
var = Selector.solve(para.variable)
parameters.append((mtype, var))
return parameters
def main():
f = open('data/person_foaf.n3')
fresnel_data = f.read()
fresnel = Fresnel( fresnel_data )
print "Created Fresnel Graph.."
rdf_graph = Graph()
print "Downloading resource.."
#f = open('data/Tim_Berners-Lee.rdf')
#rdf_graph.parse( file=f, format="xml")
#rdf_graph.parse(source="http://dbpedia.org/page/A._P._J._Abdul_Kalam")
#print "more..."
f = open('data/A._P._J._Abdul_Kalam.rdf')
rdf_graph.parse( file=f, format="xml")
#rdf_graph.parse(source="http://dbpedia.org/page/Tim_Berners-Lee")
for term in rdf_graph:
print term
print "Making selection.."
selector = Selector( fresnel , rdf_graph)
selector.select()
formatter = Formatter( selector )
formatter.format()
data = "<html>\n<head><link rel='stylesheet' type='text/css' href='style.css'></head>\n<body>\n"
for resource in formatter.result:
print resource.render()
data += resource.render()
data += "</body>\n</html>"
f = open('output/test.html', 'w')
f.write(data.encode('utf-8'))
def parse(self, response):
sel = Selector(response)
productos = sel.xpath('//div[@id="js-product-wrapper"]/article')
# sel.css también puede ser usado.
# iterar sobre todos los productos:
for i, elem in enumerate(productos):
item = ItemLoader(Producto(), elem)
item.add_xpath(
# 'imagen', './div[@class="dkt-product__gallery"]/div/div/div/div/picture/source[5]/@srcset')
# 'imagen', './div/div/div/div/div/picture/source[position()=4]/@srcset')
'imagen',
'./div[@class="dkt-product__gallery"]/div/div/div[position()=1]/div/picture/source/source/source/source/source/@srcset'
)
item.add_xpath(
'titulo',
'div[@class="dkt-product__infos-wrapper"]/div[@class="dkt-product__infos__link"]/div/div/a/h2/text()'
)
item.add_xpath(
# 'precio', './div[@class="dkt-product__infos-wrapper"]/div/div/div[@class="dkt-product__price"]/div/div/@data-price')
'precio',
'./div[@class="dkt-product__infos-wrapper"]/div/div/div[@class="dkt-product__price"]/div/div[@class="dkt-price__cartridge"]/@data-price'
)
item.add_xpath(
'precio_a',
'normalize-space(.//div[@class="dkt-price__cartridge"]/text())'
)
item.add_xpath(
'precio_b',
'normalize-space(.//div[@class="dkt-price__cartridge"]/sup/text())'
)
item.add_xpath(
'precio_previo',
'./div[@class="dkt-product__infos-wrapper"]/div/div/div[@class="dkt-product__price"]/div/span/span[position()=1]/text()'
)
item.add_xpath(
'reduccion',
'./div[@class="dkt-product__infos-wrapper"]/div/div/div[@class="dkt-product__price"]/div/span/span[position()=2]/text()'
)
item.add_xpath(
'marca',
'./div[@class="dkt-product__infos-wrapper"]/div/div/div/span/span/text()'
)
item.add_xpath(
'url',
'./div[@class="dkt-product__infos-wrapper"]/div[@class="dkt-product__infos__link"]/div/div/a/@href'
)
item.add_xpath(
'rating',
'./div[@class="dkt-product__infos-wrapper"]/div/div/span[@itemprop="ratingValue"]/text()'
)
item.add_xpath(
'review',
'./div[@class="dkt-product__infos-wrapper"]/div/div/span[@itemprop="reviewCount"]/text()'
)
item.add_value('id', i)
yield item.load_item()
def listConditions(self, category):
dialog = Selector(category) if category == 'Tags' else SingleSelector(category)
if dialog.exec_() == QDialog.Accepted:
data = dialog.retrieveData()
if category == 'Title':
self.titleEdit.setText(data)
elif category == 'Tags':
self.tagEdit.setText(data)
def testWitnessStream():
from WitnessFiltration import WitnessStream
from WeakWitnessFiltration import WeakWitnessStream
# mean = [0, 0]
# cov = [[1, 0], [0, 1]]
# matrixofpoints_inR2 = np.random.multivariate_normal(mean, cov, 100)
# p = pc.PointCloud(matrixofpoints_inR2)
filename = '/Users/naheed/PycharmProjects/PersistenceHomology/data/eight.mat'
p = pc.MatlabPointCloud(filename, 'point_cloud')
p.compute_distancematrix()
# Create Selector
pointcloud_sel = sel.PointCloudSelector(p, 100, "RandomSelector")
R = float(pointcloud_sel.get_maxdistance_landmarktoPointcloud()) / 2
# R = 0
print 'R = ', R
numdivision = 5
maxdim = 3
# ws = WitnessStream(landmarkselector=pointcloud_sel, maxdistance=R, numdivision=numdivision, maxdimension=maxdim)
ws = WeakWitnessStream(mu=2,
landmarkselector=pointcloud_sel,
maxdistance=R,
numdivision=numdivision,
maxdimension=maxdim)
ws.construct
print ws
print "Total number of Simplices in the Filtration: ", len(ws)
def buttonPath(self):
tab = self.actualMap.randomPath()
cities = self.actualMap.cities
self.canvas.delete("all")
#self.canvasText.delete("all")
self.canvas.create_rectangle(0, 0, 500, 500, fill='dark slate grey')
for i in range(len(tab)):
self.placePoint(cities[tab[i]][0],cities[tab[i]][1])
self.contenuePopu = self.nbrePopu.get()
tab = slt.selectionPath(int(self.contenuePopu,base=10),self.actualMap,25)
#self.textBestPath = Label(self.root, text = str(self.actualMap.pathLength(tab)))
self.stringVariable.set(str(self.actualMap.pathLength(tab)))
cities = self.actualMap.cities
for j in range(len(tab)):
if (j+1)<len(tab):
self.bindPoint(cities[tab[j]][0],
cities[tab[j]][1],
cities[tab[j+1]][0],
cities[tab[j+1]][1])
self.bindPoint(cities[tab[0]][0],
cities[tab[0]][1],
cities[tab[len(tab)-1]][0],
cities[tab[len(tab)-1]][1])
def post_experiment():
'''Experiment where we try generating a series of posts'''
Poster = DL.PostDownloader()
Selector = PS.PostSelector()
if not os.path.exists('experiment_posts'):
os.mkdir('experiment_posts')
for postnumber in range(100):
try:
posttype = np.random.choice(['news','general'],p=[0.2,0.8])
print('Running download')
downloaded, story, download_type, pt = Poster.rundownload(posttype=posttype)
print('Selecting post meta')
selected_post_meta = Selector.select_post(download_type=download_type)
print(downloaded,story,download_type,pt,selected_post_meta)
post_image = 'post_%03d.jpg' %postnumber
os.system('mv debug_test.png experiment_posts/%s' %post_image)
print('Done post number %i' %postnumber)
print('----------------------------------------------------------------')
except:
print('Wait 10 seconds, try again')
time.sleep(10)
def process(arguments):
for perplexity, name in Selector.generate(arguments):
# if verbose is enabled, Selector.generate already printed the word,
# so print it only in verbose is disabled
if not arguments.get('verbose'):
try:
print(("%s" % name))
except UnicodeError as e:
print(("%s" % name).encode('ascii', 'backslashreplace'))
def process( arguments ):
for perplexity, name in Selector.generate( arguments ):
# if verbose is enabled, Selector.generate already printed the word,
# so print it only in verbose is disabled
if not arguments.get('verbose'):
try:
print( ("%s" % name) )
except UnicodeError as e:
print( ("%s" % name).encode('ascii','backslashreplace') )
def Main(filename,
features_file,
Algorithms,
number_of_features,
filter_target,
list_features=None):
print("\n---------------------------------\n")
print("Load Data", end="")
DATA = Load_Data.load_data(filename,
ALGORITHMS,
filter_target=filter_target)
print(" [OK]\nLoad ELA Features", end="")
P, D, F = Load_Data.load_ELA_features(features_file)
print(" [OK]\nLink ELA Features to problems", end="")
Problem.link_all_features(DATA, P, D, F)
print(" [OK]\nInitialize Empirical Performance Model", end="")
model = epm.EmpiricalPerformanceModel(
number_of_parameters,
number_of_features,
len(ALGORITHMS),
input_type="parameters",
selector=Selector.Random_selector(probability=0.7),
list_features=list_features)
print(" [OK]\nFix Training and Testing sets", end="")
model.build_training_and_testing_sets(DATA)
print("\nNumber of problems : " + str(len(model.get_results())) + "\n")
'''
print(" [OK]\nTrain EPM",end="")
model.train_model()
print(" [OK]\nTest EPM",end="")
model.test_model()
print(" [OK]\n")
SBS=Statistic.SingleBestSolver(model)
VBS=Statistic.VirtualBestSolver(model)
RS=Statistic.RealSolver(model)
Merit=Statistic.Merit(SBS,VBS,RS)
print("SBS "+str(SBS))
print("VBS "+str(VBS))
print("RS "+str(RS))
print("Merit "+str(Merit))
'''
model.reset_model()
model.set_input_type('features')
print("Train EPM", end="")
model.train_model()
print(" [OK]\nTest EPM", end="")
model.test_model()
print(" [OK]\n")
SBS = Statistic.SingleBestSolver(model)
VBS = Statistic.VirtualBestSolver(model)
RS = Statistic.RealSolver(model)
Merit = Statistic.Merit(SBS, VBS, RS)
print("SBS " + str(SBS))
print("VBS " + str(VBS))
print("RS " + str(RS))
print("Merit " + str(Merit))
def test_parse():
assert_equals(parse('foobar'), Ref(Package(None, 'foobar'), None, None, None))
assert_equals(parse('@spekklez/foobar'), Ref(Package('spekklez', 'foobar'), None, None, None))
assert_equals(parse('@spekklez/[email protected]'), Ref(Package('spekklez', 'foobar'), Selector(Version('1.6.4')), None, None))
assert_equals(parse('@spekklez/foobar@>=1.6.4'), Ref(Package('spekklez', 'foobar'), Selector('>=1.6.4'), None, None))
assert_equals(parse('spam@~1.0.0'), Ref(Package(None, 'spam'), Selector('~1.0.0'), None, None))
assert_equals(parse('spam@~1.0.0/main'), Ref(Package(None, 'spam'), Selector('~1.0.0'), 'main', None))
assert_equals(parse('spam@~1.0.0/main:run'), Ref(Package(None, 'spam'), Selector('~1.0.0'), 'main', 'run'))
assert_equals(parse('spam@~1.0.0:run'), Ref(Package(None, 'spam'), Selector('~1.0.0'), None, 'run'))
with assert_raises(ValueError):
parse('.')
with assert_raises(ValueError):
parse('..')
with assert_raises(ValueError):
parse('/')
with assert_raises(ValueError):
parse('@/')
with assert_raises(ValueError):
parse('@')
def testMaxdist():
from DistanceMetricinput import DistanceMetricIn
d_in = DistanceMetricIn(
'/Users/naheed/PycharmProjects/PersistenceHomology/data/dmateight.mat',
varname='dist_eight',
dimension=2)
print d_in.size, d_in.dim
s = sel.MetricSelector(d_in, 4, "RandomSelector")
print s.getdistance_subsetstoPointcloud()
def findZk(self,W,ct):
Nn = Selector.selector(self.N)
WD = Nn.extractw(W,ct)
X = WD[0,0] + (WD[1,1]).conjugate()
Y = (WD[0,1]).conjugate() - WD[1,0]
k = cmath.sqrt(abs(X)**2 + abs(Y)**2).real
x = X/k
y = Y/k
Z = np.matrix([[(x).conjugate(), - (y).conjugate()] ,[y,x]])
return k,Z
def startup():
p = list(serial_ports())
p.append("Rescan")
p.append("Exit")
buttonbox = Selector(p)
if buttonbox.exec_():
a = buttonbox.getValue()
if a == len(p) - 1:
sys.exit()
elif a == len(p) - 2:
startup()
return
else:
rm.reset()
bt.initialize(p[a])
rm.showMaximized()
else:
sys.exit()
def interfaceFinder(self, filePath):
nativeMethodPath = "ClassDeclaration[name$=Proxy]>MethodDeclaration[throws*=RemoteException]"
result = Selector.Selector(filePath).query(nativeMethodPath)
if len(result) == 0:
AIDLMethodPath = "ClassDeclaration[name=Stub]>ClassDeclaration[name=Proxy]>MethodDeclaration[throws*=android.os.RemoteException]"
result = Selector.Selector(filePath).query(AIDLMethodPath)
functions = {}
for method in result:
name = Selector.solve(method.name)
parameters = self.parseParameters(method.parameters)
functions[name] = ", ".join(i[0] + " " + i[1] for i in parameters)
return functions
def testRandomSelector():
# Create PointCloud
matrixofpoints_inR2 = []
mean = [0, 0]
cov = [[1, 0], [0, 1]]
matrixofpoints_inR2 = np.random.multivariate_normal(mean, cov, 100)
p = pc.PointCloud(matrixofpoints_inR2)
# Create Selector
s = sel.PointCloudSelector(p, 5, "RandomSelector")
print s.getLandmarkPoints().points
print s.getdistance_subsetstoPointcloud()
def echo(bot, update):
file = bot.getFile(update.message.photo[-1].file_id)
# print(file)
file_url = (file.file_path)
# print(file_url)
file_name = wget.download(file_url)
# print(file_name)
tags = Tag_Getter.detect_labels(file_name)
emo = Emo_Getter.detect_faces(file_name)
meme_message = Selector.simple_function(list_to_int(emo), tags)
meme_writer(meme_message, 0, file_name)
bot.send_photo(chat_id=update.message.chat_id,
photo=open("Completed/temp.png", 'rb'))
print(tags, '\n', emo)
def __init__(self,
number_of_parameters,
numberOfFeatures,
numberOfAlgorithms,
input_type="features",
learning_method=Learning_method.classical_forestRegression(),
selector=Selector.Random_selector(probability=0.2),
list_features=None):
self.numberOfFeatures = numberOfFeatures
self.list_features = list_features
self.number_of_parameters = number_of_parameters
self.set_input_type(input_type)
self.numberOfAlgorithms = numberOfAlgorithms
self.input_type = input_type
self.selector = selector
self.learning_method = learning_method
def addSelections(self):
selector = Selector.SelectionWidget(self.names,
self) #creates a selector object
#allows for box/scatter plot btns to be pressed
selector.completed.connect(self.boxPlot.setEnabled)
selector.completed.connect(self.scatterPlot.setEnabled)
try:
self.selectWidgetList.append(
selector
) #everytime a new selector widget is created save it in a list
self.selectionsLayout.addWidget(selector) #add to our layout
except Exception as selectorListErr:
print(
"error occurred when trying to append selector widget to object list........ERROR:{}"
.format(selectorListErr))
def SuggestRecipes(menu_func=None): #TODO
'''
Suggest a set of candidate Recipes based on global state
'''
global current_chefs
global current_recipes
global current_restrictions
global ingredients_to_use
global banned_ingredients
suggestions = Selector.SuggestRecipes(Common.global_state, current_recipes,
current_restrictions, current_chefs,
ingredients_to_use,
banned_ingredients,
Common.global_state.history)
print 'How does this look?'
for s in suggestions:
print '\t' + s.name
DisplaySuggestRecipeMenu()
while True:
selection = raw_input('> ')
if selection == 'e' or selection == 'E':
break
elif selection == '1':
current_recipes = suggestions
GenerateList()
History.Update(current_recipes)
break
elif selection == '2':
current_recipes = suggestions
EditCurrentRecipesMenu()
break
elif selection == '3':
EditCurrentRecipesMenu()
SuggestRecipes()
break
elif selection == '4':
SuggestRecipes()
break
#TODO figure out what menu to show
print "done"
if menu_func:
menu_func()
return
def __init__(self):
"""
Initialize Explorer Tools, that will:
- parse arguments given by user.
- get an instance of Utilities.
- get an instance of Selector, that will fetch dbpedia resources.
"""
self.args = self.parse_arguments()
self.research_type = None
self.topic = self.set_topic()
self.chapter = self.set_chapter()
self.output_format = self.set_output_format()
# declare pyTableExtractor Utilities
self.utils = Utilities.Utilities(self.chapter, self.topic,
self.research_type)
# if user doesn't choose for single resource
if not self.args.single:
self.selector = Selector.Selector(self.utils)
def testWitnessStreamPH():
from WitnessFiltration import WitnessStream
from src.WeakWitnessFiltration import WeakWitnessStream
from src.ComputeInterval import IntervalComputation
# mean = [0, 0]
# cov = [[1, 0], [0, 1]]
# matrixofpoints_inR2 = np.random.multivariate_normal(mean, cov, 100)
# p = pc.PointCloud(matrixofpoints_inR2)
filename = '/Users/naheed/PycharmProjects/PersistenceHomology/data/eight.mat'
p = pc.MatlabPointCloud(filename, 'point_cloud')
p.compute_distancematrix()
# Create Selector
# pointcloud_sel = sel.PointCloudSelector(p, 10, "RandomSelector")
pointcloud_sel = sel.PointCloudSelector(p, 50, "MaxminSelector")
# Measuring system time elapsed since the epoch (UTC time-zone)
start_time = time()
pointcloud_sel.select()
elapsed_time = time() - start_time
R = float(pointcloud_sel.get_maxdistance_landmarktoPointcloud()) / 3
# R = 0
print 'R = ', R
numdivision = 10
maxdim = 2
# ws = WitnessStream(landmarkselector=pointcloud_sel, maxdistance=R, numdivision=numdivision, maxdimension=maxdim)
ws = WeakWitnessStream(mu=2,
landmarkselector=pointcloud_sel,
maxdistance=R,
numdivision=numdivision,
maxdimension=maxdim)
ws.construct()
print "Total number of Simplices in the Filtration: ", len(ws)
ci = IntervalComputation(ws)
ci.compute_intervals(
maxdim
) # I should check everything is ok in this function since the deg for simplices can have real value now.
ci.print_BettiNumbers()
def actualiza_objetivo(objetivo, objetivos):
"Actualiza el objetivo si se mueve"
# No hay nuevos objetivos, nos quedamos con el que se detecto
if (len(objetivos) < 1):
return objetivo
# Elimina las detecciones del tracker con area muy distinta del objetivo
x, y, w, h = objetivo
area_objetivo = w * h
rectangulos = Utiles.genera_rectangulos(objetivos)
rectangulos = Utiles.elimina_rect_irelevantes(rectangulos, area_objetivo)
# Los objetivos no eran suficientemente similares en area
if (len(rectangulos) < 1):
return objetivo
# Cambia el objetivo por el mas cercano detectado por el tracker
distancia_max = max(w, h)
p_objetivo = Utiles.centro_rectangulo(objetivo)
objetivo_alt = Selector.objetivo_prioritario(p_objetivo, rectangulos,
distancia_max)
if (len(objetivo_alt) > 0):
return objetivo_alt
# No habia un objetivo lo suficientemete cercano
return objetivo
def comparebars():
maxdim = 2
filename = '/Users/naheed/PycharmProjects/PersistenceHomology/data/eight.mat'
p = pc.MatlabPointCloud(filename, 'point_cloud')
print p.dimension, p.size
p.compute_distancematrix()
print p.getdistance(0, 1)
from src.RipsFiltration import BruteForceRips
rf = BruteForceRips(p, 100, maxdim, 0.01)
rf.construct()
from src.ComputeInterval import IntervalComputation
ci = IntervalComputation(rf)
ci.compute_intervals(maxdim)
print ci.betti_intervals
from src.WeakWitnessFiltration import WeakWitnessStream
filename = '/Users/naheed/PycharmProjects/PersistenceHomology/data/eight.mat'
p2 = pc.MatlabPointCloud(filename, 'point_cloud')
print p2.dimension, p2.size
p2.compute_distancematrix()
pointcloud_sel = sel.PointCloudSelector(p2, 200, "MaxminSelector")
pointcloud_sel.select()
R = 0.01
print "R= ", R
ws = WeakWitnessStream(mu=1,
landmarkselector=pointcloud_sel,
maxdistance=R,
numdivision=100,
maxdimension=maxdim)
ws.construct()
ci2 = IntervalComputation(ws)
ci2.compute_intervals(maxdim)
# print len(ci2.betti_intervals), len(ci.betti_intervals)
from src.Intervalviz import PersistenceViz
pv = PersistenceViz(ci.betti_intervals, replace_Inf=rf.maxfiltration_val)
pv.qual_compare_barcodes(ci2.betti_intervals, R)
def on_generate_button_clicked( self, selection ):
#TODO use cached small lexicons for name generation too.
selected_files = list( self.lexicon_files_manager.yield_filepaths_from_selection() )
self.arguments.set( selected_files, 'lexicon', '<selected_files>' )
try:
for perplexity, name in Selector.generate( self.arguments ):
self.builder.get_object("generated_names").append((name,perplexity))
except Exception as e:
import traceback
traceback.print_exc()
dialog = Gtk.MessageDialog(
self.builder.get_object("main_window"),
0,
Gtk.MessageType.ERROR,
Gtk.ButtonsType.OK,
"Generation raised exception of type \"%s\":\n\n\"%s\"" % (type(e).__name__,str(e)) )
dialog.run()
dialog.destroy()
return True
def main():
Poster = DL.PostDownloader()
Selector = PS.PostSelector()
posttype = 'general'
print('Running download')
downloaded, story, download_type, pt, fromid = Poster.rundownload(posttype=posttype)
print(download_type)
print('Selecting post meta')
selected_post_meta = Selector.select_post(download_type=download_type,fromid=fromid)
print(downloaded,story,download_type,pt,selected_post_meta)
print(selected_post_meta.columns)
#Caption can be designed with input from post_meta and story.
print ('Designing caption...')
def on_generate_button_clicked(self, selection):
#TODO use cached small lexicons for name generation too.
selected_files = list(
self.lexicon_files_manager.yield_filepaths_from_selection())
self.arguments.set(selected_files, 'lexicon', '<selected_files>')
try:
for perplexity, name in Selector.generate(self.arguments):
self.builder.get_object("generated_names").append(
(name, perplexity))
except Exception as e:
import traceback
traceback.print_exc()
dialog = Gtk.MessageDialog(
self.builder.get_object("main_window"), 0,
Gtk.MessageType.ERROR, Gtk.ButtonsType.OK,
"Generation raised exception of type \"%s\":\n\n\"%s\"" %
(type(e).__name__, str(e)))
dialog.run()
dialog.destroy()
return True
def link(self,r,t,s,alpha):
LK = np.matrix(self.UU[r][t][s])
W = self.staple(r,t,s)
Nn = Selector.selector(self.N)
Cn = Selector
V = [Cn.CI(self.N) for lt in range(Nn.count())]
ct = 0
while ct < Nn.count():
k,Z = self.findZk(W,ct)
XX = Pendelton.Pendelton(alpha,k).pendlgnr()
VD = np.dot(XX,Z)
V[ct] = Nn.expandv(VD,ct)
W = np.dot(V[ct],W)
ct = ct+1
NU = Cn.CI(self.N)
for q in range(Nn.count()):
NU = np.dot(NU,V[q])
NNU = np.dot(NU,LK)
self.UU[r][t][s] = NNU
return self.UU
from tkinter import *
from Selector import *
root = Tk()
mcanvas = Canvas(root, bg="#222222", bd=3, width=1600, height=800)
mselector = Selector(mcanvas, root)
def Loop():
mselector.LevelSprite()
root.after(10, Loop)
Loop()
mcanvas.pack()
root.mainloop()
self.Str.setBlocking(0) self.Addr = addr self.Sel = sel sel.register(self, rd=sock.fileno()) def doRead(self, fd, sel): if fd == self.Sock.fileno(): self.Str.readMore() while self.Str.msgReady(): msg = self.Str.getMsg() self.Str.send(msg,0) if self.Str.eof(): # client disconnected print 'Client ',self.Addr, 'disconnected' sel.unregister(rd=self.Sock.fileno()) self.Str = None self.Sock.close() if __name__=='__main__': from Selector import * import string import sys if len(sys.argv) < 2: print 'Usage: python ech-server.py <port>' else: sel=Selector() srv = EchoSrv(string.atoi(sys.argv[1]), sel) while 1: sel.select(10)
'-f',
'--filters',
metavar='PRESET',
action='store',
help="""Presets of filters to apply on generated names.""")
#non-options arguments (lexicon files)
argparser.add_argument('files',
metavar='FILE',
action='store',
nargs='*',
default=argparse.SUPPRESS,
help='dictonary files to learn names from')
#Merge configuration files' arguments and cli-given arguments
arguments = Selector.loadDefaultArgumentTree()
cli_arguments = argparsedToArgument(argparser.parse_args())
arguments.update(cli_arguments)
#Custom tweaking of the arguments based on some CLI values
if cli_arguments.contains("number"):
arguments.set(True, 'gui', '<autogenerate_at_start>')
#Set seed if required
seedValue = arguments.get('seed', default='auto')
if type(seedValue) == int:
import random
random.seed(int(seedValue))
#Choose to use GUI or Batch
if arguments.get('gui', 'enabled', default=False):
<html>
<head>
<title>Title of the page</title>
</head>
<body>
<h1>H1 Tag</h1>
<h2>H2 Tag with <a href="#">link</a></h2>
<p>First Paragraph</p>
<p>Second Paragraph</p>
</body>
</html>
'''
## -- End pasted text --
In [3]: sel = Selector(text=html_doc)
In [4]: sel.extract()
Out[4]: '<html>\n <head>\n <title>Title of the page</title>\n </head>\n <body>\n <h1>H1 Tag</h1>\n <h2>H2 Tag with <a href="#">link</a></h2>\n <p>First Paragraph</p>\n <p>Second Paragraph</p>\n </body>\n</html>'
In [5]: sel.xpath('/html/head/title')
Out[5]: [<Selector xpath='/html/head/title' data='<title>Title of the page</title>'>]
In [6]: sel.xpath('/html/head/title/text()')
Out[6]: [<Selector xpath='/html/head/title/text()' data='Title of the page'>]
In [7]: sel.xpath('/html/head/title/text()').extract()
Out[7]: ['Title of the page']
In [8]: sel.xpath('/html/head/title/text()').extract_first()
Out[8]: 'Title of the page'
def main():
"""
The table_extractor is a 2016 GSoC project; here you have the [first project idea]
(wiki.dbpedia.org/ideas/idea/59/the-table-extractor/), assigned to Simone Papalini(s.papalini) and developed for
DBpedia Spotlight organization.
Note: Python 2.7, RDFlib, lxml are strictly required, so please install them in your environment.
LICENSE: Refer to LICENSE file. GNU GENERAL PUBLIC LICENSE. This is a OPEN SOURCE project.
Feel free to download the project, use it, enhance functionalities, add your creativity to the project,
but please SHARE your efforts with the rest of the World.
Refer to [License file](https://github.com/dbpedia/table-extractor/blob/master/LICENSE)
CONTACTS: feel free to e-mail me at: [email protected]
pyTableExtractor is a Python script used to carve out data from tables tou can find in wiki pages and to compose a
RDF data set (.ttl file) with them.
Please refer to project's Readme, [Github project page](https://github.com/dbpedia/table-extractor) and [2016 GSoC
progress page](https://github.com/dbpedia/extraction-framework/wiki/GSoC_2016_Progress_Simone) to understand script's
usage, purposes and options.
pyTableExtractor.py represents the classical usage of classes and modules for this project;
This script shows how you can easily recall modules and classes in order to:
- Test parameters (ParamTester.py)
- Select a set of wikipedia pages/dbpedia resources you want to analyze (Selector.collect_resources())
- Analyze tables you can find in selected resources' pages (Analyzer.analyze()) once they were collected with Selector.
- Serialize the mapping result in order to make a RDF dataset (Analyzer.serialize())
- Print a final report to have metrics to measure efficiency and effectiveness of algorithm and
to have an help writing down new mapping rules. (Utilities.print_report())
Once you have called pyTableExtractor with correct parameters (see DocStrings of ParamTester.py for help), just relax
and wait for the script to serialize a RDF data set (Turtle format) and to report results in a log file.
"""
# First of all a test is run over the parameters passed to the script, see ParamTester.py
p_tester = ParamTester.ParamTester()
# The following values are used by the other classes and set from ParamTester attributes :
# language is a 2 letter string used to select the right wiki/dbpedia chapter and language to use mapping data cells
language = p_tester.lang # default: 'en'
"""
where_clause is a string representing a piece of a SELECT SPARQL query.
It is used to collect resources of a certain scope (Eg all the resources representing actors)
"""
where_clause = p_tester.where # default: "?s a <http://dbpedia.org/ontology/Election>"
"""
topic is a string representing the scope of interest, used to collect resources and to choose the right mapping
rules for data extracted from tables found in those resources.
"""
topic = p_tester.topic # default:"elections"
"""
single_res is a string containing the name of a wiki page (the same of dbpedia in most cases)
NOTE: single_res is not always set, and it is used only in the case a user want to analyze a single wiki page.
"""
single_res = p_tester.single_res # no default value
# mode is a string and could be 'html' or 'json'. User can choose to use the Json or the Html parser with this value
mode = p_tester.mode # default: 'html'
"""
Instancing a Utilities object using correct language and topic.
Utilities would be used from other classes for different purposes (Internet connection, object testing,
get time and date..) and to print a final report.
"""
utils = Utilities.Utilities(language, topic)
"""
res_list_filename is created but not set. In fact, if a user want to test a single resource, there isn't a
list of resources to collect.
"""
res_list_filename = None
# Test if the user chose a single resource
if not single_res:
"""
if not, a Selector object is created, in order to retrieve resources of interest(it depends on the topic chosen)
from dbpedia/wikipedia/jsonpedia.
Note: a selector need 4 parameters, language, where_clause, topic chosen and the instance of utilities mainly
for a statistic purpose.
"""
selector = Selector.Selector(language, where_clause, topic, utils)
# Collecting resources of given topic using collect_resources().
selector.collect_resources()
"""
Retrieve the resources list's filename which the selector has just created (a .txt file you can find in
/Resource_lists/)
"""
res_list_filename = selector.res_list_file
"""
Now we want to analyze the set of resources (or the single one) we just retrieved, so an analyzer is created.
Parameters passed: language, topic, utilities object, mode, list filename and the name of the single resource.
Note: the last two parameters (res_list_filename and single_res) are actually mutual exclusive, so in both cases,
one of them is None.
"""
analyzer = Analyzer.Analyzer(language, topic, utils, mode,
res_list_filename, single_res)
"""
To actually analyze the wiki pages and tables in them, you have to call the analyze() method.
Note: Once analyze has started, it carves out tables from every single resource passed to the Analyzer, and then
trying to apply mapping rules to every single data cells of those tables. See Mapper class to get an idea of the
decision algorithm for the mapping.
"""
analyzer.analyze()
"""
At last, you surely want to serialize the RDF graph obtained with serialize() method.
You can find the .ttl file containing the graph serialized in /Extractions/ along with
the corresponding log file.
"""
analyzer.serialize()
# Finally, print a report for the current extraction, then exits.
utils.print_report()
def testWitnessStreamPerCom():
from WitnessFiltration import WitnessStream
from src.WeakWitnessFiltration import WeakWitnessStream
from src.AbsoluteCohomologyOptimized import *
from src.ComputeInterval import IntervalComputation
# mean = [0, 0]
# cov = [[1, 0], [0, 1]]
# matrixofpoints_inR2 = np.random.multivariate_normal(mean, cov, 100)
# p = pc.PointCloud(matrixofpoints_inR2)
filename = '/Users/naheed/PycharmProjects/PersistenceHomology/data/eight.mat'
h = hpy() # guppy heap object
p = pc.MatlabPointCloud(filename, 'point_cloud')
p.compute_distancematrix()
# Create Selector
# pointcloud_sel = sel.PointCloudSelector(p, 10, "RandomSelector")
pointcloud_sel = sel.PointCloudSelector(p, 500, "MaxminSelector")
# Measuring system time elapsed since the epoch (UTC time-zone)
start_time = time()
pointcloud_sel.select()
elapsed_time = time() - start_time
R = float(pointcloud_sel.get_maxdistance_landmarktoPointcloud())
# R = 0
print 'R = ', R
numdivision = 10
maxdim = 2
# ws = WitnessStream(landmarkselector=pointcloud_sel, maxdistance=R, numdivision=numdivision, maxdimension=maxdim)
ws = WeakWitnessStream(mu=2,
landmarkselector=pointcloud_sel,
maxdistance=R,
numdivision=numdivision,
maxdimension=maxdim)
ws.construct()
print "Total number of Simplices in the Filtration: ", len(ws)
print "After ws.construct():\n", h.heap().size
start_time = time()
cohom = FiltrationArrayCohomologyComputer(filtr=ws,
maxdim=maxdim,
maxfilter=R)
total_mem_consumed = h.heap().size
cohom.compute()
total_mem_consumed = h.heap().size - total_mem_consumed
elapsed_time = time() - start_time
print "time: ", elapsed_time
print "After cohomology:\n", total_mem_consumed
start_time = time()
ci = IntervalComputation(ws)
total_mem_consumed = h.heap().size
ci.compute_intervals(
maxdim
) # I should check everything is ok in this function since the deg for simplices can have real value now.
total_mem_consumed = h.heap().size - total_mem_consumed
ci.print_BettiNumbers()
elapsed_time = time() - start_time
print "time: ", elapsed_time
print "After Homology:\n", total_mem_consumed
import Initializer, Mutator, Recombiner, Selector, Replacer, Terminator, LocalSearcher
from GeneticAlgorithm import GeneticAlgorithm
from ProblemDefinition import ProblemDefinition
from Benchmark import Benchmark
# Choose your operators
initializer = Initializer.RandomInitializer()
mutator = Mutator.RandomMutator(0.2, dynAdapt=True)
recombiner = Recombiner.CrossoverRecombiner()
selector = Selector.TournamentSelector(s=20, dynAdapt=True)
replacer = Replacer.bottomReplacer()
terminator = Terminator.maxRuntimeTerminator(10)
# Add a local searcher if you want LocalSearcher.Idle() does nothing
localSearcher = LocalSearcher.Idle()
# Set up an example problem
nrMachines, jobRuntimes = Benchmark.benchmark1()
probDef = ProblemDefinition(nrMachines, jobRuntimes)
# Set up GA parameters
popSize = 100
nrOffspring = int(popSize / 10)
# Create Genetic Algorithm instance
GA = GeneticAlgorithm(initializer, selector, recombiner, mutator, replacer,
terminator, probDef, popSize, nrOffspring, localSearcher)
bestIndividual, results = GA.run()
print("best Individuals fitness: ", bestIndividual.fitness)
flat1 = Map(0, 750, 1600, mCanvas)
flat2 = Map(80, 600, 360, mCanvas)
flat3 = Map(490, 450, 620, mCanvas)
flat4 = Map(1160, 600, 360, mCanvas)
map1 = [flat1, flat2, flat3, flat4] #check Collison 함수 사용에 용이하기 위해 리스트로 묶음
#전역변수 초기화
unit_speed = 0.5
game_speed = 10 #1ms 단위로 canvas 업데이트 gameLoop()참고
t = 0 #pattern timer 겸사 겸사 score
selected = False #page2에서 Level 골랐는지 확인
death = False
#page
#page0 = Start(root, mCanvas)
page1 = Selector(mCanvas, root)
page3 = Score(root, mCanvas)
#Selector select버튼 대신 Insert 사용
def keyInsert(event):
global selected
selected = True
#초기화 할 것들 정리
def init():
global mainphoto
#global b1
b1 = Button(mCanvas,
help="print additional information like the perplexity." )
argparser.add_argument( '-q', '--quiet', action='store_false', dest='verbose',
help="do not print additional information like the perplexity." )
argparser.add_argument( '-t', '--tokenizer', metavar = 'PRESET', action='store',
help="""algorithm or preset used to split characters/phonems from the names.""")
argparser.add_argument( '-g', '--generator', metavar = 'PRESET', action='store',
help="""algorithm or preset used to generate the names.""" )
argparser.add_argument( '-f', '--filters', metavar = 'PRESET', action='store',
help="""Presets of filters to apply on generated names.""" )
#non-options arguments (lexicon files)
argparser.add_argument( 'files', metavar='FILE', action='store', nargs='*', default=argparse.SUPPRESS,
help='dictonary files to learn names from' )
#Merge configuration files' arguments and cli-given arguments
arguments = Selector.loadDefaultArgumentTree()
cli_arguments = argparsedToArgument( argparser.parse_args() )
arguments.update( cli_arguments )
#Custom tweaking of the arguments based on some CLI values
if cli_arguments.contains("number"):
arguments.set( True, 'gui', '<autogenerate_at_start>' )
#Set seed if required
seedValue = arguments.get('seed',default='auto')
if type(seedValue) == int:
import random
random.seed(int(seedValue))
#Choose to use GUI or Batch
if arguments.get('gui','enabled',default=False):
def listConditions(self):
dialog = Selector(self.category)
if dialog.exec_() == QDialog.Accepted:
self.lineEdit.setText(dialog.retrieveData())
