def main2(self):
if (self.login()):
new = higher_level()
while (1):
#read_sockets,write_sockets,error_sockets = select.select(self.socket_list,[],[])
k = int(
input(
"Select an operation\n1.See details\n2.Create a request\n3.Choose request to fulfill\n4.Exit\n"
))
if k == 1:
print(new)
elif k == 2:
req = input("What is your request?\n")
stri = "create request"
self.c.send(stri.encode('utf-8'))
rec = str(self.c.recv(4096))
if rec[2:len(rec) - 1] == '0':
self.c.send(req.encode('utf-8'))
print("request sent,waiting to find sender")
ip = self.c.recv(4096).decode()
print("recieved something")
fn = ip.split()[1]
ip = ip.split()[0]
fn = "rec" + fn
print("Found sender, downloading...")
lin1 = link()
#time.sleep(10)
lin1.create_client(ip, fn)
else:
print("network error, try again")
elif k == 3:
stri = "show list"
self.c.send(stri.encode('utf-8'))
lst = str(self.c.recv(4096))
print(lst[2:len(lst) - 1])
path, req_num = new.choose_req()
if path != None:
stri = "choosing request"
self.c.send(stri.encode('utf-8'))
rec = str(self.c.recv(4096))
if rec[2:len(rec) - 1] == '0':
self.c.send(path.encode('utf-8'))
rec = str(self.c.recv(4096))
if rec[2:len(rec) - 1] == '0':
self.c.send(str(req_num).encode('utf-8'))
rec = self.c.recv(4096).decode('utf-8')
print(rec)
if (rec == '1'):
print("other user is not active")
elif (rec == '0'):
lin = link()
self.c.send(str('true').encode())
print("sent conf")
lin.create_server(new.ip_address, path)
elif k == 4:
self.c.send(str("close").encode())
#self.c.shutdown(self.c.SHUT_RDWR)
self.c.close()
break
def createnetwork(points, polygons):
#points : array with all the coordinates of the vertices
#polygons : array with n-integer-array for each cell that
#represent the numero of the vertices that are in this cell
vertices = []
links = []
cells = []
# Find the boundary vertices
boundary = ConvexHull(points)
#Creation of the vertices with the argument 1 for boundary and 0 for bulk
for i in range(len(points)):
if i in boundary.vertices:
vertices.append(vertice(i, points[i], 1))
else:
vertices.append(vertice(i, points[i], 0))
#Creation of the cells and adding the vertices
for i in range(len(polygons)):
cells.append(cell(i))
for j in range(len(polygons[i])):
cells[i].add_vertice(vertices[polygons[i][j]])
cells[i].orderingvertices()
#Creation of the links and adding to the cells.
for i in range(len(cells)):
for j in range(len(cells[i].vertices)):
if j != len(cells[i].vertices) - 1:
link_0 = link([cells[i].vertices[j], cells[i].vertices[j + 1]],
"membrane")
cells[i].add_link(link_0)
else:
link_0 = link([cells[i].vertices[j], cells[i].vertices[0]],
"membrane")
cells[i].add_link(link_0)
if link_0 not in links:
links.append(link_0)
for j in range(len(cells[i].vertices)):
if j == 0:
for k in range(2, len(cells[i].vertices) - 1):
link_0 = link([cells[i].vertices[j], cells[i].vertices[k]],
"inside")
cells[i].add_link(link_0)
links.append(link_0)
else:
for k in range(j + 2, len(cells[i].vertices)):
link_0 = link([cells[i].vertices[j], cells[i].vertices[k]],
"inside")
cells[i].add_link(link_0)
links.append(link_0)
return [vertices, links, cells]
def createnetwork(points, polygons):
#points : array with all the coordinates of the vertices
#polygons : array with n-integer-array for each cell that
#represent the numero of the vertices that are in this cell
vertices = []
links = []
cells = []
#Creation of the vertices with the argument 1 for boundary and 0 for bulk
for i in range(len(points)):
vertices.append(vertice(i, points[i]))
#Creation of the cells and adding the vertices
for i in range(len(polygons)):
cells.append(cell(i))
for j in range(len(polygons[i])):
cells[i].add_vertice(vertices[polygons[i][j]])
cells[i].orderingvertices()
cells[i].A0 = cells[i].area()
cells[i].A_ac = cells[i].A0
cells[i].K = 10.
# Finding the vertices on the boundary
count = np.zeros(len(vertices))
for i in range(len(cells)):
for j in range(len(cells[i].vertices)):
count[cells[i].vertices[j].n] += 1
for i in range(len(count)):
if count[i] < 3:
vertices[i].boundary = True
for i in range(len(cells)):
for j in range(len(cells[i].vertices)):
if cells[i].vertices[j].boundary == True:
cells[i].boundary = True
#Creation of the links and adding to the cells.
for i in range(len(cells)):
for j in range(len(cells[i].vertices)):
if j != len(cells[i].vertices) - 1:
link_0 = link([cells[i].vertices[j], cells[i].vertices[j + 1]])
cells[i].add_link(link_0)
else:
link_0 = link([cells[i].vertices[j], cells[i].vertices[0]])
cells[i].add_link(link_0)
if link_0 not in links:
links.append(link_0)
return [vertices, links, cells]
def run(args):
if args.input_file:
with open(args.input_file, "r") as input_file:
source = input_file.read()
else:
import sys
source = sys.stdin.read()
if args.assemble:
with open(args.output, "w") as binout:
write_binary(parse_instructions(source), binout)
else:
tree = parse(preprocess(source))
if args.print_ast:
print_tree(tree)
if not tree:
return
instructions = link(*translate(tree))
if args.print_assembly:
print_instructions(instructions)
with open(args.output, "w") as binout:
write_binary(instructions, binout)
def get(self):
userid = self.request.get("userid")[:80]
if userid == None or userid == "":
userid = str(random.randint(1, 10 ** 10))
global games, gametexts
if userid in games:
cmd = self.request.get("content")[:80]
if cmd == "restart":
del games[userid]
else:
try:
gametexts[userid] = games[userid].send(cmd).replace("\n", "<br>").replace(" ", " ")
gametexts[userid] = gametexts[userid].replace("@@img@@", "img src").replace("@@width@@", " width")
except StopIteration:
del games[userid]
if not userid in games:
games[userid] = link.link()
gametexts[userid] = games[userid].next().replace("\n", "<br>")
template_path = os.path.join(os.path.dirname(__file__), "index.html")
html = template.render(template_path, dict(text=gametexts[userid], userid=userid))
self.response.out.write(html)
def run():
poolSize = min(len(configs), os.cpu_count());
p = Pool(processes=poolSize);
for l in [link.link(k, 1, k) for k in configs]:
p.apply_async(runSpider, args=(l,));
redisClient.sadd("runingQueue", *list(configs.keys()));
p.close();
p.join();
def watch():
siteQueue = redisQueue("spiderSite");
while 1:
site = siteQueue.blockDequeue();
if site is not None:
addSiteConfig(site);
l = link.link(site["_id"], 1, site["_id"]);
p = Process(target=runSpider, args=(l,));
p.start();
def runSpider(url):
try:
reader = novelSpider(url);
reader.run();
reader.closeTransaction();
now = time.strftime('%y-%m-%d', time.localtime(time.time()));
pipe = redisClient.pipeline();
pipe.set("lastTime_" + url.host, now);
pipe.srem("runingQueue", url.host);
pipe.spop("runingQueue");
r = pipe.execute();
host = r[2];
if host is None:
return;
host = host.decode("utf-8");
if redisClient.exists(host):
url = link.link(host, 1, host);
runSpider(url);
except Exception as ex:
print("异常类型:%s,消息：%s" % (type(ex), ex));
def run(args):
if args.input_file:
with open(args.input_file, 'r') as input_file:
source = input_file.read()
else:
import sys
source = sys.stdin.read()
if args.assemble:
with open(args.output, 'w') as binout:
write_binary(parse_instructions(source), binout)
else:
tree = parse(preprocess(source))
if args.print_ast:
print_tree(tree)
if not tree:
return
instructions = link(*translate(tree))
if args.print_assembly:
print_instructions(instructions)
with open(args.output, 'w') as binout:
write_binary(instructions, binout)
def do_link(self):
return link(), noduplicates()
def do_link(self):
from link import link
self.links += [link()]
return self.links[-1]
def main(argv):
# Check command line parameters
if len(sys.argv) != 3:
print "python networksimulation.py (0,1,2) (0,1)"
print "1st arg: con_ctrl: 0 - none; 1 - TCP Reno; 2 - FAST"
print "2nd arg: verbose: 0 - no; 1 - yes"
print('\n')
sys.exit()
# Store command line parameters
con_ctrl = int(sys.argv[1])
verbose = int(sys.argv[2])
print('\n')
# Give specifications for links file and flows file and then save
# the filename in variables
print("Enter the file of links.")
print("Each line is one link in the following format: ")
print("End1 End2 Rate Delay Capacity\n")
print("Ends=host/source/destination/router ID (ex. H1,S2,T3,R4)")
print("Rate(Mbps), Delay(ms), Capacity(KB)")
linkFile = input('Enter link file: ')
print('\n')
print("Enter the file of flows.")
print("Each line is one flow in the following format: ")
print("Source Dest DataAmount StartTime\n")
print("Source/Dest = ID's (ex. H1,S2,T3,R4)")
print("DataAmount(MB), StartTime(s)")
flowFile = input('Enter flow file: ')
print('\n')
# Variables to store the different network objects
routers = {}
linksLst = []
flowsLst = []
# Try to open the files. If fail, throw an error.
try:
links = open(linkFile, 'r')
flows = open(flowFile, 'r')
except IOError:
print("Link and Flow files do not exist in this directory.")
sys.exit()
# First, make all the links
i = 0
for line in links:
# Split the line into a list, which must have 5 values
info = line.split(' ')
if(len(info) != 5):
print('Each link line must have 5 values')
sys.exit()
# Try converting the rate, delay and capacity. If fail, throw error
try:
rate, delay, capacity = float(info[2]), float(info[3]), float(info[4])
except ValueError:
print("The last 3 values must be numbers")
sys.exit()
# Either make a new entry in the routers table, or grab an entry
# First get the end1 of the link
if(info[0] not in routers.keys()):
router1 = router(info[0])
routers[info[0]] = router1
else:
router1 = routers[info[0]]
# Next get end2
if(info[1] not in routers.keys()):
router2 = router(info[1])
routers[info[1]] = router2
else:
router2 = routers[info[1]]
# Build a link with all the information and add it to the list of links.
temp = link(i, router1, router2, rate, delay, capacity)
linksLst.append(temp)
i += 1
# Now, make all the flows.
for line in flows:
# Split the line into a list, which must have 4 values
info = line.split(' ')
if(len(info) != 4):
print('Each flow line must have 4 values')
sys.exit()
# Try converting the amount and start time. If fail, throw error.
try:
amount, start = int(info[2]), float(info[3])
except ValueError:
print("The last 2 values must be numbers")
sys.exit()
# Try getting the source and dest from the dictionary. If they
# don't exist, it means no link was connected to them, so we
# can't have a flow.
try:
source = routers[info[0]]
destination = routers[info[1]]
except KeyError:
print("Network not connected")
sys.exit()
# Build a flow with all the information and add it to the list of flows.
temp = flow(source, destination, amount, start, con_ctrl)
flowsLst.append(temp)
# Make a list of all the routers using the routers dictionary.
routersLst = []
for key in routers:
routersLst.append(routers[key])
# intitialize an event queue
the_event_queue = event_queue(verbose)
# Statically initialize all the routing tables
globalTable = sP.fillTable(linksLst, flowsLst)
for fl in flowsLst:
fl.src.updateStatic(globalTable)
for rt in routersLst:
rt.updateStatic(globalTable)
# Start the simulation
run_simulation(the_event_queue, flowsLst, linksLst, routersLst, con_ctrl)
def openLinkBox(self, box):
"""Link to another note"""
link.link(self, box)
def do_atom_link(self):
from link import link
return link()
def do_link(self):
from link import link
self.links += [link()]
return self.links[-1]
from link import link
from measures import mu, sigma, p_correlation
from plot import Plotter
from copy import deepcopy
import networkx as nx
# Graph definition
n = 500
G = nx.random_geometric_graph(n, 0.05)
adj = list(G.adjacency())
# Derived structural information
means = list(map(lambda x: mu(x, n), adj))
variances = sigma(adj, means)
similarities = p_correlation(adj, means, variances)
# Hierarchical clustering
stree = link(deepcopy(similarities), max)
ctree = link(deepcopy(similarities), min)
splotter = Plotter(G, stree, "slink.html")
cplotter = Plotter(G, ctree, "clink.html")
level = 450
splotter.plot(level)
cplotter.plot(level)
if __name__ == '__main__':
pass
def do_link(self):
self.metadata()
return link(), noduplicates()
def do_link(self):
return link(), noduplicates()
def parse_html_to_postgres(input_folder,
output_html,
merge_folder,
output_words,
output_equations,
db_connect_str,
strip_tags,
ignored_file_when_link,
csv_file,
corenlp_fd,
store_into_postgres=True):
"""
Helper function for database ingestion.
:param input_folder: Location of input folder containing source HTML files.
:param output_html: Intermediate HTML files which will be consumed by the Fonduer parser.
:param merge_folder: Location of folder containing the merged HTML files.
:param output_words: Location of folder containing word coordinate json files.
:param output_equations: Location of folder containing equation coordinate json files.
:param db_connect_str: Database connection string.
:param strip_tags: Tags to be flatten.
:param ignored_file_when_link: Files to be ignored when linking.
:param csv_file: Location of the output csv file
:param corenlp_fd: Location of the CoreNLP java file.
:param store_into_postgres: Flag for whether to ingest data into Postgres.
"""
assert os.path.isabs(input_folder)
assert os.path.isabs(output_html)
assert os.path.isabs(merge_folder)
assert os.path.isabs(output_words)
"""
# 1. group files by file name
merge.stamp: pagemerger.py
rm -r -f $(merge_folder)
mkdir -p $(merge_folder)
python pagemerger.py --rawfolder $(input_folder) --outputfolder $(merge_folder)
@touch merge.stamp
"""
if os.path.exists(merge_folder):
shutil.rmtree(merge_folder)
os.makedirs(merge_folder, exist_ok=True)
pagemerger(input_folder, merge_folder)
"""
# 2. preprocess the input html and store intermediate json and html in the output folder declared above.
preprocess.stamp: preprocess.py merge.stamp
rm -r -f $(output_html)
rm -r -f $(output_words)
mkdir -p $(output_html)
mkdir -p $(output_words)
@$(foreach file,$(all_inputs),\
python preprocess.py --input $(merge_folder)$(file) --output_words $(output_words)$(file).json --output_html $(output_html)$(file);)
@touch preprocess.stamp
"""
if os.path.exists(output_html):
shutil.rmtree(output_html)
if os.path.exists(output_words):
shutil.rmtree(output_words)
if os.path.exists(output_equations):
shutil.rmtree(output_equations)
os.makedirs(output_html, exist_ok=True)
os.makedirs(output_words, exist_ok=True)
os.makedirs(output_equations, exist_ok=True)
all_inputs = [f for f in os.listdir(merge_folder)]
for html_file in all_inputs:
preprocess(
os.path.join(merge_folder, html_file),
"%s.json" % (os.path.join(output_words, html_file)),
os.path.join(output_html, html_file),
"%s.json" % (os.path.join(output_equations, html_file)),
"%s.json" % (os.path.join(output_words, 'path_info_' + html_file)),
strip_tags)
if store_into_postgres:
"""
# 3. run the fonduer parser on the generated html file. This will fill in the postgres dabase with everything
# fonduer can understand except the coordinate information.
parse.stamp: preprocess.stamp parse.py
python parse.py --html_location $(output_html) --database $(db_connect_str)
@touch parse.stamp
"""
parse(output_html, db_connect_str)
"""
# 4. run the link file to insert coordinate information into fonduer based on the information from the json output folder (aka. hocr)
link.stamp: parse.stamp link.py
python link.py --words_location $(output_words) --database $(db_connect_str)
@touch link.stamp
"""
link(output_words, db_connect_str, ignored_file_when_link)
insert_equation_tuple(db_connect_str, output_equations)
var_in_text(db_connect_str)
build_table_X(db_connect_str, corenlp_fd)
generate_csv(db_connect_str, csv_file)
def parse_html_to_postgres(input_folder,
output_html,
db_connect_str,
strip_tags,
ignored_file_when_link,
store_into_postgres=True):
"""
Helper function for database ingestion.
:param input_folder: Location of input folder containing source XML files from image segmentation.
:param output_html: Intermediate HTML files which will be consumed by the Fonduer parser.
:param db_connect_str: Database connection string.
:param strip_tags: Tags to be flatten.
:param ignored_file_when_link: Files to be ignored when linking.
:param store_into_postgres: Flag for whether to ingest data into Postgres.
"""
# assert os.path.isabs(input_folder)
# assert os.path.isabs(output_html)
# assert os.path.isabs(merge_folder)
# assert os.path.isabs(output_words)
"""
# 1. group files by file name
merge.stamp: pagemerger.py
rm -r -f $(merge_folder)
mkdir -p $(merge_folder)
python pagemerger.py --rawfolder $(input_folder) --outputfolder $(merge_folder)
@touch merge.stamp
"""
# if os.path.exists(merge_folder):
# shutil.rmtree(merge_folder)
# os.makedirs(merge_folder, exist_ok=True)
merged_file = pagemerger(input_folder)
"""
# 2. preprocess the input html and store intermediate json and html in the output folder declared above.
preprocess.stamp: preprocess.py merge.stamp
rm -r -f $(output_html)
rm -r -f $(output_words)
mkdir -p $(output_html)
mkdir -p $(output_words)
@$(foreach file,$(all_inputs),\
python preprocess.py --input $(merge_folder)$(file) --output_words $(output_words)$(file).json --output_html $(output_html)$(file);)
@touch preprocess.stamp
"""
if os.path.exists(output_html):
shutil.rmtree(output_html)
# if os.path.exists(output_words):
# shutil.rmtree(output_words)
# if os.path.exists(output_equations):
# shutil.rmtree(output_equations)
os.makedirs(output_html, exist_ok=True)
# os.makedirs(output_words, exist_ok=True)
# os.makedirs(output_equations, exist_ok=True)
# all_inputs = [f for f in os.listdir(merge_folder)]
all_words = {}
all_equations = {}
for filename, tree in merged_file.items():
words, equations = preprocess(tree,
os.path.join(output_html,
filename), strip_tags)
all_words[filename] = words
all_equations[filename] = equations
if store_into_postgres:
"""
# 3. run the fonduer parser on the generated html file. This will fill in the postgres dabase with everything
# fonduer can understand except the coordinate information.
parse.stamp: preprocess.stamp parse.py
python parse.py --html_location $(output_html) --database $(db_connect_str)
@touch parse.stamp
"""
parse(output_html, db_connect_str, parallelism=1)
"""
# 4. run the link file to insert coordinate information into fonduer based on the information from the json output folder (aka. hocr)
link.stamp: parse.stamp link.py
python link.py --words_location $(output_words) --database $(db_connect_str)
@touch link.stamp
"""
link(all_words, db_connect_str, ignored_file_when_link)
insert_equation_tuple(db_connect_str, all_equations)
import argparse
from link import link
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--products_file', type=str, default='data/products.txt')
parser.add_argument('--listings_file', type=str, default='data/listings.txt')
parser.add_argument('--output_file', type=str, default='out.txt')
return parser.parse_args()
if __name__ == '__main__':
args = parse_args()
link(args.products_file, args.listings_file, args.output_file)
def announce(func):
links = ", ".join(link(f) for f in sorted(function_index[func]))
print(f'* `{func}`: {links}')
def do_link(self): self.metadata() from link import link self.links.append(link()) return self.links[-1]
def do_atom_link(self):
self.metadata()
from link import link
self.links.append(link())
return self.links[-1]
def do_atom_link(self): from link import link return link()
def do_link(self):
self.metadata()
return link(), noduplicates()
# coding: utf-8
import os
def lstat(filename):
return os.lstat(filename)
if __name__ == '__main__':
import link
link.link('test.txt', 'test')
print(lstat('test'))
def do_link(self):
self.metadata()
from link import link
self.links += [link()]
return self.links[-1]
def do_link(self): self.metadata() from link import link self.links += [link()] return self.links[-1]
import core
import link
def main():
try:
if len(sys.argv) < 1:
raise RuntimeError("No repo supplied")
dir = sys.argv[1]
if os.path.basename(dir) != dir:
raise RuntimeError(dir + " is an invalid repo")
dir = os.path.join(os.getcwd(), dir)
if os.path.exists(dir):
raise RuntimeError(dir + " already exists!")
except Exception, err:
sys.stderr.write("ERROR: %s\n" % str(err))
return 1
try:
os.mkdir(dir)
os.chdir(dir)
core.sh("git init")
link.link(os.path.join(dir, ".git"))
return 0
except Exception, err:
sys.stderr.write("ERROR: %s\n" % str(err))
shutil.rmtree(dir)
return 1
if __name__ == "__main__":
sys.exit(main())
