def create_index():
startTime = datetime.now()
#index the testbeds
for i in range(1, 17):
index("testbeds/testbed" + str(i), i)
print("Time taken: " + str(datetime.now() - startTime))
def init():
indexDir = constants.getIndexPath()
indexFilePath = os.path.join(indexDir, constants.getDirIndexName())
if not os.path.isfile(indexFilePath):
createClassesFile()
index()
classes = constants.getClasses()
# if not classes:
# error.noClasses()
# return
for c in classes:
if not os.path.isfile(os.path.join(indexDir, c)):
index()
print(":indexed")
folders = utils.initFoldersList(indexFilePath)
files = utils.initFilesList(classes, indexDir)
return folders, files
def login(index1):
user = username.get() #获得用户名
pd = password.get() #获得密码
db = MySQLdb.connect("localhost","root","142118","shujuku")
cursor = db.cursor()
sql = "select * from USER";
try:
cursor.execute(sql)
results = cursor.fetchall()
except:
print("sql exec error!")
db.close()
word = 0
for row in results:
if((row[0] == user) & (row[1] == pd)):
word = 1
break
if(word == 0):
win1 = Toplevel()
win1.title("提示消息")
win1.geometry("400x200")
Label(win1, text="账户或者密码输入错误!").place(x = 130, y = 50)
win1.mainloop()
elif(word == 1):
win.destroy()
if(index1 == 1):
index.index(user, pd)
elif(index1 == 2):
index_stu.index_stu(user,pd)
def create_index():
startTime = datetime.now()
#index the testbeds
for i in range(1, 17):
index("testbeds/testbed"+str(i), i)
print ("Time taken: "+str(datetime.now() - startTime))
def build():
call(["clear"])
print("Building....")
functions.rm("build")
index.index()
topic.top()
posts.make_posts()
functions.cp_dir("src/templates/images", "build/images")
functions.cp_dir("src/templates/css", "build/css")
functions.cp_dir("src/templates/fonts", "build/fonts")
print("Built")
def test_index():
assert index([1, 2, 3, 4], 2) == 9
assert index([1, 3, 10, 100], 3) == 1000000
assert index([0, 1], 0) == 1
assert index([1, 2], 3) == -1
assert index([0], 0) == 1
assert index([1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 9) == 1
assert index([1, 1, 1, 1, 1, 1, 1, 1, 1, 100], 9) == 1000000000000000000
assert index([29, 82, 45, 10], 3) == 1000
assert index([6, 31], 3) == -1
assert index([75, 68, 35, 61, 9, 36, 89, 0, 30], 10) == -1
def updateUnique(connection, collection, document):
"""
Getting the data from collection matching the query
:param connection: the Solr Connection
:param collection: the Solr Collection
:param query: Solr Query
:param rows: number of rows to return
:return: the list of documents returned by Solr
"""
query = 'id:' + document['id']
index.index(connection, collection, [document])
def open_sql():
try:
sql_data = "select * from url"
cursor = Cmysql.sql.conn.cursor()
n = cursor.execute(sql_data)
cursor.scroll(0)
for row in cursor.fetchall():
index.index(row[0]) #url地址
chanpen.index_data() #生成文章
download.download_index(row[4]) #
cursor.close()
except:
pass
def Login(event=None):
Database()
if USERNAME.get() == "" or PASSWORD.get() == "":
lbl_text.config(text="Please complete the required field!", fg="red")
else:
cursor.execute("SELECT * FROM `member` WHERE `username` = ? AND `password` = ?", (USERNAME.get(), PASSWORD.get()))
if cursor.fetchone() is not None:
index()
else:
lbl_text.config(text="Invalid username or password", fg="red")
USERNAME.set("")
PASSWORD.set("")
cursor.close()
conn.close()
def search(request):
kws = []
term = ''
form = SearchForm(request.REQUEST)
term = request.REQUEST.get('search_term', '').strip()
if not term:
from index import index
return index(request)
library_group = LibraryGroup.get_or_create('official')
kws = Keyword.all().ancestor(library_group).search(term,
properties=['name', 'doc'])
offset = int(request.REQUEST.get('page', '1').strip())
limit = int(request.REQUEST.get('limit', '50').strip())
keyword_count = kws.count()
if keyword_count > limit:
page_url = "?search_term=%s&limit=%s&page=PAGE" % (term, limit)
page_nav = Paging(keyword_count, offset, page_url, limit)
else:
page_nav = None
from library import KeywordDoc
start_index, end_index = (offset -1 ) * limit, offset * limit
kws = [ KeywordDoc(e, e.library) for e in kws.fetch(limit, start_index) ]
result_page = kws and "search.html" or "not_found_keyword.html"
return render_to_response(result_page, {'form': form, 'kws': kws,
'term':term,
'keyword_count': keyword_count,
'page_nav':page_nav})
def get_index_page():
idxpg = index.index()
idxpg.contenttype="Content-Type: text/html\n"
idxpg.title="SNPDecomp - Main"
idxpg.availScores=data.getAvailScores()
idxpg.snpscorereq=snpscorereq
return idxpg
def test_full_dictionary(self):
index_object = index.index('test_data/config_file.yaml')
result_converted = convert(index_object)
self.assertEqual({'the': {'test_data/full_file_dirty.txt': [1]},
'bush': {'test_data/full_file_dirty.txt': [1]},
'as': {'test_data/full_file_dirty.txt': [1, 2]},
'bulgaria': {'test_data/full_file_dirty.txt': [1, 3]}}, result_converted)
def do_index(self, args):
"""Index papers in iota paperdir
iota> index
"""
sexps = index(self.database, self.args)
self.print_sexp(sexps)
def do_GET(self):
if self.path == "/":
self.path = "/index"
sendReply = False
if self.path.endswith("index") or self.path.endswith(
"assignments") or self.path.endswith("freelancers"):
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
# Start
self.wfile.write(
bytes(
"<html>" + "<head>" +
"<link rel=\"stylesheet\" type=\"text/css\" href=\"head.css\">"
+ "<!-- Latest compiled and minified CSS -->" +
"<link rel=\"stylesheet\" href=\"https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css\" integrity=\"sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u\" crossorigin=\"anonymous\">"
+ "<!-- Optional theme -->" +
"<link rel=\"stylesheet\" href=\"https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap-theme.min.css\" integrity=\"sha384-rHyoN1iRsVXV4nD0JutlnGaslCJuC7uwjduW9SVrLvRYooPp2bWYgmgJQIXwl/Sp\" crossorigin=\"anonymous\">"
+ "<!-- Latest compiled and minified JavaScript -->" +
"<script src=\"https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js\" integrity=\"sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa\" crossorigin=\"anonymous\"></script>"
+ "</head>" + "<body bgcolor=\"#487cbc\">", "UTF-8"))
# Document header
self.wfile.write(header())
# Document body
if self.path.endswith("index"):
self.wfile.write(index())
elif self.path.endswith("assignments"):
self.wfile.write(assignments())
elif self.path.endswith("freelancers"):
self.wfile.write(freelancers())
# End
self.wfile.write(bytes("</div></body>" + "</html>", "UTF-8"))
sendReply = False
if self.path.endswith(".jpg"):
mimetype = 'image/jpg'
sendReply = True
if self.path.endswith(".gif"):
mimetype = 'image/gif'
sendReply = True
if self.path.endswith(".png"):
mimetype = 'image/png'
sendReply = True
if self.path.endswith(".js"):
mimetype = 'application/javascript'
sendReply = True
if self.path.endswith(".css"):
mimetype = 'text/css'
sendReply = True
if sendReply == True:
#Open the static file requested and send it
f = open(curdir + sep + self.path, 'rb')
self.send_response(200)
self.send_header('Content-type', mimetype)
self.end_headers()
self.wfile.write(f.read())
f.close()
def worker_run(args):
# parse parameters
table = load_table(args['<table-file>'])
min_res = table['min_resolution']
max_res = table['max_resolution']
lattice_constants = table['lattice_constants']
centering = table['centering']
transform_matrix = calc_transform_matrix(lattice_constants)
inv_transform_matrix = np.linalg.inv(transform_matrix)
seed_pool_size = int(args['--seed-pool-size'])
seed_len_tol = float(args['--seed-len-tol'])
seed_angle_tol = float(args['--seed-angle-tol'])
seed_hkl_tol = float(args['--seed-hkl-tol'])
eval_hkl_tol = float(args['--eval-hkl-tol'])
centering_weight = float(args['--centering-weight'])
refine_mode = args['--refine-mode']
refine_cycles = int(args['--refine-cycles'])
miller_file = args['--miller-file']
nb_top = int(args['--top-solutions'])
verbose = args['--verbose']
if miller_file is not None:
miller_set = np.loadtxt(miller_file)
else:
miller_set = None
multi_index = args['--multi-index']
nb_try = int(args['--nb-try'])
stop = False
while not stop:
job = comm.recv(source=0)
for i in range(len(job)):
peaks = load_peaks(job[i]['peak path'],
min_res=min_res,
max_res=max_res)
np.savetxt('peaks.txt', peaks, fmt='%.4E')
print('worker %d working on %s' % (rank, job[i]['peak path']),
flush=True)
solutions = index(peaks,
table,
transform_matrix,
inv_transform_matrix,
seed_pool_size=seed_pool_size,
seed_len_tol=seed_len_tol,
seed_angle_tol=seed_angle_tol,
seed_hkl_tol=seed_hkl_tol,
eval_hkl_tol=eval_hkl_tol,
centering=centering,
centering_weight=centering_weight,
refine_mode=refine_mode,
refine_cycles=refine_cycles,
miller_set=miller_set,
nb_top=nb_top,
multi_index=multi_index,
nb_try=nb_try,
verbose=verbose)
job[i]['solutions'] = solutions
comm.send(job, dest=0)
stop = comm.recv(source=0)
print('worker %d is exiting' % rank)
def __init__(self, p_dSkin=DEFAULT_CFG):
self.check_cfg_file()
self.dCFG = p_dSkin.copy()
self.m_sSkinPath = os.path.join(SCRIPT_DIR, self.dCFG['style'])
self.m_oIndex = index()
self._init_pygame()
self.m_oWach = change_watcher(self.m_lLines, self.m_iLine)
self._create_threads()
self.run()
def main():
msg = '''
----------------欢迎使用奇葩信用卡专用ATM机-----------------
########################################################
** *************** **** ****
**** *************** ** ** ** **
** ** *** ** ** ** **
** ** *** ** ** ** **
********** *** ** ** ** **
************ *** ** ** ** **
** ** *** ** ** ** **
** ** *** ** ** **
** ** *** ** ** **
########################################################
'''
print(msg)
index(username)
def main(mode):
if mode == 'game':
import launchGame
launchGame.launchGame(params)
elif mode == 'games':
import listGames
listGames.listGames(params)
elif mode == 'clients':
import listClients
listClients.listClients(params)
elif mode == 'noauth':
import listClients
listClients.no_authentification(params)
elif mode == 'achievements':
import listAchievements
listAchievements.listAchievements(params)
elif mode == 'movies':
import listMovies
listMovies.listMovies(params)
elif mode == 'wakeIndex':
import wakeIndex
wakeIndex.wakeIndex()
elif mode == 'wake':
import wake
wake.wake(params)
elif mode == 'clearMacs':
try:
import cPickle as pickle
except:
import pickle
pickle.dump(set(), open(os.path.join(utils.USER_DATA, "macs.pickle"), "w+"))
elif mode == 'reset':
main("clearMacs")
utils.addon.setSetting("invalid_steam_api_key", "false")
utils.addon.setSetting("steam_api_key", "")
utils.addon.setSetting("prescript", "")
utils.addon.setSetting("postscript", "")
utils.addon.setSetting("first_run", "true")
elif mode == 'settings':
utils.addon.openSettings()
else:
import index
index.index()
def handle_one_request(self, client_address):
self.request_data = request_data = self.client_connection.recv(1024)
# Print formatted request data a la 'curl -v'
print('< Client IP: ' + str(client_address))
index.index(client_address,
self.get_ip()) # script to change index.html
print(''.join('< {line}\n'.format(line=line.decode())
for line in request_data.splitlines()))
self.parse_request(request_data)
# Construct environment dictionary using request data
env = self.get_environ()
# It's time to call our application callable and get
# back a result that will become HTTP response body
result = self.application(env, self.start_response)
# Construct a response and send it back to the client
self.finish_response(result)
def test_Find():
"Check that the Index has the right number of results"
# set up db
dbpath = tempfile.mkdtemp()
db = xapian.WritableDatabase(dbpath, xapian.DB_CREATE_OR_OPEN)
# fake arguments
Args = namedtuple('Args', ['paperdir'])
args = Args(TEST_PAPERDIR)
# do the indexing
index.index(db, args)
# do a query
Args = namedtuple('Args', ['query'])
args = Args("relativity")
results = find.find(db, args)
print(results)
def cli():
print('Welcome to my pathetic file indexer.')
help_cli()
while True:
cmd = input('>> ')
if cmd == 'list':
print('\n'.join(map(str, os.listdir(datadir))))
elif cmd == 'index':
index(datadir, indexfile)
elif cmd == 'query':
if not os.path.isfile(indexfile):
print("""
Ooops, your files seem not to be indexed.
To initiate index type <index>.
""")
else:
query_cli(indexfile)
else:
help_cli()
def test_rand(self):
def solution(array, n):
if len(array) <= n:
return -1
return (array[n]**n)
from random import randint as R
for _ in range(100):
n = R(0, 10)
a = [R(0, 20) for _ in range(R(0, 10))]
self.assertEqual(index(a[:], n), solution(a, n))
def main():
user_name = ""
tweets = ""
if request.method == "POST":
# リクエストの取得
user_name = request.form["username"]
count = request.form["count"]
# ツィートの取得
tweets = get_timeline.getUserTimeLine(user_name, count)
return index.index(user_name, tweets)
def index(self):
# d-spacings <=8
d1 = float(self.ui.d1box.toPlainText())
d2 = float(self.ui.d2box.toPlainText())
d3 = float(self.ui.d3box.toPlainText())
d4 = float(self.ui.d4box.toPlainText())
d5 = float(self.ui.d5box.toPlainText())
d6 = float(self.ui.d6box.toPlainText())
d7 = float(self.ui.d7box.toPlainText())
d8 = float(self.ui.d8box.toPlainText())
# operation = self.ui.operation.currentText()
#use_q = self.ui.use_q.currentText()
model_path = self.ui.modelPath.text()
num_d = int(self.ui.num_d.toPlainText())
# run the ANN ?
"""
try:
scaler = joblib.load(scalerPath)
except FileNotFoundError as e:
print(e)
print(e.filename)
self.ui.error_toast.setText("Scaler file not found: {}".format(e.filename))
return
"""
try:
m = utils.load_model_scaler(model_path, num_d)
except FileNotFoundError as e:
self.ui.error_toast.setText("File not found: {}".format(e.filename))
return
except utils.NoModelFoundException:
self.ui.error_toast.setText("The given file {} doesn't have a saved model for {} d_spacings.".format(model_path, num_d))
return
model_dict = m['model']
model = deep_d.SimpleNet(num_spacings=num_d)
model.load_state_dict(model_dict)
scaler = m['scaler']
result, percent_error = index.index(np.array([d1, d2, d3, d4, d5, d6, d7, d8][:num_d]).reshape(1,-1), model, scaler=scaler)
# results i an array of shape (,3)
print(result)
# output result
lattice_a = result[0]
lattice_b = result[1]
lattice_gam = np.degrees(result[2])
a_string = format_decimal(lattice_a)
self.ui.a_box.setText(a_string)
b_string = format_decimal(lattice_b)
self.ui.b_box.setText(b_string)
gam_string = format_decimal(lattice_gam)
self.ui.gam_box.setText(gam_string)
error_string = format_decimal(percent_error)
self.ui.error_box.setText(error_string)
def application(environ, start_response):
script_name = environ["PATH_INFO"]
if script_name == "/" or script_name == "/index.html":
return index.index(environ, start_response)
elif script_name == "/form":
return form.login(environ, start_response)
elif script_name == "/imageloader":
return imageloader.loadImage(environ, start_response)
elif script_name == "/authmobile":
return authmobile.auth(environ, start_response)
elif script_name == "/ressource":
return ressource.ressource(environ, start_response)
def application(environ, start_response):
script_name = environ["PATH_INFO"]
if script_name == "/" or script_name == "/index.html":
return index.index(environ, start_response)
elif script_name == "/form":
return form.login(environ, start_response)
elif script_name == "/imageloader":
return imageloader.loadImage(environ, start_response)
elif script_name == "/authmobile":
return authmobile.auth(environ, start_response)
elif script_name == "/ressource":
return ressource.ressource(environ, start_response)
def test_Index():
"Check that the Index has the right number of results"
# set up db
dbpath = tempfile.mkdtemp()
db = xapian.WritableDatabase(dbpath, xapian.DB_CREATE_OR_OPEN)
# fake arguments
Args = namedtuple('Args', ['paperdir'])
args = Args(TEST_PAPERDIR)
result = index.index(db, args)
assert result['count'] == 7
def worker_run(args):
print('worker(%d) running' % rank)
det_dist = float(args['<DET-DIST>'])
sort_by = args['--sort-by']
refine = args['--refine']
show_progress = args['--show-progress']
seed_len_tol = float(args['--seed-len-tol'])
seed_angle_tol = float(args['--seed-angle-tol'])
seed_pair_num = int(args['--seed-pair-num'])
sort_by = None if sort_by == 'none' else sort_by
refine = True if refine == 'true' else False
show_progress = True if show_progress == 'true' else False
table_file = args['<TABLE-FILE>']
photon_energy_list = list(
map(float, args['<PHOTON-ENERGY-LIST>'].split(',')))
table = load_table(table_file)
table['A0'] = calc_transform_matrix(table['lattice_constants'])
stop = False
while not stop:
try:
job = comm.recv(source=0)
except:
comm.send(None, dest=0)
stop = comm.recv(source=0)
continue
if job is None:
comm.send(None, dest=0) # send dummy response for dummy job
break
t0 = time.time()
res = index(
table, job['coords'], photon_energy_list, det_dist,
seed_len_tol=seed_len_tol,
seed_angle_tol=seed_angle_tol,
seed_pair_num=seed_pair_num,
intensity=job['intensity'],
snr=job['snr'],
pre_solution=job['pre_solution'],
sort=sort_by,
refine=refine,
show_progress=show_progress,
)
t1 = time.time()
res['time'] = t1 - t0
res['id'] = job['id']
res['nb_peak'] = job['coords'].shape[0]
comm.send(res, dest=0)
stop = comm.recv(source=0)
print('worker(%d) finished' % rank)
def test_index_database(self):
index_object = index.index('test_data/config_file.yaml')
index.build_index(index_object, 'test_data/config.ini')
db_keys = index.load_config_file('test_data/config.ini')
connection = psycopg2.connect(database=db_keys.get('database'),
user=db_keys.get('user'),
password=db_keys.get('password'))
cursor = connection.cursor()
cursor.execute("SELECT * FROM index order by 1,2;")
table_content = cursor.fetchall()
self.assertEqual([('as', 'test_data/full_file_dirty.txt', [1, 2]),
('bulgaria', 'test_data/full_file_dirty.txt', [1, 3]),
('bulgaria', 'try/try.txt', [22, 84, 99]),
('bush', 'test_data/full_file_dirty.txt', [1]),
('the', 'test_data/full_file_dirty.txt', [1])], table_content)
def evaluate_one_image(image_array, _index):
label = index.index()
with tf.Graph().as_default():
BATCH_SIZE = 1
N_CLASSES = len(label)
image = tf.cast(image_array, tf.float32)
image = tf.image.per_image_standardization(image)
image = tf.reshape(image, [1, 112, 112, 3])
logit = model_2.inference(image, BATCH_SIZE, N_CLASSES)
logit = tf.nn.softmax(logit)
x = tf.placeholder(tf.float32, shape=[112, 112, 3])
# you need to change the directories to yours.
logs_train_dir = r'D:\MyProjects\understand\save_2'
logs_train_dir = os.path.join(os.getcwd(), '..\\', 'understand',
'save_2')
print(logs_train_dir)
saver = tf.train.Saver()
with tf.Session() as sess:
print("Reading checkpoints...")
ckpt = tf.train.get_checkpoint_state(logs_train_dir)
# ckpt = tf.train.get_checkpoint_state(r'D:\MyProjects\understand\save')
# if ckpt and ckpt.model_checkpoint_path:
if True:
global_step = ckpt.model_checkpoint_path.split('\\')[-1].split(
'-')[-1]
saver.restore(sess, ckpt.model_checkpoint_path)
print('Loading success, global_step is %s' % global_step)
# else:
# print('No checkpoint file found')
prediction = sess.run(logit, feed_dict={x: image_array})
max_index = np.argmax(prediction)
for i in range(len(label)):
if max_index == i:
# result = ('这是{}的可能性为：'.format(label[str(i)]) + '%.6f' % prediction[:, i])
print(label[str(i)] + '-' + str(_index))
def images_to_html_threads(self):
for imageIndex, image in enumerate(self.images):
'''The below file open operation takes place every time
when the for loop iterates (I know doing this operation
every time in a for loop is bad). I tried it by declaring
it as global varibale. But the problem with global variable
is -- BeautifulSoup is maintaining its state even though the
global data is passed several times.
Briefly, the result which comes in the first loop is repeating
again and again through out the loop.
For this reason, we have put up the open operation in the for
loop itself'''
htt_file = open(os.path.join(self.c_dir, "slide.htt"), 'r')
soup = BeautifulSoup(htt_file)
im = ""
if imageIndex > 0 and imageIndex < (len(self.images) - 1):
previousimage = self.images[imageIndex - 1]
im = self.images[imageIndex]
nextimage = self.images[imageIndex + 1]
elif imageIndex > 0 and imageIndex == (len(self.images) - 1):
previousimage = self.images[imageIndex - 1]
im = self.images[imageIndex]
nextimage = ""
elif imageIndex > 0 and imageIndex > (len(self.images) - 1):
break
else:
previousimage = ""
im = self.images[imageIndex]
nextimage = self.images[imageIndex + 1]
new_thread = slides_image_urls(
soup, previousimage, im, nextimage,
self.album_name, self.slides_path)
new_thread.start()
self.threads.append(new_thread)
htt_file.close()
htt_file1 = open(os.path.join(self.c_dir, "index.htt"), 'r')
sp = BeautifulSoup(htt_file1)
index_create = index(sp, self.album_path, self.images, self.album_name)
index_create.index_parser()
htt_file1.close()
for t in self.threads:
t.join()
def images_to_html_threads(self):
for imageIndex, image in enumerate(self.images):
'''The below file open operation takes place every time
when the for loop iterates (I know doing this operation
every time in a for loop is bad). I tried it by declaring
it as global varibale. But the problem with global variable
is -- BeautifulSoup is maintaining its state even though the
global data is passed several times.
Briefly, the result which comes in the first loop is repeating
again and again through out the loop.
For this reason, we have put up the open operation in the for
loop itself'''
htt_file = open(os.path.join(self.c_dir, "slide.htt"), 'r')
soup = BeautifulSoup(htt_file)
im = ""
if imageIndex > 0 and imageIndex < (len(self.images) - 1):
previousimage = self.images[imageIndex - 1]
im = self.images[imageIndex]
nextimage = self.images[imageIndex + 1]
elif imageIndex > 0 and imageIndex == (len(self.images) - 1):
previousimage = self.images[imageIndex - 1]
im = self.images[imageIndex]
nextimage = ""
elif imageIndex > 0 and imageIndex > (len(self.images) - 1):
break
else:
previousimage = ""
im = self.images[imageIndex]
nextimage = self.images[imageIndex + 1]
new_thread = slides_image_urls(soup, previousimage, im, nextimage,
self.album_name, self.slides_path)
new_thread.start()
self.threads.append(new_thread)
htt_file.close()
htt_file1 = open(os.path.join(self.c_dir, "index.htt"), 'r')
sp = BeautifulSoup(htt_file1)
index_create = index(sp, self.album_path, self.images, self.album_name)
index_create.index_parser()
htt_file1.close()
for t in self.threads:
t.join()
def index_scan(dirname):
dirname = secure_filename(dirname)
path = os.path.join(app.config['STAGING_FOLDER'], dirname)
# Indexing
with INDEX_LOCK:
indexfile = os.path.join(app.config['STAGING_FOLDER'], app.config['indexfile'])
indexed = index.index({
'input': path, 'output': indexfile, 'root': app.config['STAGING_FOLDER'],
'single': True, 'append': True, 'checkCleaned': True,
'source': app.config['source'], 'datasets': app.config['datasets'],
'stages': app.config['stages'],
'includeAll': True
})
if indexed:
res = post(WEBUI + '/scans/populate?group=staging&replace=true', indexed.values(), log)
if res.get('status') == 'ok':
return res.get('response')
else:
resp = jsonify({"message": res.message})
resp.status_code = 500
return resp
else:
return 'Nothing to index'
def indexPage(): # noqa
return index.index()
def main():
init_logging() # Before anything else, initialise logging.
logger.info("Beginning process cycle at %s (UTC)", timefunc.utcnow())
socket.setdefaulttimeout(config.timeout)
global stat_re, addy_re, chain_re
stat_re = re.compile("(\w{1,12})\s+([0-9A-H?]{12}\s.*)")
addy_re = re.compile('\$remailer\{"([0-9a-z]{1,12})"\}\s\=\s"\<(.*)\>\s')
chain_re = re.compile("\((\S{1,12})\s(\S{1,12})\)")
# Are we running in testmode? Testmode implies the script was executed
# without a --live argument.
testmode = live_or_test(sys.argv)
# If not in testmode, fetch url's and process them
if not testmode:
pingers = db.pinger_names()
for row in pingers:
url = url_fetch(row[1])
if url:
url_process(row[0], url)
# Fetch pubring.mix files and write them to the DB
getkeystats()
else:
logger.debug("Running in testmode, url's will not be retreived")
# We need to do some periodic housekeeping. It's not very process
# intensive so might as well do it every time we run.
db.housekeeping(timefunc.hours_ago(config.dead_after_hours))
# For a pinger to be considered active, it must appear in tables mlist2
# and pingers. This basically means, don't create empty pinger columns
# in the index file.
active_pingers = db.active_pinger_names()
# A boolean value to rotate row colours within the index
rotate_color = 0
# The main loop. This creates individual remailer text files and
# indexing data based on database values.
for name, addy in db.distinct_rem_names():
logger.debug("Generating statsistics for remailer %s", name)
# remailer_vitals is a dictionary of standard deviation and average
# values for a specific remailer.
remailer_vitals = gen_remailer_vitals(name, addy)
# remailer_active_pings: Based on the vitals generated above, we now
# extract stats lines for pingers considered active. The up_hist
# part is used by the fail_recover routine.
remailer_active_pings = db.remailer_active_pings(remailer_vitals)
# If a remailers is perceived to be dead, timestamp it in the
# genealogy table. Likewise, if it's not dead, unstamp it.
fail_recover(name, addy, remailer_active_pings)
# Write the remailer text file that contains pinger stats and averages
logger.debug("Writing stats file for %s %s", name, addy)
write_remailer_stats(name, addy, remailer_vitals)
# Rotate the colour used in index generation.
rotate_color = not rotate_color
db.gene_find_new()
index()
genealogy()
uptimes()
chainstats()
writekeystats()
logger.info("Processing cycle completed at %s (UTC)", timefunc.utcnow())
def test_no_match_punctuation(self):
index.index('test_data/config_file.yaml')
self.assertEqual([], Search('test_data/config.ini').search("-more_videos:!"))
def indexPage(self):
return index.index()
U = U[padding_each_side :(data_size - padding_each_side), padding_each_side :(data_size - padding_each_side) ,padding_each_side :(data_size - padding_each_side) ]
V = V[padding_each_side :(data_size - padding_each_side), padding_each_side :(data_size - padding_each_side) ,padding_each_side :(data_size - padding_each_side) ]
W = W[padding_each_side :(data_size - padding_each_side), padding_each_side :(data_size - padding_each_side) ,padding_each_side :(data_size - padding_each_side) ]
#%%
data_size = round(U.size**(1/3))
print((data_size))
#%% padding for convolution operation
U = el.enlarge(U,data_size,(sum(kernal_width_total)-len(kernal_width_total)),1)
V = el.enlarge(V,data_size,(sum(kernal_width_total)-len(kernal_width_total)),1)
W = el.enlarge(W,data_size,(sum(kernal_width_total)-len(kernal_width_total)),1)
#%%
data_size = round(U.size**(1/3))
print((data_size))
#%% extract starting indices of the core cubes
ix = index.index(data_size,data_size,data_size,coresize,kernal_width_total)
#%%extract cores of filtered data
u_cores = core.allcore(U,ix,coresize,kernal_width_total)
v_cores = core.allcore(V,ix,coresize,kernal_width_total)
w_cores = core.allcore(W,ix,coresize,kernal_width_total)
#
#%%
num_of_filtered_velocity_files = len(u_cores)
#%%save data
#training data
for i in range(0,num_of_filtered_velocity_files):
print("creating training data: ",i)
with h5.File('/home/student/Documents/Gangu_project/data/u_filter_train/u_filter_'+str(i)+'.h5', 'w') as hf:
hf.create_dataset('u', data = u_cores[i])
def test_match(self):
index.index('test_data/config_file.yaml')
self.assertEqual([('bulgaria', 'try/try.txt', [22, 84, 99]),
('bulgaria', 'test_data/full_file_dirty.txt', [1, 3])],
Search('test_data/config.ini').search("bulgaria"))
def test_first_value_is_same(self):
self.assertEqual(index([1], 1), 0)
def test_04_index(self):
ndx.index()
import os import logging import index import Dbsetup as makedb import index #Setup Logging logger = logging.getLogger(__name__) #contents = os.listdir(os.getcwd())#gets the contents of the current directory makedb.makedb()#not dependent on wether or not there is actually a database index.index()
def main():
#rmtree(PATH)
#makedirs(PATH)
index(zsite_keyword_iter)
def test_first_value_not_same(self):
self.assertEqual(index([2], 1), -1)
def test_empty(self):
self.assertEqual(index([], 1), -1)
def test_second_value_is_same(self):
self.assertEqual(index([0, 1], 1), 1)
def main():
#rmtree(PATH)
#makedirs(PATH)
index(zsite_keyword_iter)
#!/usr/bin/env python
import index
i=index.index()
i.add_word("hello",1)
i.add_word("world",2)
i.index_buffer("This is a hello world")
for o in i.get_offsets():
print o.id,o.offset
##t=index.idx_new_indexing_trie()
##index.idx_add_word(t,"hello",1)
##index.idx_add_word(t,"goodbye",2)
##index.idx_index_buffer(t,"this is a test hello goodbye ")
##import struct
##result=index.get_offset_table(t)
##length=len(result)/struct.calcsize('@i')
##offsets=struct.unpack('@%si'%length,result)
##print offsets
##index.idx_free_indexing_trie(t)
def test_third_value(self): self.assertEqual(index([1, 2, 3], 3), 2)
def test_second_value(self): self.assertEqual(index([1, 2], 2), 1)
def BCH_Decoder(code_rate, dxcm, N):
kbch, nbch = kbch_nbch_selection.K_N(code_rate, N)
alpha = alpha_lookup_tables(code_rate, N)
g, Gg = gen_poly(N)
m1, m = Gg.shape
m = m - 1
n_bch = nbch #(2**(m))-1
S, s = syndromes(code_rate, N, dxcm)
t = 12
### B E R L E K A M P ' S A L G O R I T H M
e = np.zeros(nbch, dtype=int)
lmd = np.zeros(t, dtype=int)
T = np.zeros(t, dtype=int)
lmd[0] = s[0]
if s[0] == -1:
T[0] = -1
T[1] = 0
else:
T[0] = np.mod((n_bch - s[0]), n_bch)
for v in range(t - 2):
deltar = alpha[(s[2 * v]) % n_bch, :]
for j in range(v):
if lmd[j] == -1 | s[2 * v - j] == -1:
deltar = deltar
else:
deltar = np.mod(
deltar + alpha[(lmd[j] + s[2 * v - j]) % n_bch, :], 2)
delta = index(deltar, code_rate, N)
V = lmd
if delta == -1 | T[v] == -1:
lmd[v + 1] = -1
else:
lmd[v + 1] = (delta + T[v]) % n_bch
for i in range(1, v):
if delta == -1 | T[i] == -1:
lmdr = alpha[(lmd[i]) % n_bch, :]
else:
lmdr = np.mod(
alpha[lmd[i] % n_bch, :] +
alpha[(T[i - 1] + delta) % n_bch, :], 2)
lmd[i] = index(lmdr, code_rate, N)
if delta != -1:
T[0] = np.mod((n_bch - delta), n_bch)
for i in range(1, v + 1):
T[i] = np.mod((V[i - 1] - delta), n_bch)
else:
T[v + 2] = T[v]
for j in range(1, v + 1):
T[j] = 0
for i in range(nbch):
xx = np.zeros(m, dtype=int)
for j in range(t):
if lmd[j] == -1:
xx = xx
else:
xx = np.mod((xx + alpha[(lmd[j] + j * i) % n_bch, :]), 2)
if index(xx, code_rate, N) == 1:
e[np.mod(nbch - i, nbch) + 1] = 1
d = np.mod(dxcm + e, 2)
return d
def feeds(lexiDic, list_personalities, opinionDic):
print "feeds"
with open("feeds.txt",'a+') as f:
####################################################################################################
# VARIAVEIS #
####################################################################################################
listHash = []
#listHash.append('ccfaa96301a3e11a1add20e1d1ce4dedf0743a71')
####################################################################################################
# FUNCOES PARA IR BUSCAR OS FEEDS DN #
####################################################################################################
feed = raw_input("Insert the feed address please: ")
dn = feedparser.parse(feed)
for i, post in enumerate(dn.entries):
html = urllib2.urlopen(post.link).read()
soup = BeautifulSoup(html)
artigo = soup.find(id = 'Article')
summ = soup.find(id = 'NewsSummary')
print "############### Titulo noticia:", i, " ###################"
sha = hashlib.sha1(unicode(post.title)).hexdigest() #hash para certificar que nao existem 2 noticias iguais
print post.title
print "Hash da noticia:", sha, "\n"
#print post.id
#print "PUBLICADO A: ", post.published, "\n"
#print "############### RESUMO ###################"
#print summ, "\n"
#print "############### ARTIGO ################### \n"
h = open('hash_file.txt', 'a+')
r = open('hash_file.txt', 'r')
t = open('title_file.txt', 'a+')
t2 = open('title_file.txt', 'r')
# for digSha in h:
#shalist = re.split('\W+', digSha)
# listHash.append(digSha.strip())
t2read = t2.readlines()
if len(t2read) == 0:
j = 0
else:
tline = t2read[len(t2read)-1]
titleSplit = re.compile("([\W+])", re.UNICODE).split(unicode(tline, 'utf-8'))
j = int(titleSplit[0]) + 1
if sha in r.read():
print "ARTIGO JA EXISTENTE", "\n"
else:
if artigo == None: #Verficacao se o artigo esta vazio
print "Artigo Vazio", "\n"
h.write(sha + '\n')
else:
f.write(str(j) + " - " + str(artigo.div.contents[0]) + "\n") #ESCREVER O ARTIGO PARA O FICHEIRO
t.write(str(j) + " - " + post.title + "\n")
h.write(sha + '\n')
sentiments.sentimentos(j, post.title, str(artigo.div.contents[0]), lexiDic, list_personalities, opinionDic)
j = j + 1
#print artigo.div.contents, "\n"
#print listHash
h.close()
r.close()
t.close()
t2.close()
index.index("feeds.txt") #Executa a funcao de indexacao do ficheiro com os artigos
def created(self):
QtWidgets.QMessageBox.information(self, '提示', '你即将开始创建一个本地图库\n这可能会使用你大量的内存空间和时间请耐心等待')
index.index()
QtWidgets.QMessageBox.information(self, '提示', '恭喜你，图库建立完成\n请您开始使用图图搜索模块')
def test_04_index(self):
ndx.index()
def test_no_match_single_character(self):
index.index('test_data/config_file.yaml')
self.assertEqual([], Search('test_data/config.ini').search("a"))
last = len(sys.argv) - 1
poll_id = int(sys.argv[last-1])
choice_id = int(sys.argv[last])
# Announce launch
print "Starting bot for poll " \
+ str(poll_id) \
+ " and choice " \
+ str(choice_id) \
+ " ..."
# Remind of the question
print "The question for poll ID " \
+ str(poll_id) \
+ " is \"" \
+ index()[poll_id] \
+ "\"."
# Remind of the choice
print "The choice for choice ID " \
+ str(choice_id) \
+ " is \"" \
+ detail(poll_id)[choice_id]["choice"] \
+ "\" with current popularity " \
+ str(detail(poll_id)[choice_id]["votes"]) \
+ "."
# Start an infinite loop (use CTRL-C)
try:
while True:
max = choice_id
def test_no_match(self):
index.index('test_data/config_file.yaml')
self.assertEqual([], Search('test_data/config.ini').search("wrongword"))
data = fd.read(CONTEXT+len(keyword)+CONTEXT)
print "%010u: %s" % (offset,pretty_print(data[:CONTEXT],data[CONTEXT:CONTEXT+len(keyword)],data[CONTEXT+len(keyword):]))
try:
## Open the image file
fd = open(args[0],'r')
except IndexError:
print "You must specify an image file to index"
## Indexing mode
if opt_dict.has_key("-i"):
## We are in index mode - first some sanity checks:
if opt_dict.has_key("-s"):
raise IndexerException("Can not specify both search mode and index mode at the same time - you must first build the index using -i and then search it using -s")
try:
idx = index.index(opt_dict["-f"])
except KeyError:
raise IndexerException("You must specify the name of the file to use as the index")
## First process keywords given on the command line
for key,value in opts:
if key=="-W":
idx.add(value)
## Now keywords in external files
if key=="-w":
fd2 = open(value,'r')
for line in fd2:
line = line[:-1]
idx.add(line)
fd2.close()
def test_first_value_differs(self): self.assertEqual(index([1], 2), -1)
def test_third_value_is_same(self):
self.assertEqual(index([0, 1, 2], 2), 2)
