def main():
parser = argparse.ArgumentParser()
parser.add_argument('File1')
parser.add_argument('File2')
options = parser.parse_args()
#InputFiles
file1 = options.File1
file2 = options.File2
#Object1 to compare
obj1 = functions.t_read(file1)
#Object2 to compare
obj2 = functions.t_read(file2)
if obj1 == None or obj2 == None:
sys.exit("Execution aborted due to input problem!!")
#output file1
f1= open("output/tree1_out.txt","w")
#initiate compare
comp = Compare(obj1, obj2,f1)
f1.close();
#output file2
f2= open("output/tree2_out.txt","w")
comp = Compare(obj2, obj1,f2)
f2.close()
def main():
print("Input first version:")
ver1 = sys.stdin.readline().strip()
print("Input second version:")
ver2 = sys.stdin.readline().strip()
obj = Compare(ver1, ver2)
res = obj.compare()
print(res)
def test_for_nlpnet(self):
"""
Attempting to use nlpnet. This will cause an
error if the required dependencies are not
downloaded.
"""
try:
# Creating a new compare object
compare_nlpnet = Compare()
# Comparing using the nltk parser
compare_nlpnet.compare_strings(
text=["what time is it here?", "This is the cat's hat"],
pattern_detection=False,
parser="nlpnet")
# If that was successfuly, getting information
sentence_information = compare_nlpnet.get_pattern_information()
for sentence in sentence_information:
my_pattern = "[ Pattern ] : " + sentence.pattern
my_subject = "[ Subject ] : " + sentence.subject
my_verb = "[ Verb ] : " + sentence.verb
my_object = "[ Object ] : " + sentence.object[0]
my_preps = "[ Prep Phrases ] : " + str(
sentence.prepositional_phrases)
my_reliability_score = "[ Reliability Score ]: " + str(
sentence.reliability_score)
except:
# Getting nltk data path
running = Popen(['python -c "import nltk;print nltk.data.path"'],
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
shell=True)
stdin, stdout = running.communicate()
# Setting the path that the nlpnet dependency will be downloaded from
path = re.sub(
r"\'", "",
re.sub(r"\[", '', str(stdin.split('\n')[0].split(',')[0])))
path = path.split(r"/")
path = '/'.join(path[0:len(path) - 1]) + '/nlpnet_dependency/'
# Download the dependencies & extract
current_directory = os.getcwd()
os.mkdir(path)
os.chdir(path)
os.system(
"wget http://nilc.icmc.usp.br/nlpnet/data/dependency-en.tgz")
tar = tarfile.open(path + 'dependency-en.tgz', 'r:gz')
tar.extractall(path)
os.remove(path + 'dependency-en.tgz')
os.chdir(current_directory)
def best_price(self):
if self.valid_inputs():
product_a = Product("Product A", int(self.quantity_a.text),
float(self.price_a.text))
product_b = Product("Product B", int(self.quantity_b.text),
float(self.price_b.text))
compare = Compare([product_a, product_b])
best_product = compare.best()
self.highlight_best_product(best_product)
def __init__(self, **kwargs):
"""
Constructor method.
"""
# Testing the system to make sure all dependencies are installed
RunDependencyTests()
# Instantiating compare object to be used
self.compare_object = Compare()
class regex4dummies:
# Setting global version variable which contains the version of this library
__version__ = '1.4.6'
def __init__(self, **kwargs):
"""
Constructor method.
"""
# Testing the system to make sure all dependencies are installed
RunDependencyTests()
# Instantiating compare object to be used
self.compare_object = Compare()
def compare_strings(self, **kwargs):
"""
Function that is integral in communicating between a compare object and the user
This function returns a 3-tuple array containing reliability score, applicability score, and pattern
"""
# Call compare_strings of compare object & return output
if kwargs.get("pattern_detection") == "literal":
return self.compare_object.compare_strings(
text=kwargs.get("text"),
pattern_detection=True,
parser=kwargs.get("parser"))
else:
return self.compare_object.compare_strings(
text=kwargs.get("text"),
pattern_detection=False,
parser=kwargs.get("parser"))
def get_pattern_information(self):
"""
This function returns the information for each sentence/pattern that was identified.
This is only useful if semantic parsing is implemented; otherwise, {} will be returned.
"""
return self.compare_object.get_pattern_information()
def get_topics(self, **kwargs):
""" Returns the list of topics that the parsers identified """
return self.compare_object.get_pattern_topics(kwargs.get("text"))
def extract_important_information(self, **kwargs):
"""
Returns the important information within the given text.
"""
return self.compare_object.extract_important_information(
kwargs.get("text"))
def compare(self, idx1, op, idx2):
if not self.stack.is_valid(idx1) or not self.stack.is_valid(idx2):
return False
a = self.stack.get(idx1)
b = self.stack.get(idx2)
if op == CmpOp.EQ:
return Compare.eq(a, b)
elif op == CmpOp.LT:
return Compare.lt(a, b)
elif op == CmpOp.LE:
return Compare.le(a, b)
def __init__(self):
# Initiate all the necessary database and utilities
self.detector = SHDetector()
self.notice = BulletNotice()
self.compare = Compare()
self.database = Database()
self.dbname = 'sensedata.db'
# self.sample_freq = 60 # time in seconds, change to 60 for pi
# self.num_of_notice = 0 # record of notification sent today
if (not (os.path.isfile('./' + self.dbname))):
self.database.create_dbtable(
) # create db and tables if db file not existing
def run_testcase(filename):
# find destinations in folder starting_... and ending_...
finder = findDestinations(filename)
end = finder.returnDestination()
start = finder.returnStarting()
map_file = np.loadtxt('map.txt',dtype=int)
# bounding negative values for keeping it in bounds
map_file[0,:] = MIN_VALUE
map_file[:,0] = MIN_VALUE
map_file[:,len(map_file)-1]=MIN_VALUE
map_file[len(map_file)-1,:]=MIN_VALUE
# UAV map emulation
env = Map(start,end,filename,map_file)
RL = DeepQNetwork(env.n_actions, env.n_features,
learning_rate=0.01,
reward_decay=0.9,
e_greedy=0.9,
replace_target_iter=200,
memory_size=2000,
output_graph=True,
iteration=filename
)
run_map(filename,RL,env)
RL.plot_cost()
compare = Compare(filename)
#compare to given results
print("Finished iteration", filename)
class regex4dummies:
# Setting global version variable which contains the version of this library
__version__ = '1.4.6'
def __init__(self, **kwargs):
"""
Constructor method.
"""
# Testing the system to make sure all dependencies are installed
RunDependencyTests()
# Instantiating compare object to be used
self.compare_object = Compare()
def compare_strings(self, **kwargs):
"""
Function that is integral in communicating between a compare object and the user
This function returns a 3-tuple array containing reliability score, applicability score, and pattern
"""
# Call compare_strings of compare object & return output
if kwargs.get("pattern_detection") == "literal":
return self.compare_object.compare_strings(text=kwargs.get("text"), pattern_detection=True, parser=kwargs.get("parser"))
else:
return self.compare_object.compare_strings(text=kwargs.get("text"), pattern_detection=False, parser=kwargs.get("parser"))
def get_pattern_information( self ):
"""
This function returns the information for each sentence/pattern that was identified.
This is only useful if semantic parsing is implemented; otherwise, {} will be returned.
"""
return self.compare_object.get_pattern_information()
def get_topics(self, **kwargs):
""" Returns the list of topics that the parsers identified """
return self.compare_object.get_pattern_topics(kwargs.get("text"))
def extract_important_information(self, **kwargs):
"""
Returns the important information within the given text.
"""
return self.compare_object.extract_important_information(kwargs.get("text"))
def test_for_nlpnet(self):
"""
Attempting to use nlpnet. This will cause an
error if the required dependencies are not
downloaded.
"""
try:
# Creating a new compare object
compare_nlpnet = Compare()
# Comparing using the nltk parser
compare_nlpnet.compare_strings(text=["what time is it here?", "This is the cat's hat"], pattern_detection=False, parser="nlpnet")
# If that was successfuly, getting information
sentence_information = compare_nlpnet.get_pattern_information()
for sentence in sentence_information:
my_pattern = "[ Pattern ] : " + sentence.pattern
my_subject = "[ Subject ] : " + sentence.subject
my_verb = "[ Verb ] : " + sentence.verb
my_object = "[ Object ] : " + sentence.object[0]
my_preps = "[ Prep Phrases ] : " + str(sentence.prepositional_phrases)
my_reliability_score = "[ Reliability Score ]: " + str(sentence.reliability_score)
except:
# Getting nltk data path
running = Popen(['python -c "import nltk;print nltk.data.path"'], stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=True)
stdin, stdout = running.communicate()
# Setting the path that the nlpnet dependency will be downloaded from
path = re.sub(r"\'", "", re.sub(r"\[", '', str(stdin.split('\n')[0].split(',')[0])))
path = path.split(r"/")
path = '/'.join(path[0 : len(path) - 1]) + '/nlpnet_dependency/'
# Download the dependencies & extract
current_directory = os.getcwd()
os.mkdir(path)
os.chdir(path)
os.system("wget http://nilc.icmc.usp.br/nlpnet/data/dependency-en.tgz")
tar = tarfile.open(path + 'dependency-en.tgz', 'r:gz')
tar.extractall(path)
os.remove(path + 'dependency-en.tgz')
os.chdir(current_directory)
def main():
print('Executing main() ....')
print("N SAMPLES:", 30000, " N FEATURES:", 10)
X, y = make_blobs(n_samples=15000,
centers=3,
n_features=10,
random_state=500)
# plt.plot(X[:,0], X[:,1], "ko")
# plt.show()
# iris = datasets.load_iris()
# X = iris.data
# y = iris.target
# MIN_SAMPLES=4
# EPS=0.4
# DELTA=0.7
# fetch_covtype = datasets.fetch_covtype()
# X = fetch_covtype.data
# y = fetch_covtype.target
# plt.plot(X[:,0], X[:,1], "ko")
# plt.show()
# EPS=250
# fetch_olivetti_faces = datasets.fetch_olivetti_faces()
# X = fetch_olivetti_faces.data
# y = fetch_olivetti_faces.target
# plt.plot(X[:,0], X[:,1], "ko")
# plt.show()
# EPS=7.3
# DELTA=0.6
# lfw_people = datasets.fetch_lfw_people(min_faces_per_person=70, resize=0.4)
# X = lfw_people.data
# y = lfw_people.target
# plt.plot(X[:,0], X[:,1], "ko")
# plt.show()
# EPS=1400
# DELTA=0.7
compare = Compare(X, y, EPS, MIN_SAMPLES, DELTA)
compare.run()
def on_post(self, req, resp):
req_body = req.stream.read()
json_data = json.loads(req_body.decode('utf8'))
student_answer = json_data.get('student_answer')
expected_answer = json_data.get('expected_answer')
model_name = json_data.get('model', 'en')
try:
model = get_model(model_name)
compare = Compare(model, student_answer, expected_answer)
resp.body = json.dumps(compare.to_json(), sort_keys=True, indent=2)
resp.content_type = 'text/string'
resp.append_header('Access-Control-Allow-Origin', "*")
resp.status = falcon.HTTP_200
print(resp)
except Exception as e:
raise falcon.HTTPBadRequest('Dependency parsing failed',
'{}'.format(e))
resp.status = falcon.HTTP_500
def __init__(self, **kwargs):
"""
Constructor method.
"""
# Testing the system to make sure all dependencies are installed
RunDependencyTests()
# Instantiating compare object to be used
self.compare_object = Compare()
def run_testcase(test_samples):
map_file = init_map('map.txt')
case = 1
for sample in test_samples:
start = sample[0]
end = sample[1]
filename = str(case)
case += 1
# UAV map emulation
env = Map(start, end, filename, map_file)
total_epochs = 300
run_map(filename, env, True, total_epochs)
RL.plot_cost()
Compare(filename)
class Monitor:
def __init__(self):
# Initiate all the necessary database and utilities
self.detector = SHDetector()
self.notice = BulletNotice()
self.compare = Compare()
self.database = Database()
self.dbname = 'sensedata.db'
# self.sample_freq = 60 # time in seconds, change to 60 for pi
# self.num_of_notice = 0 # record of notification sent today
if (not (os.path.isfile('./' + self.dbname))):
self.database.create_dbtable(
) # create db and tables if db file not existing
def main(self):
# Check if today's status alreay exist in DAILY_REPORT DB, and return
# 0 - if not exist, 1 - if exist and is OK, 2 - if exist and is BAD
status_num = self.database.check_status_exist()
if status_num == 0: # If status not exists, pre-save a "OK" as today's status
self.database.save_daily_data('OK')
# for _ in range(0, 5):
self.detector.getSenseData()
monitor_time = self.detector.getTime()
temp = self.detector.getTemp()
humid = self.detector.getHumid()
status = self.compare.compare_data(temp,
humid) # defect code exists here
# status = "OK" # For testing purpose - This works
self.detector.sense.show_message("Status:" + status, scroll_speed=0.05)
# Save the monitored temperature and humidity with current time to DB - TEMP_HUMID table
self.database.save_dbdata(monitor_time, temp, humid)
self.detector.sense.show_message("DBSaved", scroll_speed=0.05)
# If exceed boundary - status is bad,
# and current saved status in DB is OK,
# then update status in DB and send notification (once per day)
if status != "OK" and status_num == 1:
self.database.update_daily_data(status)
self.detector.sense.show_message("DBUpdated", scroll_speed=0.05)
self.notice.send_notification(
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), status)
def btnstate(self):
if self.vecButton.isChecked():
print("Vector Selected")
fileName = self.fileDialog.getOpenFileName()
baby = Compare(fileName)
self.vecButton.deleteLater()
self.vecButton = None
self.finButton.deleteLater()
self.finButton = None
self.okButton.deleteLater()
self.okButton = None
f = open('compare.txt', 'r')
sr = f.read()
self.label.setText("Top 5 Closest Documents")
self.ans.setText(sr)
os.remove("compare.txt")
self.ans.setFont(QtGui.QFont("Coic", 13, QtGui.QFont.Bold))
elif self.finButton.isChecked():
print("Fingerprinting Selected")
fileName = self.fileDialog.getOpenFileName()
baby = Compare1(fileName)
self.vecButton.deleteLater()
self.vecButton = None
self.finButton.deleteLater()
self.finButton = None
self.okButton.deleteLater()
self.okButton = None
f = open('compare1.txt', 'r')
sr = f.read()
self.label.setText("Top 5 Closest Documents")
self.ans.setText(sr)
os.remove("compare1.txt")
self.ans.setFont(QtGui.QFont("Coic", 13, QtGui.QFont.Bold))
else:
print("Please select a method")
def main():
input = ComLine(sys.argv[1:])
#handle bayescan output
bs = BayeScan(input.args.bayescan, input.args.fdr)
bsCommand = bs.build_command()
bs.run_program(bsCommand)
print(input.args.bayescan)
#handle lositan output
los = Lositan(input.args.lositan, 0.025)
los.readLositan()
los.printSignificant()
#compare outputs
com = Compare("bayescan.outliers.txt", "lositan.positive_selection.txt")
com.readInput()
com.printResults(input.args.vcf)
def main(self):
"""
program entry point
first runs merge on the input json filenames
specified on the command line
then creates a summary file, and a kml file
as per usual convention, this returns 0 if everything goes fine
and non-zero otherwise, and specifically
(*) 1 if both output files were created but they do not match
the reference result
(*) 2 if an exception occured and prevented file creation
run with --help to see the list of available options
in particular it is possible to restrict to the ship(s) that match
one specific ship name using -s/--ship
"""
# this try/except is mostly for illustrating how
# to display an exception in your code, rather than relying
# on Python to do it for you when an exception is not handled
# in this particular instance, the try/except is not
# particularly helpful then
try:
# send the contents of all incoming files to the ship_dict instance
# this method also clears
self.merge(self.args.json_filenames)
# if --ship is specified on the command line,
# corresponding value is stored in self.args.ship_name
# as per ArgumentParser definition. So:
# if --ship is not specified
if not self.args.ship_name:
# use all ships in ship_dict
ships = self.ship_dict.all_ships()
# we use this to compute output filenames
output_name = "ALL_SHIPS"
# if --ship was specified
else:
# restrict to ships that match the selected ship name
ships = self.ship_dict.ships_by_name(self.args.ship_name)
output_name = self.args.ship_name
# sort ships once and for good
# we can't sort these objects inline as they are
# typically dict_values objects
ships = sorted(ships, key=lambda ship: ship.name)
# create summary file
summary_filename = self.write_ships_summary(ships, output_name)
kml_filename = self.write_kml_output(ships, output_name)
# if a ship name was specified, we cannot compare anything
if self.args.ship_name:
ok = True
else:
# for each of the 2 files, compare contents with the reference
# that is expected to be in this directory with a .ref extension
ok_summary = Compare(summary_filename).compare_and_print()
ok_kml = Compare(kml_filename).compare_and_print()
# is everything fine ?
ok = ok_summary and ok_kml
# if so return 0 otherwise 1
return 0 if ok else 1
# if anything goes south
except Exception as exc:
# give description of the exception
print(f'Something went wrong, {type(exc)}, {exc}')
# plus provide a snapshot of the stack at the point
# where the exception was raised
import traceback
traceback.print_exc()
# return 2 in this case
return 2
test.get_frequencies()
print test.frequencies[:10]
test.get_duration()
print test.duration
test.make_mono()
print test.frequencies[:10]
# call fft on the freqs
test.get_fft()
print test.frequencies[:10]
"""
print "Two identical files that should match:"
test1 = Compare('audio/A4/z01.wav', 'audio/A4/z01.wav', 2.7, 1)
test1.compare()
print "Two diff files that should match:"
test2 = Compare('audio/A4/z01.wav', 'audio/A4/z02.wav', 2.7, 1)
test2.compare()
print "Two diff files that should NOT match:"
test3 = Compare('audio/A4/z02.wav', 'audio/A4/z03.wav', 2.7, 1)
test3.compare()
print "Two diff files that should match:"
test4 = Compare('audio/A4/z03.wav', 'audio/A4/z04.wav', 672, 1)
test4.compare()
print "Two diff files that should NOT match:"
def test_should_deny_compare_empty_list(product_one):
with pytest.raises(ValueError):
Compare([])
if __name__ == '__main__':
from MiniImagenet import MiniImagenet
from torch.utils.data import DataLoader
from torchvision.utils import make_grid
from tensorboardX import SummaryWriter
from datetime import datetime
n_way = 5
k_shot = 5
k_query = 1 # query num per class
batchsz = 3
# Multi-GPU support
print(
'To run on single GPU, change device_ids=[0] and downsize batch size! \nmkdir ckpt if not exists!'
)
net = torch.nn.DataParallel(Compare(n_way, k_shot), device_ids=[0]).cuda()
# print(net)
mdl_file = 'ckpt/compare%d%d.mdl' % (n_way, k_shot)
if os.path.exists(mdl_file):
print('load checkpoint ...', mdl_file)
net.load_state_dict(torch.load(mdl_file))
model_parameters = filter(lambda p: p.requires_grad, net.parameters())
params = sum([np.prod(p.size()) for p in model_parameters])
print('total params:', params)
optimizer = optim.Adam(net.parameters(), lr=1e-3)
tb = SummaryWriter('runs', str(datetime.now()))
best_accuracy = 0
def test_should_return_best_price_from_three_product(product_one, product_two,
product_three):
compare = Compare([product_one, product_two, product_three])
assert compare.best() == product_two
__author__ = 'daniyuu' from compare import Compare # extractor = Extractor() # # extractor.extract_worse_img() # extractor.extract_better_img() base_name = "1" target_name = "4" cmp = Compare(base_name, target_name) cmp.extract()
#!/usr/bin/python
# -*- coding: UTF-8 -*-
from face_collect import Collect
from compare import Compare
import time
import numpy as np
import logger
import cv2
import cache
from schedule import FaceDescriptorSchedule
from concurrent.futures import ThreadPoolExecutor
collect = Collect()
compare = Compare()
threadPool = ThreadPoolExecutor(5)
face_cascade = cv2.CascadeClassifier("haarcascade_frontalface_alt2.xml")
'''
人脸特征比对，判断是否是同一个人
'''
def reg(faceImg):
_128descriptor = collect.collect(faceImg)
if _128descriptor is None:
logger.info("no face to det")
return
allDescriptors = cache.getCacheList()
if allDescriptors is None or len(allDescriptors) <= 0:
time.sleep(0.5) # 若没有数据，则等0.5秒，系统加载数据
if allDescriptors is None or len(allDescriptors) <= 0:
def webcompare():
return Compare().compare()
if topKIndex.data[0] == 129:
valueToBeSet = topKIndex.data[1]
else:
valueToBeSet = topKIndex.data[1]
b.itemset((0, value.data[0]), 1)
b.itemset((0, valueToBeSet), 1)
else:
b.itemset((0, value.data[0]), 1)'''
if (song is None):
song = b
else:
song = np.concatenate([song, b])
print(song)
compareItems.append(song)
rec = recreate.RecreateMIDI()
track = rec.recreateMIDI(song, 30)
rec.createMIDITest(
track, 'Recreated ' + str(number) + ', ' + str(numEpochs) +
' epochs,' + ' learning rate = ' + str(learningRateTitle))
print('Runtime: ' + str(t1 - t0) + " seconds")
'''Compare solo with test data'''
file = open(
'results for ' + str(numEpochs) + ' epochs, ' + 'learning rate = ' +
str(learningRateTitle) + '.txt', 'w')
comparator = Compare()
for song in compareItems:
comparator.compareData(songdata, song, file)
file.write('\n')
file.write('\n' + str(losses))
file.close()
def run(self):
cmp = Compare(self.surf_url_path, self.log_url_path)
cmp.do_compare()
self.finish.emit(str(cmp))
def compareURL(self):
'''
比对URL并更新相似度self.semblance
'''
self.semblance = Compare(self.Left_URL_Object, self.Right_URL_Object).get_score()
def __init__(self, config):
#self.fm = FileManager()
self.config = config
self.parsedlist = {}
self.compare = Compare()
self.fm = FileManager()
class Parser():
def __init__(self, config):
#self.fm = FileManager()
self.config = config
self.parsedlist = {}
self.compare = Compare()
self.fm = FileManager()
def parse(self, resources):
"""
parses each resource, calculates diff to existing resource on HD (dependent on lastdate)
"""
for resource in resources:
recent_file = self.getRecentLocalFile(resource.directory)
if resource.diff:
compresult = self.compare.compare(recent_file, resource.url)
logging.debug("Comparing resources: %s and %s" %
(recent_file, resource))
result = {}
result['url'] = resource.url
result['diff'] = compresult.diff
results[resource.url] = result
print compresult
def getRecentLocalFile(self, directory, date=None):
"""
Return newest local file (dump) in directory e.g. data/orf.at
If dump with datestamp can be found this dump will be returned
"""
try:
logging.debug("Trying to alocate file near %s" % directory)
files = reversed(sorted([f for f in os.listdir(directory)]))
lastfile = None
for file in files:
if "dump" in file:
if date != None:
compstr = file.replace(".dump", "")
if float(compstr) < float(date):
if lastfile:
return "%s/%s" % (directory, lastfile)
else:
return "%s/%s" % (directory, file)
elif float(compstr) == float(date):
return "%s/%s" % (directory, file)
elif float(compstr) > float(date):
pass
# return "%s/%s" % (directory, lastfile)
else:
return "%s/%s" % (
directory, file
) #if date is None, return last saved version of file
lastfile = file
except OSError as e:
logging.debug("%s seems like first time fetched" % e)
return None
except ValueError as e:
logging.debug(e)
return None
def checkEquals(self, resourcelist, date=None):
"""
compare duplicates with duplicates in storage
"""
timestamp = time.strftime("%Y%m%d%H%M%S", time.gmtime())
output = {}
output['data'] = {}
output['data']['date'] = timestamp
if date:
output['data']['lastdate'] = date
results = {}
#debug
# datasources = []
# for domain, resourcelist in resourcelist.iteritems():
# datasources.append(self.fm.getAllByDomain(domain, resourcelist))
import pickle
import sys
# fh=open("rdump","w+")
# pickle.dump(datasources,fh)
# fh.close()
fh = open("rdump", "r+")
datasources = pickle.load(fh)
fh.close()
for datasource in datasources:
md5list = {
} #list of md5 hashes of current datasource to avoid downloading alias URIs
for resource in datasource:
if True or "sws.geonames.org" in resource.domain:
directory = dirurl(resource.url)
resource.directory = directory
diff = False
#read content of file of same url of HD if existing
recent_file = self.fm.getRecentLocalFile(directory, date)
if recent_file:
logging.debug("recent_file found: %s" % recent_file)
rf_fh = open(recent_file)
rf_content = rf_fh.read()
if resource.md5 in md5list: #resource already downloaded in this job
logging.debug(
"Already downloaded in this job, creating symlink %s"
% resource.url)
self.fm.symlink(
os.path.join(md5list[resource.md5],
"%s.dump" % timestamp),
os.path.join(resource.directory,
"%s.dump" % timestamp))
elif recent_file == os.path.join(
resource.directory, "%s.dump" % timestamp
): #file exists on harddisk with same name (timedelta too small)
logging.debug(
"Downloaded in the minute before, skipping")
elif recent_file and md5(rf_content).digest(
) != resource.md5: #recent file found on HD, but diffs
logging.debug(
"Recent version found, but diff detected, downloading file %s"
% resource.url)
md5list[resource.md5] = directory
diff = True
self.fm.downloadFile(resource, timestamp)
elif recent_file and md5(rf_content).digest(
) == resource.md5: #recent file located and proved that same size
logging.debug(
"File found on HD with same hash: %s at %s %s %s" %
(resource.url, resource.directory, recent_file,
timestamp))
if resource.directory == os.path.dirname(
recent_file
): #only create symlink if directory differs
logging.debug(
"File already exists in same directory, no symlink created"
)
else:
logging.debug(
"File is in different directory, symlink created %s"
% resource.url)
self.fm.symlink(recent_file, resource.directory,
timestamp)
else: #no recent file found and file is not in md5list
logging.debug("No equals found, downloading file %s" %
resource.url)
md5list[resource.md5] = directory
self.fm.downloadFile(resource, timestamp)
#start diff process
if diff:
compresult = self.compare.compare(
recent_file, resource.url)
logging.debug("Comparing resources: %s and %s" %
(recent_file, resource))
result = {}
result['url'] = resource.url
result['diff'] = compresult.diff
results[resource.url] = result
print compresult
output['equals'] = results
return output
def train(rank, world_size):
print(f"Running basic DDP example on rank {rank}.")
setup(rank, world_size)
'''
超参数定义
'''
# 5 个类
n_way = 5
# 1 个shot
k_shot = 1
# 每个类的查询样本
k_query = 1
# 选择 batchsz 组 [训练集，测试集]
batchsz = 2
# 输出训练过程中最好的准确性
best_accuracy = 0
# 数据并行化
print('parameters setting over.')
# print('To run on single GPU, change device_ids=[0] and downsize batch size! \nmkdir ckpt if not exists!')
# GPU 单卡
# net = torch.nn.DataParallel(Compare(n_way, k_shot), device_ids=[0]).cuda()
# GPU 单机多卡
net = DDP(Compare(n_way, k_shot).to(rank), device_ids=[rank])
# CPU 版本
# net = Compare(n_way, k_shot)
# 打印模型
# print(net)
# 模型参数要保留的文件名
# mdl_file = '../ckpt/compare%d%d.mdl' % (n_way, k_shot)
# 如果有之前的经验，则加载，否则直接运行
# if os.path.exists(mdl_file):
# print('load checkpoint ...', mdl_file)
# net.load_state_dict(torch.load(mdl_file))
# 选择需要梯度更新的参数
print("The numbers of parameter model: ",
sum(p.numel() for p in net.parameters() if p.requires_grad))
# 创建优化器
optimizer = optim.Adam(net.parameters(), lr=1e-3)
# 开始训练
print('Train is beginning...')
since = time.time()
for epoch in range(200):
print('Begin to load data...')
# 加载训练数据集 batchsz 组支持集和查询集
mini = MiniImagenet('../mini-imagenet/',
mode='train',
n_way=n_way,
k_shot=k_shot,
k_query=k_query,
batchsz=10000,
resize=224)
# pin_memory 快速的将数据转化为 GPU 可以处理的数据
# num_workers 读取数据子线程的数量
db = DataLoader(mini,
batchsz,
shuffle=True,
num_workers=0,
pin_memory=True)
# 加载测试数据集
mini_val = MiniImagenet('../mini-imagenet/',
mode='val',
n_way=n_way,
k_shot=k_shot,
k_query=k_query,
batchsz=200,
resize=224)
db_val = DataLoader(mini_val,
batchsz,
shuffle=True,
num_workers=0,
pin_memory=True)
print('Support and Query set is entranced...')
# 训练阶段。遍历训练数据集中的每一个 batch
#
for step, batch in enumerate(db):
# for i in range(200):
# 支持集合与查询集合
support_x = Variable(batch[0]).cuda().to(rank)
support_y = Variable(batch[1]).cuda().to(rank)
query_x = Variable(batch[2]).cuda().to(rank)
query_y = Variable(batch[3]).cuda().to(rank)
# 开始训练
net.train()
# 计算 loss
# print('computing loss....')
loss = net(support_x, support_y, query_x, query_y, rank)
# print(type(loss), loss.item())
# Multi-GPU support
loss = loss.mean()
# print(loss)
# 清空优化器之前的梯度
optimizer.zero_grad()
# 反向传播
# print('backwarding ...')
loss.backward()
optimizer.step()
# 清空内存
# 开始验证
total_val_loss = 0
if step % 200 == 0 and step != 0:
total_correct = 0
total_num = 0
# 在测试集中
for j, batch_test in enumerate(db_val):
# 选出支持集和查询集
support_x = Variable(batch_test[0]).cuda().to(rank)
support_y = Variable(batch_test[1]).cuda().to(rank)
query_x = Variable(batch_test[2]).cuda().to(rank)
query_y = Variable(batch_test[3]).cuda().to(rank)
# 评价
net.eval()
pred, correct = net(support_x, support_y, query_x, query_y,
rank, False)
correct = correct.sum() # multi-gpu support
# 累计的准确率
total_correct += correct.item()
# query_y.size(0) 表示这么多组的查询集
# query_y.size(0) 每个查询集中样本的数量
# total_num 就是一共多少测试样本
total_num += query_y.size(0) * query_y.size(1)
# 计算准确率
accuracy = total_correct / total_num
# 保留最好的准确率
if accuracy > best_accuracy:
best_accuracy = accuracy
# torch.save(net.state_dict(), mdl_file)
# print('saved to checkpoint:', mdl_file)
print('<<<<>>>>accuracy:', accuracy, 'best accuracy:',
best_accuracy)
with open('../output/accuracy.txt', 'a') as f:
f.write('accuracy: {}\n'.format(accuracy))
# 每 15 轮打印一次 loss 函数
if step % 15 == 0 and step != 0:
print('%d-way %d-shot %d batch> epoch:%d step:%d, loss:%f' %
(n_way, k_shot, batchsz, epoch, step, loss.cpu().item()))
with open('../output/loss.txt', 'a') as f:
f.write('loss: {}\n'.format(loss.cpu().item()))
cleanup()
end = time.time()
print("cost time is {}".format(end - since))
def compare():
compared_files = request.form.getlist("choice")
# print(compared_files)
table = Compare([os.path.join(CONFIG_FOLDER, compared_files[0]), os.path.join(RESULT_FOLDER, compared_files[1])]).compare()
flash('Comparing')
return table
with open('config.json') as json_data_file:
data = json.load(json_data_file)
data['elasticsearch']['host'] = raw_input('Saisir une adresse IP : ')
#data['configApp']['clientID'] = raw_input('Saisir un identifiant : ')
#data['configApp']['fileTypes'] = raw_input('Saisir le type de fichiers a syncroniser [*]: ')
#data['configApp']['syncFolder'] = raw_input('Preciser le dossier a synchroniser : ')
#data['configApp']['nbrVersions'] = raw_input('Nombre de versions a garder [1]: ')
#data['configApp']['syncFreq'] = raw_input('Frequence de synchronisation Cronjob [C]/ OnChange [Ch]: [C] ')
#data['configApp']['syncFreq'] = raw_input('Periodicite de synchronisation : [1 jour] ')
with open('config.json', 'w') as outfile:
json.dump(data, outfile)
print("Config. OK")
if reponse == "1":
print(reponse)
if reponse == "2":
print(reponse)
if reponse == "3":
print(reponse)
if reponse == "4":
path_orig = "/home/superuser/Desktop/Scripting/TryOut/ORIG"
clientID = 3
Comp = Compare(clientID)
filename = "fileZ.txt"
path_bkp = "/home/superuser/Desktop/Scripting/TryOut/BKP"
absolute_path = path_orig + '/' + filename
orig_filehash = Comp.sha256_checksum(absolute_path)
print orig_filehash
Comp.compareHashesAndCopy(clientID, filename, orig_filehash, path_orig,
path_bkp)
from compare import Compare
from longpolling import LongPoll
import time
import config
clever = LongPoll(config.VK_TOKEN)
if __name__ == "__main__":
while True:
question = clever.check_question()
if question:
c = Compare(question.question, question.answers)
c.check()
time.sleep(10)
def flush():
Compare().flush()
return redirect("/index", code=302)
