def main():
print("Input first version:")
ver1 = sys.stdin.readline().strip()
print("Input second version:")
ver2 = sys.stdin.readline().strip()
obj = Compare(ver1, ver2)
res = obj.compare()
print(res)
class Parser():
def __init__(self, config):
#self.fm = FileManager()
self.config = config
self.parsedlist = {}
self.compare = Compare()
self.fm = FileManager()
def parse(self, resources):
"""
parses each resource, calculates diff to existing resource on HD (dependent on lastdate)
"""
for resource in resources:
recent_file = self.getRecentLocalFile(resource.directory)
if resource.diff:
compresult = self.compare.compare(recent_file, resource.url)
logging.debug("Comparing resources: %s and %s" %
(recent_file, resource))
result = {}
result['url'] = resource.url
result['diff'] = compresult.diff
results[resource.url] = result
print compresult
def getRecentLocalFile(self, directory, date=None):
"""
Return newest local file (dump) in directory e.g. data/orf.at
If dump with datestamp can be found this dump will be returned
"""
try:
logging.debug("Trying to alocate file near %s" % directory)
files = reversed(sorted([f for f in os.listdir(directory)]))
lastfile = None
for file in files:
if "dump" in file:
if date != None:
compstr = file.replace(".dump", "")
if float(compstr) < float(date):
if lastfile:
return "%s/%s" % (directory, lastfile)
else:
return "%s/%s" % (directory, file)
elif float(compstr) == float(date):
return "%s/%s" % (directory, file)
elif float(compstr) > float(date):
pass
# return "%s/%s" % (directory, lastfile)
else:
return "%s/%s" % (
directory, file
) #if date is None, return last saved version of file
lastfile = file
except OSError as e:
logging.debug("%s seems like first time fetched" % e)
return None
except ValueError as e:
logging.debug(e)
return None
def checkEquals(self, resourcelist, date=None):
"""
compare duplicates with duplicates in storage
"""
timestamp = time.strftime("%Y%m%d%H%M%S", time.gmtime())
output = {}
output['data'] = {}
output['data']['date'] = timestamp
if date:
output['data']['lastdate'] = date
results = {}
#debug
# datasources = []
# for domain, resourcelist in resourcelist.iteritems():
# datasources.append(self.fm.getAllByDomain(domain, resourcelist))
import pickle
import sys
# fh=open("rdump","w+")
# pickle.dump(datasources,fh)
# fh.close()
fh = open("rdump", "r+")
datasources = pickle.load(fh)
fh.close()
for datasource in datasources:
md5list = {
} #list of md5 hashes of current datasource to avoid downloading alias URIs
for resource in datasource:
if True or "sws.geonames.org" in resource.domain:
directory = dirurl(resource.url)
resource.directory = directory
diff = False
#read content of file of same url of HD if existing
recent_file = self.fm.getRecentLocalFile(directory, date)
if recent_file:
logging.debug("recent_file found: %s" % recent_file)
rf_fh = open(recent_file)
rf_content = rf_fh.read()
if resource.md5 in md5list: #resource already downloaded in this job
logging.debug(
"Already downloaded in this job, creating symlink %s"
% resource.url)
self.fm.symlink(
os.path.join(md5list[resource.md5],
"%s.dump" % timestamp),
os.path.join(resource.directory,
"%s.dump" % timestamp))
elif recent_file == os.path.join(
resource.directory, "%s.dump" % timestamp
): #file exists on harddisk with same name (timedelta too small)
logging.debug(
"Downloaded in the minute before, skipping")
elif recent_file and md5(rf_content).digest(
) != resource.md5: #recent file found on HD, but diffs
logging.debug(
"Recent version found, but diff detected, downloading file %s"
% resource.url)
md5list[resource.md5] = directory
diff = True
self.fm.downloadFile(resource, timestamp)
elif recent_file and md5(rf_content).digest(
) == resource.md5: #recent file located and proved that same size
logging.debug(
"File found on HD with same hash: %s at %s %s %s" %
(resource.url, resource.directory, recent_file,
timestamp))
if resource.directory == os.path.dirname(
recent_file
): #only create symlink if directory differs
logging.debug(
"File already exists in same directory, no symlink created"
)
else:
logging.debug(
"File is in different directory, symlink created %s"
% resource.url)
self.fm.symlink(recent_file, resource.directory,
timestamp)
else: #no recent file found and file is not in md5list
logging.debug("No equals found, downloading file %s" %
resource.url)
md5list[resource.md5] = directory
self.fm.downloadFile(resource, timestamp)
#start diff process
if diff:
compresult = self.compare.compare(
recent_file, resource.url)
logging.debug("Comparing resources: %s and %s" %
(recent_file, resource))
result = {}
result['url'] = resource.url
result['diff'] = compresult.diff
results[resource.url] = result
print compresult
output['equals'] = results
return output
print test.frequencies[:10]
test.get_duration()
print test.duration
test.make_mono()
print test.frequencies[:10]
# call fft on the freqs
test.get_fft()
print test.frequencies[:10]
"""
print "Two identical files that should match:"
test1 = Compare('audio/A4/z01.wav', 'audio/A4/z01.wav', 2.7, 1)
test1.compare()
print "Two diff files that should match:"
test2 = Compare('audio/A4/z01.wav', 'audio/A4/z02.wav', 2.7, 1)
test2.compare()
print "Two diff files that should NOT match:"
test3 = Compare('audio/A4/z02.wav', 'audio/A4/z03.wav', 2.7, 1)
test3.compare()
print "Two diff files that should match:"
test4 = Compare('audio/A4/z03.wav', 'audio/A4/z04.wav', 672, 1)
test4.compare()
print "Two diff files that should NOT match:"
test5 = Compare('audio/A4/z04.wav', 'audio/A4/z05.wav', 2.7, 1)
def update(self):
"""
Proof if online representation is newer or diffs from last downloaded file
If yes, download new version (or make symlink to already downloaded newer file), otherwise do nothing
"""
logging.debug("Updating all resources of datasource: %s" % self.domain)
finished = {
} #list of md5 hashes of current datasource to avoid downloading alias URIs
compare = Compare()
for resource in self.resources:
logging.debug("Updating Resource %s " % resource)
resource.getBody()
resource.timestamp = self.timestamp
resource.directory = formatUrl(resource.url)
if resource.online:
#read content of file of same url of HD if existing
recent_file = self.getRecentLocalFile(resource.directory,
self.lastdate)
if recent_file:
logging.debug("recent_file found: %s" % recent_file)
rf_fh = open(recent_file)
rf_content = rf_fh.read()
rf_fh.close()
if resource.md5 in finished: #resource already downloaded in this job
logging.debug(
"Already downloaded in this job, check if symlink needed %s"
% resource.url)
if recent_file and resource.directory == os.path.dirname(
recent_file
): #only create symlink if directory differs
logging.debug(
"File already exists in same directory, no symlink created"
)
else:
logging.debug(
"File is in different directory, symlink created1 %s"
% resource.url)
self.symlink(
os.path.join(finished[resource.md5]),
os.path.join(resource.directory,
"%s.dump" % self.timestamp))
resource.isDuplicate = True
elif recent_file == os.path.join(
resource.directory, "%s.dump" % self.timestamp
): #file exists on harddisk with same name (timedelta too small)
logging.debug("Downloaded just a few moments ago skipping")
elif recent_file and md5(rf_content).digest(
) != resource.md5: #recent file found on HD, but diffs
logging.debug(
"Recent version found, but diff detected, downloading file %s"
% resource.url)
resource.diff = True
resource.save()
finished[resource.md5] = recent_file
elif recent_file and md5(rf_content).digest(
) == resource.md5: #recent file located and proved that same hash
finished[resource.md5] = recent_file
logging.debug(
"File found on HD with same hash: %s at %s %s %s" %
(resource.url, resource.directory, recent_file,
self.timestamp))
if resource.directory == os.path.dirname(
recent_file
): #only create symlink if directory differs
logging.debug(
"File already exists in same directory, no symlink created"
)
else:
logging.debug(
"File is in different directory, symlink created %s"
% resource.url)
self.symlink(recent_file, resource.directory)
else: #no recent file found and file is not in finished
logging.debug("No equals found, downloading file %s" %
resource.url)
finished[resource.md5] = resource.getPath()
resource.save()
if resource.diff:
compresult = compare.compare(recent_file, resource.url)
resource.diffbody = compresult
logging.debug("Comparing resources: %s and %s" %
(compresult.file1, compresult.file2))
self.complete = 1
return self.resources
def test9():
ver1, ver2 = "1.123443.1", "1.1111111.2"
obj = Compare(ver1, ver2)
res = obj.compare()
assert (res == "1.123443.1 is smaller than 1.1111111.2")
from compare import Compare
from many_extractor import FeatureExtractor as fe
import pickle as pkl
#create compare object
compOBJ = Compare()
#import feature from compare objects
#you can find all of vector data that you need
features = compOBJ.data
# import single image by user : you can use another input methods
single_image_path = input('image path : ')
product_id = input('input product id : ')
#create object of feature extractor on many_extractor file to make vector of single image.
fe_obj = fe()
#####################################################
feature_table = fe_obj.feature_table(single_image_path, product_id, 'shape',
'texture', 'color')
ssim_all = compOBJ.compare(feature_table, 'shape', 'texture', 'color')
with open('compare_results.pkl', 'wb') as f:
pkl.dump(ssim_all, f)
def test7():
ver1, ver2 = "1.13.4.2", "1.13.15.2"
obj = Compare(ver1, ver2)
res = obj.compare()
assert (res == "1.13.4.2 is smaller than 1.13.15.2")
def test8():
ver1, ver2 = "1.13.15.2", "1.13.4.2"
obj = Compare(ver1, ver2)
res = obj.compare()
assert (res == "1.13.15.2 is greater than 1.13.4.2")
def test1():
ver1, ver2 = "1.0", "1.1"
obj = Compare(ver1, ver2)
res = obj.compare()
assert (res == "1.0 is smaller than 1.1")
def test6():
ver1, ver2 = "1.2", "1.20"
obj = Compare(ver1, ver2)
res = obj.compare()
assert (res == "1.2 is equal to 1.20")
def test4():
ver1, ver2 = "1.2.0", "1.2.0.0"
obj = Compare(ver1, ver2)
res = obj.compare()
assert (res == "1.2.0 is equal to 1.2.0.0")
def test2():
ver1, ver2 = "1.1", "1.0"
obj = Compare(ver1, ver2)
res = obj.compare()
assert (res == "1.1 is greater than 1.0")
seed=0,
group_size=1)
comparator = Compare(device)
episodic = Episodic(device)
percent_correct = 0.0
for i, (data, label) in enumerate(dataset):
print("data: ", i)
input = data.to(device)
output = label.to(device)
if i >= 1:
a = episodic.retrieve_all()
indices = comparator.compare(input, a)
prediction = episodic.resolve(indices[:, 0])
prediction_cpu = prediction.cpu()
correct = (prediction == output)
count_correct = np.sum(correct.cpu().numpy())
percent_correct = 0.99 * percent_correct + 0.01 * count_correct * 100 / batch_size
print("Truth: ", dataset.readout(label))
print("Guess: ", dataset.readout(prediction_cpu.flatten()))
print("Percent correct: ", percent_correct)
episodic.learn(input, output)
p, n = episodic.sample(output, 20, 20)
comparator.learn(input, p, n, steps=1)
img = np.reshape(data.numpy(), [-1, data.shape[2]])
cv2.imshow("sample", img)