def getTerms(self, filename, filters=[], relaxed=False):
"""Input file, output a FreqDist of terms"""
filterfname = os.path.join(os.path.dirname(filename), "filter.save")
if os.path.exists(filename + ".nps") and os.path.exists(filterfname):
f = open(filename + ".nps")
old_filters, fd = pickle.load(f)
f.close()
if old_filters == filters:
if not Filter.unstemdict:
Filter._get_stemdict(filterfname)
return fd
NPs = self.getNPs(filename)
fd = FreqDist()
for NP in NPs:
# get the possible terms for each NP
terms = self.extractPossibleTerms(NP, relaxed)
# filter each term by some given criteria
# this requires keeping case information until this point
# filt = Filter.Filter() # class containing all filters
for t in terms:
for f in filters:
t = Filter.criteria[f](t)
if t:
fd[t] += 1
f = open(filename + ".nps", "w")
pickle.dump((filters, fd), f)
f.close()
if os.path.exists(filterfname):
os.remove(filterfname)
return fd
def getGroupWide(folder='../test/patents/US_out/full/'):
"""Return a set of terms used across an entire set of files."""
parser = NPParser.NPParser()
filters = Settings.getDocumentFilters()
if 'stops' in filters:
filters.remove('stops')
termlist = []
filenames = [f for f in os.listdir(folder) if f[-4:]=='.txt']
filtfname = os.path.join(folder, 'filter.save')
if os.path.exists(filtfname):
Filter._get_stemdict(filtfname)
for f in filenames:
nps = parser.getTerms(os.path.join(folder,f), filters)
termlist.append(nps)
# if not os.path.exists(filtfname):
# Filter._save_stemdict(filtfname)
all_terms = set()
for termset in termlist:
all_terms.update(termset)
retlist = set()
for term in all_terms:
count = 0
for termset in termlist:
if term in termset:
count += 1
if count > len(filenames)*0.2:
if 'stem' in filters:
retlist.update(Filter.unstem(term))
else:
retlist.add(term)
return retlist
def run_Filter(self):
self.update_file_names('FilterComplete')
# Set up filter object
f = Filter(self.input_file_path(), self.output_file_path())
if 'inclusionListObject' in self.filter_configs:
f.inclusionListAdd(self.filter_configs["inclusionListObject"])
if 'exclusionListObject' in self.filter_configs:
f.exclusionListAdd(self.filter_configs["exclusionListObject"])
if 'inclusionDirectoryPath' in self.filter_configs:
incList = FilterLists.ListFromDirectory(
self.filter_configs["inclusionDirectoryPath"])
f.inclusionListAdd(incList)
if 'inclusionFilePath' in self.filter_configs:
incList = FilterLists.ListFromExcel(
self.filter_configs["inclusionFilePath"], 'ID',
self.filter_configs["inclusionCutoffDate"])
f.inclusionListAdd(incList)
if 'exclusionDirectoryPath' in self.filter_configs:
excList = FilterLists.ListFromDirectory(
self.filter_configs["exclusionDirectoryPath"])
f.exclusionListAdd(excList)
if 'exclusionURL' in self.filter_configs:
excList = FilterLists.ListFromURL(
self.filter_configs["exclusionURL"])
f.exclusionListAdd(excList)
f.filter()
def main():
args = get_args()
filepath = args['filepath']
method = args['method']
test_filepath = args['testfilepath']
if method != 'mean' and method != 'weighted' and method != 'knn':
print("Incorrect method choice; choose one of: mean | weighted | knn")
sys.exit(0)
matrix, jokes, users = Filter.get_ratings(filepath)
corr_matrix = Filter.get_pearson_corr(users)
test_cases = get_test_cases(matrix, test_filepath)
source_user_ids = [test_case[0] for test_case in test_cases]
total_absolute_error = 0
for test_case in test_cases:
prediction = predict(method, test_case, source_user_ids, matrix, jokes,
users, corr_matrix)
actual = matrix[test_case[0], test_case[1]]
absolute_error = abs(prediction-actual)
print("{}, {}, {}, {}, {}".format(test_case[0], test_case[1], actual,
prediction, absolute_error))
total_absolute_error += absolute_error
mae = total_absolute_error / len(test_cases)
print(mae)
def generateDOT(self, major=None, minor=None):
"""
Generate DOT declarations
"""
# Turn off persistence generating code
self.persistence = False
# turn over all our attrs
ctx = UfoFilter.DOT.Context(self)
# Add sub-contexts
for c in self.subcontexts:
subctx = UfoFilter.DOT.Context(self.subcontexts[c], c)
ctx.addContext(subctx)
tdir = self.tdir
odir = self.odir
uclassname = string.capwords(self.className, '_')
uclassname = string.replace(uclassname, '_', '')
# Generate User DOT - but only if it's not there.
template = tdir + 'ufo_' + self.templateType + '.T'
outfile = odir + '%s.dot.T' % (uclassname)
if not os.access(outfile, os.F_OK):
Filter.Parser(template, outfile, ctx)
else:
print "Will not overwrite %s" % outfile
# Generate Base DOT
template = tdir + 'ufo_' + self.templateType + '_base.T'
outfile = odir + 'generated/%s_base.dot' % (uclassname)
Filter.Parser(template, outfile, ctx)
return ctx
def getTerms(self, filename, filters=[], relaxed=False, overwrite=False):
"""Input file, output a FreqDist of terms"""
filterfname = os.path.join(os.path.dirname(filename), 'filter.save')
if os.path.exists(filename + '.nps') and os.path.exists(filterfname):
f = File(filename + '.nps').openBin(mode='r')
old_filters, fd = pickle.load(f)
f.close()
if old_filters == filters:
if not Filter.unstemdict:
Filter._get_stemdict(filterfname)
return fd
NPs = self.getNPs(filename)
fd = FreqDist()
for NP in NPs:
# get the possible terms for each NP
terms = self.extractPossibleTerms(NP, relaxed)
# filter each term by some given criteria
# this requires keeping case information until this point
# filt = Filter.Filter() # class containing all filters
for t in terms:
for f in filters:
t = Filter.criteria[f](
t) # @semanticbeeng @todo global state mutation
if t:
fd[t] += 1
if overwrite or (not os.path.isfile(filename + '.nps')):
f = File[CHUNKNPS](filename + '.nps').openBin('w')
pickle.dump((filters, fd), f)
f.close()
if os.path.exists(filterfname):
os.remove(filterfname)
return fd
def mrz_tool(filename, except_score): # Return 1 if image is success increase, 0 if failed, 2 if no need to change
mrz_file = read_mrz(filename)
method = 1
count = 1
while mrz_file == None or mrz_file.valid_score < except_score:
if method == 1: # First try to increase brightness
path, new_frame = Filter.increase_brightness(filename, 3) # return a path
method = 2
elif method == 2: # Second try to increase contrast
path, new_frame = Filter.increase_contrast(filename, 127)
method = 3
else: # Last try to increase both
path, new_frame = Filter.increase_brightness(path, 3)
method = 4
mrz_file = read_mrz(path) # Re-identify again
if mrz_file == None:
print(str(count) + ': ' + 'Failed')
count += 1
if method == 4:
return filename, 0
continue
else:
print(str(count) + ': ' + str(mrz_file.valid_score))
count += 1
if mrz_file.valid_score > except_score:
return path, 1 # return a new frame
if mrz_file.valid_score > except_score:
return filename, 2
def datnormhrs(Rdepth, seis, tseis, Vp, Vs, Rho, tlog, wav, AVO):
num = 10
if (seis.shape[1] != wav.size):
raise Exception("seismic and obj.wavletelet has unequal angles")
dat = mfun.segdat(Rdepth, tseis, seis) # to do
vp, vs, rho = mfun.seglog(Rdepth, tlog, Vp, Vs, Rho) # to do
ndat, nang = mfun.size(wav)
sc = mfun.cell(nang)
datnorm = mfun.cell(ndat, nang)
Re = Dobsn(vp, vs, rho, AVO, ang) # to do
sc = np.zeros(nang)
for i in range(nang):
synth = convm(Re[i], wav[i])
Logsc = np.sort(ft.xcorr(wav[i], synth)) # BUG
Trcsc = np.sort(ft.xcorr(wav[i], dat[i])) # BUG
wscl, lag = ft.xcorr(wav[i])
wsc = wscl[np.nonzero(lag == 0)]
Logsc = Logsc / wsc
nsL = len(Logsc)
nsT = len(Trcsc)
dLg = mfun.rms(np.concatenate(Logsc[0:num], Logsc[num:-1]))
dTr = mfun.rms(np.concatenate(Trcsc[0:num], Trcsc[num:-1]))
sc[i] = dTr / dLg
datnorm = seis[i] / sc[i]
return sc, datnorm
def main():
fs = 256000
f_min = 1000
QED_frequencies = np.arange(-3000, 5000,
2000) # Specify operating frequencies
num_qubits = np.log2(len(QED_frequencies))
(x_n, x_k) = syn.SynthesizeSignals(
fs, QED_frequencies
) # This function is used to simulate one period of a QED input signal.
# The phase is random.
# From this point onward we attempt to simulate projective filtering using the time domain signal x_n
re_psi = np.real(x_n)
im_psi = np.imag(
x_n
) # Each filter has access to the real and imaginary signals separately
# We simulate two filters FilterProj0 and FilterProj1. The configuration is decided based on the target qubit where
# qubit 0 is the least significant qubit and implies we should filter every other frequency and qubit n is the most
# significant qubit
target_qubit = 0 # choose target qubit here ( 0 or 1)
# Project onto qubit n state 0 or 1
p_n_0_prefilter = bs.BasisFunctionProj0(
x_n, fs, QED_frequencies, target_qubit
) # Apply the basis function for filter zero based on the target qubit index and return it
p_n_1_prefilter = bs.BasisFunctionProj1(
x_n, fs, QED_frequencies, target_qubit
) # Apply the basis function for filter one based on the target qubit index and return it
# Filter the signals
p_n_0 = fft.DIF_FFT(target_qubit, p_n_0_prefilter, fs, QED_frequencies)
p_n_1 = fft.DIF_FFT(target_qubit, p_n_1_prefilter, fs, QED_frequencies)
def create_tree_menu(f_menu, root):
"""
Автор: Ворожцов Михаил
Цель: Создание treeview c таблицей меню. Cоздаем scroll для этого treeview
Вход: f_type (рамка в данном разделе note_tree), root
Выход: Глобальные переменные(tree_menu)
"""
global tree_menu
#Создаем дерево
tree_menu = ttk.Treeview(f_menu, style="style.Treeview")
tree_menu.pack(side='left', fill='y')
#Присоединим scroll к tree
def create_scroll_menu():
global scroll_menu
scroll_menu = ttk.Scrollbar(f_menu,
orient="vertical",
command=tree_menu.yview)
scroll_menu.pack(side='left', fill='y')
tree_menu.configure(
yscrollcommand=scroll_menu.set) # прикрепляем scroll к tree
create_scroll_menu()
#Объявляем столбцы
tree_menu["columns"] = ("one", "two", "three", "four")
tree_menu.column("#0", width=240, minwidth=100)
tree_menu.column("#1", width=320, minwidth=130)
tree_menu.column("#2", width=270, minwidth=130)
tree_menu.column("#3", width=220, minwidth=150)
tree_menu.column("#4", width=200, minwidth=120)
tree_menu.heading("#0",
text="Код блюда",
anchor=W,
command=lambda: filtr.create_filter(
root, check_open, tree_menu, "Код блюда", "0"))
tree_menu.heading(
"#1",
text="Наименование блюда",
anchor=W,
command=lambda: filtr.create_filter(root, check_open, tree_menu,
"Наименование блюда", "1"))
tree_menu.heading("#2",
text="Стоимость",
anchor=W,
command=lambda: filtr.create_filter(
root, check_open, tree_menu, "Стоимость", "2"))
tree_menu.heading("#3",
text="Размерность",
anchor=W,
command=lambda: filtr.create_filter(
root, check_open, tree_menu, "Размерность", "3"))
tree_menu.heading("#4",
text="Тип блюда",
anchor=W,
command=lambda: filtr.create_filter(
root, check_open, tree_menu, "Тип блюда", "4"))
def check_file(self, pkg, filename):
beam = BeamFile(pkg.files()[filename].path)
if 'debug_info' not in beam.compileinfo['options']:
Filter.printWarning(pkg, "beam-compiled-without-debug_info",
filename)
if not self.source_re.match(beam.compileinfo['source'].value):
Filter.printWarning(pkg, "beam-was-not-recompiled", filename,
beam.compileinfo['source'].value)
def check_file(self, pkg, filename):
if filename.startswith('/usr/lib/debug') or pkg.isSource():
return
if not stat.S_ISREG(pkg.files()[filename].mode):
return
if len(pkg.grep(self.build_root_re, filename)):
Filter.printError(pkg, "file-contains-buildroot", filename)
def filter(self, package, notFilt=False):
if notFilt:
self.javaFiles = Parser.getAllPath(self.directory, '.java')
self.maxFiles = len(self.javaFiles)
else:
filt = Filter(self.directory, Rules.extractPkgNames(package))
self.javaFiles = filt.execute()
self.maxFiles = len(self.javaFiles)
def explore(filename, attrs, cond=False, wait=False, quiet=False):
count = 0
if quiet:
wait = False
with open(filename, "rb") as fp:
# iterate through objects in file
for obj in iterload(fp):
# print out entire object
if attrs[0] == "all":
if not quiet:
print(json.dumps(obj, indent=2))
count += 1
if wait:
raw_input()
# only print out the desired attributes
else:
# check if we are going to filter results
if cond:
# the only arg given is conditional expression
if len(attrs) == 1:
# test the current object
if Flt.filter_exp(attrs[0], obj):
if not quiet:
print(json.dumps(obj, indent=2))
count += 1
if wait:
raw_input()
# the last arg is the conditional expression
else:
if Flt.filter_exp(attrs[-1], obj):
try:
stuff = {attr: obj[attr] for attr in attrs[:-1]}
if not quiet:
print(json.dumps(stuff, indent=2))
count += 1
except KeyError:
print "ERROR: " + str(attrs[:-1]) + " contains an invalid attribute name "
exit()
if wait:
raw_input()
# don't filter the results
else:
# return only the desired attributes
try:
stuff = {attr: obj[attr] for attr in attrs}
if not quiet:
print(json.dumps(stuff, indent=2))
count += 1
except KeyError:
print "ERROR: " + str(attrs[:-1]) + " contains an invalid attribute name "
exit()
if wait:
raw_input()
print "Total objects found: " + str(count)
def generatePHP(self, major, minor):
"""
Generate PHP class
"""
# turn over all our attrs
if major == '5':
ctx = UfoFilter.PHP5.Context(self)
# Add sub-contexts
for c in self.subcontexts:
subctx = UfoFilter.PHP5.Context(self.subcontexts[c], c)
ctx.addContext(subctx)
else:
ctx = UfoFilter.PHP.Context(self)
# Add sub-contexts
for c in self.subcontexts:
subctx = UfoFilter.PHP.Context(self.subcontexts[c], c)
ctx.addContext(subctx)
self.persistenceDelegate.setContext(ctx)
tdir = self.tdir
odir = self.odir
# Generate User Class - but only if it's not there as either a .php or a .php.T file.
template = tdir + 'ufo_' + self.templateType + '.T'
outfile = odir + '%s_%s.php' % (self.prefix, self.className)
outfileT = odir + '%s_%s.php.T' % (self.prefix, self.className)
if not os.access(outfile, os.F_OK) and not os.access(
outfileT, os.F_OK):
Filter.Parser(template, outfile, ctx)
else:
print "Will not overwrite %s" % outfile
# Generate Base Class
template = tdir + 'ufo_' + self.templateType + '_base.T'
outfile = odir + '%s_%s_base.php' % (self.prefix, self.className)
Filter.Parser(template, outfile, ctx)
# generate container if requested
if self.containerName:
template = tdir + 'ufo_container.T'
outfile = odir + '%s_%s.php' % (self.prefix, self.containerName)
outfileT = odir + '%s_%s.php.T' % (self.prefix, self.containerName)
# Generate User Class - but only if it's not there as either a .php or a .php.T file.
if not os.access(outfile, os.F_OK) and not os.access(
outfileT, os.F_OK):
Filter.Parser(template, outfile, ctx)
else:
print "Will not overwrite %s" % outfile
# Generate Base Class
template = tdir + 'ufo_container_base.T'
outfile = odir + '%s_%s_base.php' % (self.prefix,
self.containerName)
Filter.Parser(template, outfile, ctx)
return ctx
def sharpen(self):
if s.is_empty():
self.open()
else:
self.make_duplicate()
path = s.peek()
flt = Filter()
flt.sharpen(path)
self.show_()
def filter_more_(self, filter_type):
if s.is_empty():
self.open()
else:
self.make_duplicate()
path = s.peek()
flt = Filter()
flt.filter_more(path, filter_type)
self.show_()
def check_file(self, pkg, filename):
beam = BeamFile(pkg.files()[filename].path)
if 'debug_info' not in beam.compileinfo['options']:
Filter.printWarning(
pkg, "beam-compiled-without-debug_info", filename)
if not self.source_re.match(Pkg.b2s(beam.compileinfo['source'].value)):
Filter.printWarning(
pkg, "beam-was-not-recompiled", filename,
beam.compileinfo['source'].value)
def __init__(self):
super().__init__()
self.initUI()
self.nega = Filter.Nega()
self.br = Filter.Brightness()
self.med = Filter.Median()
self.med.set_parameter(9)
self.dofft = Filter.DoFFT()
self.dofft.set_parameter(0.1, 1)
self.show()
def get_request(address):
# print(address)
# Try except structure for error handling
try:
r = requests.get(address)
print('connected to Pi: ' + address)
Filter.filter_content(str(r.content))
except:
print('unable to connect to Pi: ' + address)
def update_ratio(aday, pre_day):
portfolio_pool = Utils.stock_pool
if len(portfolio_pool) < 5:
print('Less than 5 stocks for portfolio!! state_dt : ' + str(aday))
pf_src = pf.get_portfolio(portfolio_pool, pre_day, Utils.year)
# 取最佳收益方向的资产组合
risk = pf_src[1][0]
weight = pf_src[1][1]
Filter.filter_main(portfolio_pool, aday, pre_day, weight)
def loopProcess(url):
r = requests.get(url)
Filter.proc(url, r.text, r.headers)
# push all url into raw queue
mats = re.findall(PATTERN_URL, r.text)
if mats is not None:
for mat in mats:
link = mat[0]
if not link.startswith('http://'):
link = url[:url.find('/', 8)] + '/' + link
URLService.pushRawUrl(link)
def playFilteredSound():
Zero.list = []
Pole.list = []
for key in entryDictionary.keys():
entryDictionary[key].complexRoot.addToList()
soundFilter = Filter(Zero.list, Pole.list)
filteredSamples = soundFilter.filterStream(originalSamples)
snd.play(getWaveBytes(inputSoundInfo, filteredSamples))
def check(self, pkg):
ghosts = pkg.ghostFiles()
for filename in pkg.files():
if filename in ghosts:
continue
if not stat.S_ISREG(pkg.files()[filename].mode):
continue
if wrong_compression(os.path.join(pkg.dirname, filename)):
Filter.printError(pkg, 'files-wrong-compression', filename)
def loopProcess(url):
r = requests.get(url)
Filter.proc(url, r.text, r.headers)
# push all url into raw queue
mats = re.findall(PATTERN_URL, r.text)
if mats is not None:
for mat in mats:
link = mat[0]
if not link.startswith('http://'):
link = url[:url.find('/', 8)] + '/' + link
URLService.pushRawUrl(link)
def __init__(self, gpio: int):
self.logger = log.get_logger("DHT11")
super(DHT11, self).__init__(self.logger)
self.delay = 2 # at least 2 seconds between measurements for DHT11!
self.gpio = gpio
self.humidity = None
self.temperature = None
self.humidity_filter = Filter.MovingAverage(10, True, 10., 2)
self.temperature_filter = Filter.MovingAverage(10, True, 10., 2)
self.lock = threading.Lock()
def check_file(self, pkg, filename):
if pkg.isSource() or not stat.S_ISREG(pkg.files()[filename].mode):
return
if pkg.grep(self.suspicious_dir, filename):
Filter.printError(pkg, "invalid-pkgconfig-file", filename)
pc_file = file(pkg.dirName() + "/" + filename, "r")
for l in pc_file:
if l.startswith('Libs:') and self.wronglib_dir.search(l):
Filter.printError(pkg, 'pkgconfig-invalid-libs-dir',
filename, l)
def check_file(self, pkg, filename):
if pkg.isSource() or not stat.S_ISREG(pkg.files()[filename].mode):
return
if pkg.grep(self.suspicious_dir, filename):
Filter.printError(pkg, "invalid-pkgconfig-file", filename)
pc_file = open(pkg.dirName() + "/" + filename, "r")
for l in pc_file:
if l.startswith('Libs:') and self.wronglib_dir.search(l):
Filter.printError(pkg, 'pkgconfig-invalid-libs-dir', filename,
l)
def GetTheUrl(url):
header = {'User-Agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'}
request = urllib2.Request(url, None, header)
try:
response = urllib2.urlopen(request, timeout=20)
except:
pass
else:
page = response.read()
file = open('iplists.html', 'w')
file.write(page)
file.close()
Filter.get_ip('iplists.html')
def saveSound():
Zero.list = []
Pole.list = []
for key in entryDictionary.keys():
entryDictionary[key].complexRoot.addToList()
soundFilter = Filter(Zero.list, Pole.list)
filteredSamples = soundFilter.filterStream(originalSamples)
filename = tkFileDialog.asksaveasfilename(filetypes=[("Waveform Audio", ".wav")],
defaultextension='.wav')
saveSoundToFile(inputSoundInfo, filteredSamples, filename)
def generateJavascript(self, major=None, minor=None):
"""
Generate Javascript class
"""
# turn over all our attrs
ctx = UfoFilter.Javascript.Context(self)
# Add sub-contexts
for c in self.subcontexts:
subctx = UfoFilter.Javascript.Context(self.subcontexts[c], c)
ctx.addContext(subctx)
# Probably want a remoting context instead.
self.persistenceDelegate.setContext( ctx )
tdir = self.tdir
odir = self.odir
uclassname = string.capwords(self.className, '_')
uclassname = string.replace(uclassname, '_', '')
# Generate User Class - but only if it's not there.
template = tdir + 'ufo_' + self.templateType + '.T'
outfile = odir + '%s.js.T' % (uclassname)
if not os.access(outfile, os.F_OK):
Filter.Parser(template, outfile, ctx)
else:
print "Will not overwrite %s" % outfile
# Generate Base Class - Disabled
template = tdir + 'ufo_' + self.templateType + '_base.T'
outfile = odir + 'generated/%s_base.js' % (uclassname)
Filter.Parser(template, outfile, ctx)
# generate container if requested
if self.containerName:
template = tdir + 'ufo_container.T'
outfile = odir + '%sContainer.js.T' % (uclassname)
# Generate User Class - but only if it's not there.
if not os.access(outfile, os.F_OK):
Filter.Parser(template, outfile, ctx)
else:
print "Will not overwrite %s" % outfile
# Generate Base Class
template = tdir + 'ufo_container_base.T'
outfile = odir + 'generated/%sContainer_base.js' % (uclassname)
Filter.Parser(template, outfile, ctx)
return ctx
def predict(method_name, test_case, source_user_ids, matrix, jokes, users,
corr_matrix):
if method_name == 'mean':
prediction = Filter.predict_mean_utility(matrix, jokes, users, test_case[0],
test_case[1])
elif method_name == 'weighted':
prediction = Filter.predict_weighted_sum(matrix, jokes, users, corr_matrix,
test_case[0], test_case[1])
else:
k = 1000
userId_to_knn = Filter.get_knn(corr_matrix, k, source_user_ids)
prediction = Filter.predict_knn_weighted_sum(matrix, jokes, users,
corr_matrix, userId_to_knn,
k, test_case[0], test_case[1])
return prediction
def grep(self, regex, filename):
"""Grep regex from a file, return matching line numbers."""
ret = []
lineno = 0
try:
with open(os.path.join(self.dirName() or '/',
filename.lstrip('/'))) as in_file:
for line in in_file:
lineno += 1
if regex.search(line):
ret.append(str(lineno))
break
except Exception as e:
Filter.printWarning(self, 'read-error', filename, e)
return ret
def grep(self, regex, filename):
"""Grep regex from a file, return matching line numbers."""
ret = []
lineno = 0
try:
with open(os.path.join(
self.dirName() or '/', filename.lstrip('/'))) as in_file:
for line in in_file:
lineno += 1
if regex.search(line):
ret.append(str(lineno))
break
except Exception as e:
Filter.printWarning(self, 'read-error', filename, e)
return ret
def run(args):
usage = 'Usage: ' + args[0] + ' path [-terms|-chem|-dna|-group]'
logging.basicConfig(level=LEVEL)
if len(args) < 2 or len(args) > 3:
print usage
return None
path = None
getFunc = None
for arg in args[1:]:
if os.path.isdir(arg):
path = arg
elif arg == '-terms':
getFunc = getPatentTerms
elif arg == '-chem':
getFunc = getChemicals
elif arg == '-dna':
getFunc = getDNA
elif arg == '-group':
getFunc = getGroupWide
if not path or not getFunc:
print usage
return None
path = os.path.abspath(path)
logging.info('RDG path: '+path)
logging.info('Get Function: '+getFunc.func_name)
if getFunc.func_name == 'getGroupWide':
terms = getFunc(path)
else:
logging.debug('Collecting File ids...')
filenames = [f for f in os.listdir(path) if f[-4:]=='.txt']
terms = []
logging.debug('Finding terms...')
filtfname = os.path.join(path, 'filter.save')
if getFunc.func_name == 'getPatentTerms' and os.path.exists(filtfname):
Filter._get_stemdict(filtfname)
for f in filenames:
logging.debug('...'+f+'...')
terms.extend(getFunc(os.path.join(path,f)))
# if getFunc.func_name == 'getPatentTerms' and not os.path.exists(filtfname):
# Filter._save_stemdict(filtfname)
logging.debug('Clean up...')
if getFunc.func_name == 'getPatentTerms':
temp = set()
for t in terms:
temp.update(Filter.unstem(t))
terms = temp
terms = set(terms)
return terms
def main():
args = get_args()
filepath = args['filepath']
method = args['method']
size = int(args['size'])
repeats = int(args['repeats'])
if method != 'mean' and method != 'weighted' and method != 'knn':
print("Incorrect method choice; choose one of: mean | weighted | knn")
sys.exit(0)
matrix, jokes, users = Filter.get_ratings(filepath)
corr_matrix = Filter.get_pearson_corr(users)
maes = []
for run in range(repeats):
print("Run: {}".format(run + 1))
test_cases = get_test_cases(matrix, size)
source_user_ids = [test_case[0] for test_case in test_cases]
total_absolute_error = 0
for test_case in test_cases:
prediction = predict(method, test_case, source_user_ids, matrix,
jokes, users, corr_matrix)
actual = matrix[test_case[0], test_case[1]]
absolute_error = abs(prediction - actual)
print("{}, {}, {}, {}, {}".format(test_case[0], test_case[1],
actual, prediction,
absolute_error))
total_absolute_error += absolute_error
mae = total_absolute_error / size
maes.append(mae)
print(mae)
print(
"-------------------------------------------------------------------"
)
print("Mean MAE: {}".format(round(statistics.mean(maes), 3)))
if len(maes) > 1:
print("Standard Deviation of MAE: {}".format(statistics.stdev(maes)))
else:
print("Not enough repeats for std of MAE")
def __init__(self, recordFactory, sensorView):
self.recordFactory = recordFactory
self.records = []
self.frecuency = 30.0
self.a = 0
self.sensorView = sensorView
self.filter = Filter.filter()
def mse(dp, testdata, p, q):
length = len(testdata)
mae = 0
rmse = 0
for car, cposition, nextposition in testdata:
rank = Predict.recommend(
dp, p, q, car, cposition) #filter approximate range.list-tuple
gridset, rank = Filter.filterByPosition(dp, cposition,
rank) # norm distribution
maxvalue = 0
for i, s in rank:
if float(s) > maxvalue:
maxvalue = float(s)
if nextposition in gridset:
for i, s in rank:
if (i == nextposition) and (maxvalue != 0):
mae += abs(1 - float(s) / maxvalue)
rmse += (1 - float(s) / maxvalue)**2
break
else:
mae += 1
rmse += 1
content = "mae:" + str((float(mae) / length)) + str('\n')
content += "rmse" + str(math.sqrt(float(rmse) / length))
dp.persitantCurrentPath(content)
def mse_car(testdata, p, q):
ccar = ""
carmse = 0
count = 0
for car, cposition, nextposition in testdata:
if ccar != car: #跳到了另外一条记录
if carmse != 0:
print(math.sqrt(carmse / count))
ccar = car
carmse = 0
count = 0
count += 1
rank = Predict.recommend(
p, q, ccar, cposition) # filter approximate range.list-tuple
gridset, rank = Filter.filterByPosition(cposition,
rank) # norm distribution
maxvalue = 0
for i, s in rank:
if float(s) > maxvalue:
maxvalue = float(s)
if nextposition in gridset and maxvalue != 0:
for i, s in rank:
if i == nextposition:
carmse += abs(1 - float(s) / maxvalue)
#carmse += (1 - float(s) / maxvalue)**2
break
else:
carmse += 1
def check_file(self, pkg, filename):
if filename.startswith('/usr/lib/debug') or pkg.isSource():
return
if not stat.S_ISREG(pkg.files()[filename].mode):
return
grep_date = pkg.grep(self.istoday, filename)
if len(grep_date):
grep_time = pkg.grep(self.looksliketime, filename)
if len(grep_time):
Filter.printError(pkg, "file-contains-date-and-time", filename)
else:
Filter.printWarning(pkg, "file-contains-current-date",
filename)
def check(self, pkg):
if pkg.isSource():
return
md5s = {}
sizes = {}
files = pkg.files()
configFiles = pkg.configFiles()
for f, pkgfile in files.items():
if f in pkg.ghostFiles():
continue
if not stat.S_ISREG(pkgfile.mode):
continue
md5s.setdefault(pkgfile.md5, set()).add(f)
sizes[pkgfile.md5] = pkgfile.size
sum = 0
for f in md5s:
duplicates = md5s[f]
if len(duplicates) == 1:
continue
one = duplicates.pop()
one_is_config = False
if one in configFiles:
one_is_config = True
partition = get_prefix(one)
st = os.stat(pkg.dirName() + '/' + one)
diff = 1 + len(duplicates) - st[stat.ST_NLINK]
if diff <= 0:
for dupe in duplicates:
if partition != get_prefix(dupe):
Filter.printError(pkg, "hardlink-across-partition",
one, dupe)
if one_is_config and dupe in configFiles:
Filter.printError(pkg, "hardlink-across-config-files",
one, dupe)
continue
for dupe in duplicates:
if partition != get_prefix(dupe):
diff = diff - 1
sum += sizes[f] * diff
if sizes[f] and diff > 0:
Filter.printWarning(pkg, 'files-duplicate', one,
":".join(duplicates))
if sum > 100000:
Filter.printError(pkg, 'files-duplicated-waste', sum)
def convert(self, raw_data):
raw_data_s = Filter.s(raw_data)
if raw_data == u'n/a':
return 0
elif raw_data == u'< 1 year':
return 1
elif raw_data == u'10+ years':
return 11
return int(raw_data[0]) + 1
def grep(self, regex, filename):
"""Grep regex from a file, return matching line numbers."""
ret = []
lineno = 0
in_file = None
try:
try:
in_file = open(self.dirName() + '/' + filename)
for line in in_file:
lineno += 1
if regex.search(line):
ret.append(str(lineno))
break
except Exception, e:
Filter.printWarning(self, 'read-error', filename, e)
finally:
if in_file:
in_file.close()
return ret
def grep(self, regex, filename):
"""Grep regex from a file, return matching line numbers."""
ret = []
lineno = 0
in_file = None
try:
try:
in_file = open(os.path.join(
self.dirName() or '/', filename.lstrip('/')))
for line in in_file:
lineno += 1
if regex.search(line):
ret.append(str(lineno))
break
except Exception:
Filter.printWarning(self, 'read-error', filename,
sys.exc_info()[1])
finally:
if in_file:
in_file.close()
return ret
def check(self, pkg):
if pkg.isSource():
return
files = pkg.files()
for f in files:
if f in pkg.ghostFiles():
continue
md5 = files[f].md5
if len(md5) and md5 in (
'c59cbaf0df9bcf35feca0d0f1fc01dae',
'cf8c4d1a5ab88db006c47ae2b51a6b30',
'5d4638159851671944108691f23e4f28',
'0d6be33865b76025c20b48bcac87adb7'):
Filter.printError(pkg, "generic-build-instructions", f)
# bnc 379919
# if len(md5) and md5 in (
# '94d55d512a9ba36caa9b7df079bae19f'):
# printError(pkg, "duplicated-file-gpl-v2", f)
# if len(md5) and md5 in (
# 'd32239bcb673463ab874e80d47fae504'):
# printError(pkg, "duplicated-file-gpl-v3", f)
# bsd causes the false positive COPYING.BSD
if (len(md5) and f.rsplit('/', 1)[1][0].lower() == 'r' and
f.rsplit('.', 1)[-1].lower() in (
'aix', 'bsd', 'dos', 'hpux', 'irix', 'os2', 'mac', 'macos',
'tru64', 'sco', 'vms', 'win32', 'win', 'solaris')):
Filter.printWarning(pkg, "non-linux-readme", f)
if (f.endswith("/Makefile.am") and f[:-3] + ".in" in files and
f in pkg.docFiles()):
if not len(pkg.grep(self.sources_am_re, f)):
Filter.printError(pkg, "makefile-junk", f)
Filter.printError(pkg, "makefile-junk", f[:-3] + ".in")
if f[:-3] in files:
Filter.printError(pkg, "makefile-junk", f[:-3])
def bandFilters(wav, skipFirst):
rangeend = len(wav)/480
frames = []
if(skipFirst): startI = 1
else: startI = 0
for i in range(rangeend):
try:
res = Filter.apply_filter(wav[i*480:(i+1)*480])[startI:]
except Exception as e:
print(e)
raise Exception('LogSumError')
frames.append(res)
return frames
def __save(self, hotels):
for hotel in hotels:
bfStr = hotel["name"] + hotel["lat"] + hotel["lon"]
# Bloom Filter
exist = Filter.isExist(bfStr)
if not exist :
#insert into database and get hotel's review
id = hotel["id"]
print id
eurl = self.eurl_prefix + str(id) +"/"
detail_code = requests.get(eurl)
soup = BeautifulSoup(detail_code.text, "html.parser")
en_name = soup.find('h1',{'itemprop':'name'}).string
if en_name is None:
en_name = hotel["name"]
en_score = soup.find('strong',{'itemprop':'ratingValue'})
if en_score is None:
en_score = '3'
else:
en_score = soup.find('strong',{'itemprop':'ratingValue'}).string
star = 4
if hotel["star"] == 'hotel_halfdiamond06':
star = 5.5
elif hotel["star"] == 'hotel_stars05':
star = 5.0
elif hotel["star"] == 'hotel_halfdiamond05':
star = 4.5
elif hotel["star"] == 'hotel_stars04':
star = 4.0
sql = "insert into hotel(name_cn,name_en,rating_cn,rating_en,starCount,latitude,longitude,originalHotelID,source) values(\"" \
+ hotel["name"] +"\",\""+ en_name +"\","+ hotel["score"] +","+ en_score + ","+ str(star) + ",\"" \
+ hotel["lat"] +"\",\""+hotel["lon"] +"\",\""+hotel["id"]+"\",\"c\")"
print sql
rowid = DB.insert(sql)
print rowid
self.__save_review(rowid,id)
Filter.addItem(bfStr)
def check(self, pkg):
if pkg.isSource():
return
files = pkg.files()
complete_size = 0
lang_size = 0
for f, pkgfile in files.items():
if stat.S_ISREG(pkgfile.mode):
complete_size += pkgfile.size
if pkgfile.lang != '':
lang_size += pkgfile.size
doc_size = 0
for f in pkg.docFiles():
if stat.S_ISREG(files[f].mode):
doc_size += files[f].size
if doc_size * 2 >= complete_size and \
doc_size > 100 * 1024 and \
(complete_size - doc_size) * 20 > complete_size and \
not ignore_pkg(pkg.name):
Filter.printWarning(pkg, "package-with-huge-docs",
("%3d%%" % (doc_size * 100 / complete_size)))
if lang_size * 2 >= complete_size \
and lang_size > 100 * 1024 and \
(complete_size - lang_size) * 20 > complete_size and \
not lang_ignore_pkg(pkg.name):
Filter.printWarning(pkg, "package-with-huge-translation",
("%3d%%" % (lang_size * 100 / complete_size)))
for f in pkg.docFiles():
mode = files[f].mode
if not stat.S_ISREG(mode) or not mode & 0o111:
continue
for ext in ['txt', 'gif', 'jpg', 'html',
'pdf', 'ps', 'pdf.gz', 'ps.gz']:
if f.endswith("." + ext):
Filter.printError(pkg, 'executable-docs', f)
for name in ['README', 'NEWS', 'COPYING', 'AUTHORS']:
if f.endswith("/" + name):
Filter.printError(pkg, 'executable-docs', f)
def simpleFilterTest(ims):
printAll(ims)
print "1.filter by FocalLength"
print "2.filter by DateTime"
f = input("Enter #: ")
if(f == 1):
newView = Filter(ims,'FocalLength')
newView.view()
elif(f == 2):
newView = Filter(ims,'DateTime')
newView.view()
def loadmovies(path):
os.chdir(path)
for file in glob.glob("*.mp4"):
vidcap = cv2.VideoCapture(file)
count = 0;
values = []
while success:
success,image = vidcap.read()
cv2.imshow('Direct Video', image)
frame_color = Filter.select_method(args.method)(frame)[:3]
values.append(frame_color)
count += 1
print (count)
# head, tail = os.path.split("/tmp/d/a.dat")
videopackage = (file,values)
json.dumps(videopackage)
json.dump(videopackage,f)
return
def filterFunc(self):
import Image as i, Filter
self.pic = i.Image("Resources/test.jpg")
if(self.radioButton_0.isChecked()):
self.pic = i.Image("Resources/test.jpg")
if(self.radioButton_1.isChecked()):
self.pic.set_img(Filter.mars(self.pic.get_img()))
if(self.radioButton_2.isChecked()):
self.pic.set_img(Filter.night_vision(self.pic.get_img()))
if(self.radioButton_3.isChecked()):
self.pic.set_img(Filter.night(self.pic.get_img()))
if(self.radioButton_4.isChecked()):
self.pic.set_img(Filter.aquamarine(self.pic.get_img()))
if(self.radioButton_5.isChecked()):
self.pic.set_img(Filter.magenta(self.pic.get_img()))
if(self.radioButton_6.isChecked()):
self.pic.set_img(Filter.pale(self.pic.get_img()))
if(self.radioButton_7.isChecked()):
self.pic.set_img(Filter.wnb(self.pic.get_img()))
if(self.radioButton_8.isChecked()):
self.pic.set_img(Filter.bnw(self.pic.get_img()))
from scipy.misc import imsave
imsave("Resources/test2.jpg",self.pic.get_img())
self.imageUpdate("Resources/test2.jpg")
def extract_domain(url):
url = urllib.unquote(urllib.unquote(urllib.unquote(url)))
if "http" not in url:
print >> sys.stderr, "Error 4, http is not in url, invalid url: %s" % url
return "-1"
url = url.split("http")[-1]
if url.startswith("://"):
url = url.strip("://")
elif url.startswith("s://"):
url = url.strip("s://")
else:
print >> sys.stderr, "Error 2, error http head, invalid url : %s" % url
return "-1"
url = "http://" + url
if url_util.is_valid_url(url):
url = url_util.regularize_url(url)
domain = Filter.domain_url(url)
if domain == "NULL":
print >> sys.stderr, "Error 3, domain is null, invalid url : %s" % url
return "-1"
return domain
else:
print >> sys.stderr, "Error 1, invalid url: %s" % url
return "-1"
if f[:-3] in files:
Filter.printError(pkg, "makefile-junk", f[:-3])
check = CommonFilesCheck()
if Config.info:
Filter.addDetails(
'generic-build-instructions',
"""Your package contains a file that contains the FSF generic
configure/make/make install instructions. Those are useless
for a binary package. Consider removing it to save 3kb of rpm size.""",
'duplicated-file-gpl-v3',
"""Your package contains a file that contains the FSF GPLv3
license. If you really have to ship it, consider symlinking it
from the licenses package.""",
'duplicated-file-gpl-v2',
"""Your package contains a file that contains the FSF GPLv2
license. If you really have to ship it, consider symlinking it
from the licenses package.""",
'non-linux-readme',
"""Your package contains a file that contains instructions
for non-linux platforms. They're most likely unneccessary bloat,
consider removing them from your package.""",
'makefile-junk',
"""Your package contains makefiles that only make sense in a
source package. Did you package a complete directory from the
tarball by using %doc? Consider removing Makefile* from this
directory at the end of your %install section to reduce bloat."""
)
not lang_ignore_pkg(pkg.name):
Filter.printWarning(pkg, "package-with-huge-translation",
("%3d%%" % (lang_size * 100 / complete_size)))
for f in pkg.docFiles():
mode = files[f].mode
if not stat.S_ISREG(mode) or not mode & 0o111:
continue
for ext in ['txt', 'gif', 'jpg', 'html',
'pdf', 'ps', 'pdf.gz', 'ps.gz']:
if f.endswith("." + ext):
Filter.printError(pkg, 'executable-docs', f)
for name in ['README', 'NEWS', 'COPYING', 'AUTHORS']:
if f.endswith("/" + name):
Filter.printError(pkg, 'executable-docs', f)
check = ExecDocsCheck()
Filter.addDetails(
'executable-docs',
"Documentation should not be executable.",
'package-with-huge-docs',
"""More than half the size of your package is documentation.
Consider splitting it into a -doc subpackage.""",
'package-with-huge-translation',
"""More than half the size of your package is language-specific.
Consider splitting it into a -lang subpackage."""
)
def check_file(self, pkg, filename):
if filename.startswith('/usr/lib/debug') or pkg.isSource():
return
if not stat.S_ISREG(pkg.files()[filename].mode):
return
grep_date = pkg.grep(self.istoday, filename)
if len(grep_date):
grep_time = pkg.grep(self.looksliketime, filename)
if len(grep_time):
Filter.printError(pkg, "file-contains-date-and-time", filename)
else:
Filter.printWarning(pkg, "file-contains-current-date",
filename)
check = BuildDateCheck()
if Config.info:
Filter.addDetails(
'file-contains-current-date',
"""Your file contains the current date, this may cause the package
to rebuild in excess.""",
'file-contains-date-and-time',
"""Your file uses __DATE and __TIME__ this causes the package to
rebuild when not needed"""
)
class ErlangCheck(AbstractCheck.AbstractFilesCheck):
def __init__(self):
AbstractCheck.AbstractFilesCheck.__init__(
self, "ErlangCheck", r'.*?\.beam$')
build_dir = rpm.expandMacro("%_builddir")
self.source_re = re.compile(build_dir)
def check_file(self, pkg, filename):
beam = BeamFile(pkg.files()[filename].path)
if 'debug_info' not in beam.compileinfo['options']:
Filter.printWarning(
pkg, "beam-compiled-without-debug_info", filename)
if not self.source_re.match(Pkg.b2s(beam.compileinfo['source'].value)):
Filter.printWarning(
pkg, "beam-was-not-recompiled", filename,
beam.compileinfo['source'].value)
check = ErlangCheck()
Filter.addDetails(
'beam-compiled-without-debug_info',
""""Your beam file indicates that it doesn't contain debug_info.
Please, make sure that you compile with +debug_info.""",
'beam-was-not-recompiled',
"""It seems that your beam file was not compiled by you, but was
just copied in binary form to destination. Please, make sure
that you really compile it from the sources.""",
)
