def find_similar(word, wordlist, hits=5):
ur"""
:param word: the word to find similar words for
:param wordlist: the word list to find similar in
:param hits: the number of hits in first match (more hits will be added than this)
:return: a set with the matches
>>> find_similar(u'rb1203', [u'Rb1203', u'rb 1203', u'gert', u'rb', u'rb1203', u'b1203', u'rb120', u'rb11', u'rb123', u'rb1203_bgfgf'], 5)
[u'rb 1203', u'b1203', u'rb120', u'Rb1203', u'rb1203_bgfgf', u'rb123', u'rb1203']
>>> find_similar(u'1', [u'2', u'3'], 5)
[u'']
>>> find_similar(None, [u'2', u'3'], 5)
[u'']
>>> find_similar(None, None, 5)
[u'']
>>> find_similar(u'1', [], 5)
[u'']
>>> find_similar(u'1', False, 5)
[u'']
>>> find_similar(False, [u'2', u'3'], 5)
[u'']
"""
if None in [word, wordlist] or not wordlist or not word:
return [u'']
matches = set(difflib.get_close_matches(word, wordlist, hits))
matches.update([x for x in wordlist if any((x.startswith(word.lower()), x.startswith(word.upper()), x.startswith(word.capitalize())))])
nr_of_hits = len(matches)
if nr_of_hits == 0:
return [u'']
#Sort again to get best hit first
matches = list(set(difflib.get_close_matches(word, matches, nr_of_hits)))
return matches
def job_info():
#LISTS FOR IDENTIFYING LEADERSHIP SECTION
majors = pd.read_csv(extdataloc+'/ListofPennMajors.csv', sep=',')
jobs = pd.read_csv(Step2output, sep=',')
wfmjobs = pd.read_csv(extdataloc+'/WorkForMoneyJobs.csv', sep=',', header=None)
topemp= pd.read_csv(extdataloc+'/TopPennEngineeringEmployers.csv', sep=',')
jobs['DEGREE'] = [deg.split(' in ', 1)[-1].strip() for deg in jobs['DEGREE']]
jobs['GuessMaj'] = [difflib.get_close_matches(x, majors['Major'], n=1)[0] if
difflib.get_close_matches(x, majors['Major'], n=1) else ''
for x in jobs['DEGREE']]
jobs = jobs.merge(majors, how='left', left_on='GuessMaj', right_on='Major')
jobs['WFM'] = [any([wfm in job.lower() for wfm in[title.lower() for title in wfmjobs[0]]]) for job in jobs['JOBTITLE']]
jobs = jobs.loc[(jobs.JOBCOMPANY != 'Not Listed') & (jobs.JOBTITLE != "Not Listed") & (jobs.Des1 != "Not Listed")]
jobs['GuessTopJob'] = [difflib.get_close_matches(x.lower(), [y.lower() for y in topemp['TopCompany']], n=1, cutoff=0.8)[0] if
difflib.get_close_matches(x.lower(), [y.lower() for y in topemp['TopCompany']], n=1, cutoff=0.8) else ''
for x in jobs['JOBCOMPANY']]
jobs.to_csv(Step3output, sep=",", index=False)
def query_form(filename="merged_table.ipac"):
table = Table.read(os.path.join(app.config['DATABASE_FOLDER'], filename), format='ascii.ipac')
tolerance=1.1
min_values=[np.round(min(table['SurfaceDensity'])/tolerance,4),np.round(min(table['VelocityDispersion'])/tolerance,4),np.round(min(table['Radius'])/tolerance,4)]
max_values=[np.round(max(table['SurfaceDensity'])*tolerance,1),np.round(max(table['VelocityDispersion'])*tolerance,1),np.round(max(table['Radius'])*tolerance,1)]
usetable = table[use_column_names]
best_matches = {difflib.get_close_matches(vcn, usetable.colnames, n=1,
cutoff=0.4)[0]: vcn
for vcn in use_column_names
if any(difflib.get_close_matches(vcn, usetable.colnames, n=1, cutoff=0.4))
}
best_column_names = [best_matches[colname] if colname in best_matches else 'Ignore'
for colname in usetable.colnames]
return render_template("query_form.html", table=table, usetable=usetable,
use_units=use_units, filename=filename,
use_column_names=use_column_names,
best_column_names=best_column_names,
min_values=min_values,
max_values=max_values
)
def handleConstituency(dom,isAC):
if isAC == 'Y':
parentAttr = 'AC_LATLONG'
stateAttr = 'ST_NAME'
constAttr = 'AC_NAME'
enumVal = 'MLA Constituency'
else:
parentAttr = 'PC_LATLONG'
stateAttr = 'ST_NAME'
constAttr = 'PC_NAME'
enumVal = 'MP Constituency'
for node in dom.getElementsByTagName(parentAttr):
if 'KARNATAKA' in node.getElementsByTagName(stateAttr)[0].toxml():
electedrep = []
ac_name = node.getElementsByTagName(constAttr)[0].toxml()
ac_name_str = ac_name[9:len(ac_name)-10].upper().strip('\'')
if isAC == 'Y':
key_str = difflib.get_close_matches(ac_name_str,ac_dict.keys())
if len(key_str) > 0:
electedrep = ac_dict[key_str[0]]
else:
key_str = difflib.get_close_matches(ac_name_str,pc_dict.keys())
if len(key_str) > 0:
electedrep = pc_dict[key_str[0]]
#KARNATAKA is already inserted and is the first record so id=1
if len(electedrep) > 0:
wf.write('insert into tb_electedrep_master (parent,const_ward_name,const_ward_type,current_elected_rep,elec_comm_code,current_elected_party) values (2,\'' \
+ ac_name_str + '\',\'' + enumVal + '\',\'' + electedrep[0] + '\',' + electedrep[1] + ',\'' + electedrep[2] + '\');\n')
def parse_args(args):
"""Parse args from command line."""
service = None
api = None
fmt = None
isbn = get_canonical_isbn(canonical(clean(args[0])))
if len(args) == 1 and isbn:
return (isbn, service, fmt, api)
if isbn:
del args[0]
providers = list(registry.services.keys())
for a in args:
match = get_close_matches(a, fmts)
if len(match) == 1:
fmt = match[0]
args.remove(a)
break
for a in args:
match = get_close_matches(a, providers)
if len(match) == 1:
service = match[0]
args.remove(a)
break
api = args[0] if args else None
return (isbn, service, fmt, api)
def uploaded_file(filename, fileformat=None):
"""
Handle an uploaded file. Takes a filename, which points to a file on disk
in the UPLOAD_FOLDER directory, and an optional file format.
If this fails, it will load the ambiguous file format loader
"""
try:
table = Table.read(os.path.join(app.config['UPLOAD_FOLDER'], filename),
format=fileformat)
except Exception as ex:
print("Did not read table with format={0}. Trying to handle ambiguous version.".format(fileformat))
return handle_ambiguous_table(filename, ex)
best_matches = {difflib.get_close_matches(vcn, table.colnames, n=1,
cutoff=0.4)[0]: vcn
for vcn in valid_column_names
if any(difflib.get_close_matches(vcn, table.colnames, n=1, cutoff=0.4))
}
best_column_names = [best_matches[colname] if colname in best_matches else 'Ignore'
for colname in table.colnames]
return render_template("parse_file.html", table=table, filename=filename,
real_column_names=valid_column_names,
best_column_names=best_column_names,
fileformat=fileformat,
)
def __expand_subject_and_group(self, subject_string):
'''
Expand subject abbreviation
strip the ellipsis
Return full subject name and group number
(or 0 for lecture) in a tuple
'''
from difflib import get_close_matches
subject_string = subject_string.strip()
candidates = get_close_matches(subject_string,
self.subject_abbreviations.values(), 2, 0.7)
if len(candidates) == 1:
return (candidates[0], 0)
if len(candidates) > 1:
print u', '.join(candidates), subject_string
raise Exception(u'Too many subject candidates %s' % candidates)
candidates = get_close_matches(
subject_string, self.subject_abbreviations.keys())
if len(candidates) == 1:
subject = self.subject_abbreviations[candidates[0]]
group = None
try:
group = int(subject_string[-1])
except ValueError:
pass
return (subject, group)
print u', '.join(candidates), subject_string
return ('O_o', 9)
def diff(titles, names, guesses):
titles = titles.lower().split()
names = names.lower().split()
guesses = guesses.lower().split()
title_match = [None] * len(titles)
name_match = [None] * len(names)
for guess in guesses:
for idx, title in enumerate(titles):
if(len(difflib.get_close_matches(guess, [title])) > 0):
title_match[idx] = guess
continue
for idx, name in enumerate(names):
if(len(difflib.get_close_matches(guess, [name])) > 0):
name_match[idx] = guess
continue
matches = 0
result = {}
for i in title_match:
if i is not None: matches = matches + 1
if(matches == len(titles)):
result['title'] = {'found': True, "close": False}
elif(matches > 0):
result['title'] = {'found': False, "close": True}
else:
result['title'] = {'found': False, "close": False}
matches = 0
for i in name_match:
if i is not None: matches = matches + 1
if(matches == len(names)):
result['artist'] = {'found': True, "close": False}
elif(matches > 0):
result['artist'] = {'found': False, "close": True}
else:
result['artist'] = {'found': False, "close": False}
return result
def fuzzy_songs(self, query):
"""
Returns songs matching a query best as possible on either artist
field, etc
"""
query = query.upper()
matched_song_titles = difflib.get_close_matches(query,
self.song_titles)
matched_song_artists = difflib.get_close_matches(query,
self.song_artists)
# if query is beautifully matched, then forget about everything else
strict_priority_title = [x for x in matched_song_titles if x == query]
strict_priority_artists = [
x for x in matched_song_artists if x == query]
if strict_priority_title:
matched_song_titles = strict_priority_title
if strict_priority_artists:
matched_song_artists = strict_priority_artists
matched_songs_bytitle = [
song for song in self.songs if song.title in matched_song_titles]
matched_songs_byartist = [
song for song in self.songs if song.artist in matched_song_artists]
matches = list(set(matched_songs_bytitle + matched_songs_byartist))
return matches
def getClosestAnime(searchText, animeList):
nameList = []
trustedNames = [] #i.e. English/default names
untrustedNames = [] #everything else (French, Italian etc)
for anime in animeList:
for title in anime['titles']:
if title['lang'].lower() in ['x-jat', 'en']:
trustedNames.append(title['title'].lower())
else:
untrustedNames.append(title['title'].lower())
closestNameFromList = difflib.get_close_matches(searchText.lower(), trustedNames, 1, 0.85)
if closestNameFromList:
for anime in animeList:
for title in anime['titles']:
if closestNameFromList[0].lower() == title['title'].lower() and title['lang'].lower() in ['x-jat', 'en']:
return anime
else:
closestNameFromList = difflib.get_close_matches(searchText.lower(), untrustedNames, 1, 0.85)
if closestNameFromList:
for anime in animeList:
for title in anime['titles']:
if closestNameFromList[0].lower() == title['title'].lower() and title['lang'].lower() not in ['x-jat', 'en']:
return anime
return None
def filterResults(self):
showName = AnimeInfoExtractor(self._filename).getName()
for entry in self._results:
if difflib.get_close_matches(showName, entry.titles, 3, 0.9) != []:
entry.matchBoost += 2
if difflib.get_close_matches(showName, entry.titles, 3, 0.8) != []:
entry.matchBoost += 0.5
def findXBMCEpFile(self,show,season,eptitle): if self.json_use_http: jrapi = jsonrpc.jsonrpcAPI(url=self.http_address + '/jsonrpc',user=self.http_user,password=self.http_pass) else: jrapi = jsonrpc.jsonrpcAPI() labels = [] ids = [] for s in jrapi.VideoLibrary.GetTVShows()['tvshows']: labels.append(s['label']) ids.append(s['tvshowid']) mshow = difflib.get_close_matches(show,labels,1,0.7) if not mshow: return mshow = mshow[0] eplist = jrapi.VideoLibrary.GetEpisodes(tvshowid=ids[labels.index(mshow)],season=season) if not 'episodes' in eplist: return labels = [] files = [] for e in eplist['episodes']: labels.append(e['label']) files.append(e['file']) mep = difflib.get_close_matches(eptitle,labels,1,0.7) if not mep: return #print mep #print labels efile = files[labels.index(mep[0])] #print efile return efile
def uploaded_file(filename, fileformat=None):
print "In uploaded_file, filename={0}, fileformat={1}".format(filename, fileformat)
print request.form
print request
try:
table = Table.read(os.path.join(app.config['UPLOAD_FOLDER'], filename),
format=fileformat)
print "Successfully read table with fileformat={0}".format(fileformat)
except Exception as ex:
print "Did not read table with format={0}. Trying to handle ambiguous version.".format(fileformat)
return handle_ambiguous_table(filename, ex)
best_matches = {difflib.get_close_matches(vcn, table.colnames, n=1,
cutoff=0.4)[0]: vcn
for vcn in valid_column_names
if any(difflib.get_close_matches(vcn, table.colnames, n=1, cutoff=0.4))
}
print best_matches
best_column_names = [best_matches[colname] if colname in best_matches else 'Ignore'
for colname in table.colnames]
print 'best_column_names:', best_column_names
return render_template("parse_file.html", table=table, filename=filename,
real_column_names=valid_column_names,
best_column_names=best_column_names,
fileformat=fileformat,
)
def evalKey(fieldname, **params):
if fieldname in FezAlarmFaxChecker().fields:
_str = FezAlarmFaxChecker().fields[fieldname][0]
else: # key not found
FezAlarmFaxChecker().fields[fieldname] = (u'----', 0)
raise
if _str == '':
FezAlarmFaxChecker().fields[fieldname] = (_str, 0)
return
keys = {k.key: k.id for k in Alarmkey.getAlarmkeys()}
try:
repl = difflib.get_close_matches(_str.strip(), keys.keys(), 1, cutoff=0.8) # default cutoff 0.6
if len(repl) == 0:
repl = difflib.get_close_matches(_str.strip(), keys.keys(), 1) # try with default cutoff
if len(repl) > 0:
k = Alarmkey.getAlarmkeys(int(keys[repl[0]]))
FezAlarmFaxChecker().fields[fieldname] = (u'{}: {}'.format(k.category, k.key), k.id)
FezAlarmFaxChecker().logger.debug(u'key: found "{}: {}"'.format(k.category, k.key))
return
FezAlarmFaxChecker().logger.info(u'key: "{}" not found in alarmkeys'.format(_str))
FezAlarmFaxChecker().fields[fieldname] = (_str, 0)
except:
FezAlarmFaxChecker().logger.error(u'key: error in key evaluation')
finally:
return
def missing_part():
filedict = {}
strlist = ['mp','mla','ward']
for each in strlist:
readfile = open(rootdir+'data/missing/' + each + '_missing_only.csv','r')
for line in readfile.readlines()[1:]:
row = line.split(',')
try:
school_id = int(row[4].strip())
if each == 'mp':
mpkey = difflib.get_close_matches(row[6].strip().upper(),mpDict.keys())
mp_id = mpDict[mpkey[0]][0] if len(mpkey) != 0 else 'null'
updatefile.write('UPDATE tb_school_electedrep set mp_const_id=' + str(mp_id) + ' where sid=' + str(school_id) + ';\n')
if each == 'mla':
mlakey = difflib.get_close_matches(row[6].strip().upper(),mlaDict.keys())
mla_id = mlaDict[mlakey[0]][0] if len(mlakey) != 0 else 'null'
updatefile.write('UPDATE tb_school_electedrep set mla_const_id=' + str(mla_id) + ' where sid=' + str(school_id) + ';\n')
if each == 'ward':
wardkey = difflib.get_close_matches(row[6].strip().upper(),wardDict.keys())
ward_id = wardDict[wardkey[0]][0] if len(wardkey) != 0 else 'null'
updatefile.write('UPDATE tb_school_electedrep set ward_id=' + str(mla_id) + ' where sid=' + str(school_id) + ';\n')
except:
print "Invalid schoolid" + row[4].strip()
print "Unexpected error:", sys.exc_info()
print "Exception in user code:"
print '-'*60
traceback.print_exc(file=sys.stdout)
print '-'*60
def migrate_informers(apps, schema_editor):
Informer = apps.get_model("neutron", "Informer")
for informer in Informer.objects.all():
known_us = None
education = None
try:
education_list = ['universidad', 'master', 'secundaria', 'primaria', 'basica']
item = difflib.get_close_matches(informer.education, education_list)
if item in education_list[0:2]:
education = Informer.KNOWN_US.university
elif item in education_list[2:3]:
education = Informer.KNOWN_US.secundaria
elif item in education_list[3:]:
education = Informer.KNOWN_US.basica
except Exception as e:
print("Error: {}".format(e))
try:
known_list = ['university', 'universidad', 'calamo & cran', 'calamo']
item = difflib.get_close_matches(informer.known_us, known_list)
if item in known_list[0:2]:
known_us = Informer.KNOWN_US.university
elif item in known_list[2:]:
known_us = Informer.KNOWN_US.calamo
except Exception as e:
print("Error: {}".format(e))
informer.known_us_choice = known_us
informer.education_choice = education
informer.save()
def find_release(artist, title, album=None):
rlist = []
try:
if not artist:
query = title
elif album:
query = "%s %s" % (artist, album)
else:
query = "%s %s" % (artist, title)
s = discogs.Search(query)
rs = [i for i in s.results()[:15] if release_filter(i)]
except discogs.HTTPError:
return []
# if album name was provided, filter out non-matching
if album:
for r in rs:
rt = r.data["title"]
if not difflib.get_close_matches(album, [rt]):
rs.remove(r)
for r in rs:
for i in range(len(r.data["tracklist"])):
t = r.data["tracklist"][i]["title"]
cm = difflib.get_close_matches(title, [t])
if cm:
rlist.append((r, i + 1))
if not rlist:
for r in rs:
for i in range(len(r.data["tracklist"])):
rlist.append((r, i + 1))
return rlist
def test_product_title_similarity(self):
from difflib import get_close_matches
titles = [
u"Молоко Простоквашино пастеризованное 2,5% 930 мл",
u"Молоко Простоквашино пастеризованное 3,2% 930 мл",
u"Молоко Домик в деревне ультрапастеризованное 2,5%, 1450г",
u"Молоко М Лианозовское ультрапастеризованное 1,5%, 950г",
u"Молоко Новая деревня отборное пастеризованное 3,5%, 1л",
u"Молоко Тевье молочник Luxury пастеризованное 3,4-4%, 1л",
u"Молоко Рузское пастеризованное 2,5%, 1000г",
u"Молоко Ясный луг ультрапастеризованное 3,2%, 1л",
u"Молоко Parmalat ультрапастеризованное 3,5%, 1л",
]
t1 = u"Молоко Простоквашино, пастеризованное, 2,5% 930 мл"
t1_1 = u"Молоко ПРОСТОКВАШИНО пастеризованное, 3,2% 930 мл".lower()
t2 = u'МОЛОКО "ЯСНЫЙ ЛУГ" ПИТЬЕВОЕ УЛЬТРАПАСТЕРИЗОВАННОЕ 3,5% 1 Л'.lower()
t3 = (u"Молоко М Лианозовское ультрапастеризованное 3,5%, 950г",)
t4 = u"Каждыйдень пастеризованное 3,2% 1л"
cut_off = 0.7
cleared_titles = [t.replace(u"Молоко", "") for t in titles]
self.assertEqual(get_close_matches(t1, cleared_titles, 1, cut_off)[0], cleared_titles[0])
self.assertEqual(get_close_matches(t1_1, cleared_titles, 1, cut_off)[0], cleared_titles[1])
self.assertEqual(get_close_matches(t2, cleared_titles, 1, cut_off)[0], cleared_titles[7])
self.assertEqual(0, len(get_close_matches(t3, cleared_titles, 1, cut_off)))
res = get_close_matches(t4, cleared_titles, 1, cut_off)
self.assertEqual(0, len(res))
def find_closest_track(self, session, track):
exact_match = (session.query(Track)
.join(Track.artist)
.join(Track.album)
.filter(
Artist.name_insensitive == track.artist_name,
Album.name_insensitive == track.album_name,
Track.name_insensitive == track.name)
.first())
if exact_match:
return exact_match
mapping = self.track_mapping(session)
track_string = '\x01'.join((track.artist_name, track.album_name, track.name))
closest_matches = get_close_matches(track_string, mapping.keys(), n=20, cutoff=0.8)
if not closest_matches:
return None
closest_track_mapping = OrderedDict((mapping[match].name, mapping[match])
for match in closest_matches)
closest_track_name = get_close_matches(track.name, closest_track_mapping.keys(), n=1)
if not closest_track_name:
return None
return closest_track_mapping[closest_track_name[0]]
def getSuggestions(commands, command, tagmap):
# get suggestions of commands that are misspelled
suggestions = difflib.get_close_matches(command, commands, cutoff=MEDIUM_MATCH)
if len(suggestions) == 0:
tags = tagmap.keys()
# see if command matches a tag of a command
tagmatches = difflib.get_close_matches(command, tags, cutoff=QUALITY_MATCH)
# for each tag this command is similar to...
for tag in tagmatches:
matchingcommands = tagmap[tag]
# for each command that has that tag...
for matchingcommand in matchingcommands:
# if not added as a suggested command, add it
if not matchingcommand in suggestions:
if command != tag:
suggestions.append(matchingcommand) # "%s(i.e., %s)" % (matchingcommand, tag))
else:
suggestions.append(matchingcommand)
for c in commands:
if command in c and c not in suggestions:
suggestions.append(c)
if len(suggestions) == 0:
return ""
suggestions = sortSuggestions(command, suggestions)
return suggestions
def answer(self, answer):
if self.session_ended:
return ["PRIVMSG $C$ :Please start a new game by typing !akinator new"]
if self.has_solution:
answer = answer.lower()
close_matches = difflib.get_close_matches(answer, self.end_answers)
if len(close_matches) > 1 and close_matches[0] != answer:
return ["PRIVMSG $C$ :Did you mean: '%s'" % "', '".join(close_matches)]
if len(close_matches) == 0:
return ["PRIVMSG $C$ :I cannot accept that as an answer."]
match = self.end_answers[close_matches[0]]
if match == 0:
self.session_ended = True
return ["PRIVMSG $C$ :Great! Guessed right one more time! I love playing with you!"]
if self.question_number == 40:
self.session_ended = True
c = urllib2.urlopen('http://en.akinator.com/liste_best.php?prio=0&partie=%s&signature=%s&nqp=40&age=23&engine=0' % (self.partie, self.signature))
self.current_page = c.read()
alts = re.findall('>([^<>]+)</a></td>', self.current_page)
alts = [("PRIVMSG $C$ :\x1f%s\x1f" % '\x1f, \x1f'.join(alts)).decode('latin-1')]
return ["PRIVMSG $C$ :Bravo! You have defeated me. Other things I was thinking of:"] + alts
url = self.next_url
self.has_solution = False
else:
answer = answer.lower()
close_matches = difflib.get_close_matches(answer, self.answers)
if len(close_matches) > 1 and close_matches[0] != answer:
return ["PRIVMSG $C$ :Did you mean: '%s'" % "', '".join(close_matches)]
if len(close_matches) == 0:
return ["PRIVMSG $C$ :I cannot accept that as an answer."]
match = close_matches[0]
self.response = self.answers[match]
url = self.next_url % self.response
c = urllib2.urlopen(url)
self.current_page = c.read()
if 'I think of ...' in self.current_page:
solution = re.search('ouvre_photo\(".+?",-?\d+,"(.+?)","(.*?)",\d+', self.current_page)
if len(solution.group(2)) > 0:
solution = solution.group(1) + " (%s)" % solution.group(2)
else:
solution = solution.group(1)
self.next_url = 'http://en.akinator.com/continue_partie.php?prio=0&partie=%s&signature=%s&nqp=%d&age=23&engine=0' % (self.partie, self.signature, self.question_number)
print self.next_url
self.has_solution = True
return [("PRIVMSG $C$ :I think of ... %s" % solution).decode('latin-1')]
self.next_url = re.search('window.location="(.+?)"', self.current_page).group(1)
self.next_url = 'http://en.akinator.com/%s%s&step_prop=-1' % (self.next_url, '%d')
question = re.search('</center>([^<]+)', self.current_page).group(1)
self.question_number = int(re.search('Question N\xb0 (\d+)', self.current_page).group(1))
return [("PRIVMSG $C$ :Question N\xb0 %d: %s" % (self.question_number, question)).decode('latin-1')]
def parse(self, keyword=None):
roles = HealthRole.objects.all()
by_keyword = dict((role.keyword, role) for role in roles)
matches = difflib.get_close_matches(keyword.upper(), by_keyword)
if not matches:
raise FormatError(
u"Did not understand the keyword: '%s'." % keyword)
keyword = matches[0]
result = {
'role': by_keyword[keyword.upper()],
}
try:
code = u"".join(pico.digits())
except:
raise FormatError(u"Expected an HMIS facility code (got: %s)." %
"".join(remaining()))
try:
facility = result['facility'] = Facility.objects.filter(code=code).get()
except Facility.DoesNotExist:
raise FormatError(u"No such HMIS facility code: %s." % code)
whitespace()
optional(pico.separator, None)
# optionally provide a sub-village group
name = "".join(remaining()).strip()
if name:
# get all (name, location) pairs of all child nodes of
# groups that report to this facility
policies = {}
policy = None
group = None
for policy in facility.policies.all().select_related():
policies[policy.group.name.upper()] = policy
for descendant in policy.group.get_descendants():
try:
group_name = descendant.name.upper()
policies[group_name] = descendant.reporting_policy
except ReportingPolicy.DoesNotExist:
pass
matches = difflib.get_close_matches(name.upper(), policies)
if matches:
name = matches[0]
group = policies[name].group
elif policy is not None:
group = policy.group.add_child(
slug="added_by_user", name='"%s"' % name)
facility.policies.create(group=group)
result['group'] = group
return result
def buildAlarmFromText(self, alarmtype, rawtext):
values = {}
if alarmtype:
sections = alarmtype.getSections()
sectionnames = {s.name: s.key for s in sections}
sectionmethods = {s.key: s.method for s in sections}
FezAlarmFaxChecker().fields['alarmtype'] = (alarmtype, 0)
else: # matching alarmtype found
return values
curr_field = ""
for l in rawtext.split(u"\n"):
field = difflib.get_close_matches(re.split(u':|=|i ', l)[0], sectionnames.keys(), 1) # test line ->:|=
if len(field) == 0 and u" " in l:
field = difflib.get_close_matches(l.split()[0], sectionnames.keys(), 1) # test line first word
if len(field) == 1:
value = u":".join(re.split(u':|=|i ', l)[1:]).strip()
else:
field = [] # field has to be empty -> prevent overwriting
value = l
if len(field) == 1:
if sectionnames[field[0]] != u'':
FezAlarmFaxChecker().fields[sectionnames[field[0]]] = (value, 0)
curr_field = field[0]
elif curr_field != '':
if sectionnames[curr_field] != u'':
if value[0] == u' ':
FezAlarmFaxChecker().fields[sectionnames[curr_field]] = (u"\n".join((FezAlarmFaxChecker().fields[sectionnames[curr_field]][0], value)), FezAlarmFaxChecker().fields[sectionnames[curr_field]][1])
else:
FezAlarmFaxChecker().fields[sectionnames[curr_field]] = (u"".join((FezAlarmFaxChecker().fields[sectionnames[curr_field]][0], value)), FezAlarmFaxChecker().fields[sectionnames[curr_field]][1])
for section in sections: # sections in order
k = section.key
if sectionmethods[k] != u'':
method = sectionmethods[k].split(u';')[0]
try: # method parameters (optional)
method_params = sectionmethods[k].split(u';')[1]
except:
method_params = ''
# execute methods
try:
getattr(self, method)(k, alarmtype=alarmtype, options=method_params.split(';')) # fieldname, parameters
except:
if u'error' not in values:
values['error'] = ''
values['error'] = u"".join((values['error'], u'error in method: {}\n'.format(method)))
FezAlarmFaxChecker().logger.error(u'error in section {} {}\n{}'.format(k, method, traceback.format_exc()))
for k in FezAlarmFaxChecker().fields:
try:
values[k] = (FezAlarmFaxChecker().fields[k][0].decode('utf-8'), FezAlarmFaxChecker().fields[k][1])
except:
values[k] = (FezAlarmFaxChecker().fields[k][0], FezAlarmFaxChecker().fields[k][1])
return values
def _find_key(field, field_list):
keys = difflib.get_close_matches(field, field_list)
if keys == []:
keys = difflib.get_close_matches(field.upper(), field_list)
if keys == []:
raise KeyError("key '{}' not found in {}".format(field, field_list))
return keys[0]
def uploaded_file(filename, fileformat=None):
"""
Handle an uploaded file. Takes a filename, which points to a file on disk
in the UPLOAD_FOLDER directory, and an optional file format.
If this fails, it will load the ambiguous file format loader
"""
try:
table = Table.read(os.path.join(app.config['UPLOAD_FOLDER'], filename),
format=fileformat)
except Exception as ex:
print("Did not read table with format={0}."
" Trying to handle ambiguous version.".format(fileformat))
return handle_ambiguous_table(filename, ex)
best_matches = {difflib.get_close_matches(vcn, table.colnames, n=1,
cutoff=0.4)[0]: vcn
for vcn in valid_column_names
if any(difflib.get_close_matches(vcn, table.colnames, n=1,
cutoff=0.4))}
best_column_names = [best_matches[colname] if colname in best_matches
else 'Ignore' for colname in table.colnames]
# column names can't have non-letters in them or javascript fails
fix_bad_colnames(table)
# read units and keywords from table
column_units = [table[cln].unit for cln in table.colnames]
tab_metadata = {'Author': '', 'ADS_ID': '', 'Publication DOI or URL': '',
'Submitter': '', 'IsObserved': False,
'IsSimulated': False, 'IsGalactic': False,
'IsExtragalactic': False,
'DataURL': False, 'synthimURL': False}
metadata_name_mapping = {'author': 'Author',
'ads': 'ADS_ID', 'ads_id': 'ADS_ID',
'doi': 'Publication DOI or URL',
'url': 'Publication DOI or URL',
'email': 'Submitter', 'submitter': 'Submitter',
'isobserved': 'IsObserved',
'issimulated': 'IsSimulated',
'isgalactic': 'IsGalactic',
'isextragalactic': 'IsExtragalactic',
'dataurl': 'DataURL', 'synthimurl': 'synthimURL'}
if len(table.meta) > 0:
for name, keyword in table.meta['keywords'].items():
if name.lower() in metadata_name_mapping:
outkey = metadata_name_mapping[name.lower()]
tab_metadata[outkey] = keyword['value']
return render_template("parse_file.html", table=table, filename=filename,
real_column_names=valid_column_names,
best_column_names=best_column_names,
additional_metadata=additional_metadata_names,
unit_mapping=unit_mapping,
default_units=column_units,
tab_metadata=tab_metadata,
fileformat=fileformat)
def update(self, efile):
self.einsatz = opp.opp(efile)
# Clear all fields
self.lbl_stichwort['text'] = ""
self.lbl_schlagwort['text'] = ""
self.lbl_txt_wo['text'] = ""
self.lbl_estrasse['text'] = ""
self.lbl_eort['text'] = ""
self.lbl_eobjekt['text'] = ""
self.lbl_ekreuz['text'] = ""
self.lbl_bem['text'] = ""
if self.einsatz.load():
# Update with new data
self.lbl_stichwort['text'] = self.einsatz.get_fld_egnd_stichw()
if (self.einsatz.get_fld_egnd_stichw().find("THL") != -1):
self.lbl_stichwort['bg'] = '#7777FF'
elif (self.einsatz.get_fld_egnd_stichw().find("B") != -1):
self.lbl_stichwort['bg'] = '#FF9999'
elif (self.einsatz.get_fld_egnd_stichw().find("Unwetter") != -1):
self.lbl_stichwort['bg'] = '#7777FF'
else:
self.lbl_stichwort['bg'] = 'gray50'
self.lbl_schlagwort['text'] = self.einsatz.get_fld_egnd_schlagw()
self.lbl_estrasse['text'] = self.einsatz.get_fld_eort_strasse() + ' ' + self.einsatz.get_fld_eort_hnummer()
self.lbl_eort['text'] = self.einsatz.get_fld_eort_ort()
self.lbl_eobjekt['text'] = self.einsatz.get_fld_eort_objekt()
self.lbl_ekreuz['text'] = self.einsatz.get_fld_eort_kreuzung()
self.lbl_bem['text'] = self.einsatz.get_fld_bem()
# Show arrows
fz_right = False
fz_left = False
for f in self.einsatz.get_lst_emittel():
if get_close_matches(f, [self.__conf.get_vehicle_0()], 1, 0.9):
fz_right = True
if get_close_matches(f, [self.__conf.get_vehicle_1()], 1, 0.9):
fz_left = True
if (fz_left):
self.cv_arrow_left.place(x=+6, y=+110)
else:
self.cv_arrow_left.place(x=-100, y=-100)
if (fz_right):
self.cv_arrow_right.place(x=970, y=+110)
else:
self.cv_arrow_right.place(x=-100, y=-100)
def post(self, request, **kwargs):
cm = get_object_or_404(CommitteeMeeting, pk=kwargs['pk'])
bill = None
request = self.request
user_input_type = request.POST.get('user_input_type')
if user_input_type == 'bill':
bill_id = request.POST.get('bill_id')
if bill_id.isdigit():
bill = get_object_or_404(Bill, pk=bill_id)
else: # not a number, maybe its p/1234
m = re.findall('\d+',bill_id)
if len(m)!=1:
raise ValueError("didn't find exactly 1 number in bill_id=%s" % bill_id)
pp = PrivateProposal.objects.get(proposal_id=m[0])
bill = pp.bill
if bill.stage in ['1','2','-2','3']: # this bill is in early stage, so cm must be one of the first meetings
bill.first_committee_meetings.add(cm)
else: # this bill is in later stages
v = bill.first_vote # look for first vote
if v and v.time.date() < cm.date: # and check if the cm is after it,
bill.second_committee_meetings.add(cm) # if so, this is a second committee meeting
else: # otherwise, assume its first cms.
bill.first_committee_meetings.add(cm)
bill.update_stage()
action.send(request.user, verb='added-bill-to-cm',
description=cm,
target=bill,
timestamp=datetime.datetime.now())
if user_input_type == 'mk':
mk_names = Member.objects.values_list('name', flat=True)
mk_name = difflib.get_close_matches(request.POST.get('mk_name'),
mk_names)[0]
mk = Member.objects.get(name=mk_name)
cm.mks_attended.add(mk)
cm.save() # just to signal, so the attended Action gets created.
action.send(request.user,
verb='added-mk-to-cm',
description=cm,
target=mk,
timestamp=datetime.datetime.now())
if user_input_type == 'remove-mk':
mk_names = Member.objects.values_list('name', flat=True)
mk_name = difflib.get_close_matches(request.POST.get('mk_name'),
mk_names)[0]
mk = Member.objects.get(name=mk_name)
cm.mks_attended.remove(mk)
cm.save() # just to signal, so the attended Action gets created.
action.send(request.user,
verb='removed-mk-to-cm',
description=cm,
target=mk,
timestamp=datetime.datetime.now())
return HttpResponseRedirect(".")
def did_you_mean(s, candidates, n=3, cutoff=0.8):
"""
When a string isn't found in a set of candidates, we can be nice
to provide a list of alternatives in the exception. This
convenience function helps to format that part of the exception.
Parameters
----------
s : str
candidates : sequence of str or dict of str keys
n : int
The maximum number of results to include. See
`difflib.get_close_matches`.
cutoff : float
In the range [0, 1]. Possibilities that don't score at least
that similar to word are ignored. See
`difflib.get_close_matches`.
Returns
-------
message : str
Returns the string "Did you mean X, Y, or Z?", or the empty
string if no alternatives were found.
"""
# The heuristic here is to first try "singularizing" the word. If
# that doesn't match anything use difflib to find close matches in
# original, lower and upper case.
matches = []
if s.endswith("s"):
if s[:-1] in candidates:
matches = [s[:-1]]
if not len(matches):
if isinstance(s, six.text_type):
s = strip_accents(s)
matches = list(
set(
difflib.get_close_matches(s, candidates, n=n, cutoff=cutoff)
+ difflib.get_close_matches(s.lower(), candidates, n=n, cutoff=cutoff)
+ difflib.get_close_matches(s.upper(), candidates, n=n, cutoff=cutoff)
)
)
if len(matches):
matches.sort()
if len(matches) == 1:
matches = matches[0]
else:
matches = ", ".join(matches[:-1]) + " or " + matches[-1]
return "Did you mean {0}?".format(matches)
return ""
def matchingSens(osmLine):
index = 0
# Looping over the stations because sometimes some of them
# are not the same depending of the direction
while index+2 < len(osmLine["sensA"])-1:
firstStationName = ots.stationName(osmLine["sensA"][index+0])
secondStationName = ots.stationName(osmLine["sensA"][index+1])
if secondStationName == firstStationName:
secondStationName = ots.stationName(osmLine["sensA"][index+2])
lineId = mbt.idForLine(osmLine["name"])
#ordered Stations list from mobitrans
lineStations = mbt.stationsForLine(lineId, 1)
stationList = [x["name"] for x in lineStations]
result1 = difflib.get_close_matches(firstStationName, stationList)
result2 = difflib.get_close_matches(secondStationName, stationList)
#second chance, looking for substrings:
if not result1:
result1 = [s for s in stationList if s in firstStationName]
if not result2:
result2 = [s for s in stationList if s in secondStationName]
# third change, doing the same but with no accent nor diacritics
if not result1:
asciiName = ''.join(c for c in unicodedata.normalize('NFD', firstStationName) if unicodedata.category(c) != 'Mn')
result1 = [s for s in stationList if s in asciiName]
if not result2:
asciiName = ''.join(c for c in unicodedata.normalize('NFD', secondStationName) if unicodedata.category(c) != 'Mn')
result2 = [s for s in stationList if s in asciiName]
if result1 and result2:
break
else:
index += 1
if not result1 or not result2:
#print(firstStationName, secondStationName, stationList)
print("\n*No match found while calculating directions for line", osmLine["name"], firstStationName, secondStationName, "")
print(stationList)
return
index1 = stationList.index(result1[0])
index2 = stationList.index(result2[0])
if index1 < index2:
sens = 1
else:
sens = 2
return sens
def load_cog(cog_name, skelecog=False, path=""):
"""Return any traditional cog as an Actor.
Args:
cog_name (str): The name of the desired cog.
skelecog (bool, optional): If `True`, a skelecog will be returned.
Defaults to `False`.
path (str, optional): The file path to the Toontown phase files.
Defaults to Panda3D's search path.
Examples:
from toontown import load_cog
Hollywood = load_cog("Mr. Hollywood")
Hollywood.loop("landing")
skelecog = load_cog("Legal Eagle", True)
skelecog.loop("neutral")
CFO = load_cog("CFO")
CFO.loop("Ff_neutral")
Returns:
An instance of Panda3D's Actor class.
Note:
This function also supports name matching; that is,
Hollywood = load_cog("hollywood")
is the same as
Hollywood = load_cog("Mr. Hollywood")
"""
cog_names = cog_dict.keys()
if cog_name not in cog_names:
if difflib.get_close_matches(cog_name, cog_names):
cog_name = difflib.get_close_matches(cog_name, cog_names)[0]
else:
for name in cog_names:
if cog_name.lower() in name.lower():
cog_name = name
args = cog_dict[cog_name]
if cog_name in ("VP", "CFO", "CJ", "CEO"):
args.insert(2, path)
cog = make_boss(*args)
elif skelecog:
cog = make_skelecog(args[1], args[3], path)
cog.setScale(cog_dict[cog_name][5])
else:
args.insert(5, path)
cog = make_cog(*args[0:6])
cog.setScale(cog_dict[cog_name][6])
return cog
def handle_error(self, e):
got_request_exception.send(current_app._get_current_object(),
exception=e)
if not hasattr(e, 'code') and current_app.propagate_exceptions:
exc_type, exc_value, tb = sys.exc_info()
if exc_value is e:
raise
else:
raise e
code = getattr(e, 'code', 500)
data = getattr(e, 'data', error_data(code))
headers = {}
if code >= 500:
if sys.exc_info() == (None, None, None):
current_app.logger.error("Internal Error")
else:
current_app.logger.exception("Internal Error")
help_on_404 = current_app.config.get("ERROR_404_HELP", True)
if code == 404 and help_on_404 and ('message' not in data
or data['message']
== HTTP_STATUS_CODES[404]):
rules = dict([(re.sub('(<.*>)', '', rule.rule), rule.rule)
for rule in current_app.url_map.iter_rules()])
close_matches = difflib.get_close_matches(request.path,
rules.keys())
if close_matches:
# If we already have a message, add punctuation and continue it.
if "message" in data:
data["message"] += ". "
else:
data["message"] = ""
data['message'] += 'You have requested this URI [' + request.path + \
'] but did you mean ' + \
' or '.join((
rules[match] for match in close_matches)
) + ' ?'
if code == 405:
headers['Allow'] = e.valid_methods
error_cls_name = type(e).__name__
if error_cls_name in self.errors:
custom_data = self.errors.get(error_cls_name, {})
code = custom_data.get('status', 500)
data.update(custom_data)
if code == 406 and self.default_mediatype is None:
supported_mediatypes = list(self.representations.keys())
fallback_mediatype = supported_mediatypes[
0] if supported_mediatypes else "text/plain"
resp = self.make_response(
data,
code,
headers,
fallback_mediatype=fallback_mediatype,
)
else:
if code == 400 and current_app.config.get(
'CHANGE_400_TO_200', self.default_change_400_to_200):
code = 200
resp = self.make_response(data, code, headers)
if code == 401:
resp = self.unauthorized(resp)
return resp
def diff(f, s):
return len(get_close_matches(f, [s])) > 0
import json
import difflib
a = json.load(open("original.json"))
b = input("enter the word to be searched ")
flag = 0
b = b.lower()
for key in a:
if (b == key.lower()):
flag = 1
if (type(a[key]) == list):
for x in a[key]:
print(x)
else:
print(a[key])
if (flag == 0):
d = []
d = difflib.get_close_matches(b, a, 1)
if (type(d) == list):
for x in d:
print(x)
print(a[x])
flag = 1
else:
print(x)
print(a[x])
if (flag == 0):
print("Word not found")
def normalize_match_sample():
fuzzyw = 'ape'
kwlist = ['apple', 'peach', 'pear']
print difflib.get_close_matches(fuzzyw, kwlist)
def update(self, template, no_fuzzy_matching=False):
"""Update the catalog based on the given template catalog.
>>> from babel.messages import Catalog
>>> template = Catalog()
>>> template.add('green', locations=[('main.py', 99)])
<Message ...>
>>> template.add('blue', locations=[('main.py', 100)])
<Message ...>
>>> template.add(('salad', 'salads'), locations=[('util.py', 42)])
<Message ...>
>>> catalog = Catalog(locale='de_DE')
>>> catalog.add('blue', u'blau', locations=[('main.py', 98)])
<Message ...>
>>> catalog.add('head', u'Kopf', locations=[('util.py', 33)])
<Message ...>
>>> catalog.add(('salad', 'salads'), (u'Salat', u'Salate'),
... locations=[('util.py', 38)])
<Message ...>
>>> catalog.update(template)
>>> len(catalog)
3
>>> msg1 = catalog['green']
>>> msg1.string
>>> msg1.locations
[('main.py', 99)]
>>> msg2 = catalog['blue']
>>> msg2.string
u'blau'
>>> msg2.locations
[('main.py', 100)]
>>> msg3 = catalog['salad']
>>> msg3.string
(u'Salat', u'Salate')
>>> msg3.locations
[('util.py', 42)]
Messages that are in the catalog but not in the template are removed
from the main collection, but can still be accessed via the `obsolete`
member:
>>> 'head' in catalog
False
>>> catalog.obsolete.values()
[<Message 'head' (flags: [])>]
:param template: the reference catalog, usually read from a POT file
:param no_fuzzy_matching: whether to use fuzzy matching of message IDs
"""
messages = self._messages
remaining = messages.copy()
self._messages = odict()
# Prepare for fuzzy matching
fuzzy_candidates = []
if not no_fuzzy_matching:
fuzzy_candidates = dict([
(self._key_for(msgid), messages[msgid].context)
for msgid in messages if msgid and messages[msgid].string
])
fuzzy_matches = set()
def _merge(message, oldkey, newkey):
message = message.clone()
fuzzy = False
if oldkey != newkey:
fuzzy = True
fuzzy_matches.add(oldkey)
oldmsg = messages.get(oldkey)
if isinstance(oldmsg.id, basestring):
message.previous_id = [oldmsg.id]
else:
message.previous_id = list(oldmsg.id)
else:
oldmsg = remaining.pop(oldkey, None)
message.string = oldmsg.string
if isinstance(message.id, (list, tuple)):
if not isinstance(message.string, (list, tuple)):
fuzzy = True
message.string = tuple([message.string] +
([u''] * (len(message.id) - 1)))
elif len(message.string) != self.num_plurals:
fuzzy = True
message.string = tuple(message.string[:len(oldmsg.string)])
elif isinstance(message.string, (list, tuple)):
fuzzy = True
message.string = message.string[0]
message.flags |= oldmsg.flags
if fuzzy:
message.flags |= set([u'fuzzy'])
self[message.id] = message
for message in template:
if message.id:
key = self._key_for(message.id, message.context)
if key in messages:
_merge(message, key, key)
else:
if no_fuzzy_matching is False:
# do some fuzzy matching with difflib
if isinstance(key, tuple):
matchkey = key[0] # just the msgid, no context
else:
matchkey = key
matches = get_close_matches(matchkey.lower().strip(),
fuzzy_candidates.keys(), 1)
if matches:
newkey = matches[0]
newctxt = fuzzy_candidates[newkey]
if newctxt is not None:
newkey = newkey, newctxt
_merge(message, newkey, key)
continue
self[message.id] = message
self.obsolete = odict()
for msgid in remaining:
if no_fuzzy_matching or msgid not in fuzzy_matches:
self.obsolete[msgid] = remaining[msgid]
# Make updated catalog's POT-Creation-Date equal to the template
# used to update the catalog
self.creation_date = template.creation_date
def getCommandLineArgs():
# Allow for fileToCompile to be written to
global fileToCompile,usingLibrariesString,VerboseMode,compiler_flags,outputFileName
global DebugMode,selectedCompiler,compilerStandard,includeDirectories,libraryDirectories
argsFound = sys.argv
if(len(argsFound) == 1):
runArgsMenue()
exit(1)
# Check if --help was called or no arguments were given
if("-h" in argsFound or "--help" in argsFound):
runHelpMenue()
exit(1)
# Check if -Debug was called to put the program in debug mode
if("-Debug" in argsFound):
DebugMode = True
# Set the verbosity mode
if("-V" in argsFound or "--Verbose" in argsFound):
VerboseMode = True
# Set the filename
if("-o" in argsFound or "--outFile" in argsFound):
indexOfOutFileFlag = argsFound.index("-o")
if(indexOfOutFileFlag == len(argsFound)-1):
print("Error: name requred after using -o or --outFile.")
exit(1)
if("-" in argsFound[indexOfOutFileFlag+1]):
print("Error: name requred after using -o or --outFile.")
exit(1)
outputFileName = argsFound[indexOfOutFileFlag+1]
if("-W" in argsFound or "--CompilerWarnings" in argsFound):
compiler_flags+= "-w"
verboseModePrint("Compiler Warnings Disabled")
# Dealing with Compile target assignment
# Make sure the compile target file was specified and done correctly
if("-C" in argsFound or "--CompileTarget" in argsFound):
# Find the index of the compile target flag and add 1 to it to find the arguemnts location
indexOfCompileFileArg = None
if ("-C" in argsFound):
indexOfCompileFileArg = argsFound.index("-C")+1
elif("--CompileTarget" in argsFound):
indexOfCompileFileArg = argsFound.index("--CompileTarget")+1
# TODO: Add an additional check if the indexOfCompileFileArg is still None. Is kind of redundant through
# Check if there is a .cpp file after the flag
if(indexOfCompileFileArg > len(argsFound)-1):
print("Error: No source file (file.cpp) was given after --CompileTarget or -C.")
exit(1)
# Make sure the next thing is not a flag
if("-" != argsFound[indexOfCompileFileArg][0]):
if (".cpp" == argsFound[indexOfCompileFileArg][-4:]):
fileToCompile = argsFound[indexOfCompileFileArg]
else:
print("Error: File name: " + argsFound[indexOfCompileFileArg] + " is not a C++ source file (Example: file.cpp)")
exit(1)
else:
print("Error: No source file (file.cpp) was given after --CompileTarget or -C.")
exit(1)
# if the file we found eariler does not exist error out
if(not os.path.isfile(fileToCompile)):
print("Error: " + fileToCompile + " could not be found in directory:", os.getcwd() + "/")
# Make a sugestion on files they could have ment
possableFiles = os.listdir(".")
possableFiles = [ x for x in possableFiles if "." in x ]
sugestions = difflib.get_close_matches(fileToCompile,possableFiles)
if(len(sugestions) > 0):
print("Did you mean " + sugestions[0] + "?")
elif(len(sugestions) > 1):
print("Did you mean " + sugestions[0] + " or " + sugestions[1] + "?")
exit(1)
else:
print("Error: Compile Target not specified with -C or --CompileTarget.")
exit(1)
# Check if any librarys have been specified
if ("-l" in argsFound or "--Libs" in argsFound):
indexOfLibsArg = None
if("-l" in argsFound):
indexOfLibsArg = argsFound.index("-l")+1
elif("--Libs" in argsFound):
indexOfLibsArg = argsFound.index("--Libs")+1
# TODO: Add an additional check if the indexOfCompileFileArg is still None. Is kind of redundant through
# Do some Checks to make sure there is an argument given after the flag
if (indexOfLibsArg > len(argsFound)-1):
print("Error: No arguments found for -l or --Libs Flag!")
exit(1)
if("-" == argsFound[indexOfLibsArg][0]):
print("Error: No arguments found for -l or --Libs Flag!")
exit(1)
# Parse the input string into a list of librarys
libsString = ""
unformatedLibStrings = argsFound[indexOfLibsArg]
# If multiple librarys were specified
if(" " in unformatedLibStrings):
# Split up the librarys into a list
unformatedLibStrings = unformatedLibStrings.split(" ")
verboseModePrint("Adding Libraries")
stringToPrint = " + "
# Add -l to all of the libNames so they are in the correct format for selected compiler when linking
for lib in unformatedLibStrings:
# If we are in verbose mode tell the user a library was added to the link list
# print(" + "+lib)
stringToPrint += lib
if (unformatedLibStrings.index(lib) != len(unformatedLibStrings)-1):
stringToPrint += ","
libsString += "-l" + lib + " "
if (VerboseMode):
print(stringToPrint)
# Take away the last space which was added in the last itteration
libsString.strip()
# If a single library was specified
else:
# If we are in verbose mode tell the user a library was added to the link list
verboseModePrint("Library Added: "+unformatedLibStrings)
libsString = "-l" + unformatedLibStrings
usingLibrariesString = libsString
# Process command line inputs for compiler optimization level
if ("-O" in argsFound or "--CompilerOptimization" in argsFound):
# Check for the index of each type of optimization flag then add 1 to get the arguments location
indexOfCompileFileArg = None
if("-O" in argsFound):
indexOfOptimizationLevel = argsFound.index("-O")+1
elif("--CompilerOptimization" in argsFound):
indexOfOptimizationLevel = argsFound.index("--CompilerOptimization")+1
# Do some Checks to make sure there is an argument given after the flag
if (indexOfOptimizationLevel > len(argsFound)-1):
print("Error: No Optimization Level found after -O or --CompilerOptimization flag!")
exit(1)
if("-" == argsFound[indexOfOptimizationLevel][0]):
print("Error: No Optimization Level found after -O or --CompilerOptimization flag!")
exit(1)
if (argsFound[indexOfOptimizationLevel] == "g"):
compiler_flags += " -g"
verboseModePrint("Compiler Optimization Set to: g")
elif (argsFound[indexOfOptimizationLevel] == "1" or argsFound[indexOfOptimizationLevel] == "2" or argsFound[indexOfOptimizationLevel] == "3"):
compiler_flags += " -O" + argsFound[indexOfOptimizationLevel]
verboseModePrint("Compiler Optimization Set to: " + "-O" + argsFound[indexOfOptimizationLevel])
else:
print("Warning: Optimization Level: " + argsFound[indexOfOptimizationLevel] + " is an unrecognised input.")
ans = input("Would you like to override? (y or n) ")
if(ans != "y" and ans != "yes"):
print("exiting...")
exit(1)
else:
compiler_flags += "-O" + argsFound[indexOfOptimizationLevel]
verboseModePrint("Compiler Optimization Set to: " + "-O" + argsFound[indexOfOptimizationLevel])
# Other compiler has been selected
if ("--Clang" in argsFound and "--gcc" not in argsFound):
selectedCompiler = "clang"
elif ("--gcc" in argsFound and "--Clang" not in argsFound):
selectedCompiler = "gcc"
elif ("--gcc" not in argsFound and "--Clang" not in argsFound):
pass
else:
print("Error: Only one compiler can be chosen at once.")
exit(1)
# Set the c++ standard
if ("-std" in argsFound):
indexOfStdArg = argsFound.index("-std")
if(indexOfStdArg == len(argsFound)-1):
print("Error: No C++ standard given with -std flag.")
exit(1)
stdFound = argsFound[indexOfStdArg+1]
if ("-" in stdFound):
print("Error: No C++ standard given with -std flag.")
exit(1)
if("c++" not in stdFound and "C++" not in stdFound):
print("Error: Incorrectly formated C++ standard!")
print("Correct exaple: -std c++11 or -std C++11")
print("Given:",stdFound)
exit(1)
verboseModePrint("Compiler Standard Set to: " + stdFound)
compiler_flags += " -std="+stdFound
# Add library directories
if ("-L" in argsFound or "--LibDirectories" in argsFound):
indexOfLibDirsArg = argsFound.index("-L")
if (indexOfLibsArg == len(argsFound)-1):
print("Error: Library Directories must be specified when using -L")
exit(1)
if("-" in argsFound[indexOfLibDirsArg+1]):
print("Error: Library Directories must be specified when using -L")
exit(1)
if("/" not in argsFound[indexOfLibDirsArg+1]):
print("Warning: The input directories for -L dont contain any \"/\" and therefore might be incorrect.")
print("Given:",argsFound[indexOfLibDirsArg+1])
ans = input("Continue anyway? (y or n) ")
if("y" not in ans):
print("exiting...")
exit(1)
libraryDirectories = "".join([" -L "+x for x in argsFound[indexOfLibDirsArg+1].split(" ")])
verboseModePrint("Adding Library Directories:")
if(VerboseMode):
for incDir in argsFound[indexOfLibDirsArg+1].split(" "):
print(" +",incDir)
compiler_flags += libraryDirectories
# Add include directories
if ("-I" in argsFound or "--IncludeDirectories" in argsFound):
indexOfIncDirsArg = argsFound.index("-I")
if (indexOfIncDirsArg == len(argsFound)-1):
print("Error: Include Directories must be specified when using -L")
exit(1)
if("-" in argsFound[indexOfIncDirsArg+1]):
print("Error: Include Directories must be specified when using -L")
exit(1)
if("/" not in argsFound[indexOfIncDirsArg+1]):
print("Warning: The input directories for -I dont contain any \"/\" and therefore might be incorrect.")
print("Given:",argsFound[indexOfIncDirsArg+1])
ans = input("Continue anyway? (y or n) ")
if("y" not in ans):
print("exiting...")
exit(1)
includeDirectories = "".join([" -I "+x for x in argsFound[indexOfIncDirsArg+1].split(" ")])
verboseModePrint("Adding Include Directories:")
if(VerboseMode):
for incDir in argsFound[indexOfIncDirsArg+1].split(" "):
print(" +",incDir)
compiler_flags += includeDirectories
# Set the name of the output executable
if ("-o" in argsFound or "--outFile" in argsFound):
return 0
def get_part_html_tree(dist,
pn,
extra_search_terms='',
url=None,
descend=2,
local_part_html=None,
scrape_retries=2):
'''@brief Find the Mouser HTML page for a part number and return the URL and parse tree.
@param dist
@param pn Part number `str()`.
@param extra_search_terms
@param url
@param descend
@param local_part_html
@param scrape_retries `int` Quantity of retries in case of fail.
@return (html `str()` of the page, url)
'''
# Use the part number to lookup the part using the site search function, unless a starting url was given.
if url is None:
url = distributor_dict['mouser']['site'][
'url'] + '/Search/Refine.aspx?Keyword=' + urlquote(
pn + ' ' + extra_search_terms, safe='')
elif url[0] == '/':
url = distributor_dict['mouser']['site']['url'] + url
elif url.startswith('..'):
url = distributor_dict['mouser']['site']['url'] + '/Search/' + url
# Open the URL, read the HTML from it, and parse it into a tree structure.
req = FakeBrowser(url)
# do we actually need this cookie?
#req.add_header('Cookie', 'preferences=ps=www2&pl=en-US&pc_www2=USDe')
for _ in range(scrape_retries):
try:
response = urlopen(req)
html = response.read()
break
except WEB_SCRAPE_EXCEPTIONS:
logger.log(
DEBUG_DETAILED,
'Exception while web-scraping {} from {}'.format(pn, dist))
pass
else: # Couldn't get a good read from the website.
logger.log(DEBUG_OBSESSIVE,
'No HTML page for {} from {}'.format(pn, dist))
raise PartHtmlError
# Abort if the part number isn't in the HTML somewhere.
# (Only use the numbers and letters to compare PN to HTML.)
if re.sub('[\W_]', '',
str.lower(pn)) not in re.sub('[\W_]', '', str.lower(str(html))):
logger.log(DEBUG_OBSESSIVE,
'No part number {} in HTML page from {}'.format(pn, dist))
raise PartHtmlError
try:
tree = BeautifulSoup(html, 'lxml')
except Exception:
logger.log(DEBUG_OBSESSIVE,
'No HTML tree for {} from {}'.format(pn, dist))
raise PartHtmlError
# If the tree contains the tag for a product page, then just return it.
if tree.find('div', id='pdpPricingAvailability') is not None:
return tree, url
# If the tree is for a list of products, then examine the links to try to find the part number.
if tree.find('div', id='searchResultsTbl') is not None:
logger.log(DEBUG_OBSESSIVE,
'Found product table for {} from {}'.format(pn, dist))
if descend <= 0:
logger.log(DEBUG_OBSESSIVE,
'Passed descent limit for {} from {}'.format(pn, dist))
raise PartHtmlError
else:
# Look for the table of products.
products = tree.find('table',
class_='SearchResultsTable').find_all(
'tr',
class_=('SearchResultsRowOdd',
'SearchResultsRowEven'))
# Extract the product links for the part numbers from the table.
product_links = [
p.find('div', class_='mfrDiv').a for p in products
]
# Extract all the part numbers from the text portion of the links.
part_numbers = [l.text for l in product_links]
# Look for the part number in the list that most closely matches the requested part number.
match = difflib.get_close_matches(pn, part_numbers, 1, 0.0)[0]
# Now look for the link that goes with the closest matching part number.
for l in product_links:
if l.text == match:
# Get the tree for the linked-to page and return that.
logger.log(
DEBUG_OBSESSIVE,
'Selecting {} from product table for {} from {}'.
format(l.text, pn, dist))
return get_part_html_tree(dist,
pn,
extra_search_terms,
url=l.get('href', ''),
descend=descend - 1,
scrape_retries=scrape_retries)
# I don't know what happened here, so give up.
logger.log(DEBUG_OBSESSIVE,
'Unknown error for {} from {}'.format(pn, dist))
raise PartHtmlError
def handle_error(self, e):
"""Error handler for the API transforms a raised exception into a Flask
response, with the appropriate HTTP status code and body.
:param e: the raised Exception object
:type e: Exception
"""
got_request_exception.send(current_app._get_current_object(), exception=e)
if not isinstance(e, HTTPException) and current_app.propagate_exceptions:
exc_type, exc_value, tb = sys.exc_info()
if exc_value is e:
raise
else:
raise e
if isinstance(e, HTTPException):
code = e.code
default_data = {
'message': getattr(e, 'description', http_status_message(code))
}
else:
code = 500
default_data = {
'message': http_status_message(code),
}
data = getattr(e, 'data', default_data)
headers = {}
if code >= 500:
# There's currently a bug in Python3 that disallows calling
# logging.exception() when an exception hasn't actually be raised
if sys.exc_info() == (None, None, None):
current_app.logger.error("Internal Error")
else:
current_app.logger.exception("Internal Error")
help_on_404 = current_app.config.get("ERROR_404_HELP", True)
if code == 404 and help_on_404:
rules = dict([(re.sub('(<.*>)', '', rule.rule), rule.rule)
for rule in current_app.url_map.iter_rules()])
close_matches = difflib.get_close_matches(request.path, rules.keys())
if close_matches:
# If we already have a message, add punctuation and continue it.
if "message" in data:
data["message"] = data["message"].rstrip('.') + '. '
else:
data["message"] = ""
data['message'] += 'You have requested this URI [' + request.path + \
'] but did you mean ' + \
' or '.join((
rules[match] for match in close_matches)
) + ' ?'
if code == 405:
headers['Allow'] = e.valid_methods
error_cls_name = type(e).__name__
if error_cls_name in self.errors:
custom_data = self.errors.get(error_cls_name, {})
code = custom_data.get('status', 500)
data.update(custom_data)
if code == 406 and self.default_mediatype is None:
# if we are handling NotAcceptable (406), make sure that
# make_response uses a representation we support as the
# default mediatype (so that make_response doesn't throw
# another NotAcceptable error).
supported_mediatypes = list(self.representations.keys())
fallback_mediatype = supported_mediatypes[0] if supported_mediatypes else "text/plain"
resp = self.make_response(
data,
code,
headers,
fallback_mediatype = fallback_mediatype
)
else:
resp = self.make_response(data, code, headers)
if code == 401:
resp = self.unauthorized(resp)
return resp
def getMangaURL(searchText, authorName=None):
try:
if authorName:
html = req.get(BASE_URL + "/manga/all?name=" +
searchText.replace(" ", "%20") + '&author=' +
authorName.replace(" ", "%20"),
timeout=10)
req.close()
if "No results found" in html.text:
rearrangedAuthorNames = collections.deque(
authorName.split(' '))
rearrangedAuthorNames.rotate(-1)
rearrangedName = ' '.join(rearrangedAuthorNames)
html = req.get(BASE_URL + "/manga/all?name=" +
searchText.replace(" ", "%20") + '&author=' +
rearrangedName.replace(" ", "%20"),
timeout=10)
req.close()
else:
html = req.get(BASE_URL + "/manga/all?name=" +
searchText.replace(" ", "%20"),
timeout=10)
req.close()
ap = pq(html.text)
mangaList = []
#If it's taken us to the search page
if ap.find('.cardDeck.pure-g.cd-narrow[data-type="manga"]'):
for entry in ap.find('.card.pure-1-6'):
entryTitle = pq(entry).find('h4').text()
entryURL = pq(entry).find('a').attr('href')
manga = {}
manga['title'] = entryTitle
manga['url'] = BASE_URL + entryURL
mangaList.append(manga)
if authorName:
authorName = authorName.lower()
authorName = authorName.split(' ')
for manga in mangaList:
manga['title'] = manga['title'].lower()
for name in authorName:
manga['title'] = manga['title'].replace(name, '')
manga['title'] = manga['title'].replace('(', '').replace(
')', '').strip()
closestName = difflib.get_close_matches(
searchText.lower(), [x['title'].lower() for x in mangaList], 1,
0.85)[0]
closestURL = ''
for manga in mangaList:
if manga['title'].lower() == closestName:
return manga['url']
#Else if it's taken us right to the series page, get the url from the meta tag
else:
return ap.find("meta[property='og:url']").attr('content')
return None
except:
req.close()
return None
def add_to_database(work_rows, shelf_rows):
""" Take two lists of tuples describing works (work_rows) and shelves (shelf_rows) and add then to the 'works'
and 'shelves' tables of database goodreads_db.
"""
# Set up database connections
username = os.environ['rds_username']
password = os.environ['rds_password']
host = os.environ['rds_host']
dbname = 'goodreads_db'
port = '5432'
engine = create_engine('postgresql://{}:{}@{}:{}/{}'.format(
username, password, host, port, dbname))
con = psycopg2.connect(database=dbname,
user=username,
host=host,
password=password)
cur = con.cursor()
# create the database if it doesn't exist yet
if not database_exists(engine.url):
create_database(engine.url)
# Generate a list of every time an author won or was nominated
all_hugos_data = get_all_hugos_winners()
prevdict = {}
for author in all_hugos_data['Author'].unique():
prevdict[author] = all_hugos_data[all_hugos_data['Author'] ==
author]['Year'].values
# Now, crossmatch with the hugos data -- both winners and authors
hugos_data = get_hugos_data()
hid = list(hugos_data['ID'])
for row in work_rows:
if int(row[0]) in hid:
hd = hugos_data.iloc[hid.index(int(row[0]))]
row += [hd['Winner'], hd['Score']]
else:
row += [None, 0.0]
if row[9]:
if row[3] in prevdict:
row += [np.sum(prevdict[row[3]] <= row[9])]
else:
match = difflib.get_close_matches(row[3],
all_hugos_data['Author'],
cutoff=0.9)
if match:
match = match[0]
row += [np.sum(prevdict[match] <= row[9])]
else:
row += [0]
else:
row += [0]
# Then crossmatch with the ISFDB. Get both title+author if we can, but just the author info is helpful otherwise.
isfdb_data = get_isfdb_data()
for row in work_rows:
title_mask = row[1] == isfdb_data['pub_title']
author_mask = row[4] == isfdb_data['author_canonical']
if any(title_mask & author_mask):
irow = isfdb_data[title_mask & author_mask]
else:
if any(author_mask):
tisfdb_data = isfdb_data[author_mask]
else:
author_mask = difflib.get_close_matches(
row[1], isfdb_data['author_canonical'], cutoff=0.9)
if author_mask:
tisfdb_data = isfdb_data[isfdb_data['author_canonical'] ==
author_mask[0]]
else:
tisfdb_data = None
if tisfdb_data is not None:
title_mask = difflib.get_close_matches(
row[1], tisfdb_data['pub_title'])
if title_mask:
irow = tisfdb_data[tisfdb_data['pub_title'] ==
title_mask[0]]
else:
irow = {
'author_annualviews':
tisfdb_data['author_annualviews'].values[0],
'author_lastname':
tisfdb_data['author_lastname'].values[0],
'pub_id':
-1,
'pub_ctype':
'NOVEL',
'pub_isbn':
-1,
'author_id':
tisfdb_data['author_id'].values[0],
'author_canonical':
tisfdb_data['author_canonical'].values[0],
'title_id':
-1,
'title_storylen':
None,
'title_graphic':
'No',
'title_annualviews':
None,
'debut_year':
tisfdb_data['debut_year'].values[0],
'identifier_value_amazon':
None,
'identifier_value_goodreads':
None,
'identifier_value_oclc':
None,
'identifier_value_bn':
None,
'identifier_value_audible':
None
}
if row[9] is not None:
irow['pub_year'] = datetime.date(int(row[9]), 1, 1)
else:
irow['pub_year'] = None
else:
irow = {
'author_annualviews': None,
'author_lastname': row[4].split()[-1],
'pub_id': -1,
'pub_ctype': 'NOVEL',
'pub_isbn': -1,
'author_id': -1,
'author_canonical': row[4],
'title_id': -1,
'title_storylen': None,
'title_graphic': 'No',
'title_annualviews': None,
'debut_year': None,
'identifier_value_amazon': None,
'identifier_value_goodreads': None,
'identifier_value_oclc': None,
'identifier_value_bn': None,
'identifier_value_audible': None
}
if row[9] is not None:
irow['pub_year'] = datetime.date(int(row[9]), 1, 1)
else:
irow['pub_year'] = None
# Sometimes these are lists, sometimes single values
if hasattr(irow['pub_year'], 'len'):
row += [
irow['pub_year'].values[0],
irow['author_annualviews'].values[0],
irow['author_lastname'].values[0], irow['pub_id'].values[0],
irow['pub_ctype'].values[0], irow['pub_isbn'].values[0],
irow['author_id'].values[0],
irow['author_canonical'].values[0], irow['title_id'].values[0],
irow['title_storylen'].values[0],
irow['title_graphic'].values[0],
irow['title_annualviews'].values[0],
irow['debut_year'].values[0],
irow['identifier_value_amazon'].values[0],
irow['identifier_value_goodreads'].values[0],
irow['identifier_value_oclc'].values[0],
irow['identifier_value_bn'].values[0],
irow['identifier_value_audible'].values[0]
]
else:
row += [
irow['pub_year'], irow['author_annualviews'],
irow['author_lastname'], irow['pub_id'], irow['pub_ctype'],
irow['pub_isbn'], irow['author_id'], irow['author_canonical'],
irow['title_id'], irow['title_storylen'],
irow['title_graphic'], irow['title_annualviews'],
irow['debut_year'], irow['identifier_value_amazon'],
irow['identifier_value_goodreads'],
irow['identifier_value_oclc'], irow['identifier_value_bn'],
irow['identifier_value_audible']
]
main_df = pd.DataFrame(
work_rows,
columns=[
'id', 'title', 'is_series', 'author', 'all_authors', 'rating',
'nratings', 'nreviews', 'blurb', 'pubyear', 'language', 'reviews',
'to_read', 'favs', 'dnf', 'comics', 'short_stories', 'fantasy',
'ya', 'sf', 'horror', 'dystopian', 'romance', 'adventure',
'urban_fantasy', 'children', 'mystery', 'historical',
'high_fantasy', 'mythology', 'humor', 'literature', 'time_travel',
'space', 'lgbt', 'winner', 'true_score', 'nprev', 'pub_date',
'author_annualviews', 'author_lastname', 'pub_id', 'pub_ctype',
'pub_isbn', 'author_id', 'author_canonical', 'title_id',
'title_storylen', 'title_graphic', 'title_annualviews',
'debut_year', 'identifier_value_amazon',
'identifier_value_goodreads', 'identifier_value_oclc',
'identifier_value_bn', 'identifier_value_audible'
]).set_index('id')
pubyear = main_df['pub_date']
def pyr(py):
try:
return py.year
except:
return 3000
pubyear = [pyr(pb) for pb in pubyear]
main_df['pubyear'] = np.min([main_df['pubyear'], pubyear], axis=0)
shelf_df = pd.DataFrame(shelf_rows,
columns=['work_id', 'shelf', 'nshelves'])
try:
old_data = list(
pd.read_sql_query("SELECT id FROM works", engine)['id'])
old_data = [o for o in old_data if o in main_df['id']]
cur.execute_batch("DELETE FROM works WHERE id=%s", old_data)
cur.execute_batch("DELETE FROM shelves WHERE id=%s", old_data)
con.commit()
except pd.io.sql.DatabaseError:
# This error just means the table doesn't exist, typically.
pass
main_df.to_sql('works', engine, if_exists='append')
shelf_df.to_sql('shelves', engine, if_exists='append')
con.close()
engine.close()
def tile_list_glob(base_dir: str,
mul_pan_glob: List[str] = [],
mul_pan_str: List[str] = [],
psh_glob: List[str] = [],
extensions: List[str] = []):
"""
Glob through specified directories for (1) pairs of multispectral and panchromatic rasters or (2) pansharp rasters.
Save as csv and/or return as list.
:param base_dir: str
Base directory where globbing will occur.
:param mul_pan_glob: list of str
List of list of patterns linking multispectral and panchrom. rasters. Patterns are a two-item list:
(1) glob pattern to reach multispectral raster (excluding extension);
(2) pattern to panchrom. raster from multispectral raster,
e.g.: ["**/*_MUL/*-M*_P00?", "../*_PAN"].
:param mul_pan_str: list of str
List of list of string sections that identify multispectral and panchrom. rasters inside filename,
e.g. [['-M', '-P'],["_MSI", "_PAN"]].
:param psh_glob: list of str
List of glob patterns to find panchromatic rasters.
:param extensions: list of str
List of extensions (suffixes) the raster files may bear, e.g. ["tif", "ntf"].
:return:
list of lists (rows) containing info about files found, output pansharp name (if applies) and more.
"""
assert len(mul_pan_glob) == len(
mul_pan_str
), "Missing info about multispectral and panchromatic images"
# Reorganize mul/pan glob and str info as list of lists each containing a tuple.
# e.g. [('Sherbrooke/**/*_MUL/*-M*_P00?', '../*_PAN'), ('-M', '-P')]. See pansharp_glob()'s docstring for more info.
mul_pan_info_list = [[tuple(mul_pan_glob[x]),
tuple(mul_pan_str[x])] for x in mul_pan_glob]
os.chdir(base_dir) # Work in base directory
import logging.config
out_log_path = Path("./logs")
out_log_path.mkdir(exist_ok=True)
logging.basicConfig(filename='logs/prep_glob.log', level=logging.DEBUG)
logging.info("Started")
base_dir_res = Path(base_dir).resolve(
) # Resolved path is useful in section 2 (search for panchromatic).
glob_output_list = []
# 1. GLOB to all multispectral images in base directory using inputted pattern. Create generator from glob search.
################################################################################
for mul_pan_info, ext in product(
mul_pan_info_list, extensions
): # FIXME: if list is empty, Nonetype will cause TypeError
mul_glob_pattern = mul_pan_info[0][0] + "." + ext
# FIXME: there may be compatibilty issues with glob's case sensitivity in Linux. Working ok on Windows.
# More info: https://jdhao.github.io/2019/06/24/python_glob_case_sensitivity/
mul_glob = base_dir_res.glob(mul_glob_pattern)
# Loop through glob generator object and retrieve xml in multispectral folder
for mul_xml in tqdm(mul_glob,
desc='Iterating through multispectral xml'
): # mul_raster being a Path object
mul_rel = Path(mul_xml).relative_to(
base_dir_res) # Use only relative paths from here
image_folder = mul_rel.parents[1]
mul_rel = Path(mul_xml).relative_to(base_dir_res / image_folder)
err_mgs = []
length_err = "Check absolute path length. May exceed 260 characters."
if not validate_file_exists(image_folder / mul_rel):
err_mgs.append(length_err)
# get tile list from xml
lst_mul_tiles = get_tiles_from_xml(mul_xml)
# 2. Find panchromatic image with relative glob pattern from multispectral pattern
################################################################################
pan_glob_pattern = mul_pan_info[0][1] + "/*." + ext
# assume panchromatic file has same extension as multispectral
pan_glob = sorted(
(image_folder / mul_rel.parent).glob(pan_glob_pattern))
if len(pan_glob) == 0:
missing_pan = f"The provided glob pattern {pan_glob_pattern} could not locate a potential" \
f"panchromatic raster to match {mul_rel} in image folder {image_folder}."
logging.warning(missing_pan)
err_mgs.append(missing_pan)
continue
# Replace string that identifies the raster as a multispectral for one identifying panchromatic raster
pan_best_guess = str(mul_rel.name).replace(mul_pan_info[1][0],
mul_pan_info[1][1])
# Guess the panchromatic image's path using directory from glob results above. This file may not exist.
pan_best_guess_rel_path = (pan_glob[0].parent.resolve() /
pan_best_guess).relative_to(
base_dir_res / image_folder)
# Make a list of strings from paths given by glob results above.
pan_str = []
for potential_pan in pan_glob:
# Resolve paths to avoid path length problems in Windows,
# i.e. discard all relative references (ex.: "mul_dir/../pan_dir") making path longer
pot_pan_dir = potential_pan.parent.resolve()
pot_pan_rel = pot_pan_dir.joinpath(
potential_pan.name).relative_to(base_dir_res /
image_folder)
pan_str.append(str(pot_pan_rel))
# Get closest match between guessed name for panchromatic image and glob file names
pan_rel = Path(
get_close_matches(str(pan_best_guess_rel_path), pan_str)[0])
if validate_file_exists(image_folder / pan_rel):
lst_pan_tiles = get_tiles_from_xml(image_folder / pan_rel)
else:
no_panchro_err = f"Panchromatic xml not found to match multispectral xml {mul_rel}"
logging.warning(no_panchro_err)
err_mgs.append(no_panchro_err)
continue
# Check both mul and pan lists are the same length.
if len(lst_mul_tiles) != len(lst_pan_tiles):
xml_err = f"The number of tiles in multispectral and panchromatic xmls do not match for image {image_folder}."
logging.warning(xml_err)
err_mgs.append(xml_err)
continue
process_steps = ['psh']
if len(lst_mul_tiles) > 1:
process_steps.append('merge')
elif len(lst_mul_tiles) == 0:
xml_err = f"Could not find any tile in xmls for image {image_folder}."
logging.warning(xml_err)
try:
with rasterio_raster_reader(
str(mul_xml.parent / Path(lst_mul_tiles[0]))
) as src: # Set output dtype as original multispectral dtype
dtype = src.meta["dtype"]
except rasterio.errors.RasterioIOError as e:
logging.warning(e)
continue
logging.debug(f"\nMultispectral: {mul_rel}\n"
f"Panchromatic: {pan_rel}\n"
f"Multispectral datatype: {dtype}\n")
# Determine output path
p = re.compile('_M\w\w')
output_path = Path(p.sub('_PREP', str(mul_rel.parent)))
output_prep_path = Path(base_dir) / image_folder / output_path
output_prep_path.mkdir(exist_ok=True)
if not output_prep_path.is_dir():
raise ValueError(f"Could not create folder {output_prep_path}")
if dtype != 'uint8':
process_steps.append('scale')
mul_tile_list = [
Path(base_dir) / image_folder / mul_rel.parent / Path(elem)
for elem in lst_mul_tiles
]
pan_tile_list = [
Path(base_dir) / image_folder / pan_rel.parent / Path(elem)
for elem in lst_pan_tiles
]
im_name = get_img_name_from_img_folder(
str(image_folder).split('/')[0])
# create new row and append to existing records in glob_output_list.
img_info = ImageInfo(parent_folder=Path(base_dir),
image_folder=image_folder,
im_name=im_name,
prep_folder=output_path,
mul_tile_list=mul_tile_list,
pan_tile_list=pan_tile_list,
mul_xml=mul_rel,
pan_xml=pan_rel,
mul_pan_info=mul_pan_info,
process_steps=process_steps,
dtype=dtype)
glob_output_list.append(img_info)
mul_pan_pairs_ct = len(glob_output_list)
logging.info(
f"Found {mul_pan_pairs_ct} pair(s) of multispectral and panchromatic rasters with provided parameters"
)
# 3. Find already pansharped images with a certain name pattern
################################################################################
if psh_glob: # if config file contains any search pattern, glob.
for psh_glob_item, ext in product(psh_glob, extensions):
psh_glob_pattern = psh_glob_item + "." + ext
psh_xml_glob = base_dir_res.glob(psh_glob_pattern)
for psh_xml in tqdm(
psh_xml_glob,
desc="Iterating through already pansharped images"):
psh_rel = Path(psh_xml).relative_to(
base_dir_res) # Use only relative paths
image_folder = psh_rel.parents[1]
psh_rel = Path(psh_xml).relative_to(base_dir_res /
image_folder)
if validate_file_exists(psh_xml):
lst_psh_tiles = get_tiles_from_xml(psh_xml)
else:
no_xml_err = f"No XML file found in {psh_xml}"
logging.warning(no_xml_err)
continue
process_steps = []
if len(lst_psh_tiles) > 1:
process_steps.append('merge')
elif len(lst_psh_tiles) == 0:
xml_err = f"Could not find any tile in xmls for image {image_folder}."
logging.warning(xml_err)
try:
with rasterio_raster_reader(
str(psh_xml.parent /
Path(lst_psh_tiles[0]))) as src:
psh_dtype = src.meta["dtype"]
except rasterio.errors.RasterioIOError as e:
logging.warning(e)
continue
# Determine output path
output_path = Path(
'_'.join(str(psh_rel.parent).split('_')[:-1]) + '_PREP')
output_prep_path = Path(base_dir) / image_folder / output_path
output_prep_path.mkdir(exist_ok=True)
logging.debug(f"\nPansharp image found: {psh_rel}\n")
if psh_dtype != 'uint8':
process_steps.append('scale')
im_name = get_img_name_from_img_folder(
str(image_folder).split('/')[0])
psh_tile_list = [
Path(base_dir) / image_folder / psh_rel.parent / Path(elem)
for elem in lst_psh_tiles
]
img_info = ImageInfo(parent_folder=Path(base_dir),
image_folder=image_folder,
im_name=im_name,
prep_folder=output_path,
psh_tile_list=psh_tile_list,
dtype=psh_dtype,
psh_xml=psh_xml,
process_steps=process_steps,
mul_pan_info=psh_glob_pattern)
glob_output_list.append(img_info)
psh_ct = len(glob_output_list) - mul_pan_pairs_ct
logging.info(
f'Found {psh_ct} pansharped raster(s) with provided parameters')
return glob_output_list
help="Overwrite hyperparameter (e.g. learning_rate:0.01 train_freq:10)",
)
parser.add_argument("-uuid", "--uuid", action="store_true", default=False, help="Ensure that the run has a unique ID")
args = parser.parse_args()
# Going through custom gym packages to let them register in the global registory
for env_module in args.gym_packages:
importlib.import_module(env_module)
env_id = args.env
registered_envs = set(gym.envs.registry.env_specs.keys()) # pytype: disable=module-attr
# If the environment is not found, suggest the closest match
if env_id not in registered_envs:
try:
closest_match = difflib.get_close_matches(env_id, registered_envs, n=1)[0]
except IndexError:
closest_match = "'no close match found...'"
raise ValueError(f"{env_id} not found in gym registry, you maybe meant {closest_match}?")
# Unique id to ensure there is no race condition for the folder creation
uuid_str = f"_{uuid.uuid4()}" if args.uuid else ""
if args.seed < 0:
# Seed but with a random one
args.seed = np.random.randint(2 ** 32 - 1, dtype="int64").item()
set_random_seed(args.seed)
# Setting num threads to 1 makes things run faster on cpu
if args.num_threads > 0:
if args.verbose > 1:
def _prep_metadata(md_sect, path):
"""Process & verify the metadata from a config file
- Pull out the module name we're packaging.
- Read description-file and check that it's valid rst
- Convert dashes in key names to underscores
(e.g. home-page in config -> home_page in metadata)
"""
if not set(md_sect).issuperset(metadata_required_fields):
missing = metadata_required_fields - set(md_sect)
raise ConfigError("Required fields missing: " + '\n'.join(missing))
res = LoadedConfig()
res.module = md_sect.get('module')
if not str.isidentifier(res.module):
raise ConfigError("Module name %r is not a valid identifier" %
res.module)
md_dict = res.metadata
# Description file
if 'description-file' in md_sect:
desc_path = md_sect.get('description-file')
res.referenced_files.append(desc_path)
desc_content, mimetype = description_from_file(desc_path, path.parent)
md_dict['description'] = desc_content
md_dict['description_content_type'] = mimetype
if 'urls' in md_sect:
project_urls = md_dict['project_urls'] = []
for label, url in sorted(md_sect.pop('urls').items()):
project_urls.append("{}, {}".format(label, url))
for key, value in md_sect.items():
if key in {'description-file', 'module'}:
continue
if key not in metadata_allowed_fields:
closest = difflib.get_close_matches(key,
metadata_allowed_fields,
n=1,
cutoff=0.7)
msg = "Unrecognised metadata key: {!r}".format(key)
if closest:
msg += " (did you mean {!r}?)".format(closest[0])
raise ConfigError(msg)
k2 = key.replace('-', '_')
md_dict[k2] = value
if key in metadata_list_fields:
if not isinstance(value, list):
raise ConfigError(
'Expected a list for {} field, found {!r}'.format(
key, value))
if not all(isinstance(a, str) for a in value):
raise ConfigError(
'Expected a list of strings for {} field'.format(key))
elif key == 'requires-extra':
if not isinstance(value, dict):
raise ConfigError(
'Expected a dict for requires-extra field, found {!r}'.
format(value))
if not all(isinstance(e, list) for e in value.values()):
raise ConfigError(
'Expected a dict of lists for requires-extra field')
for e, reqs in value.items():
if not all(isinstance(a, str) for a in reqs):
raise ConfigError(
'Expected a string list for requires-extra. (extra {})'
.format(e))
else:
if not isinstance(value, str):
raise ConfigError(
'Expected a string for {} field, found {!r}'.format(
key, value))
# What we call requires in the ini file is technically requires_dist in
# the metadata.
if 'requires' in md_dict:
md_dict['requires_dist'] = md_dict.pop('requires')
# And what we call dist-name is name in the metadata
if 'dist_name' in md_dict:
md_dict['name'] = md_dict.pop('dist_name')
# Move dev-requires into requires-extra
reqs_noextra = md_dict.pop('requires_dist', [])
res.reqs_by_extra = md_dict.pop('requires_extra', {})
dev_requires = md_dict.pop('dev_requires', None)
if dev_requires is not None:
if 'dev' in res.reqs_by_extra:
raise ConfigError(
'dev-requires occurs together with its replacement requires-extra.dev.'
)
else:
log.warning(
'"dev-requires = ..." is obsolete. Use "requires-extra = {"dev" = ...}" instead.'
)
res.reqs_by_extra['dev'] = dev_requires
# Add requires-extra requirements into requires_dist
md_dict['requires_dist'] = \
reqs_noextra + list(_expand_requires_extra(res.reqs_by_extra))
md_dict['provides_extra'] = sorted(res.reqs_by_extra.keys())
# For internal use, record the main requirements as a '.none' extra.
res.reqs_by_extra['.none'] = reqs_noextra
return res
def get_nice_attr_error(obj, item):
class_name, cls, handler, has_describe = get_obj_metadata(obj)
if item.startswith('_'):
# don't handle 'private' attribute access
return "{} instance has no attribute '{}'".format(class_name, item)
else:
field_names = obj._fields.keys()
field_dict = defaultdict(lambda: [])
for f in field_names:
field_dict[f.lower()].append(f)
obj_namespace = {x for x in dir(cls) if not x.startswith('_')}
inherited = {x for x in obj_namespace if x not in cls.__dict__}
methods = {
x
for x in obj_namespace
if x in cls.__dict__ and callable(cls.__dict__[x])
}
props = obj_namespace - methods - inherited
item_lower = item.lower()
field_matches = difflib.get_close_matches(item_lower, field_dict, n=5)
inherited_matches = difflib.get_close_matches(item_lower,
inherited,
n=5)
method_matches = difflib.get_close_matches(item_lower, methods, n=5)
prop_matches = difflib.get_close_matches(item_lower, props, n=5)
s = [
"{} instance has no field, method, or property '{}'".format(
class_name, item)
]
if any(
[field_matches, method_matches, prop_matches, inherited_matches]):
s.append('\n Did you mean:')
if field_matches:
l = []
for f in field_matches:
l.extend(field_dict[f])
word = plural('field', len(l))
s.append('\n Data {}: {}'.format(
word, ', '.join(handler(f) for f in l)))
if method_matches:
word = plural('method', len(method_matches))
s.append('\n {} {}: {}'.format(
class_name, word,
', '.join("'{}'".format(m) for m in method_matches)))
if prop_matches:
word = plural('property', len(prop_matches), 'properties')
s.append('\n {} {}: {}'.format(
class_name, word,
', '.join("'{}'".format(p) for p in prop_matches)))
if inherited_matches:
word = plural('inherited method', len(inherited_matches))
s.append('\n {} {}: {}'.format(
class_name, word,
', '.join("'{}'".format(m) for m in inherited_matches)))
elif has_describe:
s.append(
"\n Hint: use 'describe()' to show the names of all data fields."
)
return ''.join(s)
def get_similar(topic): return sorted(difflib.get_close_matches(topic.lower(), PRIMARY_TOPICS, len(PRIMARY_TOPICS)))
def close_match(wordKey, data):
if(len(get_close_matches(wordKey, data.Name.values, n = 1, cutoff = 0.6)) == 0 ):
print("NO")
closeMatch = get_close_matches(wordKey, data.Name.values, n = 1, cutoff = 0.6)
print(closeMatch[0])
infos[s[0]] = s[1]
infos["link"] = item["link"]
if len(infos["link"]) < 10:
continue
quality = re.search("\[([^\]]*)\]",item["title"])
if quality:
quality = set(quality.group(1).upper().split(" - "))
infos["quality"] = quality
infos["HD"] = isHD(infos["quality"])
# Now we have in infos : show name, episode, season, url, quality
if not infos.has_key(u"Show Name") or not infos.has_key(u"Episode") or not infos.has_key(u"Season"):
continue
else:
infos["Episode"] = int(infos["Episode"])
infos["Season"] = int(infos["Season"])
res = difflib.get_close_matches(infos[u"Show Name"],tvshows.keys(),1)
if len(res) == 1:
#print "We found a match"
infos["Show"] = res[0]
if not matches.has_key(res[0]):
matches[res[0]] = []
matches[res[0]] += [infos,]
# Analyse of every match we found to download only accurates ones
final = []
for show,result in matches.items():
if len(result) == 0:
pass
elif len(result) == 1:
# We got only one result, let's download it
if WANT_HD or not result[0]["HD"]:
def menu(queue: List[str] = None):
"""Report Menu."""
report_controller = ReportController(queue)
an_input = "HELP_ME"
while True:
# There is a command in the queue
if report_controller.queue and len(report_controller.queue) > 0:
# If the command is quitting the menu we want to return in here
if report_controller.queue[0] in ("q", "..", "quit"):
print("")
if len(report_controller.queue) > 1:
return report_controller.queue[1:]
return []
# Consume 1 element from the queue
an_input = report_controller.queue[0]
report_controller.queue = report_controller.queue[1:]
# Print the current location because this was an instruction
# and we want user to know what was the action
if an_input and an_input.split(
" ")[0] in report_controller.CHOICES:
print(f"{get_flair()} /reports/ $ {an_input}")
# Get input command from user
else:
# Display help menu when entering on this menu from a level above
if an_input == "HELP_ME":
report_controller.print_help()
# Get input from user using auto-completion
if session and gtff.USE_PROMPT_TOOLKIT and report_controller.completer:
an_input = session.prompt(
f"{get_flair()} /reports/ $ ",
completer=report_controller.completer,
search_ignore_case=True,
)
# Get input from user without auto-completion
else:
an_input = input(f"{get_flair()} /reports/ $ ")
try:
# Process the input command
report_controller.queue = report_controller.switch(an_input)
except SystemExit:
print(
f"\nThe command '{an_input}' doesn't exist on the /reports menu.",
end="",
)
similar_cmd = difflib.get_close_matches(
an_input.split(" ")[0] if " " in an_input else an_input,
report_controller.CHOICES,
n=1,
cutoff=0.7,
)
if similar_cmd:
if " " in an_input:
candidate_input = (
f"{similar_cmd[0]} {' '.join(an_input.split(' ')[1:])}"
)
if candidate_input == an_input:
an_input = ""
report_controller.queue = []
print("\n")
continue
an_input = candidate_input
else:
an_input = similar_cmd[0]
print(f" Replacing by '{an_input}'.")
report_controller.queue.insert(0, an_input)
else:
print("\n")
def setModel(self, originalRoiNameDict, formalizedRTStructs, leftRightContours, extraTextContours, oarContours, tvContours, tvMultipleContours):
"""Fills the entries of the table with the ROI names and checkboxes for the default names
"""
self.__originalRoiNameDict = originalRoiNameDict
self.__originalRoiNumberList = originalRoiNameDict.keys()
self.__formalizedRTStructs = formalizedRTStructs
self._extraTextContours = extraTextContours
self._leftRightContours = leftRightContours
self._oarContours = oarContours
self._tvContours = tvContours
self._tvMultipleContours = tvMultipleContours
# Each ROI will be presented on separate row
Nrows = len(self.__originalRoiNameDict.values())
Ncols = 6
# Set number of rows and columns and set widths and titles of rows
self.tableWidget.setRowCount(Nrows)
self.tableWidget.setColumnCount(Ncols)
self.tableWidget.setColumnWidth(0, 200)
self.tableWidget.setColumnWidth(1, 200)
self.tableWidget.setColumnWidth(2, 200)
self.tableWidget.setColumnWidth(3, 120)
self.tableWidget.setColumnWidth(4, 120)
self.tableWidget.setColumnWidth(5, 200)
self.tableWidget.setHorizontalHeaderLabels(["Original Name", "Replace Name" ,"Assigned Name", "Additional Info", "Margin (mm)", "Prescription dose (cGy)"])
# Full table: standardised combined string
self.txtStandardisedList = []
# Fill table: original ROI name on the left, default name comboboxes on the right
self.comboBoxList = []
# Fill the table: additional info txt box
self.cmbExtraList = []
# Fill the table: margin txt box
self.cmbMarginList = []
# Fill the table: dose txt box
self.cmbDoseList = []
# Create rows
for i in xrange(Nrows):
# Key for Original ROI Name dictionary - it is the number of original ROI in DCM
key = self.__originalRoiNumberList[i]
# Standard
txtStandard = QtGui.QLabel()
txtStandard.setStyleSheet(self.greenStyle)
# Assigned name
combobox = ExtendedCombo()
combobox.setStyleSheet(self.orangeStyle)
# Organ laterality and extra
cmbExtra = QtGui.QComboBox()
cmbExtra.setStyleSheet(self.orangeStyle)
cmbExtra.setEnabled(False)
cmbExtra.setEditable(False)
# Margin
cmbMargin = QtGui.QComboBox()
cmbMargin.setStyleSheet(self.orangeStyle)
cmbMargin.setEnabled(False)
cmbMargin.setEditable(False)
# Dose
cmbDose = QtGui.QComboBox()
cmbDose.setStyleSheet(self.orangeStyle)
cmbDose.setEnabled(False)
cmbDose.setEditable(False)
# Use formalized RTStructs as data set for combobox
rtsNames = []
for item in self.__formalizedRTStructs:
combobox.addItem(item.name, item.identifier)
rtsNames.append(item.name)
# Put original ROI name into first column
self.tableWidget.setItem(
i,
0,
QtGui.QTableWidgetItem(
self.__originalRoiNameDict[key][0])
)
# Original name is not editable
self.tableWidget.item(i, 0).setFlags(QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled)
self.tableWidget.item(i, 0).setBackgroundColor(QtGui.QColor(gui.colours.RED))
# Put combo boxes to extra columns
self.tableWidget.setCellWidget(i, 1, txtStandard)
self.tableWidget.setCellWidget(i, 2, combobox)
self.tableWidget.setCellWidget(i, 3, cmbExtra)
self.tableWidget.setCellWidget(i, 4, cmbMargin)
self.tableWidget.setCellWidget(i, 5, cmbDose)
# Automatic matching of RTStructs names is configurable
j = 0 # Select first otherwise
if ConfigDetails().autoRTStructMatch:
# Find most promising agreement (first) to ROIName in default_values
closest = get_close_matches(\
self.__originalRoiNameDict[key][0],\
rtsNames, 1, 0)[0]
# Find position of closest name in
j = rtsNames.index(closest)
combobox.setCurrentIndex(j)
self.logger.debug(self.__originalRoiNameDict[key][0] + \
" initally mapped to: " + \
self.formalizedRTStructs[j].identifier.encode("utf-8"))
# Save presselected options to RoiNameDic
self.__originalRoiNameDict[self.__originalRoiNumberList[i]].\
append(self.formalizedRTStructs[j].identifier)
# Show initial mapping in GUI
txtStandard.setText(self.formalizedRTStructs[j].identifier.encode("utf-8"))
# Enable additional info for L/R
if self.__formalizedRTStructs[j].name in self._leftRightContours:
cmbExtra.setEnabled(True)
cmbExtra.setEditable(False)
cmbExtra.clear()
for lrItem in self._leftRightModel:
cmbExtra.addItem(lrItem[0], lrItem[1])
# Enable additional info form multiple TV
elif self.__formalizedRTStructs[j].name in self._tvMultipleContours:
cmbExtra.setEnabled(True)
cmbExtra.setEditable(False)
cmbExtra.clear()
for multiItem in self._tvMultipleModel:
cmbExtra.addItem(multiItem[0], multiItem[1])
# Enable non standard additional info for contours of common type
elif self.__formalizedRTStructs[j].name in self._extraTextContours:
cmbExtra.setEnabled(True)
cmbExtra.setEditable(True)
cmbExtra.clear()
else:
cmbExtra.setEnabled(False)
cmbExtra.setEditable(False)
cmbExtra.clear()
# Enable margin info for organ at risk OAR as well as TV
if self.__formalizedRTStructs[j].name in self._oarContours:
cmbMargin.setEnabled(True)
cmbMargin.setEditable(True)
cmbMargin.setValidator(QtGui.QIntValidator(0, 99, cmbMargin))
cmbMargin.clear()
for marginItem in self._oarMarginModel:
cmbMargin.addItem(marginItem[0], marginItem[1])
elif self.__formalizedRTStructs[j].name in self._tvContours:
cmbMargin.setEnabled(True)
cmbMargin.setEditable(True)
cmbMargin.setValidator(QtGui.QIntValidator(0, 99, cmbMargin))
cmbMargin.clear()
else:
cmbMargin.setEnabled(False)
cmbMargin.setEditable(False)
cmbMargin.clear()
# Enable dose info for target volume TV
if self.__formalizedRTStructs[j].name in self._tvContours:
cmbDose.setEnabled(True)
cmbDose.setEditable(True)
cmbDose.setValidator(QtGui.QIntValidator(0, 90000, cmbDose))
cmbDose.clear()
else:
cmbDose.setEnabled(False)
cmbDose.setEditable(False)
cmbDose.clear()
# Handler
self.connect(combobox, QtCore.SIGNAL("currentIndexChanged(QString)"), self.updateMapping)
self.connect(cmbExtra, QtCore.SIGNAL("editTextChanged(QString)"), self.updateDetailsMappingText)
self.connect(cmbExtra, QtCore.SIGNAL("currentIndexChanged(int)"), self.updateDetailsMappingCombo)
self.connect(cmbMargin, QtCore.SIGNAL("currentIndexChanged(int)"), self.updateMarginMappingCombo)
self.connect(cmbMargin, QtCore.SIGNAL("editTextChanged(QString)"), self.updateMarginMappingText)
self.connect(cmbDose, QtCore.SIGNAL("editTextChanged(QString)"), self.updateDoseMappingText)
# Append combobox to list
self.txtStandardisedList.append(txtStandard)
self.comboBoxList.append(combobox)
self.cmbExtraList.append(cmbExtra)
self.cmbMarginList.append(cmbMargin)
self.cmbDoseList.append(cmbDose)
def get_extension_folder(ext):
maybe = list(map(lambda x: x[0], submodules[__version__]))
folder = difflib.get_close_matches(ext, maybe)[0]
return folder
def postcode():
print("-----------")
if request.method == "POST":
input_data = request.form
days = "all"
postcode = input_data["postcode"]
try:
source = input_data['source']
days = input_data['days']
postcode = input_data["postcode"]
except:
source = 'list_view'
days_set = days
print(source)
if postcode.lower() not in validation_list:
close_options = get_close_matches(postcode.lower(), validation_list)
in_options = [x for x in validation_list if postcode in x]
all_options = [x.title() for x in set(in_options + close_options)]
if len(all_options) == 0:
other_options = "no"
else:
other_options = "yes"
if source == 'maps':
return render_template(
"postcode_error.html",
postcode=postcode.title(),
other_options=other_options,
all_options=all_options,
validation_set=all_postcode_suburb_list,
view_mode="map",
days_set=days_set,
last_update=last_update
)
else:
return render_template(
"postcode_error.html",
postcode=postcode.title(),
other_options=other_options,
all_options=all_options,
validation_set=all_postcode_suburb_list,
days_set=days_set,
last_update=last_update
)
# Need to find if the search is not a post code so we can convert it
if not postcode.startswith("2"):
postcode = suburb_to_postcode_dict[postcode.lower()]
if source == "list_view":
close_postcodes_try = [
int(postcode) - 1,
int(postcode) - 2,
int(postcode) + 1,
int(postcode) + 2,
]
close_postcodes = [
str(x) for x in close_postcodes_try if str(x) in all_postcodes
]
close_postcodes_all_data = {x: cases_data[x] for x in close_postcodes}
close_postcodes_high_level = get_high_level(close_postcodes_all_data)
return render_template(
"postcode.html",
postcode=postcode,
days_set=days_set,
validation_set=all_postcode_suburb_list,
last_update=last_update,
close_postcodes_data=close_postcodes_high_level,
postcode_data=cases_data[postcode]
)
print(days_set)
if days == "one":
if source == 'maps':
temp_map_name = random_map_name()
map_data = all_map('one_day', postcode_zoom=postcode,zoom=13)
map_data.save(temp_map_name)
remove_bootstrap3(temp_map_name)
just_the_file = os.path.split(temp_map_name)[-1]
map_template = f"temp_maps/{just_the_file}"
return render_template('map_direct.html', days_set="one", view_mode="map", temp_map_name=map_template, temp_map = 'true', last_update=last_update)
if days == "seven":
if source == 'maps':
temp_map_name = random_map_name()
map_data = all_map('seven_days', postcode_zoom=postcode,zoom=13)
map_data.save(temp_map_name)
remove_bootstrap3(temp_map_name)
just_the_file = os.path.split(temp_map_name)[-1]
map_template = f"temp_maps/{just_the_file}"
return render_template('map_direct.html', days_set="seven", view_mode="map", temp_map_name=map_template, temp_map = 'true', last_update=last_update)
elif days == "fourteen":
if source == 'maps':
temp_map_name = random_map_name()
map_data = all_map('fourteen_days', postcode_zoom=postcode,zoom=13)
map_data.save(temp_map_name)
remove_bootstrap3(temp_map_name)
just_the_file = os.path.split(temp_map_name)[-1]
map_template = f"temp_maps/{just_the_file}"
return render_template('map_direct.html', days_set="fourteen", view_mode="map", temp_map_name=map_template, temp_map = 'true', last_update=last_update)
elif days == "all":
if source == 'maps':
temp_map_name = random_map_name()
map_data = all_map('all_days', postcode_zoom=postcode,zoom=13)
map_data.save(temp_map_name)
remove_bootstrap3(temp_map_name)
just_the_file = os.path.split(temp_map_name)[-1]
map_template = f"temp_maps/{just_the_file}"
return render_template('map_direct.html', days_set="all", view_mode="map", temp_map_name=map_template, temp_map='true', last_update=last_update)
else:
temp_map_name = random_map_name()
map_data = all_map('active_cases', postcode_zoom=postcode,zoom=13)
map_data.save(temp_map_name)
remove_bootstrap3(temp_map_name)
just_the_file = os.path.split(temp_map_name)[-1]
map_template = f"temp_maps/{just_the_file}"
return render_template('map_direct.html', days_set="active", view_mode="map", temp_map_name=map_template, temp_map='true', last_update=last_update)
else:
print(request.args["location"])
postcode = request.args["location"]
try:
days_set = request.args["days"]
except:
days_set = "all"
if not postcode.startswith("2"):
postcode = suburb_to_postcode_dict[postcode.lower()]
close_postcodes_try = [
int(postcode) - 1,
int(postcode) - 2,
int(postcode) + 1,
int(postcode) + 2,
]
close_postcodes = [
str(x) for x in close_postcodes_try if str(x) in all_postcodes
]
close_postcodes_all_data = {x: cases_data[x] for x in close_postcodes}
close_postcodes_high_level = get_high_level(close_postcodes_all_data)
return render_template(
"postcode.html",
postcode=postcode,
days_set=days_set,
validation_set=all_postcode_suburb_list,
last_update=last_update,
close_postcodes_data=close_postcodes_high_level,
postcode_data=cases_data[postcode]
)
def get_part_html_tree(dist, pn, extra_search_terms='', url=None, descend=2, local_part_html=None, scrape_retries=2):
'''@brief Find the Newark HTML page for a part number and return the URL and parse tree.
@param dist
@param pn Part number `str()`.
@param extra_search_terms
@param url
@param descend
@param local_part_html
@param scrape_retries `int` Quantity of retries in case of fail.
@return (html `str()` of the page, url)
'''
# Use the part number to lookup the part using the site search function, unless a starting url was given.
if url is None:
url = 'http://www.newark.com/webapp/wcs/stores/servlet/Search?catalogId=15003&langId=-1&storeId=10194&gs=true&st=' + urlquote(
pn + ' ' + extra_search_terms,
safe='')
elif url[0] == '/':
url = 'http://www.newark.com' + url
elif url.startswith('..'):
url = 'http://www.newark.com/Search/' + url
# Open the URL, read the HTML from it, and parse it into a tree structure.
for _ in range(scrape_retries):
try:
req = FakeBrowser(url)
response = urlopen(req)
html = response.read()
break
except WEB_SCRAPE_EXCEPTIONS:
logger.log(DEBUG_DETAILED,'Exception while web-scraping {} from {}'.format(pn, dist))
pass
else: # Couldn't get a good read from the website.
logger.log(DEBUG_OBSESSIVE,'No HTML page for {} from {}'.format(pn, dist))
raise PartHtmlError
try:
tree = BeautifulSoup(html, 'lxml')
except Exception:
logger.log(DEBUG_OBSESSIVE,'No HTML tree for {} from {}'.format(pn, dist))
raise PartHtmlError
# Abort if the part number isn't in the HTML somewhere.
# (Only use the numbers and letters to compare PN to HTML.)
if re.sub('[\W_]','',str.lower(pn)) not in re.sub('[\W_]','',str.lower(str(html))):
logger.log(DEBUG_OBSESSIVE,'No part number {} in HTML page from {}'.format(pn, dist))
raise PartHtmlError
# If the tree contains the tag for a product page, then just return it.
if tree.find('div', class_='productDisplay', id='page') is not None:
return tree, url
# If the tree is for a list of products, then examine the links to try to find the part number.
if tree.find('table', class_='productLister', id='sProdList') is not None:
logger.log(DEBUG_OBSESSIVE,'Found product table for {} from {}'.format(pn, dist))
if descend <= 0:
logger.log(DEBUG_OBSESSIVE,'Passed descent limit for {} from {}'.format(pn, dist))
raise PartHtmlError
else:
# Look for the table of products.
products = tree.find('table',
class_='productLister',
id='sProdList').find('tbody').find_all('tr')
# Extract the product links for the part numbers from the table.
product_links = []
for p in products:
try:
product_links.append(
p.find('td', class_='mftrPart').find('a'))
except AttributeError:
continue
# Extract all the part numbers from the text portion of the links.
part_numbers = [l.text for l in product_links]
# Look for the part number in the list that most closely matches the requested part number.
try:
match = difflib.get_close_matches(pn, part_numbers, 1, 0.0)[0]
except IndexError:
raise PartHtmlError
# Now look for the link that goes with the closest matching part number.
for l in product_links:
if l.text == match:
# Get the tree for the linked-to page and return that.
logger.log(DEBUG_OBSESSIVE,'Selecting {} from product table for {} from {}'.format(l.text, pn, dist))
return get_part_html_tree(dist, pn, extra_search_terms,
url=l.get('href', ''),
descend=descend-1,
scrape_retries=scrape_retries)
# I don't know what happened here, so give up.
logger.log(DEBUG_OBSESSIVE,'Unknown error for {} from {}'.format(pn, dist))
raise PartHtmlError
def buildResponse(comment, result=''):
comm = comment
start = comm.index('[[')
end = comm.index(']]')
length = len(comm)
param = comm[start + 2:end - length]
url = '[' + param + '](https://cards.eternalwarcry.com/cards/full/' + param.replace(
' ', '_') + '.png) '
link = 'link not found'
cardFound = False
correctlyNamed = False
matches = []
for x in range(0, len(fullNames)):
itername = fullNames[x][0].lower()
if itername == param.lower():
link = fullNames[x][1]
url = fullNames[x][2]
cardFound = True
correctlyNamed = True
elif param.lower() in itername:
matches.append(itername)
cardName = ''
if not cardFound and not matches:
bestGuess = difflib.get_close_matches(param.lower(), names)
if not not bestGuess:
cardName = bestGuess[0]
cardFound = True
elif not cardFound:
possibleNames = []
for x in matches:
if ',' in x:
possibleNames.append(x)
if not possibleNames:
bestGuess = difflib.get_close_matches(param.lower(), matches, 1, 0)
cardName = bestGuess[0]
cardFound = True
else:
bestGuess = difflib.get_close_matches(param.lower(), possibleNames,
1, 0)
cardName = bestGuess[0]
cardFound = True
for x in range(0, len(fullNames)):
itername = fullNames[x][0].lower()
if itername == cardName.lower():
link = fullNames[x][1]
url = fullNames[x][2]
param2 = '[' + param + '](' + url + ') ' if correctlyNamed else '[' + capwords(
cardName) + '](' + url + ') '
param3 = ' - ([EWC](' + link + ')) \n \n'
if cardFound:
newResult = result + param2 + param3
else:
newResult = result
rString = comm[end - length + 1:]
if '[[' in rString and ']]' in rString:
newResult = buildResponse(rString, newResult)
return newResult
def menu(self, custom_path_menu_above: str = ""):
an_input = "HELP_ME"
while True:
# There is a command in the queue
if self.queue and len(self.queue) > 0:
# If the command is quitting the menu we want to return in here
if self.queue[0] in ("q", "..", "quit"):
console.print("")
# Go back to the root in order to go to the right directory because
# there was a jump between indirect menus
if custom_path_menu_above:
self.queue.insert(1, custom_path_menu_above)
if len(self.queue) > 1:
return self.queue[1:]
if gtff.ENABLE_EXIT_AUTO_HELP:
return ["help"]
return []
# Consume 1 element from the queue
an_input = self.queue[0]
self.queue = self.queue[1:]
# Print location because this was an instruction and we want user to know the action
if an_input and an_input.split(
" ")[0] in self.controller_choices:
console.print(f"{get_flair()} {self.PATH} $ {an_input}")
# Get input command from user
else:
# Display help menu when entering on this menu from a level above
if an_input == "HELP_ME":
self.print_help()
try:
# Get input from user using auto-completion
if session and gtff.USE_PROMPT_TOOLKIT:
an_input = session.prompt(
f"{get_flair()} {self.PATH} $ ",
completer=self.completer,
search_ignore_case=True,
)
# Get input from user without auto-completion
else:
an_input = input(f"{get_flair()} {self.PATH} $ ")
except KeyboardInterrupt:
# Exit in case of keyboard interrupt
an_input = "exit"
try:
# Process the input command
self.queue = self.switch(an_input)
except SystemExit:
console.print(
f"\nThe command '{an_input}' doesn't exist on the {self.PATH} menu.",
end="",
)
similar_cmd = difflib.get_close_matches(
an_input.split(" ")[0] if " " in an_input else an_input,
self.controller_choices,
n=1,
cutoff=0.7,
)
if similar_cmd:
if " " in an_input:
candidate_input = (
f"{similar_cmd[0]} {' '.join(an_input.split(' ')[1:])}"
)
if candidate_input == an_input:
an_input = ""
self.queue = []
console.print("\n")
continue
an_input = candidate_input
else:
an_input = similar_cmd[0]
console.print(f" Replacing by '{an_input}'.")
self.queue.insert(0, an_input)
else:
console.print("\n")
def detect_text():
countries_names = []
country_codes = []
official_names = []
full_length = []
loss = []
with open(root + "/" + filename, 'rb') as image:
base64_image = base64.b64encode(image.read()).decode()
#print(base64_image)
url = 'https://vision.googleapis.com/v1/images:annotate?key=AIzaSyAOztXTencncNtoRENa1E3I0jdgTR7IfL0'
header = {'Content-Type': 'application/json'}
body = {
'requests': [{
'image': {
'content': base64_image,
},
'features': [{
'type': 'DOCUMENT_TEXT_DETECTION',
'maxResults': 10,
}]
#"imageContext": {
#"languageHints": ["en-t-i0-handwrit"]
#}
}]
}
response = requests.post(url, headers=header, json=body).json()
text = response['responses'][0]['textAnnotations'][0][
'description'] if len(response['responses'][0]) > 0 else ''
list_of_countries = list(pycountry.countries)
#print(list_of_countries)
match = re.search(r'\d{2}/\d{2}/\d{4}', text)
#print("matched date:",match)
newlist = re.findall(r'-?\d+\.?\d*', text)
block = str(text).split('\n')
offence = []
#print("blocks data:",block[::-2])
print("last two lines of passport:", block[::-1])
for x in block[::-1]:
if (len(x) >= 25):
full_length.append(x)
print("full_length:", full_length)
appie = block[::-1]
ca = re.sub(r'\s+', '', str(appie[2]))
loss.append(ca)
a = re.sub(r'\s+', '', str(appie[1]))
#print("replaced data:",a)
loss.append(a)
mine = re.sub(r'\s+', '', str(full_length[1]))
offence.append(mine)
dug = re.sub(r'\s+', '', str(full_length[0]))
offence.append(dug)
#print(tuple(loss))
#print("appie data:",str(appie[2]),str(appie[1]))
#print(len(ca),len(a))
first = loss[0]
second = loss[1]
#first=offence[0]
#second=offence[1]
length = (len(ca) + len(a))
#print("length of mrz code:",length)
if (first[0] == 'P'):
passport_type = ('\n'.join(tuple(loss)))
# print("datafh:",passport_type)
mrz_td3 = passport_type
type = first[0]
#print("type:",type)
country_code = re.sub('\ |\?|\.|\!|\/|\;|\:|\<', ' ',
first[2:5])
for key, value in country_with_codes.items():
#print(key,value)
luck = country_code.strip(' ')
if (key == luck):
print("===============", key, value)
fullname_with_symbols = first[5:45]
#print("kkk",pic)
dan = fullname_with_symbols.strip('<')
hat = dan.split('<<')
surname = re.sub('\ |\?|\.|\!|\/|\;|\:|\<', ' ', hat[0])
if (len(hat) == 2):
mrx = re.sub('\ |\?|\.|\!|\/|\;|\:|\<', ' ', hat[1])
givenname = mrx
else:
givenname = ''
#print("given_name:",givenname)
document_no = second[0:9]
passport_no = re.sub(r'[^\w]', ' ', document_no)
#print("passport_number:",passport_no)
nationality = re.sub('\ |\?|\.|\!|\/|\;|\:|\<', ' ',
second[10:13])
#print("nationality:",nationality)
birthdate = second[13:19]
date_of_birth = '/'.join(
[birthdate[:2], birthdate[2:4], birthdate[4:]])
#print("date of birth:",date_of_birth)
abc = re.findall(
r'\s([0-9][0-9] [a-zA-Z]+ \d{4}|\d{2}/\d{2}/\d{4}|\d{2}.\d{2}.\d{4}|\d{2} \w+/\w+ \d{4}|\d{2} \d{2} \d{4}|\d{2}-d{2}-d{4}|\d{2} \w+ /\w+ \d{2}|\d{2} \w+ \d{2}|\d{2} \w+/\w+ \d{2}|\d{2} \w+ \w+ \d{2}|\d{2} \w \/ \w+ \d{2})',
text)
print("date of birth:", abc)
Date_of_issue = abc[1]
sex = second[20]
#print("sex:",sex)
expiry_date = second[21:27]
date_of_expiry = '/'.join(
[expiry_date[:2], expiry_date[2:4], expiry_date[4:]])
data = {
"Passport_Document_Type": type,
"country_code": country_code,
"FamilyName": surname,
"Given_Name": givenname,
"Passport_Document_No": passport_no,
"Nationality": nationality,
"Date_of_Birth": date_of_birth,
"Gender": sex,
"Date_of_issue": Date_of_issue,
"Date_of_Expiry": date_of_expiry
}
print(data)
with open('/home/caratred/copy/new_csv/' + filename + '.csv',
'w') as csv_file:
writer = csv.writer(csv_file)
for key, value in data.items():
writer.writerow([key, value])
return data
elif (first[0] == 'V'):
#if(length==88):
act = ('\n'.join(tuple(loss)))
print("datafh:", act)
#print(MRVACodeChecker(act))
type = first[0]
print("document type:", type)
print("issued state or country:", first[1])
types_of_visas = [
'B-2', 'B-3', 'B-1X', 'B-1', 'E-1', 'MEDX', 'T-1', 'T- 1',
'EMPLOYMENT', 'T', 'U', 'B', 'X-ENTRY', 'E', 'E-3', 'x',
'TOURIST'
]
for x in block[::-1]:
for y in x:
#print("cols:",get_close_matches(y,types_of_visas))
if x in types_of_visas:
print("type of visa:", x)
entries = ['MULTIPLE', 'SINGLE', 'DOUBLE']
entry = ' '
for y in block[::-1]:
a = get_close_matches(y, entries)
print(a)
if len(a) == 1:
entry = a[0]
print("no of entries:", a[0])
#elif y in entries:
# print("no of entries:",y)
if (first[1].isalpha()):
abc = re.findall(
r'\s([0-9][0-9] [a-zA-Z]+ \d{4}|\d{2}/\d{2}/\d{4}|\d{2}.\d{2}.\d{4}|\d{2} \w+/\w+ \d{4}|\d{2} \d{2} \d{4}|\d{2}-d{2}-d{4}|\d{2} \w+ /\w+ \d{2}|\d{2} \w+/\w+ \d{4} \w+)',
text)
print("date of issue:", abc, first[1])
great = min(abc[0][6:10], abc[1][6:10])
for x in abc:
if great in x:
Date_of_issue = x
break
print("great value:", x)
elif (first[1] == '<'):
abc = re.findall(
r'\s([0-9][0-9] [a-zA-Z]+ \d{4}|\d{2}/\d{2}/\d{4}|\d{2}.\d{2}.\d{4}|\d{2} \w+/\w+ \d{4}|\d{2} \d{2} \d{4}|\d{2}-d{2}-d{4}|\d{2} \w+ /\w+ \d{2}|\d{2} \w+/\w+ \d{4} \w+)',
text)
if (len(abc) == 2):
print("date of issue:", abc[1])
Date_of_issue = abc[1]
else:
Date_of_issue = abc[0]
issuingcountry = first[2:5]
print("issuing country :", issuingcountry)
ad = (first[5:]).strip('<')
bc = ad.split('<<')
surname = bc[0]
print("sur_name:", surname)
givenname = bc[1]
print("first_name:", givenname)
print("name of the person:", first[5:])
visa_number = second[0:9]
print("visa_number:", visa_number)
nationality = second[10:13]
print("nationality:", nationality)
birthdate = second[13:19]
print(birthdate)
date_of_birth = '/'.join(
[birthdate[:2], birthdate[2:4], birthdate[4:]])
print("date of birth:", date_of_birth)
sex = second[20]
print("sex:", sex)
expiry_date = second[21:27]
date_of_expiry = '/'.join(
[expiry_date[:2], expiry_date[2:4], expiry_date[4:]])
print("expiry_date:", date_of_expiry)
optional_data = second[28:]
print("optional_data:", optional_data)
data = {
"type": type,
"issued_country": issuingcountry,
"visa_no_of_entries": entry,
"sur_name": surname,
"given_name": givenname,
"Date_of_issue": Date_of_issue,
"visa_no": visa_number,
"nationality": nationality,
"date_of_birth": date_of_birth,
"gender": sex,
"visa_expirydate": date_of_expiry
}
print("person_visa_details:", data)
with open('/home/caratred/copy/new_csv/' + filename + '.csv',
'w') as csv_file:
writer = csv.writer(csv_file)
for key, value in data.items():
writer.writerow([key, value])
return data
def desired_map():
data = pandas.read_csv("country-code.csv")
country_name = input("\n Enter Country Name : ")
while(1):
if(country_name in data.Name.values):
ccode = list(data.loc[data['Name'] == country_name, 'Code'])[0]
city_data = pandas.read_csv("world-cities_csv.csv")
city_name = input("\n Enter City Name : ")
while(1):
if((city_name in city_data.name.values)):
location = city_name +" "+ ccode
print("\n Desired Location : {}\n".format(location))
load_animation(" Please Wait...","|/-\\",100,0.075,0)
location_coordinates = geocoder.osm(location)
sys.stdout.write("\033[F")
try:
ln = location_coordinates.osm['x']
lt = location_coordinates.osm['y']
except TypeError as e:
error_page()
map = folium.Map(location = (lt,ln), zoom_start = 12)
feature_group = folium.FeatureGroup(name = "My Map")
la = owm.weather_at_coords(lt,ln)
w = la.get_weather()
message = [location,str(w.get_status()),str(w.get_temperature(unit='celsius')['temp'])+"° C"]
feature_group.add_child(folium.Marker(location=[float(lt), float(ln)], popup = ",\n".join(message), icon = folium.Icon(color="red", icon = "glyphicon glyphicon-map-marker")))
save_map(feature_group, map)
break
else:
if(len(get_close_matches(city_name, city_data.name.values, n = 5, cutoff = 0.4)) == 0):
print("\n NO RESULTS FOUND. TRY AGAIN.\n")
city_name = input(" Enter City Name : ")
else:
city_name = list(get_close_matches(city_name, city_data.name.values, n = 5, cutoff = 0.4))
print("\n {0} SIMILAR RESULTS FOUND.\n CHOOSE FROM BELOW.".format(len(city_name)))
dash = '-' * 61
print(" " + dash)
print('{:<52s}{:>6s}'.format(' | Options'," | Code |"))
print(" " + dash)
print('{:<52s}{:>6s}'.format(' | Retry'," | 0 |"))
for i in range(len(city_name)):
nm = " | "+city_name[i]
print('{:<53s}|{:^7d}|'.format(nm,i+1))
print(" " + dash)
city_response = int(input("\n Enter Response : "))
if((city_response-1) != -1) and (city_response <= len(city_name)):
city_name = city_name[city_response-1]
else:
city_name = input(" Enter City Name : ")
break
else:
if(len(get_close_matches(country_name, data.Name.values, n = 1, cutoff = 0.4)) == 0 ):
print("\n NO RESULTS FOUND. TRY AGAIN.\n")
country_name = input(" Enter Country Name : ")
else:
country_name = list(get_close_matches(country_name, data.Name.values, n = 1, cutoff = 0.4))
print("\n {0} SIMILAR RESULTS FOUND.\n CHOOSE FROM BELOW.".format(len(country_name)))
dash = '-' * 61
print(" " + dash)
print('{:<52s}{:>6s}'.format(' | Options'," | Code |"))
print(" " + dash)
print('{:<52s}{:>6s}'.format(' | Retry'," | 0 |"))
for i in range(len(country_name)):
nm = " | "+country_name[i]
print('{:<53s}|{:^7d}|'.format(nm,i+1))
print(" " + dash)
country_response = int(input("\n Enter Response : "))
if((country_response -1) != -1) and (country_response <= len(country_name)):
country_name = country_name[country_response-1]
else:
country_name = input(" Enter Country Name : ")
def region_table_body(data):
"""Generate data rows for region table."""
region_names = (
'Andaman and Nicobar Islands',
'Andhra Pradesh',
'Arunachal Pradesh',
'Assam',
'Bihar',
'Chandigarh',
'Chhattisgarh',
'Dadra and Nagar Haveli and Daman and Diu',
'Delhi',
'Goa',
'Gujarat',
'Haryana',
'Himachal Pradesh',
'Jammu and Kashmir',
'Jharkhand',
'Karnataka',
'Kerala',
'Ladakh',
'Lakshadweep',
'Madhya Pradesh',
'Maharashtra',
'Manipur',
'Meghalaya',
'Mizoram',
'Nagaland',
'Odisha',
'Puducherry',
'Punjab',
'Rajasthan',
'Sikkim',
'Tamil Nadu',
'Telangana',
'Tripura',
'Uttarakhand',
'Uttar Pradesh',
'West Bengal',
)
out = []
for i, name in enumerate(region_names, 1):
matches = difflib.get_close_matches(name, list(data.regions), 1)
key = None
if len(matches) != 0:
key = matches[0]
elif name == 'Dadra and Nagar Haveli and Daman and Diu':
candidates = ['Dadar Nagar Haveli', 'Dadra and Nagar Haveli']
for candidate in candidates:
if candidate in data.regions:
key = candidate
break
if key is None:
total, active, cured, death = 0, 0, 0, 0
else:
total, active, cured, death = data.regions[key]
total, active, cured, death = (gldg(total), gldg(active),
gldg(cured), gldg(death))
if name == 'Assam':
total = str(total) + open('layout/fn1.txt').read().strip()
elif name == 'Kerala':
death = str(death) + open('layout/fn2.txt').read().strip()
out.append('|-')
out.append('! scope="row" |{}'.format(markup_region(name)))
out.append('|{}'.format(markup_num(total)))
out.append('|{}'.format(markup_num(death)))
out.append('|{}'.format(markup_num(cured)))
out.append('|{}'.format(markup_num(active)))
out = '\n'.join(out)
return out
def the_jazz_page_midi_files(name, refresh_cache=False, verbose_out=None):
"""
The Jazz Page has quite a few of midi files on a single page.
There's only one of most songs and it's not exactly a huge database,
but they claim to be high quality.
By default, the list of midi files and song names will be cached
the first time it's fetched. Subsequently, songs will just be
looked up in the local database. Use C{refresh_cache=True} to
force the list to be re-read from the site.
To refresh the cache without doing a search, see
L{refresh_the_jazz_page_cache}.
@return: list of (midi-file,song-name) pairs each containing a
L{midi.EventStream} for each file. Unlikely to be more than one.
"""
from jazzparser.settings import LOCAL_DATA_DIR
from jazzparser.utils.csv import UnicodeCsvReader
from urllib2 import urlopen, HTTPError, URLError
from midi import read_midifile, MidiReadError
import os, difflib
from cStringIO import StringIO
def _verbose(message):
if verbose_out is not None:
print >>verbose_out, message
_verbose("<<<<<<<<<< The Jazz Page midi search >>>>>>>>>>>>>>")
# Try reading the cache to see if we've fetched the data before
cache_filename = os.path.join(LOCAL_DATA_DIR, "the_midi_site_cache")
cached = os.path.exists(cache_filename)
# Recreate the cache if it doesn't exist or it's being forced
if not cached or refresh_cache:
refresh_the_jazz_page_cache()
# Now read in the cache from the file
cache_file = open(cache_filename, 'r')
try:
reader = UnicodeCsvReader(cache_file)
song_names = {}
# Index the links by song name (allowing for multiple songs with
# the same name)
for link,song_name in reader:
song_names.setdefault(song_name,[]).append(link)
finally:
cache_file.close()
# Fetch each file for songs whose names are close to the search term
_verbose("Searching for song name: %s" % name)
matches = difflib.get_close_matches(name.lower(), song_names.keys(), cutoff=0.8)
if len(matches) == 0:
_verbose("No matches found")
files = []
for name in matches:
for link in song_names[name]:
_verbose("Fetching midi file %s" % link)
file_data = urlopen(link).read()
files.append((file_data, link, name))
# Check that each one is actually a MIDI file (these should be good
# quality in general)
ok_files = []
for data,link,name in files:
try:
mid = read_midifile(StringIO(data))
except MidiReadError, err:
_verbose("Invalid midi file: %s (%s)" % (link, err))
# Skip this file
pass
else:
ok_files.append((data,name))
def _complete_gallery_conf(sphinx_gallery_conf, src_dir, plot_gallery,
abort_on_example_error, lang='python',
builder_name='html', app=None, check_keys=True):
gallery_conf = copy.deepcopy(DEFAULT_GALLERY_CONF)
options = sorted(gallery_conf)
extra_keys = sorted(set(sphinx_gallery_conf) - set(options))
if extra_keys and check_keys:
msg = 'Unknown key(s) in sphinx_gallery_conf:\n'
for key in extra_keys:
options = get_close_matches(key, options, cutoff=0.66)
msg += repr(key)
if len(options) == 1:
msg += ', did you mean %r?' % (options[0],)
elif len(options) > 1:
msg += ', did you mean one of %r?' % (options,)
msg += '\n'
raise ConfigError(msg.strip())
gallery_conf.update(sphinx_gallery_conf)
if sphinx_gallery_conf.get('find_mayavi_figures', False):
logger.warning(
"Deprecated image scraping variable `find_mayavi_figures`\n"
"detected, use `image_scrapers` instead as:\n\n"
" image_scrapers=('matplotlib', 'mayavi')",
type=DeprecationWarning)
gallery_conf['image_scrapers'] += ('mayavi',)
gallery_conf.update(plot_gallery=plot_gallery)
gallery_conf.update(abort_on_example_error=abort_on_example_error)
# XXX anything that can only be a bool (rather than str) should probably be
# evaluated this way as it allows setting via -D on the command line
for key in ('run_stale_examples',):
gallery_conf[key] = _bool_eval(gallery_conf[key])
gallery_conf['src_dir'] = src_dir
gallery_conf['app'] = app
# Check capture_repr
capture_repr = gallery_conf['capture_repr']
supported_reprs = ['__repr__', '__str__', '_repr_html_']
if isinstance(capture_repr, tuple):
for rep in capture_repr:
if rep not in supported_reprs:
raise ConfigError("All entries in 'capture_repr' must be one "
"of %s, got: %s" % (supported_reprs, rep))
else:
raise ConfigError("'capture_repr' must be a tuple, got: %s"
% (type(capture_repr),))
# Check ignore_repr_types
if not isinstance(gallery_conf['ignore_repr_types'], str):
raise ConfigError("'ignore_repr_types' must be a string, got: %s"
% (type(gallery_conf['ignore_repr_types']),))
# deal with show_memory
gallery_conf['memory_base'] = 0.
if gallery_conf['show_memory']:
if not callable(gallery_conf['show_memory']): # True-like
try:
from memory_profiler import memory_usage # noqa
except ImportError:
logger.warning("Please install 'memory_profiler' to enable "
"peak memory measurements.")
gallery_conf['show_memory'] = False
else:
def call_memory(func):
mem, out = memory_usage(func, max_usage=True, retval=True,
multiprocess=True)
try:
mem = mem[0] # old MP always returned a list
except TypeError: # 'float' object is not subscriptable
pass
return mem, out
gallery_conf['call_memory'] = call_memory
gallery_conf['memory_base'] = _get_memory_base(gallery_conf)
else:
gallery_conf['call_memory'] = gallery_conf['show_memory']
if not gallery_conf['show_memory']: # can be set to False above
def call_memory(func):
return 0., func()
gallery_conf['call_memory'] = call_memory
assert callable(gallery_conf['call_memory'])
# deal with scrapers
scrapers = gallery_conf['image_scrapers']
if not isinstance(scrapers, (tuple, list)):
scrapers = [scrapers]
scrapers = list(scrapers)
for si, scraper in enumerate(scrapers):
if isinstance(scraper, str):
if scraper in _scraper_dict:
scraper = _scraper_dict[scraper]
else:
orig_scraper = scraper
try:
scraper = import_module(scraper)
scraper = getattr(scraper, '_get_sg_image_scraper')
scraper = scraper()
except Exception as exp:
raise ConfigError('Unknown image scraper %r, got:\n%s'
% (orig_scraper, exp))
scrapers[si] = scraper
if not callable(scraper):
raise ConfigError('Scraper %r was not callable' % (scraper,))
gallery_conf['image_scrapers'] = tuple(scrapers)
del scrapers
# Here we try to set up matplotlib but don't raise an error,
# we will raise an error later when we actually try to use it
# (if we do so) in scrapers.py.
# In principle we could look to see if there is a matplotlib scraper
# in our scrapers list, but this would be backward incompatible with
# anyone using or relying on our Agg-setting behavior (e.g., for some
# custom matplotlib SVG scraper as in our docs).
# Eventually we can make this a config var like matplotlib_agg or something
# if people need us not to set it to Agg.
try:
_import_matplotlib()
except (ImportError, ValueError):
pass
# compress_images
compress_images = gallery_conf['compress_images']
if isinstance(compress_images, str):
compress_images = [compress_images]
elif not isinstance(compress_images, (tuple, list)):
raise ConfigError('compress_images must be a tuple, list, or str, '
'got %s' % (type(compress_images),))
compress_images = list(compress_images)
allowed_values = ('images', 'thumbnails')
pops = list()
for ki, kind in enumerate(compress_images):
if kind not in allowed_values:
if kind.startswith('-'):
pops.append(ki)
continue
raise ConfigError('All entries in compress_images must be one of '
'%s or a command-line switch starting with "-", '
'got %r' % (allowed_values, kind))
compress_images_args = [compress_images.pop(p) for p in pops[::-1]]
if len(compress_images) and not _has_optipng():
logger.warning(
'optipng binaries not found, PNG %s will not be optimized'
% (' and '.join(compress_images),))
compress_images = ()
gallery_conf['compress_images'] = compress_images
gallery_conf['compress_images_args'] = compress_images_args
# deal with resetters
resetters = gallery_conf['reset_modules']
if not isinstance(resetters, (tuple, list)):
resetters = [resetters]
resetters = list(resetters)
for ri, resetter in enumerate(resetters):
if isinstance(resetter, str):
if resetter not in _reset_dict:
raise ConfigError('Unknown module resetter named %r'
% (resetter,))
resetters[ri] = _reset_dict[resetter]
elif not callable(resetter):
raise ConfigError('Module resetter %r was not callable'
% (resetter,))
gallery_conf['reset_modules'] = tuple(resetters)
if not isinstance(gallery_conf['reset_modules_order'], str):
raise ConfigError('reset_modules_order must be a str, '
'got %r' % gallery_conf['reset_modules_order'])
if gallery_conf['reset_modules_order'] not in ['before', 'after', 'both']:
raise ConfigError("reset_modules_order must be in"
"['before', 'after', 'both'], "
'got %r' % gallery_conf['reset_modules_order'])
lang = lang if lang in ('python', 'python3', 'default') else 'python'
gallery_conf['lang'] = lang
del resetters
# Ensure the first cell text is a string if we have it
first_cell = gallery_conf.get("first_notebook_cell")
if (not isinstance(first_cell, str)) and (first_cell is not None):
raise ConfigError("The 'first_notebook_cell' parameter must be type "
"str or None, found type %s" % type(first_cell))
# Ensure the last cell text is a string if we have it
last_cell = gallery_conf.get("last_notebook_cell")
if (not isinstance(last_cell, str)) and (last_cell is not None):
raise ConfigError("The 'last_notebook_cell' parameter must be type str"
" or None, found type %s" % type(last_cell))
# Check pypandoc
pypandoc = gallery_conf['pypandoc']
if not isinstance(pypandoc, (dict, bool)):
raise ConfigError("'pypandoc' parameter must be of type bool or dict,"
"got: %s." % type(pypandoc))
gallery_conf['pypandoc'] = dict() if pypandoc is True else pypandoc
has_pypandoc, version = _has_pypandoc()
if isinstance(gallery_conf['pypandoc'], dict) and has_pypandoc is None:
logger.warning("'pypandoc' not available. Using Sphinx-Gallery to "
"convert rst text blocks to markdown for .ipynb files.")
gallery_conf['pypandoc'] = False
elif isinstance(gallery_conf['pypandoc'], dict):
logger.info("Using pandoc version: %s to convert rst text blocks to "
"markdown for .ipynb files" % (version,))
else:
logger.info("Using Sphinx-Gallery to convert rst text blocks to "
"markdown for .ipynb files.")
if isinstance(pypandoc, dict):
accepted_keys = ('extra_args', 'filters')
for key in pypandoc:
if key not in accepted_keys:
raise ConfigError("'pypandoc' only accepts the following key "
"values: %s, got: %s."
% (accepted_keys, key))
# Make it easy to know which builder we're in
gallery_conf['builder_name'] = builder_name
gallery_conf['titles'] = {}
# Ensure 'backreferences_dir' is str, pathlib.Path or None
backref = gallery_conf['backreferences_dir']
if (not isinstance(backref, (str, pathlib.Path))) and \
(backref is not None):
raise ConfigError("The 'backreferences_dir' parameter must be of type "
"str, pathlib.Path or None, "
"found type %s" % type(backref))
# if 'backreferences_dir' is pathlib.Path, make str for Python <=3.5
# compatibility
if isinstance(backref, pathlib.Path):
gallery_conf['backreferences_dir'] = str(backref)
# binder
gallery_conf['binder'] = check_binder_conf(gallery_conf['binder'])
if not isinstance(gallery_conf['css'], (list, tuple)):
raise ConfigError('gallery_conf["css"] must be list or tuple, got %r'
% (gallery_conf['css'],))
for css in gallery_conf['css']:
if css not in _KNOWN_CSS:
raise ConfigError('Unknown css %r, must be one of %r'
% (css, _KNOWN_CSS))
if gallery_conf['app'] is not None: # can be None in testing
gallery_conf['app'].add_css_file(css + '.css')
return gallery_conf
async def plugins_registry(self, ctx, *, plugin_name: typing.Union[int, str] = None):
"""
Shows a list of all approved plugins.
Usage:
`{prefix}plugin registry` Details about all plugins.
`{prefix}plugin registry plugin-name` Details about the indicated plugin.
`{prefix}plugin registry page-number` Jump to a page in the registry.
"""
await self.populate_registry()
embeds = []
registry = sorted(self.registry.items(), key=lambda elem: elem[0])
if isinstance(plugin_name, int):
index = plugin_name - 1
if index < 0:
index = 0
if index >= len(registry):
index = len(registry) - 1
else:
index = next((i for i, (n, _) in enumerate(registry) if plugin_name == n), 0)
if not index and plugin_name is not None:
embed = discord.Embed(
color=self.bot.error_color,
description=f'Could not find a plugin with name "{plugin_name}" within the registry.',
)
matches = get_close_matches(plugin_name, self.registry.keys())
if matches:
embed.add_field(
name="Perhaps you meant:", value="\n".join(f"`{m}`" for m in matches)
)
return await ctx.send(embed=embed)
for name, details in registry:
details = self.registry[name]
user, repo = details["repository"].split("/", maxsplit=1)
branch = details.get("branch")
plugin = Plugin(user, repo, name, branch)
embed = discord.Embed(
color=self.bot.main_color,
description=details["description"],
url=plugin.link,
title=details["repository"],
)
embed.add_field(
name="Installation", value=f"```{self.bot.prefix}plugins add {name}```"
)
embed.set_author(
name=details["title"], icon_url=details.get("icon_url"), url=plugin.link
)
if details.get("thumbnail_url"):
embed.set_thumbnail(url=details.get("thumbnail_url"))
if details.get("image_url"):
embed.set_image(url=details.get("image_url"))
if plugin in self.loaded_plugins:
embed.set_footer(text="This plugin is currently loaded.")
else:
required_version = details.get("bot_version", False)
if required_version and self.bot.version < parse_version(required_version):
embed.set_footer(
text="Your bot is unable to install this plugin, "
f"minimum required version is v{required_version}."
)
else:
embed.set_footer(text="Your bot is able to install this plugin.")
embeds.append(embed)
paginator = EmbedPaginatorSession(ctx, *embeds)
paginator.current = index
await paginator.run()
