def run_threads(self):
"""
设置线程池，并启动线程
"""
args_dict = {}
args_dict['output_dir'] = self.output_dir
args_dict['crawl_interval'] = self.crawl_interval
args_dict['crawl_timeout'] = self.crawl_timeout
args_dict['url_pattern'] = self.url_pattern
args_dict['max_depth'] = self.max_depth
args_dict['tag_dict'] = self.tag_dict
for index in xrange(self.thread_count):
thread_name = 'thread - %d' % index
thread = crawl_thread.CrawlerThread(thread_name,
self.process_request,
self.process_response,
args_dict)
thread.setDaemon(True)
thread.start()
print termcolor.colored(("第%s个线程开始工作") % index, 'yellow')
logging.info(("第%s个线程开始工作") % index)
self.checking_url.join()
self.program_end('normal exits ')
def _exit_():
print colored('[INFO] Exiting...', 'green')
try:
os.remove(merged_script_path)
except Exception as e:
pass
sys.exit(1)
def highlightList(self, taskName):
# get highlighted
sql_command = """SELECT * FROM "%s" WHERE taskName="%s" """ % (self.table, taskName)
self.cursor.execute(sql_command)
result = self.cursor.fetchall()
desired = str(result[0][self.taskName_position])
# get list
rows = self.retrieveList()
# display
self.displayHeader()
for row in rows:
if row[self.taskName_position] == desired:
text = self.myFormat(row)
print colored(text, "blue", attrs=["reverse", "blink"])
else:
text = self.myFormat(row)
self.printByPriority(row)
def getData(self, url):
data = Data()
try:
req = urllib2.Request(url, headers={'User-Agent' : "Magic Browser"})
request = urllib2.urlopen(req)
mime = request.info().getheader('Content-Type')
code = request.code
print(colored('[' + mime + '] ' + url, 'yellow'))
if code is 200:
if 'text/html' in mime:
html = request.read()
data = self.parse(html, url)
else:
#ANALYSIS TYPE
data.url = url
data.type = mime
elif code is 400:
data.broke = True
except UnicodeEncodeError as e :
print(colored(e, 'red'))
data.broke = True
return data
def server():
global connection
global board
global newTiles
global WAIT
# Create a TCP/IP socket
server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Bind the socket to the port
if int(sys.argv[1]) == 1:
server_address = ('localhost', PORT_NUMBER_2)
else:
server_address = ('localhost', PORT_NUMBER_1)
print >>sys.stderr, 'starting up on %s port %s' % server_address
server_sock.bind(server_address)
# Listen for incoming connections
server_sock.listen(1)
while True:
# Wait for a connection
print >>sys.stderr, 'waiting for a connection'
connection, client_address = server_sock.accept()
try:
print >>sys.stderr, 'connection from', client_address
state = "NORMAL"
# Receive the data in small chunks and retransmit it
while True:
data = connection.recv(1000)
# print >>sys.stderr, 'received "%s"' % data
if data:
if data == "BOARD" or data == "INITIAL" or data == "NEW_TILES":
state = data
print "STATE: %s"%state
else:
if state == "BOARD":
board = pickle.loads(data)
print colored("The other player has made a move",'red')
printBoard()
print "received board"
WAIT = False
elif state == "INITIAL":
board = pickle.loads(data)
print "received initial"
printBoard()
elif state == "NEW_TILES":
newTiles = pickle.loads(data)
print newTiles
print "received new tiles"
state = "NORMAL"
print "STATE: NORMAL"
else:
break
except KeyboardInterrupt:
print "Closed connection on server"
connection.close()
break
finally:
# Clean up the connection
print "Closed connection on server"
connection.close()
break
def _make_context(quiet=False):
# Start by adding the session instance to the namespace.
namespace = {'db': db, 'session': db.session}
if not quiet:
# Let the user know.
print(colored('from alchemist import db', 'green'))
print(colored('session = db.session', 'green'))
# Iterate through all models and add those to the namespace.
module = {}
for name, models in application.models.items():
for model in models:
if model.__module__ not in module:
module[model.__module__] = []
module[model.__module__].append(model.__name__)
namespace[model.__name__] = model
if not quiet:
# Let the user know.
for name, models in module.items():
items = ', '.join(models)
print(colored('from %s import %s' % (name, items), 'green'))
# Return the constructed namespace.
return namespace
def hash_cracker(dict_file_name, hash_file_name):
found = 0
file_dict = open(dict_file_name, "r")
passwords = file_dict.read().split('\n')
file_hash = open(hash_file_name, "r")
enc_hashes = file_hash.read().split('\n')
print enc_hashes
for pwd in passwords:
try:
for rec in enc_hashes:
# print pwd.split(':')[1]
test = sha1(pwd.split(':')[1] + rec.split(':')[1]).hexdigest()
if pwd.split(':')[1] == 'admin':
print rec.split(':')[0]
print test
if test == rec.split(':')[0]:
print colored("[-] Password Found for password_sha " + rec.split(':')[0] + " : " + pwd)
found = 1
break
else:
pass
if found == 1:
break
except:
pass
def server_thread(port):
global server_run
server_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind(('0.0.0.0', port))
while server_run:
try:
with Timeout(2, False):
(data, from_ip) = server_socket.recvfrom(1024)
if data is None:
continue
ping = ping_parse(data)
if ping is None:
continue
msg = 'Received - Seq: %d, Timestamp: %d' % (ping.seq, ping.timestamp)
print colored(msg, 'green')
pong = make_pong(ping)
server_socket.sendto(pong, from_ip)
except timeout:
continue
except KeyboardInterrupt:
server_run = False
def one(expected, champs, display, shuffle = False):
global count, total
vm = VM()
vm.add(champs)
if shuffle:
vm.shuffle()
if display:
util.tprint('Battle', '%s / %s' % (colored('%d' % (count), 'yellow'), colored('%d' % (total), 'yellow')))
util.tprint('Champions', ', '.join([colored(champ.champion.name, 'blue', attrs = ['bold']) for champ in vm.champions]))
count += 1
winner = vm.launch(stdout = PIPE)
report = None
if winner:
if display:
if winner.champion.filename == expected.filename:
color = 'green'
else:
color = 'red'
util.tprint('Winner', colored(winner.champion.name, color, attrs = ['bold']))
report = Report(champs, winner.champion)
if display:
print ''
return report
def list_latest(limit, vb=False, output=False):
'''list words by latest time you add to database.'''
conn = sqlite3.connect(os.path.join(DEFAULT_PATH, 'word.db'))
curs = conn.cursor()
try:
if not vb:
curs.execute('SELECT name, pr, addtime FROM Word ORDER by datetime(addtime) DESC LIMIT %d' % limit)
else:
curs.execute('SELECT expl, pr, addtime FROM Word ORDER by datetime(addtime) DESC LIMIT %d' % limit)
except Exception as e:
print(e)
print(colored('something\'s wrong, please set the limit', 'red'))
else:
for line in curs.fetchall():
expl = line[0]
pr = line[1]
print('\n' + '=' * 40 + '\n')
if not output:
print(colored('★ ' * pr, 'red'), colored('☆ ' * (5 - pr), 'yellow'), sep='')
colorful_print(expl)
else:
print('★ ' * pr + '☆ ' * (5 - pr))
normal_print(expl)
finally:
curs.close()
conn.close()
def process(self, msg, kwargs):
if self.extra is None:
return u'{}'.format(msg), kwargs
if 'module' in self.extra.keys():
if len(self.extra['module']) > 8:
self.extra['module'] = self.extra['module'][:8] + '...'
#If the logger is being called when hooking the 'options' module function
if len(self.extra) == 1 and ('module' in self.extra.keys()):
return u'{:<64} {}'.format(colored(self.extra['module'], 'cyan', attrs=['bold']), msg), kwargs
#If the logger is being called from CMEServer
if len(self.extra) == 2 and ('module' in self.extra.keys()) and ('host' in self.extra.keys()):
return u'{:<24} {:<39} {}'.format(colored(self.extra['module'], 'cyan', attrs=['bold']), self.extra['host'], msg), kwargs
#If the logger is being called from a protocol
if 'module' in self.extra.keys():
module_name = colored(self.extra['module'], 'cyan', attrs=['bold'])
else:
module_name = colored(self.extra['protocol'], 'blue', attrs=['bold'])
return u'{:<24} {:<15} {:<6} {:<16} {}'.format(module_name,
self.extra['host'],
self.extra['port'],
self.extra['hostname'].decode('utf-8') if self.extra['hostname'] else 'NONE',
msg), kwargs
def search_database(word):
'''offline search.'''
conn = sqlite3.connect(os.path.join(DEFAULT_PATH, 'word.db'))
curs = conn.cursor()
curs.execute(r'SELECT expl, pr FROM Word WHERE name LIKE "%s%%"' % word)
res = curs.fetchall()
if res:
print(colored(word + ' 在数据库中存在', 'white', 'on_green'))
print()
print(colored('★ ' * res[0][1], 'red'), colored('☆ ' * (5 - res[0][1]), 'yellow'), sep='')
colorful_print(res[0][0])
else:
print(colored(word + ' 不在本地，从有道词典查询', 'white', 'on_red'))
search_online(word)
input_msg = '若存入本地，请输入优先级(1~5) ，否则 Enter 跳过\n>>> '
if sys.version_info[0] == 2:
add_in_db_pr = raw_input(input_msg)
else:
add_in_db_pr = input(input_msg)
if add_in_db_pr and add_in_db_pr.isdigit():
if(int(add_in_db_pr) >= 1 and int(add_in_db_pr) <= 5):
add_word(word, int(add_in_db_pr))
print(colored('单词 {word} 已加入数据库中'.format(word=word), 'white', 'on_red'))
curs.close()
conn.close()
def list_letter(aset, vb=False, output=False):
'''list words by letter, from a-z (ingore case).'''
conn = sqlite3.connect(os.path.join(DEFAULT_PATH, 'word.db'))
curs = conn.cursor()
try:
if not vb:
curs.execute('SELECT name, pr FROM Word WHERE aset = "%s"' % aset)
else:
curs.execute('SELECT expl, pr FROM Word WHERE aset = "%s"' % aset)
except Exception as e:
print(colored('something\'s wrong, catlog is from A to Z', 'red'))
print(e)
else:
if not output:
print(colored(format(aset, '-^40s'), 'green'))
else:
print(format(aset, '-^40s'))
for line in curs.fetchall():
expl = line[0]
pr = line[1]
print('\n' + '=' * 40 + '\n')
if not output:
print(colored('★ ' * pr, 'red', ), colored('☆ ' * (5 - pr), 'yellow'), sep='')
colorful_print(expl)
else:
print('★ ' * pr + '☆ ' * (5 - pr))
normal_print(expl)
finally:
curs.close()
conn.close()
def informer(self,_type,msg,debug=True,log=False,color='green',type_color="cyan",attrs=["bold"]):
if debug :print '[ '+colored(_type,type_color,attrs=attrs)+' ]'" :"+colored(msg,color,attrs=attrs)
if log:
try:
self.log_file.write(msg)
except Exception,e:
print colored(e,'red','on_white',attrs=['bold'])
def main(username):
google_api = vault.get_key('google_api')
if google_api != None:
API_SERVICE_NAME = "youtube"
API_VERSION = "v3"
max_results = 50
video_ids = []
service = build(API_SERVICE_NAME, API_VERSION,
developerKey=google_api)
channel_id = find_channel_by_username(service,
part='snippet',
maxResults=max_results,
q=username)
if channel_id is not None:
channel_details = get_channel_details(username,
service,
part='snippet,contentDetails,statistics',
id=channel_id['Channel ID'])
channel_analysis = analyze_activity(service,
part='snippet,contentDetails',
channelId=channel_id['Channel ID'],
maxResults=max_results)
return [ channel_id, channel_details, channel_analysis ]
else:
return [ colored(style.BOLD +'[!] Error: Channel not found for ' +
username + '\n' + style.END, 'red') ]
else:
return [ colored(style.BOLD + '[!] Error: No Google API key found. Skipping' +
style.END, 'red') ]
def clone(self, url):
"""
Creates a local cloudlet as a git repo on disk
from the supplied remote git URL.
"""
try:
temp_repo = tempfile.mkdtemp()
validate = Repo.init(temp_repo, bare=True)
validate.git.ls_remote(url, heads=True)
except Exception as e:
print colored("Error: ", "red") + "Invalid or inaccessible remote repository URL.\n"
print e
exit(1)
else:
try:
repo = Repo.clone_from(url, self.path)
repo.submodule_update(init=True)
except Exception as e:
print "Cloudlet install failed."
print e
exit(1)
else:
print "Cloudlet installed: %s" % (self.path)
finally:
rmtree(temp_repo)
def search_replace_with_prompt(fpath, txt1, txt2, force=False):
""" Search and replace all txt1 by txt2 in the file with confirmation"""
from termcolor import colored
with open(fpath, 'r') as f:
content = f.readlines()
tmp = []
for c in content:
if c.find(txt1) != -1:
print fpath
print colored(txt1, 'red').join(c[:-1].split(txt1))
a = ''
if force:
c = c.replace(txt1, txt2)
else:
while a.lower() not in ['y', 'n', 'skip']:
a = raw_input('Do you want to Change [y/n/skip]?')
if a.lower() == 'y':
c = c.replace(txt1, txt2)
elif a.lower() == 'skip':
return 'skip'
tmp.append(c)
with open(fpath, 'w') as f:
f.write(''.join(tmp))
print colored('Updated', 'green')
def unpack_logs(args, logs):
successful = []
for zipped_file in logs:
try:
if os.path.isfile(zipped_file):
if args.verbose:
sys.stderr.write(colored('Starting unpack of {0}'.format(zipped_file), 'magenta') + '\n')
file_in = gzip.open(zipped_file, 'rb')
unzipped = zipped_file.replace('.gz', '.log')
file_out = open(unzipped, 'wb')
file_out.write(file_in.read())
file_out.close()
file_in.close
os.remove(zipped_file)
if args.verbose:
sys.stderr.write(colored('Unpacked ', 'green') + colored(zipped_file, 'white') + '\n')
else:
sys.stderr.write(colored('.', 'green'))
successful.append(unzipped)
except Exception as e:
print e
if os.path.isfile(zipped_file):
os.remove(zipped_file)
sys.stderr.write(colored('Could not unpack {0}'.format(zipped_file), 'red') + '\n')
continue
sys.stderr.write('\n')
return successful
def _print_status(self, service, success):
""" Print status message for a specified service """
status = 'UNDEF'
if success is None:
status = colored("NOOP", "yellow", attrs=['bold'])
elif success:
status = colored("OK", "green", attrs=['bold'])
else:
status = colored("FAIL", "red", attrs=['bold'])
status_msg = "%-50s[ %s ]" % (service, status)
# Extended output requested ?
if self._extended:
pid = self._find_pid(service)
if pid:
status_msg += "\n\tPID: %d" % pid
ports = self._find_ports(pid)
if ports:
status_msg += "\n\tPORTS:\n\t\t%s" % "\n\t\t".join(ports)
logs = self._find_logs(pid)
if logs:
status_msg += "\n\tLOGS:\n\t\t%s" % "\n\t\t".join(logs)
# Last \n to separate different services
status_msg += "\n"
print(status_msg)
def remap_keys(old_filename, new_filename, output_folder):
"""
Create a script to remap bibtex keys from one .bib file to another.
This function uses the edit distance on filenames generated from a bitex entry
to compare entries together, and greedily matches old entries to new entries.
"""
old_db = utils.read_bib_file(old_filename, homogenize=False)
new_db = utils.read_bib_file(new_filename, homogenize=False)
old_list = {}
new_list = {}
subst = {}
for entry in new_db.entries:
name = nomenclature.gen_filename(entry)
new_list[name] = entry['ID']
for entry in old_db.entries:
name = nomenclature.gen_filename(entry)
if name in new_list.keys():
subst[entry['ID']] = new_list[name]
del new_list[name]
else:
old_list[name] = entry['ID']
for name, bibkey in new_list.items():
match, score = utils.most_similar_filename(name, old_list.keys())
if score < 0.90:
print(termcolor.colored("Warning: potentially incorrect substitution:", 'yellow', attrs=["bold"]))
print(termcolor.colored('-', 'red') + match)
print(termcolor.colored('+', 'green') + name)
subst[old_list[match]] = bibkey
utils.write_remap_script(subst, output_folder)
def judge_lastest(self):
lastest = self.fetch_lastest()
if lastest is None:
# logging.info("No submissions")
return
submission_id = lastest["run_id"]
team_id = lastest["team_id"]
problem_id = lastest["pid"]
solution_path = "../" + lastest["output"]
logging.info("Fetching run_id %s, %s, problem %s" % (submission_id, team_id, problem_id))
result = "YES" if self.differ[problem_id].judge("", "", solution_path, self.answer[problem_id]) else "NO"
query = "UPDATE %s SET result = '%s' WHERE run_id = %s" % (self.db_status_tb, result, submission_id)
cur = self.db.cursor()
cur.execute(query)
updated = cur.rowcount != 0
self.db.commit()
if result == "YES":
result = colored(result, "green")
else:
result = colored(result, "red")
logging.info("Response of run_id %s : %s (%s)" % (submission_id, result, str(updated)))
def play(self):
self.lrc_dict = {} # 歌词清空
if not self.loop:
self.douban.get_song()
if self.is_muted: # 静音状态
subprocess.Popen('echo "mute {mute}" > {fifo}'.format(fifo=self.mplayer_controller, mute=1), shell=True, stdin=subprocess.PIPE)
song = self.douban.playingsong
self.song_time = song['length']
# 是否是红心歌曲
if song['like'] == 1:
love = self.love
else:
love = ''
title = colored(song['title'], 'green')
albumtitle = colored(song['albumtitle'], 'yellow')
artist = colored(song['artist'], 'white')
self.SUFFIX_SELECTED = (love + ' ' + title + ' • ' + albumtitle + ' • ' + artist + ' ' + song['public_time']).replace('\\', '')
cmd = 'mplayer -slave -input file={fifo} {song_url} >/dev/null 2>&1'
self.p = subprocess.Popen(cmd.format(fifo=self.mplayer_controller, song_url=song['url']), shell=True, stdin=subprocess.PIPE) # subprocess.PIPE防止继承父进程
self.pause = False
self.display()
self.notifySend()
if self.lrc_display: # 获取歌词
self.lrc_dict = self.douban.get_lrc()
if not self.lrc_dict: # 歌词获取失败,关闭歌词界面
self.lrc_display = 0
self.start = 1
def bye():
global flag
global op_stop
flag = False
op_stop = True
print termcolor.colored("Bye", "cyan")
print termcolor.colored("有任何建议欢迎与我联系: [email protected]", "cyan")
def __init__(self):
# Load the trained models
print(colored('Loading CLF models','magenta'))
self.topicHasher = texthasher.safe_load(TEXT_VECTORIZER_PATH,stop_words=[])
self.tagHasher = taghasher.safe_load(TAG_HASHER_PATH,n_feature=256)
self.clf = cluster.safe_load(CLF_PATH,method=None,n_features=256)
print(colored('[DONE] ','green'),'CLF models loaded')
def loadsession():
global session
try:
session.cookies.load(ignore_discard="true")
except:
termcolor.colored("加载异常", "red")
pass
def show_telemetry(tel):
output = []
for k, v in tel.iteritems():
if k == 'register':
color = 'blue'
elif k == 'density':
color = 'red'
elif k == 'harmonicity':
color = 'green'
elif k == 'roughness':
color = 'cyan'
elif k == 'pace':
color = 'yellow'
else:
color = 'white'
if k == 'name':
output += [ colored(' '.join(v), color) ]
else:
output += [ colored('%s: %.2f' % (k[:3], v), color) ]
output = ' | '.join(output)
dsp.log(output)
def pty_based_auth(auth_backend, pty):
"""
Show a username/password login prompt.
Return username when authentication succeeded.
"""
tries = 0
while True:
# Password authentication required for this session?
sys.stdout.write('\033[2J\033[0;0H') # Clear screen
sys.stdout.write(colored('Please authenticate\r\n\r\n', 'cyan'))
if tries:
sys.stdout.write(colored(' Authentication failed, try again\r\n', 'red'))
try:
console = Console(pty)
username = console.input('Username', False)
password = console.input('Password', True)
except NoInput:
raise NotAuthenticated
if auth_backend.authenticate(username, password):
sys.stdout.write(colored(' ' * 40 + 'Authentication successful\r\n\r\n', 'green'))
return username
else:
tries += 1
if tries == 3:
raise NotAuthenticated
def display_time(self):
length = len(self.TITLE)
while True:
if self.q == 1: # 退出
break
if self.song_time >= 0 and self.douban.playingsong:
minute = int(self.song_time) / 60
sec = int(self.song_time) % 60
show_time = str(minute).zfill(2) + ':' + str(sec).zfill(2)
self.volume = self.get_volume() # 获取音量
self.TITLE = self.TITLE[:length - 1] + ' ' + self.douban.playingsong['kbps'] + 'kbps ' + colored(show_time, 'cyan') + ' rate: ' + colored(self.rate[int(round(self.douban.playingsong['rating_avg'])) - 1], 'red') + ' vol: '
if self.is_muted:
self.TITLE += '✖'
else:
self.TITLE += self.volume.strip() + '%'
if self.loop:
self.TITLE += ' ' + colored('↺', 'red')
else:
self.TITLE += ' ' + colored('→', 'red')
self.TITLE += '\r'
self.display()
if not self.pause:
self.song_time -= 1
else:
self.TITLE = self.TITLE[:length]
time.sleep(1)
def signal_handler(signal, frame):
print colored('You just pressed Ctrl+C! Please wait while the program is terminating...', 'red')
#TO BE IMPLEMENTED
#rollback()
time.sleep(2)
# Exit
sys.exit(0)
def xap_doc_sim_for_pmids(pmids,n_terms=100):
enq=xapian.Enquire(database)
rset=xapian.RSet()
for a in pmids:
try:
database.get_document(a)
except xapian.DocNotFoundError:
continue
rset.add_document(a)
if rset.empty():
print colored("No documents for set %s"%(str(pmids)),'red')
return {},[]
eset=enq.get_eset(n_terms,rset) # Number of terms in the eset
# q=xapian.Query(xapian.Query.OP_OR,eset.begin(),eset.end())
terms=[x[0] for x in eset.items]
print ";".join(terms[:10])
q=xapian.Query(xapian.Query.OP_OR,terms)
enq.set_query(q)
res=enq.get_mset(0,database.get_doccount())
weighted_docs=[(r.docid,r.weight) for r in res]
#normalize the weights to [0,1]
max_w=max([x[1] for x in weighted_docs])
weighted_docs=[(x[0],x[1]*1.0/max_w) for x in weighted_docs]
sorted_results=dict(weighted_docs)
return sorted_results,terms
def testCoverage(self):
print('CTEST_FULL_OUTPUT')
prefixPath = os.environ['QGIS_PREFIX_PATH']
docPath = os.path.join(prefixPath, '..', 'doc', 'api', 'xml')
parser = DoxygenParser(docPath)
# first look for objects without any bindings
objects = set([m[0] for m in parser.bindable_members])
missing_objects = []
bound_objects = {}
for o in objects:
try:
if '::' in o:
bound_objects[o] = getattr(globals()[o.split('::')[0]], o.split('::')[1])
else:
bound_objects[o] = globals()[o]
except:
missing_objects.append(o)
missing_objects.sort()
# next check for individual members
parser.bindable_members.sort()
missing_members = []
for m in parser.bindable_members:
if m[0] in bound_objects:
obj = bound_objects[m[0]]
if "::" in m[0] and m[0].split("::")[1] == m[1]:
# skip constructors of nested classes
continue
# try two different methods of checking for member existence
try:
if hasattr(obj, m[1]):
continue
except:
pass
try:
if m[1] in dir(obj):
continue
except:
printImportant("SIP coverage test: something strange happened in {}.{}, obj={}".format(m[0], m[1], obj))
missing_members.append('{}.{}'.format(m[0], m[1]))
missing_members.sort()
if missing_objects:
print("---------------------------------")
print((colored('Missing classes:', 'yellow')))
print((' ' + '\n '.join([colored(obj, 'yellow', attrs=['bold']) for obj in missing_objects])))
if missing_members:
print("---------------------------------")
print((colored('Missing members:', 'yellow')))
print((' ' + '\n '.join([colored(mem, 'yellow', attrs=['bold']) for mem in missing_members])))
# print summaries
missing_class_count = len(missing_objects)
present_count = len(objects) - missing_class_count
coverage = 100.0 * present_count / len(objects)
print("---------------------------------")
printImportant("{} total bindable classes".format(len(objects)))
printImportant("{} total have bindings".format(present_count))
printImportant("Binding coverage by classes {}%".format(coverage))
printImportant("---------------------------------")
printImportant("{} classes missing bindings".format(missing_class_count))
print("---------------------------------")
missing_member_count = len(missing_members)
present_count = len(parser.bindable_members) - missing_member_count
coverage = 100.0 * present_count / len(parser.bindable_members)
print("---------------------------------")
printImportant("{} total bindable members".format(len(parser.bindable_members)))
printImportant("{} total have bindings".format(present_count))
printImportant("Binding coverage by members {}%".format(coverage))
printImportant("---------------------------------")
printImportant("{} members missing bindings".format(missing_member_count))
self.assertEqual(missing_class_count, 0, """\n\nFAIL: new unbound classes have been introduced, please add SIP bindings for these classes
If these classes are not suitable for the Python bindings, please add the Doxygen tag
"\\note not available in Python bindings" to the CLASS Doxygen comments""")
self.assertEqual(missing_member_count, 0, """\n\nFAIL: new unbound members have been introduced, please add SIP bindings for these members
If these members are not suitable for the Python bindings, please add the Doxygen tag
"\\note not available in Python bindings" to the MEMBER Doxygen comments""")
def warn(msg):
if Logging.flag == True:
print "".join([
termcolor.colored("WARN", "yellow"), ": ",
termcolor.colored(msg, "white")
])
def error(msg):
if Logging.flag == True:
print "".join([
termcolor.colored("ERROR", "red"), ": ",
termcolor.colored(msg, "white")
])
def _color_arg(arg: Any) -> Union[str, float, int]:
if isinstance(arg, (int, float)):
# In case of %d or %f formatting
return arg
return colored(str(arg), **ARGS) # type: ignore
def execute(self, sql, *args, **kwargs):
"""Execute a SQL statement."""
# Lazily import
import decimal
import re
import sqlalchemy
import sqlparse
import termcolor
import warnings
# Parse statement, stripping comments and then leading/trailing whitespace
statements = sqlparse.parse(
sqlparse.format(sql, strip_comments=True).strip())
# Allow only one statement at a time, since SQLite doesn't support multiple
# https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor.execute
if len(statements) > 1:
raise RuntimeError("too many statements at once")
elif len(statements) == 0:
raise RuntimeError("missing statement")
# Ensure named and positional parameters are mutually exclusive
if len(args) > 0 and len(kwargs) > 0:
raise RuntimeError(
"cannot pass both named and positional parameters")
# Infer command from (unflattened) statement
for token in statements[0]:
if token.ttype in [
sqlparse.tokens.Keyword.DDL, sqlparse.tokens.Keyword.DML
]:
command = token.value.upper()
break
# Begin a new session, if transaction opened by caller (not using autocommit)
elif token.value.upper() in ["BEGIN", "START"]:
if self._in_transaction:
raise RuntimeError("transaction already open")
self._in_transaction = True
else:
command = None
# Flatten statement
tokens = list(statements[0].flatten())
# Validate paramstyle
placeholders = {}
paramstyle = None
for index, token in enumerate(tokens):
# If token is a placeholder
if token.ttype == sqlparse.tokens.Name.Placeholder:
# Determine paramstyle, name
_paramstyle, name = _parse_placeholder(token)
# Remember paramstyle
if not paramstyle:
paramstyle = _paramstyle
# Ensure paramstyle is consistent
elif _paramstyle != paramstyle:
raise RuntimeError("inconsistent paramstyle")
# Remember placeholder's index, name
placeholders[index] = name
# If more placeholders than arguments
if len(args) == 1 and len(placeholders) > 1:
# If user passed args as list or tuple, explode values into args
if isinstance(args[0], (list, tuple)):
args = args[0]
# If user passed kwargs as dict, migrate values from args to kwargs
elif len(kwargs) == 0 and isinstance(args[0], dict):
kwargs = args[0]
args = []
# If no placeholders
if not paramstyle:
# Error-check like qmark if args
if args:
paramstyle = "qmark"
# Error-check like named if kwargs
elif kwargs:
paramstyle = "named"
# In case of errors
_placeholders = ", ".join(
[str(tokens[index]) for index in placeholders])
_args = ", ".join([str(self._escape(arg)) for arg in args])
# qmark
if paramstyle == "qmark":
# Validate number of placeholders
if len(placeholders) != len(args):
if len(placeholders) < len(args):
raise RuntimeError(
"fewer placeholders ({}) than values ({})".format(
_placeholders, _args))
else:
raise RuntimeError(
"more placeholders ({}) than values ({})".format(
_placeholders, _args))
# Escape values
for i, index in enumerate(placeholders.keys()):
tokens[index] = self._escape(args[i])
# numeric
elif paramstyle == "numeric":
# Escape values
for index, i in placeholders.items():
if i >= len(args):
raise RuntimeError(
"missing value for placeholder (:{})".format(
i + 1, len(args)))
tokens[index] = self._escape(args[i])
# Check if any values unused
indices = set(range(len(args))) - set(placeholders.values())
if indices:
raise RuntimeError("unused {} ({})".format(
"value" if len(indices) == 1 else "values", ", ".join([
str(self._escape(args[index])) for index in indices
])))
# named
elif paramstyle == "named":
# Escape values
for index, name in placeholders.items():
if name not in kwargs:
raise RuntimeError(
"missing value for placeholder (:{})".format(name))
tokens[index] = self._escape(kwargs[name])
# Check if any keys unused
keys = kwargs.keys() - placeholders.values()
if keys:
raise RuntimeError("unused values ({})".format(
", ".join(keys)))
# format
elif paramstyle == "format":
# Validate number of placeholders
if len(placeholders) != len(args):
if len(placeholders) < len(args):
raise RuntimeError(
"fewer placeholders ({}) than values ({})".format(
_placeholders, _args))
else:
raise RuntimeError(
"more placeholders ({}) than values ({})".format(
_placeholders, _args))
# Escape values
for i, index in enumerate(placeholders.keys()):
tokens[index] = self._escape(args[i])
# pyformat
elif paramstyle == "pyformat":
# Escape values
for index, name in placeholders.items():
if name not in kwargs:
raise RuntimeError(
"missing value for placeholder (%{}s)".format(name))
tokens[index] = self._escape(kwargs[name])
# Check if any keys unused
keys = kwargs.keys() - placeholders.values()
if keys:
raise RuntimeError("unused {} ({})".format(
"value" if len(keys) == 1 else "values", ", ".join(keys)))
# For SQL statements where a colon is required verbatim, as within an inline string, use a backslash to escape
# https://docs.sqlalchemy.org/en/13/core/sqlelement.html?highlight=text#sqlalchemy.sql.expression.text
for index, token in enumerate(tokens):
# In string literal
# https://www.sqlite.org/lang_keywords.html
if token.ttype in [
sqlparse.tokens.Literal.String,
sqlparse.tokens.Literal.String.Single
]:
token.value = re.sub("(^'|\s+):", r"\1\:", token.value)
# In identifier
# https://www.sqlite.org/lang_keywords.html
elif token.ttype == sqlparse.tokens.Literal.String.Symbol:
token.value = re.sub("(^\"|\s+):", r"\1\:", token.value)
# Join tokens into statement
statement = "".join([str(token) for token in tokens])
# Connect to database (for transactions' sake)
if self._session is None:
self._session = self._Session()
# Set up a Flask app teardown function to close session at teardown
try:
# Infer whether Flask is installed
import flask
# Infer whether app is defined
assert flask.current_app
# Disconnect later - but only once
if not hasattr(self, "_teardown_appcontext_added"):
self._teardown_appcontext_added = True
@flask.current_app.teardown_appcontext
def shutdown_session(exception=None):
"""Close any existing session on app context teardown."""
self._close_session()
except (ModuleNotFoundError, AssertionError):
pass
# Catch SQLAlchemy warnings
with warnings.catch_warnings():
# Raise exceptions for warnings
warnings.simplefilter("error")
# Prepare, execute statement
try:
# Join tokens into statement, abbreviating binary data as <class 'bytes'>
_statement = "".join([
str(bytes)
if token.ttype == sqlparse.tokens.Other else str(token)
for token in tokens
])
# If COMMIT or ROLLBACK, turn on autocommit mode
if command in ["COMMIT", "ROLLBACK"
] and "TO" not in (token.value
for token in tokens):
if not self._in_transaction:
raise RuntimeError(
"transactions must be opened with BEGIN or START TRANSACTION"
)
self._in_transaction = False
# Execute statement
result = self._session.execute(sqlalchemy.text(statement))
# Return value
ret = True
# If SELECT, return result set as list of dict objects
if command == "SELECT":
# Coerce types
rows = [dict(row) for row in result.fetchall()]
for row in rows:
for column in row:
# Coerce decimal.Decimal objects to float objects
# https://groups.google.com/d/msg/sqlalchemy/0qXMYJvq8SA/oqtvMD9Uw-kJ
if type(row[column]) is decimal.Decimal:
row[column] = float(row[column])
# Coerce memoryview objects (as from PostgreSQL's bytea columns) to bytes
elif type(row[column]) is memoryview:
row[column] = bytes(row[column])
# Rows to be returned
ret = rows
# If INSERT, return primary key value for a newly inserted row (or None if none)
elif command == "INSERT":
if self._engine.url.get_backend_name() in [
"postgres", "postgresql"
]:
try:
result = self._session.execute("SELECT LASTVAL()")
ret = result.first()[0]
except sqlalchemy.exc.OperationalError: # If lastval is not yet defined in this session
ret = None
else:
ret = result.lastrowid if result.rowcount == 1 else None
# If DELETE or UPDATE, return number of rows matched
elif command in ["DELETE", "UPDATE"]:
ret = result.rowcount
# If autocommit is on, commit
if not self._in_transaction:
self._session.commit()
# If constraint violated, return None
except sqlalchemy.exc.IntegrityError as e:
self._logger.debug(termcolor.colored(statement, "yellow"))
e = RuntimeError(e.orig)
e.__cause__ = None
raise e
# If user errror
except sqlalchemy.exc.OperationalError as e:
self._logger.debug(termcolor.colored(statement, "red"))
e = RuntimeError(e.orig)
e.__cause__ = None
raise e
# Return value
else:
self._logger.debug(termcolor.colored(_statement, "green"))
return ret
def format(self, record):
msg = super().format(record)
return msg if not self.use_color else termcolor.colored(
msg, getattr(record, "color", self.COLORS.get(record.levelname)))
def optimize_class_model(args, logger=None):
"""
加载中文分类模型
:param args:
:param num_labels:
:param logger:
:return:
"""
if not logger:
logger = set_logger(
colored('CLASSIFICATION_MODEL, Lodding...', 'cyan'), args.verbose)
pass
try:
# 如果PB文件已经存在则，返回PB文件的路径，否则将模型转化为PB文件，并且返回存储PB文件的路径
if args.model_pb_dir is None:
tmp_file = args.model_dir
# # 获取当前的运行路径
# tmp_file = os.path.join(os.getcwd(), 'predict_optimizer')
# if not os.path.exists(tmp_file):
# os.mkdir(tmp_file)
else:
tmp_file = args.model_pb_dir
#latest_checkpoint = tf.train.latest_checkpoint(args.model_dir)
pb_file = os.path.join(tmp_file, 'classification_model.pb')
if os.path.exists(pb_file):
print('pb_file exits', pb_file)
return pb_file
#import tensorflow as tf
graph = tf.Graph()
with graph.as_default():
with tf.Session() as sess:
input_ids = tf.placeholder(tf.int32, (None, args.max_seq_len),
'input_ids')
input_mask = tf.placeholder(tf.int32, (None, args.max_seq_len),
'input_mask')
bert_config = modeling.BertConfig.from_json_file(
os.path.join(args.bert_model_dir, 'bert_config.json'))
#from bert_base.train.models import create_classification_model
#from models import create_classification_model
loss, per_example_loss, logits, probabilities = create_classification_model(
bert_config=bert_config,
is_training=False,
input_ids=input_ids,
input_mask=input_mask,
segment_ids=None,
labels=None,
num_labels=args.num_labels)
# pred_ids = tf.argmax(probabilities, axis=-1, output_type=tf.int32, name='pred_ids')
# pred_ids = tf.identity(pred_ids, 'pred_ids')
probabilities = tf.identity(probabilities, 'pred_prob')
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
latest_checkpoint = tf.train.latest_checkpoint(args.model_dir)
logger.info('loading... %s ' % latest_checkpoint)
saver.restore(sess, latest_checkpoint)
logger.info('freeze...')
from tensorflow.python.framework import graph_util
tmp_g = graph_util.convert_variables_to_constants(
sess, graph.as_graph_def(), ['pred_prob'])
logger.info('predict cut finished !!!')
# 存储二进制模型到文件中
logger.info('write graph to a tmp file: %s' % pb_file)
with tf.gfile.GFile(pb_file, 'wb') as f:
f.write(tmp_g.SerializeToString())
return pb_file
except Exception as e:
logger.error('fail to optimize the graph! %s' % e, exc_info=True)
def colorize_comment(self, comment):
"""Return a string with color markup for a given comment."""
if self.mark_re is None:
return comment
return self.mark_re.sub(
colored(r'\g<0>', None, None, ['reverse', 'bold']), comment)
def yellow(content): return termcolor.colored(str(content),"yellow",attrs=["bold"]) def magenta(content): return termcolor.colored(str(content),"magenta",attrs=["bold"])
def colorize_attr(attr, value, marked):
"""Return a string with color markup for a given attr and its value."""
color = COLOR_OF.get(attr, None)
return colored(value, color, None,
['reverse', 'bold'] if marked else None)
def blue(content): return termcolor.colored(str(content),"blue",attrs=["bold"]) def cyan(content): return termcolor.colored(str(content),"cyan",attrs=["bold"])
def magenta(content): return termcolor.colored(str(content),"magenta",attrs=["bold"]) def get_time(sec):
def red(content): return termcolor.colored(str(content),"red",attrs=["bold"]) def green(content): return termcolor.colored(str(content),"green",attrs=["bold"])
def cyan(content): return termcolor.colored(str(content),"cyan",attrs=["bold"]) def yellow(content): return termcolor.colored(str(content),"yellow",attrs=["bold"])
def print_cyan(cls, msg):
color_format = {"color": "cyan"}
colore_msg = colored(msg, **color_format)
print colore_msg
def green(content): return termcolor.colored(str(content),"green",attrs=["bold"]) def blue(content): return termcolor.colored(str(content),"blue",attrs=["bold"])
def print_ok(cls, msg):
color_format = {"color": "green"}
head = colored("[ OK ]", **color_format)
body = "{}:{}".format(head, msg)
print body
def print_blue(cls, msg):
color_format = {"color": "blue"}
colore_msg = colored(msg, **color_format)
print colore_msg
def print_white(cls, msg):
color_format = {"color": "white"}
colore_msg = colored(msg, **color_format)
print colore_msg
def print_war(cls, msg):
color_format = {"color": "yellow"}
head = colored("[ WARNING ]", **color_format)
body = "{}:{}".format(head, msg)
print body
def print_green(cls, msg):
color_format = {"color": "green"}
colore_msg = colored(msg, **color_format)
print colore_msg
def print_error(cls, msg):
color_format = {"color": "red"}
head = colored("[ ERROR ]", **color_format)
body = "{}:{}".format(head, msg)
print body
def subfailure(name):
print('{} {}'.format(colored(" ->", 'red', attrs=['bold']),
colored(name, 'red', attrs=['bold'])))
def print_title(cls, msg):
color_format = {"color": "green"}
msg = msg.center(86, '-')
colore_msg = colored(msg, **color_format)
print colore_msg
def subtask(name):
print('{} {}'.format(colored(" ->", 'blue', attrs=['bold']),
colored(name, attrs=['bold'])))
def subprompt(name):
print('{} {}'.format(colored(" ->", 'yellow', attrs=['bold']),
colored(name, attrs=['bold'])),
end="")
def section(name):
print("\n{} {}".format(colored("::", 'blue', attrs=['bold']),
colored(name, attrs=['bold'])))
def failure(name):
print('{} {}'.format(colored("==> ERROR:", 'red', attrs=['bold']),
colored(name, attrs=['bold'])))
import os
import numpy as np
from termcolor import colored
results = []
tests = 'oscillator_tst','lfo_tst','adsr_tst','mixer_tst','mov_avg_tst','sound_gen_tst'
for test_name in tests:
return_value = os.system('python '+ ' mainPrj.py -b nucleo -p '+ test_name + '/')
results.append([test_name,return_value])
print '\n-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*\n'
print '\t|SUMMARY OF TESTS:'
print '\t------------------------------------------------'
for result in results:
result_string = ('OK' if result[1] == 0 else 'FAIL')
out_color = ('green' if result[1] == 0 else 'red')
print colored('\t' + result[0] + ' ' + '\t ---> ' + result_string,out_color)
print '\n'
def task(name):
print('{} {}'.format(colored("==>", 'green', attrs=['bold']),
colored(name, attrs=['bold'])))