Beispiel #1
0
    def run_threads(self):
        """
        设置线程池,并启动线程
        """
        args_dict = {}
        args_dict['output_dir'] = self.output_dir
        args_dict['crawl_interval'] = self.crawl_interval
        args_dict['crawl_timeout'] = self.crawl_timeout
        args_dict['url_pattern'] = self.url_pattern
        args_dict['max_depth'] = self.max_depth
        args_dict['tag_dict'] = self.tag_dict

        for index in xrange(self.thread_count):
            thread_name = 'thread - %d' % index
            thread = crawl_thread.CrawlerThread(thread_name,
                                                self.process_request,
                                                self.process_response,
                                                args_dict)

            thread.setDaemon(True)
            thread.start()
            print termcolor.colored(("第%s个线程开始工作") % index, 'yellow')
            logging.info(("第%s个线程开始工作") % index)

        self.checking_url.join()
        self.program_end('normal exits ')
Beispiel #2
0
def _exit_():
	print colored('[INFO] Exiting...', 'green')
	try:
		os.remove(merged_script_path)
	except Exception as e:
		pass
	sys.exit(1)
Beispiel #3
0
    def highlightList(self, taskName):

        # get highlighted
        sql_command = """SELECT * FROM "%s" WHERE taskName="%s" """ % (self.table, taskName)

        self.cursor.execute(sql_command)

        result = self.cursor.fetchall()

        desired = str(result[0][self.taskName_position])

        # get list
        rows = self.retrieveList()

        # display
        self.displayHeader()

        for row in rows:

            if row[self.taskName_position] == desired:
                text = self.myFormat(row)
                print colored(text, "blue", attrs=["reverse", "blink"])
            else:
                text = self.myFormat(row)
                self.printByPriority(row)
	def getData(self, url):
		
		data = Data()

		try:
			req = urllib2.Request(url, headers={'User-Agent' : "Magic Browser"})
			request = urllib2.urlopen(req)
			mime = request.info().getheader('Content-Type')		
			code = request.code

			print(colored('[' + mime + '] ' + url, 'yellow'))

			if code is 200:
				if 'text/html' in mime:

					html = request.read()
					data = self.parse(html, url)
					
				else:
					#ANALYSIS TYPE
					data.url = url
					data.type = mime

			elif code is 400:
				data.broke = True

		except UnicodeEncodeError as e :

			print(colored(e, 'red'))
			data.broke = True

		return data 
def server():
	global connection
	global board
	global newTiles
	global WAIT
	# Create a TCP/IP socket
	server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	# Bind the socket to the port
	if int(sys.argv[1]) == 1:
		server_address = ('localhost', PORT_NUMBER_2)
	else:
		server_address = ('localhost', PORT_NUMBER_1)
	print >>sys.stderr, 'starting up on %s port %s' % server_address
	server_sock.bind(server_address)
	# Listen for incoming connections
	server_sock.listen(1)

	while True:
	    # Wait for a connection
	    print >>sys.stderr, 'waiting for a connection'
	    connection, client_address = server_sock.accept()
	    try:
	        print >>sys.stderr, 'connection from', client_address
	        state = "NORMAL"
	        # Receive the data in small chunks and retransmit it
	        while True:
	            data = connection.recv(1000)
	            # print >>sys.stderr, 'received "%s"' % data
	            if data:
	            	if data == "BOARD" or data == "INITIAL" or data == "NEW_TILES":
	            		state = data
	            		print "STATE: %s"%state
	            	else:
		            	if state == "BOARD":
		            		board = pickle.loads(data)
		            		print colored("The other player has made a move",'red')
		            		printBoard()
		            		print "received board"
		            		WAIT = False
		            	elif state == "INITIAL":
		            		board = pickle.loads(data)
		            		print "received initial"
		            		printBoard()
		            	elif state == "NEW_TILES":
		            		newTiles = pickle.loads(data)
		            		print newTiles
		            		print "received new tiles"
		            	state = "NORMAL"
		            	print "STATE: NORMAL"
	            else:
	                break
	    except KeyboardInterrupt:
	    	print "Closed connection on server"
	    	connection.close()
	    	break     
	    finally:
	        # Clean up the connection
	    	print "Closed connection on server"
	        connection.close()
	        break
Beispiel #6
0
def _make_context(quiet=False):
    # Start by adding the session instance to the namespace.
    namespace = {'db': db, 'session': db.session}

    if not quiet:
        # Let the user know.
        print(colored('from alchemist import db', 'green'))
        print(colored('session = db.session', 'green'))

    # Iterate through all models and add those to the namespace.
    module = {}
    for name, models in application.models.items():
        for model in models:
            if model.__module__ not in module:
                module[model.__module__] = []

            module[model.__module__].append(model.__name__)
            namespace[model.__name__] = model

    if not quiet:
        # Let the user know.
        for name, models in module.items():
            items = ', '.join(models)
            print(colored('from %s import %s' % (name, items), 'green'))

    # Return the constructed namespace.
    return namespace
def hash_cracker(dict_file_name, hash_file_name):
    found = 0
    file_dict = open(dict_file_name, "r")
    passwords = file_dict.read().split('\n')
    file_hash = open(hash_file_name, "r")
    enc_hashes = file_hash.read().split('\n')
    print enc_hashes
    for pwd in passwords:
        try:
            for rec in enc_hashes:
                # print pwd.split(':')[1]
                test = sha1(pwd.split(':')[1] + rec.split(':')[1]).hexdigest()
                if pwd.split(':')[1] == 'admin':
                    print rec.split(':')[0]
                    print test
                if test == rec.split(':')[0]:
                    print colored("[-] Password Found for password_sha " + rec.split(':')[0] + " : " + pwd)
                    found = 1
                    break
                else:
                    pass
                    if found == 1:
                        break
        except:
            pass
Beispiel #8
0
def server_thread(port):
    global server_run

    server_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    server_socket.bind(('0.0.0.0', port))

    while server_run:
        try:
            with Timeout(2, False):
                (data, from_ip) = server_socket.recvfrom(1024)

            if data is None:
                continue

            ping = ping_parse(data)
            if ping is None:
                continue

            msg = 'Received - Seq: %d, Timestamp: %d' % (ping.seq, ping.timestamp)
            print colored(msg, 'green')

            pong = make_pong(ping)

            server_socket.sendto(pong, from_ip)

        except timeout:
            continue
        except KeyboardInterrupt:
            server_run = False
def one(expected, champs, display, shuffle = False):
    global count, total
    vm = VM()
    vm.add(champs)

    if shuffle:
        vm.shuffle()

    if display:
        util.tprint('Battle', '%s / %s' % (colored('%d' % (count), 'yellow'), colored('%d' % (total), 'yellow')))
        util.tprint('Champions', ', '.join([colored(champ.champion.name, 'blue', attrs = ['bold']) for champ in vm.champions]))
    count += 1
    winner = vm.launch(stdout = PIPE)
    report = None
    if winner:
        if display:
            if winner.champion.filename == expected.filename:
                color = 'green'
            else:
                color = 'red'
            util.tprint('Winner', colored(winner.champion.name, color, attrs = ['bold']))
        report = Report(champs, winner.champion)

    if display:
        print ''
    return report
Beispiel #10
0
def list_latest(limit, vb=False, output=False):
    '''list words by latest time you add to database.'''

    conn = sqlite3.connect(os.path.join(DEFAULT_PATH, 'word.db'))
    curs = conn.cursor()
    try:
        if not vb:
            curs.execute('SELECT name, pr, addtime FROM Word ORDER by datetime(addtime) DESC LIMIT  %d' % limit)
        else:
            curs.execute('SELECT expl, pr, addtime FROM Word ORDER by datetime(addtime) DESC LIMIT  %d' % limit)
    except Exception as e:
        print(e)
        print(colored('something\'s wrong, please set the limit', 'red'))
    else:
        for line in curs.fetchall():
            expl = line[0]
            pr = line[1]
            print('\n' + '=' * 40 + '\n')
            if not output:
                print(colored('★ ' * pr, 'red'), colored('☆ ' * (5 - pr), 'yellow'), sep='')
                colorful_print(expl)
            else:
                print('★ ' * pr + '☆ ' * (5 - pr))
                normal_print(expl)
    finally:
        curs.close()
        conn.close()
Beispiel #11
0
    def process(self, msg, kwargs):
        if self.extra is None:
            return u'{}'.format(msg), kwargs

        if 'module' in self.extra.keys():
            if len(self.extra['module']) > 8:
                self.extra['module'] = self.extra['module'][:8] + '...'

        #If the logger is being called when hooking the 'options' module function
        if len(self.extra) == 1 and ('module' in self.extra.keys()):
            return u'{:<64} {}'.format(colored(self.extra['module'], 'cyan', attrs=['bold']), msg), kwargs

        #If the logger is being called from CMEServer
        if len(self.extra) == 2 and ('module' in self.extra.keys()) and ('host' in self.extra.keys()):
            return u'{:<24} {:<39} {}'.format(colored(self.extra['module'], 'cyan', attrs=['bold']), self.extra['host'], msg), kwargs

        #If the logger is being called from a protocol
        if 'module' in self.extra.keys():
            module_name = colored(self.extra['module'], 'cyan', attrs=['bold'])
        else:
            module_name = colored(self.extra['protocol'], 'blue', attrs=['bold'])

        return u'{:<24} {:<15} {:<6} {:<16} {}'.format(module_name,
                                                    self.extra['host'],
                                                    self.extra['port'],
                                                    self.extra['hostname'].decode('utf-8') if self.extra['hostname'] else 'NONE',
                                                    msg), kwargs
Beispiel #12
0
def search_database(word):
    '''offline search.'''

    conn = sqlite3.connect(os.path.join(DEFAULT_PATH, 'word.db'))
    curs = conn.cursor()
    curs.execute(r'SELECT expl, pr FROM Word WHERE name LIKE "%s%%"' % word)
    res = curs.fetchall()
    if res:
        print(colored(word + ' 在数据库中存在', 'white', 'on_green'))
        print()
        print(colored('★ ' * res[0][1], 'red'), colored('☆ ' * (5 - res[0][1]), 'yellow'), sep='')
        colorful_print(res[0][0])
    else:
        print(colored(word + ' 不在本地,从有道词典查询', 'white', 'on_red'))
        search_online(word)
        input_msg = '若存入本地,请输入优先级(1~5) ,否则 Enter 跳过\n>>> '
        if sys.version_info[0] == 2:
            add_in_db_pr = raw_input(input_msg)
        else:
            add_in_db_pr = input(input_msg)

        if add_in_db_pr and add_in_db_pr.isdigit():
            if(int(add_in_db_pr) >= 1 and int(add_in_db_pr) <= 5):
                add_word(word, int(add_in_db_pr))
                print(colored('单词 {word} 已加入数据库中'.format(word=word), 'white', 'on_red'))
    curs.close()
    conn.close()
Beispiel #13
0
def list_letter(aset, vb=False, output=False):
    '''list words by letter, from a-z (ingore case).'''

    conn = sqlite3.connect(os.path.join(DEFAULT_PATH, 'word.db'))
    curs = conn.cursor()
    try:
        if not vb:
            curs.execute('SELECT name, pr FROM Word WHERE aset = "%s"' % aset)
        else:
            curs.execute('SELECT expl, pr FROM Word WHERE aset = "%s"' % aset)
    except Exception as e:
        print(colored('something\'s wrong, catlog is from A to Z', 'red'))
        print(e)
    else:
        if not output:
            print(colored(format(aset, '-^40s'), 'green'))
        else:
            print(format(aset, '-^40s'))

        for line in curs.fetchall():
            expl = line[0]
            pr = line[1]
            print('\n' + '=' * 40 + '\n')
            if not output:
                print(colored('★ ' * pr, 'red', ), colored('☆ ' * (5 - pr), 'yellow'), sep='')
                colorful_print(expl)
            else:
                print('★ ' * pr + '☆ ' * (5 - pr))
                normal_print(expl)
    finally:
        curs.close()
        conn.close()
 def informer(self,_type,msg,debug=True,log=False,color='green',type_color="cyan",attrs=["bold"]):
     if debug :print '[ '+colored(_type,type_color,attrs=attrs)+' ]'" :"+colored(msg,color,attrs=attrs)
     if log:
         try:
             self.log_file.write(msg)
         except Exception,e:
             print colored(e,'red','on_white',attrs=['bold'])
def main(username):
    google_api = vault.get_key('google_api')
    if google_api != None:
        API_SERVICE_NAME = "youtube"
        API_VERSION = "v3"
        max_results = 50
        video_ids = []
        service = build(API_SERVICE_NAME, API_VERSION,
                        developerKey=google_api)
        channel_id = find_channel_by_username(service,
                                              part='snippet',
                                              maxResults=max_results,
                                              q=username)
        if channel_id is not None:
            channel_details = get_channel_details(username,
                                                  service,
                                                  part='snippet,contentDetails,statistics',
                                                  id=channel_id['Channel ID'])

            channel_analysis = analyze_activity(service,
                                                part='snippet,contentDetails',
                                                channelId=channel_id['Channel ID'],
                                                maxResults=max_results)
            return [ channel_id, channel_details, channel_analysis ]
        else:
            return [ colored(style.BOLD +'[!] Error: Channel not found for ' +
                             username + '\n' + style.END, 'red') ]
    else:
        return [ colored(style.BOLD + '[!] Error: No Google API key found. Skipping' +
                         style.END, 'red') ]
Beispiel #16
0
 def clone(self, url):
     """
     Creates a local cloudlet as a git repo on disk
      from the supplied remote git URL.
     """
     try:
         temp_repo = tempfile.mkdtemp()
         validate = Repo.init(temp_repo, bare=True)
         validate.git.ls_remote(url, heads=True)
     except Exception as e:
         print colored("Error: ", "red") + "Invalid or inaccessible remote repository URL.\n"
         print e
         exit(1)
     else:
         try:
             repo = Repo.clone_from(url, self.path)
             repo.submodule_update(init=True)
         except Exception as e:
             print "Cloudlet install failed."
             print e
             exit(1)
         else:
             print "Cloudlet installed: %s" % (self.path)
     finally:
         rmtree(temp_repo)
Beispiel #17
0
def search_replace_with_prompt(fpath, txt1, txt2, force=False):
	""" Search and replace all txt1 by txt2 in the file with confirmation"""

	from termcolor import colored
	with open(fpath, 'r') as f:
		content = f.readlines()

	tmp = []
	for c in content:
		if c.find(txt1) != -1:
			print fpath
			print  colored(txt1, 'red').join(c[:-1].split(txt1))
			a = ''
			if force:
				c = c.replace(txt1, txt2)
			else:
				while a.lower() not in ['y', 'n', 'skip']:
					a = raw_input('Do you want to Change [y/n/skip]?')
				if a.lower() == 'y':
					c = c.replace(txt1, txt2)
				elif a.lower() == 'skip':
					return 'skip'
		tmp.append(c)

	with open(fpath, 'w') as f:
		f.write(''.join(tmp))
	print colored('Updated', 'green')
Beispiel #18
0
def unpack_logs(args, logs):
  successful = []
  for zipped_file in logs:
    try:
      if os.path.isfile(zipped_file):
        if args.verbose:
          sys.stderr.write(colored('Starting unpack of {0}'.format(zipped_file), 'magenta') + '\n')
        file_in = gzip.open(zipped_file, 'rb')
        unzipped = zipped_file.replace('.gz', '.log')
        file_out = open(unzipped, 'wb')
        file_out.write(file_in.read())
        file_out.close()
        file_in.close
        os.remove(zipped_file)
        if args.verbose:
          sys.stderr.write(colored('Unpacked ', 'green') + colored(zipped_file, 'white') + '\n')
        else:
          sys.stderr.write(colored('.', 'green'))
        successful.append(unzipped)
    except Exception as e:
      print e
      if os.path.isfile(zipped_file):
        os.remove(zipped_file)
      sys.stderr.write(colored('Could not unpack {0}'.format(zipped_file), 'red') + '\n')
      continue
  sys.stderr.write('\n')
  return successful
Beispiel #19
0
    def _print_status(self, service, success):
        """ Print status message for a specified service """
        status = 'UNDEF'

        if success is None:
            status = colored("NOOP", "yellow", attrs=['bold'])
        elif success:
            status = colored("OK", "green", attrs=['bold'])
        else:
            status = colored("FAIL", "red", attrs=['bold'])
        
        status_msg = "%-50s[ %s ]" % (service, status)

        # Extended output requested ?
        if self._extended:
            pid = self._find_pid(service)

            if pid:
                status_msg += "\n\tPID: %d" % pid

                ports = self._find_ports(pid)
                if ports:
                    status_msg += "\n\tPORTS:\n\t\t%s" % "\n\t\t".join(ports)

                logs = self._find_logs(pid)
                if logs:
                    status_msg += "\n\tLOGS:\n\t\t%s" % "\n\t\t".join(logs)

                # Last \n to separate different services
                status_msg += "\n"

        print(status_msg)
Beispiel #20
0
def remap_keys(old_filename, new_filename, output_folder):
    """
    Create a script to remap bibtex keys from one .bib file to another.

    This function uses the edit distance on filenames generated from a bitex entry
    to compare entries together, and greedily matches old entries to new entries.
    """
    old_db = utils.read_bib_file(old_filename, homogenize=False)
    new_db = utils.read_bib_file(new_filename, homogenize=False)
    old_list = {}
    new_list = {}
    subst = {}
    for entry in new_db.entries:
        name = nomenclature.gen_filename(entry)
        new_list[name] = entry['ID']
    for entry in old_db.entries:
        name = nomenclature.gen_filename(entry)
        if name in new_list.keys():
            subst[entry['ID']] = new_list[name]
            del new_list[name]
        else:
            old_list[name] = entry['ID']
    for name, bibkey in new_list.items():
        match, score = utils.most_similar_filename(name, old_list.keys())
        if score < 0.90:
            print(termcolor.colored("Warning: potentially incorrect substitution:", 'yellow', attrs=["bold"]))
            print(termcolor.colored('-', 'red') + match)
            print(termcolor.colored('+', 'green') + name)
        subst[old_list[match]] = bibkey
    utils.write_remap_script(subst, output_folder)
Beispiel #21
0
    def judge_lastest(self):
        lastest = self.fetch_lastest()
        if lastest is None:
            # logging.info("No submissions")
            return

        submission_id = lastest["run_id"]
        team_id = lastest["team_id"]
        problem_id = lastest["pid"]
        solution_path = "../" + lastest["output"]

        logging.info("Fetching run_id %s, %s, problem %s" % (submission_id, team_id, problem_id))
        result = "YES" if self.differ[problem_id].judge("", "", solution_path, self.answer[problem_id]) else "NO"

        query = "UPDATE %s SET result = '%s' WHERE run_id = %s" % (self.db_status_tb, result, submission_id)

        cur = self.db.cursor()
        cur.execute(query)
        updated = cur.rowcount != 0
        self.db.commit()

        if result == "YES":
            result = colored(result, "green")
        else:
            result = colored(result, "red")
        logging.info("Response of run_id %s : %s (%s)" % (submission_id, result, str(updated)))
Beispiel #22
0
    def play(self):
        self.lrc_dict = {}  # 歌词清空
        if not self.loop:
            self.douban.get_song()
        if self.is_muted:  # 静音状态
            subprocess.Popen('echo "mute {mute}" > {fifo}'.format(fifo=self.mplayer_controller, mute=1), shell=True, stdin=subprocess.PIPE)
        song = self.douban.playingsong
        self.song_time = song['length']
        # 是否是红心歌曲
        if song['like'] == 1:
            love = self.love
        else:
            love = ''
        title = colored(song['title'], 'green')
        albumtitle = colored(song['albumtitle'], 'yellow')
        artist = colored(song['artist'], 'white')
        self.SUFFIX_SELECTED = (love + ' ' + title + ' • ' + albumtitle + ' • ' + artist + ' ' + song['public_time']).replace('\\', '')

        cmd = 'mplayer -slave -input file={fifo} {song_url} >/dev/null 2>&1'
        self.p = subprocess.Popen(cmd.format(fifo=self.mplayer_controller, song_url=song['url']), shell=True, stdin=subprocess.PIPE)  # subprocess.PIPE防止继承父进程
        self.pause = False
        self.display()
        self.notifySend()
        if self.lrc_display:  # 获取歌词
            self.lrc_dict = self.douban.get_lrc()
            if not self.lrc_dict:  # 歌词获取失败,关闭歌词界面
                self.lrc_display = 0
        self.start = 1
Beispiel #23
0
def bye():
    global flag
    global op_stop
    flag = False
    op_stop = True
    print termcolor.colored("Bye", "cyan")
    print termcolor.colored("有任何建议欢迎与我联系: [email protected]", "cyan")
Beispiel #24
0
 def __init__(self):
   # Load the trained models
   print(colored('Loading CLF models','magenta'))
   self.topicHasher = texthasher.safe_load(TEXT_VECTORIZER_PATH,stop_words=[])
   self.tagHasher   = taghasher.safe_load(TAG_HASHER_PATH,n_feature=256)
   self.clf         = cluster.safe_load(CLF_PATH,method=None,n_features=256)
   print(colored('[DONE] ','green'),'CLF models loaded')
Beispiel #25
0
def loadsession():
    global session
    try:
        session.cookies.load(ignore_discard="true")
    except:
        termcolor.colored("加载异常", "red")
        pass
Beispiel #26
0
def show_telemetry(tel):
    output = [] 

    for k, v in tel.iteritems():
        if k == 'register':
            color = 'blue'

        elif k == 'density':
            color = 'red'

        elif k == 'harmonicity':
            color = 'green'

        elif k == 'roughness':
            color = 'cyan'

        elif k == 'pace':
            color = 'yellow'

        else:
            color = 'white'

        if k == 'name':
            output += [ colored(' '.join(v), color) ]
        else:
            output += [ colored('%s: %.2f' % (k[:3], v), color) ]

    output = ' | '.join(output)
    dsp.log(output)
def pty_based_auth(auth_backend, pty):
    """
    Show a username/password login prompt.
    Return username when authentication succeeded.
    """
    tries = 0
    while True:
        # Password authentication required for this session?
        sys.stdout.write('\033[2J\033[0;0H') # Clear screen
        sys.stdout.write(colored('Please authenticate\r\n\r\n', 'cyan'))

        if tries:
            sys.stdout.write(colored('  Authentication failed, try again\r\n', 'red'))

        try:
            console = Console(pty)
            username = console.input('Username', False)
            password = console.input('Password', True)
        except NoInput:
            raise NotAuthenticated

        if auth_backend.authenticate(username, password):
            sys.stdout.write(colored(' ' * 40 + 'Authentication successful\r\n\r\n', 'green'))
            return username
        else:
            tries += 1
            if tries == 3:
                raise NotAuthenticated
Beispiel #28
0
    def display_time(self):
        length = len(self.TITLE)
        while True:
            if self.q == 1:  # 退出
                break
            if self.song_time >= 0 and self.douban.playingsong:
                minute = int(self.song_time) / 60
                sec = int(self.song_time) % 60
                show_time = str(minute).zfill(2) + ':' + str(sec).zfill(2)

                self.volume = self.get_volume()  # 获取音量
                self.TITLE = self.TITLE[:length - 1] + '  ' + self.douban.playingsong['kbps'] + 'kbps  ' + colored(show_time, 'cyan') + '  rate: ' + colored(self.rate[int(round(self.douban.playingsong['rating_avg'])) - 1], 'red') + '  vol: '
                if self.is_muted:
                    self.TITLE += '✖'
                else:
                    self.TITLE += self.volume.strip() + '%'
                if self.loop:
                    self.TITLE += '  ' + colored('↺', 'red')
                else:
                    self.TITLE += '  ' + colored('→', 'red')
                self.TITLE += '\r'
                self.display()
                if not self.pause:
                    self.song_time -= 1
            else:
                self.TITLE = self.TITLE[:length]
            time.sleep(1)
Beispiel #29
0
def signal_handler(signal, frame):
    print colored('You just pressed Ctrl+C! Please wait while the program is terminating...', 'red')
    #TO BE IMPLEMENTED
    #rollback()
    time.sleep(2)
    # Exit
    sys.exit(0)
def xap_doc_sim_for_pmids(pmids,n_terms=100):
	enq=xapian.Enquire(database)
	rset=xapian.RSet()
	for a in pmids:
		try: 
			database.get_document(a)
		except xapian.DocNotFoundError:
			continue
		rset.add_document(a)
	if rset.empty():
		print colored("No documents for set %s"%(str(pmids)),'red')
		return {},[]
	eset=enq.get_eset(n_terms,rset) # Number of terms in the eset
	# q=xapian.Query(xapian.Query.OP_OR,eset.begin(),eset.end())
	terms=[x[0] for x in eset.items]
	print ";".join(terms[:10])
	q=xapian.Query(xapian.Query.OP_OR,terms)
	enq.set_query(q)
	res=enq.get_mset(0,database.get_doccount())
	weighted_docs=[(r.docid,r.weight) for r in res]
	#normalize the weights to [0,1]
	max_w=max([x[1] for x in weighted_docs])

	weighted_docs=[(x[0],x[1]*1.0/max_w) for x in weighted_docs]

	sorted_results=dict(weighted_docs)
	return sorted_results,terms
Beispiel #31
0
    def testCoverage(self):
        print('CTEST_FULL_OUTPUT')
        prefixPath = os.environ['QGIS_PREFIX_PATH']
        docPath = os.path.join(prefixPath, '..', 'doc', 'api', 'xml')
        parser = DoxygenParser(docPath)

        # first look for objects without any bindings
        objects = set([m[0] for m in parser.bindable_members])
        missing_objects = []
        bound_objects = {}
        for o in objects:
            try:
                if '::' in o:
                    bound_objects[o] = getattr(globals()[o.split('::')[0]], o.split('::')[1])
                else:
                    bound_objects[o] = globals()[o]
            except:
                missing_objects.append(o)

        missing_objects.sort()

        # next check for individual members
        parser.bindable_members.sort()
        missing_members = []
        for m in parser.bindable_members:
            if m[0] in bound_objects:
                obj = bound_objects[m[0]]
                if "::" in m[0] and m[0].split("::")[1] == m[1]:
                    # skip constructors of nested classes
                    continue

                # try two different methods of checking for member existence
                try:
                    if hasattr(obj, m[1]):
                        continue
                except:
                    pass

                try:
                    if m[1] in dir(obj):
                        continue
                except:
                    printImportant("SIP coverage test: something strange happened in {}.{}, obj={}".format(m[0], m[1], obj))

                missing_members.append('{}.{}'.format(m[0], m[1]))

        missing_members.sort()

        if missing_objects:
            print("---------------------------------")
            print((colored('Missing classes:', 'yellow')))
            print(('  ' + '\n  '.join([colored(obj, 'yellow', attrs=['bold']) for obj in missing_objects])))
        if missing_members:
            print("---------------------------------")
            print((colored('Missing members:', 'yellow')))
            print(('  ' + '\n  '.join([colored(mem, 'yellow', attrs=['bold']) for mem in missing_members])))

        # print summaries
        missing_class_count = len(missing_objects)
        present_count = len(objects) - missing_class_count
        coverage = 100.0 * present_count / len(objects)

        print("---------------------------------")
        printImportant("{} total bindable classes".format(len(objects)))
        printImportant("{} total have bindings".format(present_count))
        printImportant("Binding coverage by classes {}%".format(coverage))
        printImportant("---------------------------------")
        printImportant("{} classes missing bindings".format(missing_class_count))
        print("---------------------------------")

        missing_member_count = len(missing_members)
        present_count = len(parser.bindable_members) - missing_member_count
        coverage = 100.0 * present_count / len(parser.bindable_members)

        print("---------------------------------")
        printImportant("{} total bindable members".format(len(parser.bindable_members)))
        printImportant("{} total have bindings".format(present_count))
        printImportant("Binding coverage by members {}%".format(coverage))
        printImportant("---------------------------------")
        printImportant("{} members missing bindings".format(missing_member_count))

        self.assertEqual(missing_class_count, 0, """\n\nFAIL: new unbound classes have been introduced, please add SIP bindings for these classes
If these classes are not suitable for the Python bindings, please add the Doxygen tag
"\\note not available in Python bindings" to the CLASS Doxygen comments""")

        self.assertEqual(missing_member_count, 0, """\n\nFAIL: new unbound members have been introduced, please add SIP bindings for these members
If these members are not suitable for the Python bindings, please add the Doxygen tag
"\\note not available in Python bindings" to the MEMBER Doxygen comments""")
Beispiel #32
0
 def warn(msg):
     if Logging.flag == True:
         print "".join([
             termcolor.colored("WARN", "yellow"), ": ",
             termcolor.colored(msg, "white")
         ])
Beispiel #33
0
 def error(msg):
     if Logging.flag == True:
         print "".join([
             termcolor.colored("ERROR", "red"), ": ",
             termcolor.colored(msg, "white")
         ])
 def _color_arg(arg: Any) -> Union[str, float, int]:
     if isinstance(arg, (int, float)):
         # In case of %d or %f formatting
         return arg
     return colored(str(arg), **ARGS)  # type: ignore
Beispiel #35
0
    def execute(self, sql, *args, **kwargs):
        """Execute a SQL statement."""

        # Lazily import
        import decimal
        import re
        import sqlalchemy
        import sqlparse
        import termcolor
        import warnings

        # Parse statement, stripping comments and then leading/trailing whitespace
        statements = sqlparse.parse(
            sqlparse.format(sql, strip_comments=True).strip())

        # Allow only one statement at a time, since SQLite doesn't support multiple
        # https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor.execute
        if len(statements) > 1:
            raise RuntimeError("too many statements at once")
        elif len(statements) == 0:
            raise RuntimeError("missing statement")

        # Ensure named and positional parameters are mutually exclusive
        if len(args) > 0 and len(kwargs) > 0:
            raise RuntimeError(
                "cannot pass both named and positional parameters")

        # Infer command from (unflattened) statement
        for token in statements[0]:
            if token.ttype in [
                    sqlparse.tokens.Keyword.DDL, sqlparse.tokens.Keyword.DML
            ]:
                command = token.value.upper()
                break

            # Begin a new session, if transaction opened by caller (not using autocommit)
            elif token.value.upper() in ["BEGIN", "START"]:
                if self._in_transaction:
                    raise RuntimeError("transaction already open")
                self._in_transaction = True
        else:
            command = None

        # Flatten statement
        tokens = list(statements[0].flatten())

        # Validate paramstyle
        placeholders = {}
        paramstyle = None
        for index, token in enumerate(tokens):

            # If token is a placeholder
            if token.ttype == sqlparse.tokens.Name.Placeholder:

                # Determine paramstyle, name
                _paramstyle, name = _parse_placeholder(token)

                # Remember paramstyle
                if not paramstyle:
                    paramstyle = _paramstyle

                # Ensure paramstyle is consistent
                elif _paramstyle != paramstyle:
                    raise RuntimeError("inconsistent paramstyle")

                # Remember placeholder's index, name
                placeholders[index] = name

        # If more placeholders than arguments
        if len(args) == 1 and len(placeholders) > 1:

            # If user passed args as list or tuple, explode values into args
            if isinstance(args[0], (list, tuple)):
                args = args[0]

            # If user passed kwargs as dict, migrate values from args to kwargs
            elif len(kwargs) == 0 and isinstance(args[0], dict):
                kwargs = args[0]
                args = []

        # If no placeholders
        if not paramstyle:

            # Error-check like qmark if args
            if args:
                paramstyle = "qmark"

            # Error-check like named if kwargs
            elif kwargs:
                paramstyle = "named"

        # In case of errors
        _placeholders = ", ".join(
            [str(tokens[index]) for index in placeholders])
        _args = ", ".join([str(self._escape(arg)) for arg in args])

        # qmark
        if paramstyle == "qmark":

            # Validate number of placeholders
            if len(placeholders) != len(args):
                if len(placeholders) < len(args):
                    raise RuntimeError(
                        "fewer placeholders ({}) than values ({})".format(
                            _placeholders, _args))
                else:
                    raise RuntimeError(
                        "more placeholders ({}) than values ({})".format(
                            _placeholders, _args))

            # Escape values
            for i, index in enumerate(placeholders.keys()):
                tokens[index] = self._escape(args[i])

        # numeric
        elif paramstyle == "numeric":

            # Escape values
            for index, i in placeholders.items():
                if i >= len(args):
                    raise RuntimeError(
                        "missing value for placeholder (:{})".format(
                            i + 1, len(args)))
                tokens[index] = self._escape(args[i])

            # Check if any values unused
            indices = set(range(len(args))) - set(placeholders.values())
            if indices:
                raise RuntimeError("unused {} ({})".format(
                    "value" if len(indices) == 1 else "values", ", ".join([
                        str(self._escape(args[index])) for index in indices
                    ])))

        # named
        elif paramstyle == "named":

            # Escape values
            for index, name in placeholders.items():
                if name not in kwargs:
                    raise RuntimeError(
                        "missing value for placeholder (:{})".format(name))
                tokens[index] = self._escape(kwargs[name])

            # Check if any keys unused
            keys = kwargs.keys() - placeholders.values()
            if keys:
                raise RuntimeError("unused values ({})".format(
                    ", ".join(keys)))

        # format
        elif paramstyle == "format":

            # Validate number of placeholders
            if len(placeholders) != len(args):
                if len(placeholders) < len(args):
                    raise RuntimeError(
                        "fewer placeholders ({}) than values ({})".format(
                            _placeholders, _args))
                else:
                    raise RuntimeError(
                        "more placeholders ({}) than values ({})".format(
                            _placeholders, _args))

            # Escape values
            for i, index in enumerate(placeholders.keys()):
                tokens[index] = self._escape(args[i])

        # pyformat
        elif paramstyle == "pyformat":

            # Escape values
            for index, name in placeholders.items():
                if name not in kwargs:
                    raise RuntimeError(
                        "missing value for placeholder (%{}s)".format(name))
                tokens[index] = self._escape(kwargs[name])

            # Check if any keys unused
            keys = kwargs.keys() - placeholders.values()
            if keys:
                raise RuntimeError("unused {} ({})".format(
                    "value" if len(keys) == 1 else "values", ", ".join(keys)))

        # For SQL statements where a colon is required verbatim, as within an inline string, use a backslash to escape
        # https://docs.sqlalchemy.org/en/13/core/sqlelement.html?highlight=text#sqlalchemy.sql.expression.text
        for index, token in enumerate(tokens):

            # In string literal
            # https://www.sqlite.org/lang_keywords.html
            if token.ttype in [
                    sqlparse.tokens.Literal.String,
                    sqlparse.tokens.Literal.String.Single
            ]:
                token.value = re.sub("(^'|\s+):", r"\1\:", token.value)

            # In identifier
            # https://www.sqlite.org/lang_keywords.html
            elif token.ttype == sqlparse.tokens.Literal.String.Symbol:
                token.value = re.sub("(^\"|\s+):", r"\1\:", token.value)

        # Join tokens into statement
        statement = "".join([str(token) for token in tokens])

        # Connect to database (for transactions' sake)
        if self._session is None:
            self._session = self._Session()

        # Set up a Flask app teardown function to close session at teardown
        try:

            # Infer whether Flask is installed
            import flask

            # Infer whether app is defined
            assert flask.current_app

            # Disconnect later - but only once
            if not hasattr(self, "_teardown_appcontext_added"):
                self._teardown_appcontext_added = True

                @flask.current_app.teardown_appcontext
                def shutdown_session(exception=None):
                    """Close any existing session on app context teardown."""
                    self._close_session()

        except (ModuleNotFoundError, AssertionError):
            pass

        # Catch SQLAlchemy warnings
        with warnings.catch_warnings():

            # Raise exceptions for warnings
            warnings.simplefilter("error")

            # Prepare, execute statement
            try:

                # Join tokens into statement, abbreviating binary data as <class 'bytes'>
                _statement = "".join([
                    str(bytes)
                    if token.ttype == sqlparse.tokens.Other else str(token)
                    for token in tokens
                ])

                # If COMMIT or ROLLBACK, turn on autocommit mode
                if command in ["COMMIT", "ROLLBACK"
                               ] and "TO" not in (token.value
                                                  for token in tokens):
                    if not self._in_transaction:
                        raise RuntimeError(
                            "transactions must be opened with BEGIN or START TRANSACTION"
                        )
                    self._in_transaction = False

                # Execute statement
                result = self._session.execute(sqlalchemy.text(statement))

                # Return value
                ret = True

                # If SELECT, return result set as list of dict objects
                if command == "SELECT":

                    # Coerce types
                    rows = [dict(row) for row in result.fetchall()]
                    for row in rows:
                        for column in row:

                            # Coerce decimal.Decimal objects to float objects
                            # https://groups.google.com/d/msg/sqlalchemy/0qXMYJvq8SA/oqtvMD9Uw-kJ
                            if type(row[column]) is decimal.Decimal:
                                row[column] = float(row[column])

                            # Coerce memoryview objects (as from PostgreSQL's bytea columns) to bytes
                            elif type(row[column]) is memoryview:
                                row[column] = bytes(row[column])

                    # Rows to be returned
                    ret = rows

                # If INSERT, return primary key value for a newly inserted row (or None if none)
                elif command == "INSERT":
                    if self._engine.url.get_backend_name() in [
                            "postgres", "postgresql"
                    ]:
                        try:
                            result = self._session.execute("SELECT LASTVAL()")
                            ret = result.first()[0]
                        except sqlalchemy.exc.OperationalError:  # If lastval is not yet defined in this session
                            ret = None
                    else:
                        ret = result.lastrowid if result.rowcount == 1 else None

                # If DELETE or UPDATE, return number of rows matched
                elif command in ["DELETE", "UPDATE"]:
                    ret = result.rowcount

                # If autocommit is on, commit
                if not self._in_transaction:
                    self._session.commit()

            # If constraint violated, return None
            except sqlalchemy.exc.IntegrityError as e:
                self._logger.debug(termcolor.colored(statement, "yellow"))
                e = RuntimeError(e.orig)
                e.__cause__ = None
                raise e

            # If user errror
            except sqlalchemy.exc.OperationalError as e:
                self._logger.debug(termcolor.colored(statement, "red"))
                e = RuntimeError(e.orig)
                e.__cause__ = None
                raise e

            # Return value
            else:
                self._logger.debug(termcolor.colored(_statement, "green"))
                return ret
Beispiel #36
0
 def format(self, record):
     msg = super().format(record)
     return msg if not self.use_color else termcolor.colored(
         msg, getattr(record, "color", self.COLORS.get(record.levelname)))
Beispiel #37
0
def optimize_class_model(args, logger=None):
    """
    加载中文分类模型
    :param args:
    :param num_labels:
    :param logger:
    :return:
    """

    if not logger:
        logger = set_logger(
            colored('CLASSIFICATION_MODEL, Lodding...', 'cyan'), args.verbose)
        pass
    try:
        # 如果PB文件已经存在则,返回PB文件的路径,否则将模型转化为PB文件,并且返回存储PB文件的路径
        if args.model_pb_dir is None:
            tmp_file = args.model_dir
        #    # 获取当前的运行路径
        #    tmp_file = os.path.join(os.getcwd(), 'predict_optimizer')
        #    if not os.path.exists(tmp_file):
        #        os.mkdir(tmp_file)
        else:
            tmp_file = args.model_pb_dir

        #latest_checkpoint = tf.train.latest_checkpoint(args.model_dir)
        pb_file = os.path.join(tmp_file, 'classification_model.pb')
        if os.path.exists(pb_file):
            print('pb_file exits', pb_file)
            return pb_file
        #import tensorflow as tf

        graph = tf.Graph()
        with graph.as_default():
            with tf.Session() as sess:
                input_ids = tf.placeholder(tf.int32, (None, args.max_seq_len),
                                           'input_ids')
                input_mask = tf.placeholder(tf.int32, (None, args.max_seq_len),
                                            'input_mask')

                bert_config = modeling.BertConfig.from_json_file(
                    os.path.join(args.bert_model_dir, 'bert_config.json'))
                #from bert_base.train.models import create_classification_model
                #from models import create_classification_model
                loss, per_example_loss, logits, probabilities = create_classification_model(
                    bert_config=bert_config,
                    is_training=False,
                    input_ids=input_ids,
                    input_mask=input_mask,
                    segment_ids=None,
                    labels=None,
                    num_labels=args.num_labels)
                # pred_ids = tf.argmax(probabilities, axis=-1, output_type=tf.int32, name='pred_ids')
                # pred_ids = tf.identity(pred_ids, 'pred_ids')
                probabilities = tf.identity(probabilities, 'pred_prob')
                saver = tf.train.Saver()

            with tf.Session() as sess:
                sess.run(tf.global_variables_initializer())
                latest_checkpoint = tf.train.latest_checkpoint(args.model_dir)
                logger.info('loading... %s ' % latest_checkpoint)
                saver.restore(sess, latest_checkpoint)
                logger.info('freeze...')
                from tensorflow.python.framework import graph_util
                tmp_g = graph_util.convert_variables_to_constants(
                    sess, graph.as_graph_def(), ['pred_prob'])
                logger.info('predict cut finished !!!')
        # 存储二进制模型到文件中
        logger.info('write graph to a tmp file: %s' % pb_file)
        with tf.gfile.GFile(pb_file, 'wb') as f:
            f.write(tmp_g.SerializeToString())
        return pb_file
    except Exception as e:
        logger.error('fail to optimize the graph! %s' % e, exc_info=True)
Beispiel #38
0
 def colorize_comment(self, comment):
     """Return a string with color markup for a given comment."""
     if self.mark_re is None:
         return comment
     return self.mark_re.sub(
         colored(r'\g<0>', None, None, ['reverse', 'bold']), comment)
def yellow(content): return termcolor.colored(str(content),"yellow",attrs=["bold"])
def magenta(content): return termcolor.colored(str(content),"magenta",attrs=["bold"])
Beispiel #40
0
 def colorize_attr(attr, value, marked):
     """Return a string with color markup for a given attr and its value."""
     color = COLOR_OF.get(attr, None)
     return colored(value, color, None,
                    ['reverse', 'bold'] if marked else None)
def blue(content): return termcolor.colored(str(content),"blue",attrs=["bold"])
def cyan(content): return termcolor.colored(str(content),"cyan",attrs=["bold"])
def magenta(content): return termcolor.colored(str(content),"magenta",attrs=["bold"])

def get_time(sec):
def red(content): return termcolor.colored(str(content),"red",attrs=["bold"])
def green(content): return termcolor.colored(str(content),"green",attrs=["bold"])
def cyan(content): return termcolor.colored(str(content),"cyan",attrs=["bold"])
def yellow(content): return termcolor.colored(str(content),"yellow",attrs=["bold"])
Beispiel #45
0
 def print_cyan(cls, msg):
     color_format = {"color": "cyan"}
     colore_msg = colored(msg, **color_format)
     print colore_msg
def green(content): return termcolor.colored(str(content),"green",attrs=["bold"])
def blue(content): return termcolor.colored(str(content),"blue",attrs=["bold"])
Beispiel #47
0
 def print_ok(cls, msg):
     color_format = {"color": "green"}
     head = colored("[ OK ]", **color_format)
     body = "{}:{}".format(head, msg)
     print body
Beispiel #48
0
 def print_blue(cls, msg):
     color_format = {"color": "blue"}
     colore_msg = colored(msg, **color_format)
     print colore_msg
Beispiel #49
0
 def print_white(cls, msg):
     color_format = {"color": "white"}
     colore_msg = colored(msg, **color_format)
     print colore_msg
Beispiel #50
0
 def print_war(cls, msg):
     color_format = {"color": "yellow"}
     head = colored("[ WARNING ]", **color_format)
     body = "{}:{}".format(head, msg)
     print body
Beispiel #51
0
 def print_green(cls, msg):
     color_format = {"color": "green"}
     colore_msg = colored(msg, **color_format)
     print colore_msg
Beispiel #52
0
    def print_error(cls, msg):
        color_format = {"color": "red"}

        head = colored("[ ERROR ]", **color_format)
        body = "{}:{}".format(head, msg)
        print body
Beispiel #53
0
def subfailure(name):
    print('{} {}'.format(colored("  ->", 'red', attrs=['bold']),
                         colored(name, 'red', attrs=['bold'])))
Beispiel #54
0
 def print_title(cls, msg):
     color_format = {"color": "green"}
     msg = msg.center(86, '-')
     colore_msg = colored(msg, **color_format)
     print colore_msg
Beispiel #55
0
def subtask(name):
    print('{} {}'.format(colored("  ->", 'blue', attrs=['bold']),
                         colored(name, attrs=['bold'])))
Beispiel #56
0
def subprompt(name):
    print('{} {}'.format(colored("  ->", 'yellow', attrs=['bold']),
                         colored(name, attrs=['bold'])),
          end="")
Beispiel #57
0
def section(name):
    print("\n{} {}".format(colored("::", 'blue', attrs=['bold']),
                           colored(name, attrs=['bold'])))
Beispiel #58
0
def failure(name):
    print('{} {}'.format(colored("==> ERROR:", 'red', attrs=['bold']),
                         colored(name, attrs=['bold'])))
Beispiel #59
0
import os
import numpy as np
from termcolor import colored

results = []

tests = 'oscillator_tst','lfo_tst','adsr_tst','mixer_tst','mov_avg_tst','sound_gen_tst'

for test_name in tests:
    return_value = os.system('python '+ ' mainPrj.py -b nucleo -p '+ test_name + '/')
    results.append([test_name,return_value])
    
print '\n-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*\n'
print '\t|SUMMARY OF TESTS:'
print '\t------------------------------------------------'
for result in results:
    result_string = ('OK' if result[1] == 0 else 'FAIL')
    out_color = ('green' if result[1] == 0 else 'red')
    print colored('\t' + result[0] + ' ' + '\t ---> ' + result_string,out_color)
print '\n'
Beispiel #60
0
def task(name):
    print('{} {}'.format(colored("==>", 'green', attrs=['bold']),
                         colored(name, attrs=['bold'])))