示例#1
0
    def FileUploadCompleted(self, request, context):

        log_info(
            "################################### FILE_UPLOAD_COMPLETED_ARRIVED!!! #################################")
        # pprint.pprint(request)
        log_info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$")

        if Globals.NODE_STATE == NodeState.LEADER:
            chunk_id = request.chunkUploadInfo.chunkId
            lst_dc = [(request.chunkUploadInfo.uploadedDatacenter.ip, request.chunkUploadInfo.uploadedDatacenter.port)]

            if not _send_heartbeat_to_check_majority_consensus():
                return

            Tables.insert_file_chunk_info_to_file_log(request.fileName, chunk_id, lst_dc,
                                                      raft_proto.Uploaded if request.isSuccess else raft_proto.UploadFaied)
            _send_heartbeat()

            # pprint.pprint(Tables.TABLE_FILE_INFO)
            log_info("###########################################################################")
        else:
            client = get_leader_client()
            if client:
                my_reply = client._FileUploadCompleted(request)
                return my_reply
            else:
                return raft_proto.Empty()
        return raft_proto.Empty()
示例#2
0
def db_viterbi_alignment(es, fs,
                         transfrom=2,
                         transto=1,
                         db="sqlite:///:memory:",
                         init_val=1.0e-10):
    """
    Calculating viterbi_alignment using specified database.

    Arguments:
        trans:
            it can take "en2ja" or "ja2en"
    """
    engine = create_engine(db)
    # create session
    Session = sessionmaker(bind=engine)
    session = Session()

    # tablenames
    table_prefix = "from{0}to{1}".format(transfrom, transto)
    wordprob_tablename = table_prefix + "_" + "wordprob"
    wordalign_tablename = table_prefix + "_" + "wordalign"
    # tables
    WordProbability = Tables().get_wordprobability_table(wordprob_tablename)
    WordAlignment = Tables().get_wordalignment_table(wordalign_tablename)

    def get_wordprob(e, f, init_val=1.0e-10):

        query = session.query(WordProbability).filter_by(transto=e,
                                                         transfrom=f)
        try:
            return query.one().prob
        except sqlalchemy.orm.exc.NoResultFound:
            return init_val

    def get_wordalign(i, j, l_e, l_f, init_val=1.0e-10):

        query = session.query(WordAlignment).filter_by(from_pos=i,
                                                       to_pos=j,
                                                       to_len=l_e,
                                                       from_len=l_f)
        try:
            return query.one().prob
        except sqlalchemy.orm.exc.NoResultFound:
            return init_val

    # algorithm
    max_a = collections.defaultdict(float)
    l_e = len(es)
    l_f = len(fs)
    for (j, e) in enumerate(es, 1):
        current_max = (0, -1)
        for (i, f) in enumerate(fs, 1):
            val = get_wordprob(e, f, init_val=init_val) *\
                get_wordalign(i, j, l_e, l_f, init_val=init_val)
            # select the first one among the maximum candidates
            if current_max[1] < val:
                current_max = (i, val)
        max_a[j] = current_max[0]
    return max_a
示例#3
0
 def assignPickMeUpDrinkPreference(screen):
     person = tables.handleSingleSelectTable(
         screen, "People", State._people, "", 0,
         "Select a person to assign drink to")
     drink = tables.handleSingleSelectTable(
         screen, "Drinks", State._drinks, "", 0,
         f"Select drink to assign to {person.displayName}")
     person.PMUDrink = drink
示例#4
0
 def preparations(self):
     """ methods to make a json file, get datas, create tables and fill them ."""
     os.system('clear')
     print('get datas and create a json file...')
     self.datas.mkjsonfile()
     print('create and fille tables...')
     Tables.creation()
     Tables.fill_tables()
示例#5
0
 def createData(self):
     t = Tables()
     sql = t.terms()
     #try:
     conn = sqlite3.connect(self.dbs)
     c = conn.cursor()
     c.execute(sql)
     conn.commit()
     conn.close()
示例#6
0
 def getAllIpAddress(self):
     table = Tables()
     table.createTable("ipAddress")
     iplines = []
     try:
         ipfile = open("chinaiplist.txt")
         iplines = ipfile.readlines()
     except Exception,e:
         self._logger.error("error occured when open file")
         return None
示例#7
0
 def __init__(self):
     self._logger = Logger(__file__)
     #profile = FirefoxProfile()
     #profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
     #self._browser = webdriver.Firefox(profile)
     self._browser = webdriver.Firefox()
     self.baidu = Baidu(self._browser)
     self.map = BaiduMap()
     self.ak = "sh0wDYRg1LnB5OYTefZcuHu3zwuoFeOy"
     self.table = Tables();
示例#8
0
def create_train_db(transfrom=2,
                    transto=1,
                    lang1method=lambda x: x,
                    lang2method=lambda x: x,
                    db="sqlite:///:memory:",
                    limit=None,
                    loop_count=1000):
    engine = create_engine(db)
    # create session
    Session = sessionmaker(bind=engine)
    session = Session()

    # tablenames
    table_prefix = "from{0}to{1}".format(transfrom, transto)
    wordprob_tablename = table_prefix + "_" + "wordprob"
    wordalign_tablename = table_prefix + "_" + "wordalign"
    # tables
    WordProbability = Tables().get_wordprobability_table(wordprob_tablename)
    WordAlignment = Tables().get_wordalignment_table(wordalign_tablename)
    # create table for word probability
    WordProbability.__table__.drop(engine, checkfirst=True)
    WordProbability.__table__.create(engine)
    print("created table: {0}to{1}_wordprob".format(transfrom, transto))

    # create table for alignment probability
    WordAlignment.__table__.drop(engine, checkfirst=True)
    WordAlignment.__table__.create(engine)
    print("created table: {0}to{1}_wordalign".format(transfrom, transto))

    # IBM learning
    with ProgressLine(0.12, title='IBM Model learning...'):
        # check arguments for carete_corpus
        corpus = create_corpus(db=db, limit=limit,
                               lang1method=lang1method,
                               lang2method=lang2method)
        sentences = [(item["lang{0}".format(transto)],
                      item["lang{0}".format(transfrom)])
                     for item in corpus]
        t, a = ibmmodel2.train(sentences=sentences,
                               loop_count=loop_count)
    # insert
    with ProgressLine(0.12, title='Inserting items into database...'):
        for (_to, _from), prob in t.items():
            session.add(WordProbability(transto=_to,
                                        transfrom=_from,
                                        prob=float(prob)))
        for (from_pos, to_pos, to_len, from_len), prob in a.items():
            session.add(WordAlignment(from_pos=from_pos,
                                      to_pos=to_pos,
                                      to_len=to_len,
                                      from_len=from_len,
                                      prob=float(prob)))
        session.commit()
示例#9
0
    def AddFileLog(self, request, context):
        if Globals.NODE_STATE == NodeState.LEADER:
            request.log_index = Globals.get_next_log_index()
            Tables.FILE_LOGS.append(request)
            log_info("LOG ADDED")

            # Update Table_log and File_info_table
            Tables.set_table_log(Tables.FILE_LOGS)

        ack = raft_proto.Ack()
        ack.id = 1
        return ack
示例#10
0
    def RequestFileUpload(self, request, context):

        if Globals.NODE_STATE == NodeState.LEADER:
            my_reply = file_transfer_proto.ProxyList()

            file_name = request.fileName
            file_size = request.fileSize
            total_chunks = math.ceil(file_size / CHUNK_SIZE)

            if Tables.is_file_exists(file_name):
                return my_reply

            dcs = Tables.get_all_available_dc()

            if not _send_heartbeat_to_check_majority_consensus():
                return my_reply

            for chunk_id in range(total_chunks):
                random_dcs = [get_rand_hashing_node(dcs, file_name, chunk_id)]
                Tables.insert_file_chunk_info_to_file_log(file_name, chunk_id, random_dcs, raft_proto.UploadRequested)

            _send_heartbeat()

            # pprint.pprint("TABLE_FILE_INFO")
            # pprint.pprint(Tables.TABLE_FILE_INFO)
            # pprint.pprint(Tables.TABLE_DC_INFO)

            lst_proxies = Tables.get_all_available_proxies()
            lst_proxy_info = []
            for ip, port in lst_proxies:
                proxy_info = file_transfer_proto.ProxyInfo()
                proxy_info.ip = ip
                proxy_info.port = port
                lst_proxy_info.append(proxy_info)

            log_info("LST_PROXIES:")
            log_info(my_reply.lstProxy)
            my_reply.lstProxy.extend(lst_proxy_info)

            log_info("Replied to :")
            # pprint.pprint(request)
            # pprint.pprint(my_reply)
            log_info("############################")
            return my_reply

        else:
            client = get_leader_client()
            if client:
                my_reply = client._RequestFileUpload(request)
                return my_reply
            else:
                return file_transfer_proto.ProxyList()
示例#11
0
 def __init__(self):
     """ """
     self.host = HOST
     self.user = USER
     self.password = PASSWORD
     self.db_name = 'PureBeurre'
     self.tables = Tables()
示例#12
0
    def test_find_people_by_team(self):
        screen = initializeScreen()
        #Arrange
        initiator = Mock(Person)
        initiator.team = "Academy"

        member1 = Mock(Person)
        member1.team = "Academy"

        member2 = Mock(Person)
        member2.team = "Academy"

        member3 = Mock(Person)
        member3.team = "Hermes"

        member4 = Mock(Person)
        member4.team = "Sainsburys Bank"

        testPeople = [initiator, member1, member2, member3, member4]
        expected_output = [initiator, member1, member2]
        #Act
        actual_output = tables.findPeopleByTeam(initiator.team, testPeople)
        #Assert
        self.assertEqual(len(expected_output), len(actual_output))
        self.assertEqual(expected_output, actual_output)
        deinitializeScreen()
 def __init__(self, version):
     if version == 128:
         self.encrypt = self._encrypt_128
         self.decrypt = self._decrypt_128
         self.encrypt_str = self._encrypt_128_str
         self.decrypt_str = self._decrypt_128_str
         self.test_key = [symbol for symbol in range(16)]
         self.N_k = 4
         self.N_b = 4
         self.N_r = 10
         self.tables = Tables()
     elif version == 192:
         pass
     elif version == 256:
         pass
     else:
         pass
示例#14
0
    def RaftHeartbeat(self, request, context):

        log_info("heartbeat arrived: ", len(Tables.FILE_LOGS))

        ack = raft_proto.Ack()

        if len(request.tableLog) > len(Tables.FILE_LOGS):

            Globals.NODE_STATE = NodeState.FOLLOWER
            Globals.CURRENT_CYCLE = request.cycle_number
            Globals.HAS_CURRENT_VOTED = False
            Globals.NUMBER_OF_VOTES = 0
            Globals.LEADER_PORT = request.leader_port
            Globals.LEADER_IP = request.leader_ip

        elif len(request.tableLog) == len(Tables.FILE_LOGS) and request.cycle_number > Globals.CURRENT_CYCLE:

            Globals.NODE_STATE = NodeState.FOLLOWER
            Globals.CURRENT_CYCLE = request.cycle_number
            Globals.HAS_CURRENT_VOTED = False
            Globals.NUMBER_OF_VOTES = 0
            Globals.LEADER_PORT = request.leader_port
            Globals.LEADER_IP = request.leader_ip

        elif request.leader_ip != Globals.LEADER_IP or request.leader_port != Globals.LEADER_PORT:
            ack.id = -1
            return ack

        random_timer.reset()
        # print("MY Leader: ", Globals.LEADER_IP, Globals.LEADER_PORT, len(Tables.FILE_LOGS))

        # Update Table_log and File_info_table
        Tables.set_table_log(request.tableLog)

        ack.id = len(Tables.FILE_LOGS)

        # log_info("Tables.TABLE_FILE_INFO")
        # pprint.pprint(Tables.TABLE_FILE_INFO)
        log_info("Tables.TABLE_PROXY_INFO")
        pprint.pprint(Tables.TABLE_PROXY_INFO)
        log_info("Tables.TABLE_DC_INFO")
        pprint.pprint(Tables.TABLE_DC_INFO)

        log_info("###########################################################")

        return ack
示例#15
0
class Sessions:
    proxy = proxy_io(api_key=CONFIG.proxyio["key"],
                     session=AIOHTTP.ClientSession)
    websocket = WebSocket()
    webhook = WebhookSend()
    steam = Steam()
    database = Tables().database
    server = Server().load()
    cdn = Cdn().load()
示例#16
0
 def __init__(self):
     self._logger = Logger(__file__)
     # the entry point of grabing 
     self.base="http://scenic.cthy.com"
     self.provinces = []
     #self._browser = webdriver.PhantomJS()
     self._browser = webdriver.Firefox()
     self.tabopera = Tables();
     self.record = open("record.txt","a+")
     self.fdate = open("date.txt","a+")
     self.fprice = open("price.txt","a+")
     self.sprovince = 0
     self.spage = 1
     self.snum = 0
     self.picturenum = 10
     self.baidu = Baidu(self._browser)
     self.map = BaiduMap()
     self.ak = "sh0wDYRg1LnB5OYTefZcuHu3zwuoFeOy"
示例#17
0
    def __init__(self):
        """ This client assumes the developer has taken
            the initiative to correctly initialize the needed sessions.
        """

        self.routes = Routes(obj=self)
        self.middlewares = Middlewares(obj=self)
        self.api = Api(obj=self)
        self.tables = Tables(obj=self)
示例#18
0
 def test_filter_table_1(self):
     #Arrange
     obj1 = SimpleNamespace(displayName='David')
     obj2 = SimpleNamespace(displayName='Dave')
     obj3 = SimpleNamespace(displayName='Henry')
     test_peeps = [obj1, obj2, obj3]
     #Act
     filtered_peeps = tables.filterTable(test_peeps, "D")
     #Assert
     self.assertEqual(2, len(filtered_peeps))
示例#19
0
    def AddProxy(self, request, context):
        if Globals.NODE_STATE == NodeState.LEADER:
            if not _send_heartbeat_to_check_majority_consensus():
                ack = raft_proto.Ack()
                ack.id = -1
                return ack

            Tables.register_proxy(request.ip, request.port)
            _send_heartbeat()
            return raft_proto.Ack()
        else:
            client = get_leader_client()
            if client:
                my_reply = client._AddProxy(request)
                return my_reply
            else:
                ack = raft_proto.Ack()
                ack.id = -1
                return ack
示例#20
0
def get_file_lists(request):
    lst_files = []
    if request.isClient:
        lst_files = request_file_list_from_other_raft_nodes(request)

    lst_files = lst_files + Tables.get_all_available_file_list()
    my_reply = file_transfer_proto.FileList()
    # pprint.pprint("lst_files")
    # pprint.pprint(lst_files)
    if len(lst_files) > 0:
        my_reply.lstFileNames.extend(lst_files)
    return my_reply
示例#21
0
    def __init__(self, application=None):
        if application == None:
            sys.exit(0)
        application.config.from_object(config)

        self.db = SQLAlchemy(application)
        if self.db == None:
            sys.exit(-1)

        from tables import Tables
        #初始化表对象
        self.tables = Tables(self.db)
        self.create_all()
示例#22
0
def create_phrase_db(limit=None,
                     lang1method=lambda x: x,
                     lang2method=lambda x: x,
                     init_val=1.0e-10,
                     db="sqlite:///:memory:"):
    engine = create_engine(db)
    # create session
    Session = sessionmaker(bind=engine)
    session = Session()
    # tables
    Sentence = Tables().get_sentence_table()
    Phrase = Tables().get_phrase_table()

    # create table for word probability
    Phrase.__table__.drop(engine, checkfirst=True)
    Phrase.__table__.create(engine)
    print("created table: phrase")

    query = session.query(Sentence)[:limit] if limit \
        else session.query(Sentence)

    with ProgressLine(0.12, title='extracting phrases...'):
        for item in query:
            lang1 = item.lang1
            lang2 = item.lang2
            print("  ", lang1, lang2)
            phrases = db_phrase_extract(lang1, lang2,
                                        lang1method=lang1method,
                                        lang2method=lang2method,
                                        init_val=init_val,
                                        db=db)
            for lang1ps, lang2ps in phrases:
                lang1p = u" ".join(lang1ps)
                lang2p = u" ".join(lang2ps)
                ph = Phrase(lang1p=lang1p, lang2p=lang2p)
                session.add(ph)
        session.commit()
示例#23
0
def create_phrase_prob(db=":memory:"):
    """
    """
    # create phrase_prob table
    table_name = "phraseprob"
    engine = create_engine("sqlite:///{0}".format(db))
    # create session
    Session = sessionmaker(bind=engine)
    session = Session()
    # tables
    TransPhraseProb = Tables().get_transphraseprob_table()

    # create table for word probability
    TransPhraseProb.__table__.drop(engine, checkfirst=True)
    TransPhraseProb.__table__.create(engine)
    session.commit()
    print("created table: {0}".format(table_name))

    con = sqlite3.connect(db)
    cur = con.cursor()
    cur_sel = con.cursor()
    #cur_rec = con.cursor()
    cur.execute("select lang1p, lang2p, count from phrasecount")
    with ProgressLine(0.12, title='phrase learning...'):
        for lang1p, lang2p, count in cur:
            # for p2_1
            cur_sel.execute(u"""select count
                            from lang1_phrasecount where
                            langp=?""",
                            (lang1p,))
            count2_1 = list(cur_sel)
            count2_1 = count2_1[0][0]
            p2_1 = count / count2_1
            # for p1_2
            cur_sel.execute(u"""select count
                            from lang2_phrasecount where
                            langp=?""",
                            (lang2p,))
            count1_2 = list(cur_sel)
            count1_2 = count1_2[0][0]
            p1_2 = count / count1_2
            # insert item
            transphraseprob = TransPhraseProb(lang1p=lang1p,
                                              lang2p=lang2p,
                                              p1_2=math.log(p1_2),
                                              p2_1=math.log(p2_1))
            session.add(transphraseprob)
            print(u"  added phraseprob: {0} <=> {1} ".format(lang1p, lang2p))
        session.commit()
示例#24
0
    def __init__(self):
        """
        Creates an Agent object.
        
        The Agent chooses the actions to take according to some policy. It
        also keeps track of q-value estimates, visits, and which actions do 
        not change the robot/stack locations.

        Returns
        -------
        None.

        """
        self.tables = Tables()
        return
示例#25
0
 def __init__(
     self,
     phase_1_corner: int = 0,
     phase_1_edge: int = 0,
     phase_1_ud_slice: int = 0,
     phase_2_corner: int = 0,
     phase_2_edge: int = 0,
     phase_2_ud_slice: int = 0,
 ):
     self.phase_1_corner = phase_1_corner
     self.phase_1_edge = phase_1_edge
     self.phase_1_ud_slice = phase_1_ud_slice
     self.phase_2_corner = phase_2_corner
     self.phase_2_edge = phase_2_edge
     self.phase_2_ud_slice = phase_2_ud_slice
     self.tables = Tables()
示例#26
0
    def __init__(self, cube: Cube):
        super().__init__(cube)
        self.max_moves_length = 29  # Upper bound of the kociemba algorithm
        self.moves = []
        self.start = 0
        self.end = 0
        self.time_to_solve = 0
        self.tables = Tables()

        self.face_cube = self.cube.to_face_cube()
        self.cubie_cube = self.face_cube.to_cubie_cube()
        self.coord_cube = CoordCube.from_cubie_cube(self.cubie_cube)

        # Validate cube
        self._validate_cube()

        # `moves_face` stores the moves' face whereas `moves_turn` stores the moves' number of
        # quarter turns (i.e., 1 for clockwise, 2 for clockwise twice, and 3 for counterclockwise).
        # These are used to generate the list of strings for `moves` once solved.
        self.moves_face = [0 for i in range(self.max_moves_length)]
        self.moves_turn = [0 for i in range(self.max_moves_length)]

        # Coordinates needed for phase 1
        self.phase_1_corner = [0 for i in range(self.max_moves_length)]
        self.phase_1_corner[0] = self.coord_cube.phase_1_corner
        self.phase_1_edge = [0 for i in range(self.max_moves_length)]
        self.phase_1_edge[0] = self.coord_cube.phase_1_edge
        self.phase_1_ud_slice = [0 for i in range(self.max_moves_length)]
        self.phase_1_ud_slice[0] = self.coord_cube.phase_1_ud_slice

        # Coordinates needed for phase 2
        self.phase_2_corner = [0 for i in range(self.max_moves_length)]
        self.phase_2_corner[0] = self.coord_cube.phase_2_corner
        self.phase_2_edge = [0 for i in range(self.max_moves_length)]
        self.phase_2_edge[0] = self.coord_cube.phase_2_edge
        self.phase_2_ud_slice = [0 for i in range(self.max_moves_length)]
        self.phase_2_ud_slice[0] = self.coord_cube.phase_2_ud_slice

        # This stores the minimum number of moves required to complete phase 1 or 2 after n moves
        # and are derived from the pruning tables.
        self.phase_1_min_distance = [0 for i in range(self.max_moves_length)]
        self.phase_1_min_distance[0] = self._phase_1_heuristic(0)
        self.phase_2_min_distance = [0 for i in range(self.max_moves_length)]

        # Used for finding out which moves were calculated in phase 1 and phase 2
        self.phase_1_moves_index = 0
示例#27
0
def create_corpus(db="sqlite:///:memory:",
                  lang1method=lambda x: x,
                  lang2method=lambda x: x,
                  limit=None):
    engine = create_engine(db)
    # create session
    Session = sessionmaker(bind=engine)
    session = Session()

    Sentence = Tables().get_sentence_table()

    query = session.query(Sentence)[:limit] if limit \
        else session.query(Sentence)

    for item in query:
        yield {"lang1": lang1method(item.lang1),
               "lang2": lang2method(item.lang2)}
示例#28
0
 def addNewPerson(screen, name="", team="", favDrink=None, PMUDrink="N/A"):
     UI.clearScreen(screen)
     curses.nocbreak()
     screen.keypad(False)
     curses.echo()
     if not name:
         name = UI.cursedInput(screen, "Enter name of person: ")
     if not team:
         team = UI.cursedInput(screen, "Enter name of team person is in: ")
     if not favDrink:
         if State._drinks != []:
             favDrink = tables.handleSingleSelectTable(
                 screen, "Drinks", State._drinks, "", 0,
                 "Select this persons favorite drink")
         elif favDrink == None:
             UI.clearScreen(screen)
             screen.addstr("No drinks found. You can assign drink later...")
             screen.getch()
     State._people.append(Person(name, team, favDrink, PMUDrink))
示例#29
0
    def GetFileLocation(self, request, context):
        my_reply = file_transfer_proto.FileLocationInfo()
        file_name = request.fileName
        is_file_found = True
        if file_name not in Tables.TABLE_FILE_INFO.keys():
            # is_file_found = False
            return my_reply

        max_chunks = len(Tables.TABLE_FILE_INFO[file_name].keys())
        log_info("max_chunks from raft ", max_chunks)
        lst_proxies = Tables.get_all_available_proxies()
        lst_proxy_info = []
        for ip, port in lst_proxies:
            proxy_info = file_transfer_proto.ProxyInfo()
            proxy_info.ip = ip
            proxy_info.port = port
            lst_proxy_info.append(proxy_info)

        my_reply.fileName = file_name
        my_reply.maxChunks = max_chunks
        my_reply.lstProxy.extend(lst_proxy_info)
        my_reply.isFileFound = is_file_found
        return my_reply
示例#30
0
文件: setup.py 项目: thouis/icon
        i += 1
        if i > 20:
            break

    # store the project
    DB.storeProject( project )


#---------------------------------------------------------------------------
# Entry point to the main function of the program.
#---------------------------------------------------------------------------
if __name__ == '__main__':
    print 'Icon database (installation interface)'

    # start in a blank slate
    Tables.drop();

    # install the tables
    Tables.create()

    # install mlp project
    p             = Project( id='testmlp', type='MLP')
    p.batchSize   = 16
    p.patchSize   = 39
    p.hiddenUnits = [500,500,500]
    install( p )

    # install cnn project	
    cnn              = Project( id='testcnn', type='CNN')
    cnn.trainTime    = 30
    cnn.learningRate = 0.1
示例#31
0
 def createConducts(self, a):
     self.a = a
     t = Tables()
     sql = t.schoolConducts(self.a)
     self.logConnect(sql)