def TestNN(self): options = {} options['defaultextension'] = '.txt' options['filetypes'] = [('all files', '.*'), ('text files', '.txt')] options['initialdir'] = os.getcwd() options['initialfile'] = 'myfile.txt' options['parent'] = self.root options['title'] = 'This is a title' p = askopenfile(**options) Net = NET(1, 1) Net.LoadNet(p.name) d = DialogTestNN(self.root) res = IntListToTrans_52(d.value) #print len(res) t = Net.TestNet(res) print("sizeinput = %s\n out = %s" % (Net.inputsize, t)) t1 = t.tolist() s1 = [] for i in xrange(len(t1)): s1.append((float(t1[i]), i + 1)) dd = dict(s1) s = dd.keys() s = np.sort(np.array(s, dtype="float")) r = [] for i in range(1, 7): r.append(dd[s[-i]]) print(dd[s[-i]], s[-i]) k = "%s\n%s" % (TransToIntList(t.tolist()), r) self.TestText.set("%s" % k)
def start(subjects, generations, trials, games, xoro, testNet=False, goodNet=0): PERCENTAGE = .95 nets = [NET.Network([9, 30, 9]) for x in range(0, subjects)] old_maximum = -1 maximum = 0 maximum_wins = -1 cost = 0 best = 0 start_time = time.time() for b in range(0, generations): print b maximum_wins = -1 for x in nets: if testNet: #If network to test against wins = x.evaluate(trails, games, xoro, True, goodNet) else: #If playing against random guesser wins = x.evaluate(trials, games, xoro) if wins[0][0] > maximum_wins: maximum_wins = wins[0][0] best = copy.deepcopy(x) print maximum_wins, float(maximum_wins) / float(games), round( (time.time() - start_time), 5), b nets = [best] for amount in range(0, subjects): nets.append(procreate(best, PERCENTAGE)) #Create a new generation targetw = open("bestw", 'w') targetb = open("bestb", 'w') pickle.dump(best.biases, targetb) pickle.dump(best.weights, targetw) targetw.close() targetb.close() return best
def answering_who(cleansedQuestion, stop_words_free_question, sentence_list): # Declaring globals to be used in this function wordmatch_score_list = [] sent_containing_person_score_list = [] sent_containing_name_score_list = [] sent_containing_person_and_name_score_list = [] sent_containing_person_or_name_score_list = [] master_person_list = [] sent_score_list = [] #print 'Question is :',cleansedQuestion snowball_stemmer = SnowballStemmer('english') for i in range(0, len(sentence_list)): #print 'Sentence is :', sentence_list[i] score = 0 # 1. Score using word match rule wordmatch_score_list.append( WM.stemWordMatch(cleansedQuestion, sentence_list[i])) score = score + WM.stemWordMatch(cleansedQuestion, sentence_list[i]) # 2. If question does not contain name but the answer contains NAME then you are confident(+6) q_person_list, org_list, loc_list, time_list, prof_list = NET.named_entity_tagging( cleansedQuestion) if q_person_list == []: sent_plist, sent_olist, sent_llist, sent_tlist, sent_proflist = NET.named_entity_tagging( sentence_list[i]) master_person_list.append((sent_plist, i)) if sent_plist != []: score = score + 6 * len(sent_plist) # 3. If question does not contain a name and answer contains the word "name" then good_clue (+4) temp = sentence_list[i].split() for k in range(0, len(temp)): if snowball_stemmer.stem(temp[k].lower()) == 'name': score = score + 4 else: #Question has a name, and if the sentence contains the same name, then it is a good clue. # 4. Awards points to all sentences that contain a name or reference to a human sent_plist, sent_olist, sent_llist, sent_tlist, sent_proflist = NET.named_entity_tagging( sentence_list[i]) master_person_list.append(sent_plist) if sent_plist == q_person_list: score = score + 4 * len(sent_plist) elif sent_plist != [] or "name" in sentence_list[i]: score = score + 4 '''if sent_plist==[] and "name" in sentence_list[i]: sent_containing_name_score_list.append(4) else: sent_containing_name_score_list.append(0)''' sent_score_list.append(score) #print 'Sent score list is :',sent_score_list #print 'Master person list is:',master_person_list # Selecting the sentence that has the maximum score. If it is a tie, we choose the sentence that appears first # Preference is given to sentences which have a person name in them. If there is only one such sentence that is the answer candidate_list = [] final_result_set = [] temp_list = [] max_score_value = max(sent_score_list) #print 'Max score is :',max_score_value for i in range(0, len(sentence_list)): if sent_score_list[i] == max_score_value: candidate_list.append((sentence_list[i], i)) #print 'Candidate list is :',candidate_list if len(candidate_list) == 1: q_plist, q_olist, q_llist, q_tlist, q_proflist = NET.named_entity_tagging( stop_words_free_question) #If the question has a profession but not name of person, then the answer sentence should/would most probably #the name of a person #print 'Question Person List',q_plist if q_plist == [] or q_proflist != []: #temp_result=master_person_list[candidate_list[0][1]][0] s_plist, s_olist, s_llist, s_tlist, s_proflist = NET.named_entity_tagging( candidate_list[0][0]) result = ' '.join(s_plist) print 'Answer: ', result + '\n' #print '\n' return result elif q_plist != [] or q_proflist != []: #print candidate_list[0][1] s_plist, s_olist, s_llist, s_tlist, s_proflist = NET.named_entity_tagging( candidate_list[0][0]) result = ' '.join(s_plist) print 'Answer: ', result + '\n' #print '\n' return result elif q_plist != [] or q_proflist == []: # Implies question has a name. So pick a sentence which has the same name in sentence which is present in question # result = candidate_list[0][0] print 'Answer: ', result + '\n' #print '\n' return result else: # There are more than one candidate sentences. Print the first sentence for k in range(0, len(candidate_list)): val = candidate_list[k][0] #print 'val is :',val index = candidate_list[k][1] #print 'index is :', index temp_list.append(index) break #result=' '.join(temp_list) x = master_person_list[temp_list[0]] #print 'x is :', x result2 = temp_list[0] #for i in range(0,len(x)): if x != []: temp = ' '.join(x[0]) if temp not in stop_words_free_question: final_result_set.append(temp) else: final_result_set.append(val) if final_result_set != []: print 'Answer: ', ' '.join(final_result_set) + '\n' #print '\n' #print 'Result 2 is :',result2 return ' '.join(final_result_set) else: print 'Answer: ', temp + '\n' #print '\n' return temp #' '.join(x) # Checking to see if the question contains profession name. If so the answer should be a sentence containing a name and higher weights # is given for the score from Rule 2. Else Rule 1 and Rule 2 are given equal weightage. '''q_plist,q_olist,q_llist,q_tlist,q_proflist=NET.named_entity_tagging(stop_words_free_question)
def answering_who(cleansedQuestion,stop_words_free_question,sentence_list): # Declaring globals to be used in this function wordmatch_score_list=[] sent_containing_person_score_list=[] sent_containing_name_score_list=[] sent_containing_person_and_name_score_list=[] sent_containing_person_or_name_score_list=[] master_person_list=[] sent_score_list=[] #print 'Question is :',cleansedQuestion snowball_stemmer = SnowballStemmer('english') for i in range(0, len(sentence_list)): #print 'Sentence is :', sentence_list[i] score=0 # 1. Score using word match rule wordmatch_score_list.append(WM.stemWordMatch(cleansedQuestion,sentence_list[i])) score=score + WM.stemWordMatch(cleansedQuestion,sentence_list[i]) # 2. If question does not contain name but the answer contains NAME then you are confident(+6) q_person_list,org_list,loc_list,time_list,prof_list = NET.named_entity_tagging(cleansedQuestion) if q_person_list==[]: sent_plist,sent_olist,sent_llist,sent_tlist,sent_proflist=NET.named_entity_tagging(sentence_list[i]) master_person_list.append((sent_plist,i)) if sent_plist !=[]: score=score + 6*len(sent_plist) # 3. If question does not contain a name and answer contains the word "name" then good_clue (+4) temp= sentence_list[i].split() for k in range(0,len(temp)): if snowball_stemmer.stem(temp[k].lower())=='name': score=score +4 else: #Question has a name, and if the sentence contains the same name, then it is a good clue. # 4. Awards points to all sentences that contain a name or reference to a human sent_plist,sent_olist,sent_llist,sent_tlist,sent_proflist=NET.named_entity_tagging(sentence_list[i]) master_person_list.append(sent_plist) if sent_plist==q_person_list: score=score+4*len(sent_plist) elif sent_plist != [] or "name" in sentence_list[i]: score=score+4 '''if sent_plist==[] and "name" in sentence_list[i]: sent_containing_name_score_list.append(4) else: sent_containing_name_score_list.append(0)''' sent_score_list.append(score) #print 'Sent score list is :',sent_score_list #print 'Master person list is:',master_person_list # Selecting the sentence that has the maximum score. If it is a tie, we choose the sentence that appears first # Preference is given to sentences which have a person name in them. If there is only one such sentence that is the answer candidate_list=[] final_result_set=[] temp_list=[] max_score_value=max(sent_score_list) #print 'Max score is :',max_score_value for i in range(0, len(sentence_list)): if sent_score_list[i]==max_score_value: candidate_list.append((sentence_list[i],i)) #print 'Candidate list is :',candidate_list if len(candidate_list)==1: q_plist,q_olist,q_llist,q_tlist,q_proflist=NET.named_entity_tagging(stop_words_free_question) #If the question has a profession but not name of person, then the answer sentence should/would most probably #the name of a person #print 'Question Person List',q_plist if q_plist == [] or q_proflist != []: #temp_result=master_person_list[candidate_list[0][1]][0] s_plist,s_olist,s_llist,s_tlist,s_proflist=NET.named_entity_tagging(candidate_list[0][0]) result= ' '.join(s_plist) print 'Answer: ',result+'\n' #print '\n' return result elif q_plist != [] or q_proflist != []: #print candidate_list[0][1] s_plist,s_olist,s_llist,s_tlist,s_proflist=NET.named_entity_tagging(candidate_list[0][0]) result= ' '.join(s_plist) print 'Answer: ',result+'\n' #print '\n' return result elif q_plist != [] or q_proflist == []: # Implies question has a name. So pick a sentence which has the same name in sentence which is present in question # result=candidate_list[0][0] print 'Answer: ',result+'\n' #print '\n' return result else: # There are more than one candidate sentences. Print the first sentence for k in range(0, len(candidate_list)): val=candidate_list[k][0] #print 'val is :',val index=candidate_list[k][1] #print 'index is :', index temp_list.append(index) break #result=' '.join(temp_list) x= master_person_list[temp_list[0]] #print 'x is :', x result2 = temp_list[0] #for i in range(0,len(x)): if x != []: temp=' '.join(x[0]) if temp not in stop_words_free_question: final_result_set.append(temp) else: final_result_set.append(val) if final_result_set != []: print 'Answer: ',' '.join(final_result_set)+'\n' #print '\n' #print 'Result 2 is :',result2 return ' '.join(final_result_set) else: print 'Answer: ',temp+'\n' #print '\n' return temp #' '.join(x) # Checking to see if the question contains profession name. If so the answer should be a sentence containing a name and higher weights # is given for the score from Rule 2. Else Rule 1 and Rule 2 are given equal weightage. '''q_plist,q_olist,q_llist,q_tlist,q_proflist=NET.named_entity_tagging(stop_words_free_question)
def answering_when(cleansedQuestion, stop_words_free_question, sentence_list, dateline): # Declaring globals to be used in this function candidate_sent_list = [] sent_score_list = {} final_sent_list = [] when_year_verbs = ['play', 'fought'] #'win','lose','victorius'] when_time_values = [ 'january', 'jan', 'february', 'feb', 'march', 'mar', 'april', 'apr', 'may', 'may', 'june', 'jun', 'july', 'jul', 'august', 'aug', 'september', 'sep', 'october', 'oct', 'november', 'nov', 'december', 'dec', '1400', '1401', '1402', '1403', '1404', '1405', '1406', '1407', '1408', '1409', '1410', '1411', '1412', '1413', '1414', '1415', '1416', '1417', '1418', '1419', '1420', '1421', '1422', '1423', '1424', '1425', '1426', '1427', '1428', '1429', '1430', '1431', '1432', '1433', '1434', '1435', '1436', '1437', '1438', '1439', '1440', '1441', '1442', '1443', '1444', '1445', '1446', '1447', '1448', '1449', '1450', '1451', '1452', '1453', '1454', '1455', '1456', '1457', '1458', '1459', '1460', '1461', '1462', '1463', '1464', '1465', '1466', '1467', '1468', '1469', '1470', '1471', '1472', '1473', '1474', '1475', '1476', '1477', '1478', '1479', '1480', '1481', '1482', '1483', '1484', '1485', '1486', '1487', '1488', '1489', '1490', '1491', '1492', '1493', '1494', '1495', '1496', '1497', '1498', '1499', '1500', '1501', '1502', '1503', '1504', '1505', '1506', '1507', '1508', '1509', '1510', '1511', '1512', '1513', '1514', '1515', '1516', '1517', '1518', '1519', '1520', '1521', '1522', '1523', '1524', '1525', '1526', '1527', '1528', '1529', '1530', '1531', '1532', '1533', '1534', '1535', '1536', '1537', '1538', '1539', '1540', '1541', '1542', '1543', '1544', '1545', '1546', '1547', '1548', '1549', '1550', '1551', '1552', '1553', '1554', '1555', '1556', '1557', '1558', '1559', '1560', '1561', '1562', '1563', '1564', '1565', '1566', '1567', '1568', '1569', '1570', '1571', '1572', '1573', '1574', '1575', '1576', '1577', '1578', '1579', '1580', '1581', '1582', '1583', '1584', '1585', '1586', '1587', '1588', '1589', '1590', '1591', '1592', '1593', '1594', '1595', '1596', '1597', '1598', '1599', '1600', '1601', '1602', '1603', '1604', '1605', '1606', '1607', '1608', '1609', '1610', '1611', '1612', '1613', '1614', '1615', '1616', '1617', '1618', '1619', '1620', '1621', '1622', '1623', '1624', '1625', '1626', '1627', '1628', '1629', '1630', '1631', '1632', '1633', '1634', '1635', '1636', '1637', '1638', '1639', '1640', '1641', '1642', '1643', '1644', '1645', '1646', '1647', '1648', '1649', '1650', '1651', '1652', '1653', '1654', '1655', '1656', '1657', '1658', '1659', '1660', '1661', '1662', '1663', '1664', '1665', '1666', '1667', '1668', '1669', '1670', '1671', '1672', '1673', '1674', '1675', '1676', '1677', '1678', '1679', '1680', '1681', '1682', '1683', '1684', '1685', '1686', '1687', '1688', '1689', '1690', '1691', '1692', '1693', '1694', '1695', '1696', '1697', '1698', '1699', '1700', '1701', '1702', '1703', '1704', '1705', '1706', '1707', '1708', '1709', '1710', '1711', '1712', '1713', '1714', '1715', '1716', '1717', '1718', '1719', '1720', '1721', '1722', '1723', '1724', '1725', '1726', '1727', '1728', '1729', '1730', '1731', '1732', '1733', '1734', '1735', '1736', '1737', '1738', '1739', '1740', '1741', '1742', '1743', '1744', '1745', '1746', '1747', '1748', '1749', '1750', '1751', '1752', '1753', '1754', '1755', '1756', '1757', '1758', '1759', '1760', '1761', '1762', '1763', '1764', '1765', '1766', '1767', '1768', '1769', '1770', '1771', '1772', '1773', '1774', '1775', '1776', '1777', '1778', '1779', '1780', '1781', '1782', '1783', '1784', '1785', '1786', '1787', '1788', '1789', '1790', '1791', '1792', '1793', '1794', '1795', '1796', '1797', '1798', '1799', '1800', '1801', '1802', '1803', '1804', '1805', '1806', '1807', '1808', '1809', '1810', '1811', '1812', '1813', '1814', '1815', '1816', '1817', '1818', '1819', '1820', '1821', '1822', '1823', '1824', '1825', '1826', '1827', '1828', '1829', '1830', '1831', '1832', '1833', '1834', '1835', '1836', '1837', '1838', '1839', '1840', '1841', '1842', '1843', '1844', '1845', '1846', '1847', '1848', '1849', '1850', '1851', '1852', '1853', '1854', '1855', '1856', '1857', '1858', '1859', '1860', '1861', '1862', '1863', '1864', '1865', '1866', '1867', '1868', '1869', '1870', '1871', '1872', '1873', '1874', '1875', '1876', '1877', '1878', '1879', '1880', '1881', '1882', '1883', '1884', '1885', '1886', '1887', '1888', '1889', '1890', '1891', '1892', '1893', '1894', '1895', '1896', '1897', '1898', '1899', '1900', '1901', '1902', '1903', '1904', '1905', '1906', '1907', '1908', '1909', '1910', '1911', '1912', '1913', '1914', '1915', '1916', '1917', '1918', '1919', '1920', '1921', '1922', '1923', '1924', '1925', '1926', '1927', '1928', '1929', '1930', '1931', '1932', '1933', '1934', '1935', '1936', '1937', '1938', '1939', '1940', '1941', '1942', '1943', '1944', '1945', '1946', '1947', '1948', '1949', '1950', '1951', '1952', '1953', '1954', '1955', '1956', '1957', '1958', '1959', '1960', '1961', '1962', '1963', '1964', '1965', '1966', '1967', '1968', '1969', '1970', '1971', '1972', '1973', '1974', '1975', '1976', '1977', '1978', '1979', '1980', '1981', '1982', '1983', '1984', '1985', '1986', '1987', '1988', '1989', '1990', '1991', '1992', '1993', '1994', '1995', '1996', '1997', '1998', '1999' ] #print 'Question is :',cleansedQuestion # 1. Check if the sentence contains "TIME" expression for i in range(0, len(sentence_list)): score = 0 person_list, org_list, loc_list, time_list, prof_list = NET.named_entity_tagging( sentence_list[i]) if time_list != []: # Sentence contains a time expression candidate_sent_list.append(sentence_list[i]) # Now compute the wordmatch score score = score + 4 + WM.stemWordMatch(cleansedQuestion, sentence_list[i]) #sent_score_list.append((score,i)) # 2. Check if the Question contains "the last" and sentence contains any of "first,last,since,ago", then score+= slam_dunk for word in cleansedQuestion: if word.lower() in ['the', 'last']: for sent in sentence_list[i]: if sent in ['first', 'last', 'since', 'ago']: score = score + 20 # 3. If the question contains {start,begin} and sentence contains {start,begin,since,year} if word.lower() in ['start', 'begin']: for sent in sentence_list[i]: if sent in ['start', 'begin', 'since', 'year']: score = score + 20 sent_score_list[i] = score #print 'Candidate sentences list is :',candidate_sent_list #print 'Sent score list is :', sent_score_list # For when and where questions the answer to the question could also be from the timeline of the story dateline_score = 0 for i in range(0, len(cleansedQuestion)): # 1. If question contains "happen", it is a good clue that timeline could be answer if cleansedQuestion[i].lower() == 'happen': dateline_score = dateline_score + 4 # 2. If question contains "take place", it is a good clue that timeline could be answer if i != len(cleansedQuestion) - 1 and cleansedQuestion[i].lower( ) == 'take' and cleansedQuestion[i + 1].lower() == 'place': dateline_score = dateline_score + 4 # 3. If question contains "this", it is slam_dunk that timeline could be answer if cleansedQuestion[i].lower() == 'this': dateline_score = dateline_score + 12 # 4. If question contains "story", it is slam_dunk that timeline could be answer if cleansedQuestion[i].lower() == 'story': dateline_score = dateline_score + 12 #print 'Date line score for the question is :',dateline_score # Selecting the sentence that has the maximum score. If the dateline score is greater than max of sent_score choose # dateline_score else choose the maximum score from sent_score_list max_score_index = max(sent_score_list, key=lambda i: sent_score_list[i]) score_values = sent_score_list.values() max_score_value = max(score_values) #print 'Max value is :', max_score_value # Checking which of the scores is greater. IF score from sent_Score_list is greater than dateline score, then we find # the corresponding sentences and choose the best among them. Else we return the dateline as the result. if max_score_value > dateline_score: # Now we have to choose the best sentence among the sentences in candidate list # First step is to parse the stop-words free question and look for words in the question which might help us find #the answer #print 'Stopwords free question :', stop_words_free_question '''for i in stop_words_free_question: if i in when_year: final_sent_list.append(''' # Giving preference to sentences which contain a year value # for i in sent_score_list.keys(): '''temp=sentence_list[i].split() for j in range(0, len(temp)): if j in when_year: print 'Year is true' #final_sent_list.append(sentence_list[i]) final_sent_list.append(j)''' # If none of the sentences contain a year, then choose the one with maximum value if sent_score_list[i] == max_score_value: final_sent_list.append(sentence_list[i]) #print 'Final sentence list is:',final_sent_list # Now from the sentences extracting out the years or the date /time values alone and representing them final_temp_list = [] if len(final_sent_list) == 1: temp = nltk.word_tokenize(final_sent_list[0]) for j in range(0, len(temp)): if temp[j].lower() in when_time_values: #print 'year true' final_temp_list.append(temp[j]) if final_temp_list != []: result = ' '.join(final_temp_list) print 'Answer: ', result + '\n' #print '\n' return result else: print 'Answer: ', final_sent_list[0] + '\n' #print '\n' return final_sent_list[0] else: for i in range(0, len(final_sent_list)): temp = nltk.word_tokenize(final_sent_list[i]) for j in range(0, len(temp)): if temp[j].lower() in when_time_values: #print 'year true' final_temp_list.append(temp[j]) if final_temp_list != []: result = ' '.join(final_temp_list) print 'Answer: ', result + '\n' #print '\n' return result else: print 'Answer: ', ' '.join(final_sent_list) + '\n' #print '\n' return ' '.join(final_sent_list) else: result = dateline print 'Answer: ', result + '\n' #print '\n' return result
def answering_what(cleansedQuestion,stop_words_free_question,complete_sentence_list,sentence_list,dateline): # Declaring globals to be used in this function candidate_sent_list=[] sent_score_list=[] final_sent_list=[] master_loc_list=[] location_prepositions=['in','at','near','inside','on','behind','above','under','next to','below','between','around', 'outside','among','on the right', 'across','front','opposite','before','beneath','beside','against'] what_year=['1400', '1401', '1402', '1403', '1404', '1405', '1406', '1407', '1408', '1409', '1410', '1411', '1412', '1413', '1414', '1415', '1416', '1417', '1418', '1419', '1420', '1421', '1422', '1423', '1424', '1425', '1426', '1427', '1428', '1429', '1430', '1431', '1432', '1433', '1434', '1435', '1436', '1437', '1438', '1439', '1440', '1441', '1442', '1443', '1444', '1445', '1446', '1447', '1448', '1449', '1450', '1451', '1452', '1453', '1454', '1455', '1456', '1457', '1458', '1459', '1460', '1461', '1462', '1463', '1464', '1465', '1466', '1467', '1468', '1469', '1470', '1471', '1472', '1473', '1474', '1475', '1476', '1477', '1478', '1479', '1480', '1481', '1482', '1483', '1484', '1485', '1486', '1487', '1488', '1489', '1490', '1491', '1492', '1493', '1494', '1495', '1496', '1497', '1498', '1499', '1500', '1501', '1502', '1503', '1504', '1505', '1506', '1507', '1508', '1509', '1510', '1511', '1512', '1513', '1514', '1515', '1516', '1517', '1518', '1519', '1520', '1521', '1522', '1523', '1524', '1525', '1526', '1527', '1528', '1529', '1530', '1531', '1532', '1533', '1534', '1535', '1536', '1537', '1538', '1539', '1540', '1541', '1542', '1543', '1544', '1545', '1546', '1547', '1548', '1549', '1550', '1551', '1552', '1553', '1554', '1555', '1556', '1557', '1558', '1559', '1560', '1561', '1562', '1563', '1564', '1565', '1566', '1567', '1568', '1569', '1570', '1571', '1572', '1573', '1574', '1575', '1576', '1577', '1578', '1579', '1580', '1581', '1582', '1583', '1584', '1585', '1586', '1587', '1588', '1589', '1590', '1591', '1592', '1593', '1594', '1595', '1596', '1597', '1598', '1599', '1600', '1601', '1602', '1603', '1604', '1605', '1606', '1607', '1608', '1609', '1610', '1611', '1612', '1613', '1614', '1615', '1616', '1617', '1618', '1619', '1620', '1621', '1622', '1623', '1624', '1625', '1626', '1627', '1628', '1629', '1630', '1631', '1632', '1633', '1634', '1635', '1636', '1637', '1638', '1639', '1640', '1641', '1642', '1643', '1644', '1645', '1646', '1647', '1648', '1649', '1650', '1651', '1652', '1653', '1654', '1655', '1656', '1657', '1658', '1659', '1660', '1661', '1662', '1663', '1664', '1665', '1666', '1667', '1668', '1669', '1670', '1671', '1672', '1673', '1674', '1675', '1676', '1677', '1678', '1679', '1680', '1681', '1682', '1683', '1684', '1685', '1686', '1687', '1688', '1689', '1690', '1691', '1692', '1693', '1694', '1695', '1696', '1697', '1698', '1699', '1700', '1701', '1702', '1703', '1704', '1705', '1706', '1707', '1708', '1709', '1710', '1711', '1712', '1713', '1714', '1715', '1716', '1717', '1718', '1719', '1720', '1721', '1722', '1723', '1724', '1725', '1726', '1727', '1728', '1729', '1730', '1731', '1732', '1733', '1734', '1735', '1736', '1737', '1738', '1739', '1740', '1741', '1742', '1743', '1744', '1745', '1746', '1747', '1748', '1749', '1750', '1751', '1752', '1753', '1754', '1755', '1756', '1757', '1758', '1759', '1760', '1761', '1762', '1763', '1764', '1765', '1766', '1767', '1768', '1769', '1770', '1771', '1772', '1773', '1774', '1775', '1776', '1777', '1778', '1779', '1780', '1781', '1782', '1783', '1784', '1785', '1786', '1787', '1788', '1789', '1790', '1791', '1792', '1793', '1794', '1795', '1796', '1797', '1798', '1799', '1800', '1801', '1802', '1803', '1804', '1805', '1806', '1807', '1808', '1809', '1810', '1811', '1812', '1813', '1814', '1815', '1816', '1817', '1818', '1819', '1820', '1821', '1822', '1823', '1824', '1825', '1826', '1827', '1828', '1829', '1830', '1831', '1832', '1833', '1834', '1835', '1836', '1837', '1838', '1839', '1840', '1841', '1842', '1843', '1844', '1845', '1846', '1847', '1848', '1849', '1850', '1851', '1852', '1853', '1854', '1855', '1856', '1857', '1858', '1859', '1860', '1861', '1862', '1863', '1864', '1865', '1866', '1867', '1868', '1869', '1870', '1871', '1872', '1873', '1874', '1875', '1876', '1877', '1878', '1879', '1880', '1881', '1882', '1883', '1884', '1885', '1886', '1887', '1888', '1889', '1890', '1891', '1892', '1893', '1894', '1895', '1896', '1897', '1898', '1899', '1900', '1901', '1902', '1903', '1904', '1905', '1906', '1907', '1908', '1909', '1910', '1911', '1912', '1913', '1914', '1915', '1916', '1917', '1918', '1919', '1920', '1921', '1922', '1923', '1924', '1925', '1926', '1927', '1928', '1929', '1930', '1931', '1932', '1933', '1934', '1935', '1936', '1937', '1938', '1939', '1940', '1941', '1942', '1943', '1944', '1945', '1946', '1947', '1948', '1949', '1950', '1951', '1952', '1953', '1954', '1955', '1956', '1957', '1958', '1959', '1960', '1961', '1962', '1963', '1964', '1965', '1966', '1967', '1968', '1969', '1970', '1971', '1972', '1973', '1974', '1975', '1976', '1977', '1978', '1979', '1980', '1981', '1982', '1983', '1984', '1985', '1986', '1987', '1988', '1989', '1990', '1991', '1992', '1993', '1994', '1995', '1996', '1997', '1998', '1999'] what_month=['january','jan', 'february', 'feb', 'march', 'mar', 'april', 'apr', 'may','may', 'june', 'jun', 'july', 'jul','august','aug','september','sep','october','oct','november','nov','december','dec'] date_expression_list=['yesterday','today','tomorrow','last week','this week','next week','an hour ago','now','in an hour', 'recently','soon','a little while ago','at this moment','in the near future','a long time ago','these days', 'those days','future','present','past','nowadays','eventually','morning', 'evening','night','midnight','dawn','dusk','afternoon','noon','midday', 'am','pm','sunrise','sunset','lunchtime','teatime','dinnertime','interval','twilight', 'hourly','nightly','daily','monthly','weekly','quarterly','yearly'] #print 'Question is :',cleansedQuestion snowball_stemmer = SnowballStemmer('english') # 1. Find score for each sentence using word march score first for i in range(0,len(complete_sentence_list)): score=0 score = score + WM.stemWordMatch(cleansedQuestion,sentence_list[i]) #2. Check if the question contains a month of the year and sentence contains date expression,then it is a clue temp=cleansedQuestion.split() temp=nltk.word_tokenize(stop_words_free_question) flag=0 for j in range(0, len(temp)): if temp[j].lower() in what_month: temp2=sentence_list[i].split() for k in range(0,len(temp2)): if temp2[k] in date_expression_list: count=count+4 # 3. What "kind" questions. Sentences containing "call" or "from" if temp[j].lower() =='kind': temp2=sentence_list[i].split() for k in range(0,len(temp2)): if snowball_stemmer.stem(temp2[k]) in ['call','from']: count=count+6 # 4. If question contains "name" and the sentence contains {name,call,known} if temp[j].lower() =='name': temp2=complete_sentence_list[i].split() for k in range(0,len(temp2)): if snowball_stemmer.stem(temp2[k]) in ['name','call','known']: score=score+20 #5. If question contains name + PP and contains(S,ProperNoun) and Head PP if j != len(temp) -1 and temp[j]=='name' and temp[j+1] in ['of','for']: person_list,org_list,loc_list,time_list,prof_list = NET.named_entity_tagging(sentence_list[i]) if person_list != []: #TODO Check if it also contains (proper_noun,head(PP)) score=score +20 # If the question contains "sport" related terms, answer should also have sport related terms '''if temp[j].lower() in ['sports','games','olympics']: temp2=sentence_list[i].split() for k in range(0,len(temp2)): if snowball_stemmer.stem(temp2[k]) in ['soccer','hockey','baseball','cricket','rugby','ultimate']: score=score+6''' # If the sentence contains a "country" name and the sentence contains a LOCATION, then it is confident score if temp[j].lower() in ['country','countries','olympics']: person_list,org_list,loc_list,time_list,prof_list = NET.named_entity_tagging(sentence_list[i]) if loc_list != []: score=score + 6*len(loc_list) # Confidence score increases with increasing number of countries appearing in the sentence. sent_score_list.append(score) #print 'Sent score list values are:',sent_score_list # Selecting the sentence that has the maximum score. max_score_value =max(sent_score_list) #print 'Max value is :', max_score_value # Now we have to choose the best sentence among the sentences in candidate list.Choosing sentences # which have both maximum value and present in candidate list for i in range(0, len(sent_score_list)): if sent_score_list[i]==max_score_value: final_sent_list.append(complete_sentence_list[i]) #print 'Final list is:', final_sent_list temp_solution=[] answer_loc=[] if len(final_sent_list) == 1: print 'Answer: ',final_sent_list[0] +'\n' #print '\n' return final_sent_list[0] else: for i in range(0,len(final_sent_list)): temp=final_sent_list[i] break #result=' '.join(final_sent_list) result=temp print 'Answer: ', result +'\n' #print '\n' return result
def __init__(self,netPath = cfg.MODEL_PATH): super().__init__() self.net = NET.MainNet(cfg.CLASS_NUM).to(cfg.device) self.net.load_state_dict(torch.load(netPath)) self.net.eval()
loss_obj_cls = cls_loss_fn(output_obj[:, 5:], target_obj[:, 5].long()) loss_obj = loss_obj_conf + loss_obj_center + loss_obj_wh + loss_obj_cls #负样本 los_noobj = con_loss_fn(output_noobj[:, 4], target_noobj[:, 4]) return alpaha * loss_obj + (1 - alpaha) * los_noobj if __name__ == '__main__': data_sets = dateset.MyDataset() train_loader = DataLoader(data_sets, batch_size=4, shuffle=True) writer = SummaryWriter() net = NET.MainNet(cfg.CLASS_NUM).to(cfg.device) if os.path.exists(cfg.MODEL_PATH): print("netLoad") net.load_state_dict(torch.load(cfg.MODEL_PATH)) # for name, value in net.named_parameters(): # if name[:8] == "trunk_13": # print(value) opt = torch.optim.Adam(net.parameters()) loss = None for epoch in range(100): for target_13, target_26, target_52, imgData in train_loader: ouput_13, ouput_26, ouput_52 = net(imgData)
def answering_what(cleansedQuestion, stop_words_free_question, complete_sentence_list, sentence_list, dateline): # Declaring globals to be used in this function candidate_sent_list = [] sent_score_list = [] final_sent_list = [] master_loc_list = [] location_prepositions = [ 'in', 'at', 'near', 'inside', 'on', 'behind', 'above', 'under', 'next to', 'below', 'between', 'around', 'outside', 'among', 'on the right', 'across', 'front', 'opposite', 'before', 'beneath', 'beside', 'against' ] what_year = [ '1400', '1401', '1402', '1403', '1404', '1405', '1406', '1407', '1408', '1409', '1410', '1411', '1412', '1413', '1414', '1415', '1416', '1417', '1418', '1419', '1420', '1421', '1422', '1423', '1424', '1425', '1426', '1427', '1428', '1429', '1430', '1431', '1432', '1433', '1434', '1435', '1436', '1437', '1438', '1439', '1440', '1441', '1442', '1443', '1444', '1445', '1446', '1447', '1448', '1449', '1450', '1451', '1452', '1453', '1454', '1455', '1456', '1457', '1458', '1459', '1460', '1461', '1462', '1463', '1464', '1465', '1466', '1467', '1468', '1469', '1470', '1471', '1472', '1473', '1474', '1475', '1476', '1477', '1478', '1479', '1480', '1481', '1482', '1483', '1484', '1485', '1486', '1487', '1488', '1489', '1490', '1491', '1492', '1493', '1494', '1495', '1496', '1497', '1498', '1499', '1500', '1501', '1502', '1503', '1504', '1505', '1506', '1507', '1508', '1509', '1510', '1511', '1512', '1513', '1514', '1515', '1516', '1517', '1518', '1519', '1520', '1521', '1522', '1523', '1524', '1525', '1526', '1527', '1528', '1529', '1530', '1531', '1532', '1533', '1534', '1535', '1536', '1537', '1538', '1539', '1540', '1541', '1542', '1543', '1544', '1545', '1546', '1547', '1548', '1549', '1550', '1551', '1552', '1553', '1554', '1555', '1556', '1557', '1558', '1559', '1560', '1561', '1562', '1563', '1564', '1565', '1566', '1567', '1568', '1569', '1570', '1571', '1572', '1573', '1574', '1575', '1576', '1577', '1578', '1579', '1580', '1581', '1582', '1583', '1584', '1585', '1586', '1587', '1588', '1589', '1590', '1591', '1592', '1593', '1594', '1595', '1596', '1597', '1598', '1599', '1600', '1601', '1602', '1603', '1604', '1605', '1606', '1607', '1608', '1609', '1610', '1611', '1612', '1613', '1614', '1615', '1616', '1617', '1618', '1619', '1620', '1621', '1622', '1623', '1624', '1625', '1626', '1627', '1628', '1629', '1630', '1631', '1632', '1633', '1634', '1635', '1636', '1637', '1638', '1639', '1640', '1641', '1642', '1643', '1644', '1645', '1646', '1647', '1648', '1649', '1650', '1651', '1652', '1653', '1654', '1655', '1656', '1657', '1658', '1659', '1660', '1661', '1662', '1663', '1664', '1665', '1666', '1667', '1668', '1669', '1670', '1671', '1672', '1673', '1674', '1675', '1676', '1677', '1678', '1679', '1680', '1681', '1682', '1683', '1684', '1685', '1686', '1687', '1688', '1689', '1690', '1691', '1692', '1693', '1694', '1695', '1696', '1697', '1698', '1699', '1700', '1701', '1702', '1703', '1704', '1705', '1706', '1707', '1708', '1709', '1710', '1711', '1712', '1713', '1714', '1715', '1716', '1717', '1718', '1719', '1720', '1721', '1722', '1723', '1724', '1725', '1726', '1727', '1728', '1729', '1730', '1731', '1732', '1733', '1734', '1735', '1736', '1737', '1738', '1739', '1740', '1741', '1742', '1743', '1744', '1745', '1746', '1747', '1748', '1749', '1750', '1751', '1752', '1753', '1754', '1755', '1756', '1757', '1758', '1759', '1760', '1761', '1762', '1763', '1764', '1765', '1766', '1767', '1768', '1769', '1770', '1771', '1772', '1773', '1774', '1775', '1776', '1777', '1778', '1779', '1780', '1781', '1782', '1783', '1784', '1785', '1786', '1787', '1788', '1789', '1790', '1791', '1792', '1793', '1794', '1795', '1796', '1797', '1798', '1799', '1800', '1801', '1802', '1803', '1804', '1805', '1806', '1807', '1808', '1809', '1810', '1811', '1812', '1813', '1814', '1815', '1816', '1817', '1818', '1819', '1820', '1821', '1822', '1823', '1824', '1825', '1826', '1827', '1828', '1829', '1830', '1831', '1832', '1833', '1834', '1835', '1836', '1837', '1838', '1839', '1840', '1841', '1842', '1843', '1844', '1845', '1846', '1847', '1848', '1849', '1850', '1851', '1852', '1853', '1854', '1855', '1856', '1857', '1858', '1859', '1860', '1861', '1862', '1863', '1864', '1865', '1866', '1867', '1868', '1869', '1870', '1871', '1872', '1873', '1874', '1875', '1876', '1877', '1878', '1879', '1880', '1881', '1882', '1883', '1884', '1885', '1886', '1887', '1888', '1889', '1890', '1891', '1892', '1893', '1894', '1895', '1896', '1897', '1898', '1899', '1900', '1901', '1902', '1903', '1904', '1905', '1906', '1907', '1908', '1909', '1910', '1911', '1912', '1913', '1914', '1915', '1916', '1917', '1918', '1919', '1920', '1921', '1922', '1923', '1924', '1925', '1926', '1927', '1928', '1929', '1930', '1931', '1932', '1933', '1934', '1935', '1936', '1937', '1938', '1939', '1940', '1941', '1942', '1943', '1944', '1945', '1946', '1947', '1948', '1949', '1950', '1951', '1952', '1953', '1954', '1955', '1956', '1957', '1958', '1959', '1960', '1961', '1962', '1963', '1964', '1965', '1966', '1967', '1968', '1969', '1970', '1971', '1972', '1973', '1974', '1975', '1976', '1977', '1978', '1979', '1980', '1981', '1982', '1983', '1984', '1985', '1986', '1987', '1988', '1989', '1990', '1991', '1992', '1993', '1994', '1995', '1996', '1997', '1998', '1999' ] what_month = [ 'january', 'jan', 'february', 'feb', 'march', 'mar', 'april', 'apr', 'may', 'may', 'june', 'jun', 'july', 'jul', 'august', 'aug', 'september', 'sep', 'october', 'oct', 'november', 'nov', 'december', 'dec' ] date_expression_list = [ 'yesterday', 'today', 'tomorrow', 'last week', 'this week', 'next week', 'an hour ago', 'now', 'in an hour', 'recently', 'soon', 'a little while ago', 'at this moment', 'in the near future', 'a long time ago', 'these days', 'those days', 'future', 'present', 'past', 'nowadays', 'eventually', 'morning', 'evening', 'night', 'midnight', 'dawn', 'dusk', 'afternoon', 'noon', 'midday', 'am', 'pm', 'sunrise', 'sunset', 'lunchtime', 'teatime', 'dinnertime', 'interval', 'twilight', 'hourly', 'nightly', 'daily', 'monthly', 'weekly', 'quarterly', 'yearly' ] #print 'Question is :',cleansedQuestion snowball_stemmer = SnowballStemmer('english') # 1. Find score for each sentence using word march score first for i in range(0, len(complete_sentence_list)): score = 0 score = score + WM.stemWordMatch(cleansedQuestion, sentence_list[i]) #2. Check if the question contains a month of the year and sentence contains date expression,then it is a clue temp = cleansedQuestion.split() temp = nltk.word_tokenize(stop_words_free_question) flag = 0 for j in range(0, len(temp)): if temp[j].lower() in what_month: temp2 = sentence_list[i].split() for k in range(0, len(temp2)): if temp2[k] in date_expression_list: count = count + 4 # 3. What "kind" questions. Sentences containing "call" or "from" if temp[j].lower() == 'kind': temp2 = sentence_list[i].split() for k in range(0, len(temp2)): if snowball_stemmer.stem(temp2[k]) in ['call', 'from']: count = count + 6 # 4. If question contains "name" and the sentence contains {name,call,known} if temp[j].lower() == 'name': temp2 = complete_sentence_list[i].split() for k in range(0, len(temp2)): if snowball_stemmer.stem( temp2[k]) in ['name', 'call', 'known']: score = score + 20 #5. If question contains name + PP and contains(S,ProperNoun) and Head PP if j != len(temp) - 1 and temp[j] == 'name' and temp[j + 1] in [ 'of', 'for' ]: person_list, org_list, loc_list, time_list, prof_list = NET.named_entity_tagging( sentence_list[i]) if person_list != []: #TODO Check if it also contains (proper_noun,head(PP)) score = score + 20 # If the question contains "sport" related terms, answer should also have sport related terms '''if temp[j].lower() in ['sports','games','olympics']: temp2=sentence_list[i].split() for k in range(0,len(temp2)): if snowball_stemmer.stem(temp2[k]) in ['soccer','hockey','baseball','cricket','rugby','ultimate']: score=score+6''' # If the sentence contains a "country" name and the sentence contains a LOCATION, then it is confident score if temp[j].lower() in ['country', 'countries', 'olympics']: person_list, org_list, loc_list, time_list, prof_list = NET.named_entity_tagging( sentence_list[i]) if loc_list != []: score = score + 6 * len( loc_list ) # Confidence score increases with increasing number of countries appearing in the sentence. sent_score_list.append(score) #print 'Sent score list values are:',sent_score_list # Selecting the sentence that has the maximum score. max_score_value = max(sent_score_list) #print 'Max value is :', max_score_value # Now we have to choose the best sentence among the sentences in candidate list.Choosing sentences # which have both maximum value and present in candidate list for i in range(0, len(sent_score_list)): if sent_score_list[i] == max_score_value: final_sent_list.append(complete_sentence_list[i]) #print 'Final list is:', final_sent_list temp_solution = [] answer_loc = [] if len(final_sent_list) == 1: print 'Answer: ', final_sent_list[0] + '\n' #print '\n' return final_sent_list[0] else: for i in range(0, len(final_sent_list)): temp = final_sent_list[i] break #result=' '.join(final_sent_list) result = temp print 'Answer: ', result + '\n' #print '\n' return result
def answering_where(cleansedQuestion, stop_words_free_question, complete_sentence_list, sentence_list, dateline): # Declaring globals to be used in this function candidate_sent_list = [] sent_score_list = [] final_sent_list = [] master_loc_list = [] location_prepositions = [ 'in', 'at', 'near', 'inside', 'on', 'behind', 'above', 'under', 'next to', 'below', 'between', 'around', 'outside', 'among', 'on the right', 'across', 'front', 'opposite', 'before', 'beneath', 'beside', 'against' ] when_year_verbs = ['play', 'fought'] #'win','lose','victorius'] when_year = [ '1400', '1401', '1402', '1403', '1404', '1405', '1406', '1407', '1408', '1409', '1410', '1411', '1412', '1413', '1414', '1415', '1416', '1417', '1418', '1419', '1420', '1421', '1422', '1423', '1424', '1425', '1426', '1427', '1428', '1429', '1430', '1431', '1432', '1433', '1434', '1435', '1436', '1437', '1438', '1439', '1440', '1441', '1442', '1443', '1444', '1445', '1446', '1447', '1448', '1449', '1450', '1451', '1452', '1453', '1454', '1455', '1456', '1457', '1458', '1459', '1460', '1461', '1462', '1463', '1464', '1465', '1466', '1467', '1468', '1469', '1470', '1471', '1472', '1473', '1474', '1475', '1476', '1477', '1478', '1479', '1480', '1481', '1482', '1483', '1484', '1485', '1486', '1487', '1488', '1489', '1490', '1491', '1492', '1493', '1494', '1495', '1496', '1497', '1498', '1499', '1500', '1501', '1502', '1503', '1504', '1505', '1506', '1507', '1508', '1509', '1510', '1511', '1512', '1513', '1514', '1515', '1516', '1517', '1518', '1519', '1520', '1521', '1522', '1523', '1524', '1525', '1526', '1527', '1528', '1529', '1530', '1531', '1532', '1533', '1534', '1535', '1536', '1537', '1538', '1539', '1540', '1541', '1542', '1543', '1544', '1545', '1546', '1547', '1548', '1549', '1550', '1551', '1552', '1553', '1554', '1555', '1556', '1557', '1558', '1559', '1560', '1561', '1562', '1563', '1564', '1565', '1566', '1567', '1568', '1569', '1570', '1571', '1572', '1573', '1574', '1575', '1576', '1577', '1578', '1579', '1580', '1581', '1582', '1583', '1584', '1585', '1586', '1587', '1588', '1589', '1590', '1591', '1592', '1593', '1594', '1595', '1596', '1597', '1598', '1599', '1600', '1601', '1602', '1603', '1604', '1605', '1606', '1607', '1608', '1609', '1610', '1611', '1612', '1613', '1614', '1615', '1616', '1617', '1618', '1619', '1620', '1621', '1622', '1623', '1624', '1625', '1626', '1627', '1628', '1629', '1630', '1631', '1632', '1633', '1634', '1635', '1636', '1637', '1638', '1639', '1640', '1641', '1642', '1643', '1644', '1645', '1646', '1647', '1648', '1649', '1650', '1651', '1652', '1653', '1654', '1655', '1656', '1657', '1658', '1659', '1660', '1661', '1662', '1663', '1664', '1665', '1666', '1667', '1668', '1669', '1670', '1671', '1672', '1673', '1674', '1675', '1676', '1677', '1678', '1679', '1680', '1681', '1682', '1683', '1684', '1685', '1686', '1687', '1688', '1689', '1690', '1691', '1692', '1693', '1694', '1695', '1696', '1697', '1698', '1699', '1700', '1701', '1702', '1703', '1704', '1705', '1706', '1707', '1708', '1709', '1710', '1711', '1712', '1713', '1714', '1715', '1716', '1717', '1718', '1719', '1720', '1721', '1722', '1723', '1724', '1725', '1726', '1727', '1728', '1729', '1730', '1731', '1732', '1733', '1734', '1735', '1736', '1737', '1738', '1739', '1740', '1741', '1742', '1743', '1744', '1745', '1746', '1747', '1748', '1749', '1750', '1751', '1752', '1753', '1754', '1755', '1756', '1757', '1758', '1759', '1760', '1761', '1762', '1763', '1764', '1765', '1766', '1767', '1768', '1769', '1770', '1771', '1772', '1773', '1774', '1775', '1776', '1777', '1778', '1779', '1780', '1781', '1782', '1783', '1784', '1785', '1786', '1787', '1788', '1789', '1790', '1791', '1792', '1793', '1794', '1795', '1796', '1797', '1798', '1799', '1800', '1801', '1802', '1803', '1804', '1805', '1806', '1807', '1808', '1809', '1810', '1811', '1812', '1813', '1814', '1815', '1816', '1817', '1818', '1819', '1820', '1821', '1822', '1823', '1824', '1825', '1826', '1827', '1828', '1829', '1830', '1831', '1832', '1833', '1834', '1835', '1836', '1837', '1838', '1839', '1840', '1841', '1842', '1843', '1844', '1845', '1846', '1847', '1848', '1849', '1850', '1851', '1852', '1853', '1854', '1855', '1856', '1857', '1858', '1859', '1860', '1861', '1862', '1863', '1864', '1865', '1866', '1867', '1868', '1869', '1870', '1871', '1872', '1873', '1874', '1875', '1876', '1877', '1878', '1879', '1880', '1881', '1882', '1883', '1884', '1885', '1886', '1887', '1888', '1889', '1890', '1891', '1892', '1893', '1894', '1895', '1896', '1897', '1898', '1899', '1900', '1901', '1902', '1903', '1904', '1905', '1906', '1907', '1908', '1909', '1910', '1911', '1912', '1913', '1914', '1915', '1916', '1917', '1918', '1919', '1920', '1921', '1922', '1923', '1924', '1925', '1926', '1927', '1928', '1929', '1930', '1931', '1932', '1933', '1934', '1935', '1936', '1937', '1938', '1939', '1940', '1941', '1942', '1943', '1944', '1945', '1946', '1947', '1948', '1949', '1950', '1951', '1952', '1953', '1954', '1955', '1956', '1957', '1958', '1959', '1960', '1961', '1962', '1963', '1964', '1965', '1966', '1967', '1968', '1969', '1970', '1971', '1972', '1973', '1974', '1975', '1976', '1977', '1978', '1979', '1980', '1981', '1982', '1983', '1984', '1985', '1986', '1987', '1988', '1989', '1990', '1991', '1992', '1993', '1994', '1995', '1996', '1997', '1998', '1999' ] #print 'Question is :',cleansedQuestion # 1. Find score for each sentence using word march score first for i in range(0, len(sentence_list)): score = 0 score = score + WM.stemWordMatch(cleansedQuestion, sentence_list[i]) #2. Check if the sentence contains location preposition, then it is a good clue temp = complete_sentence_list[i].split() flag = 0 for j in range(0, len(temp)): if temp[j] in location_prepositions: flag = 1 if flag == 1: score = score + 4 # 3. Check if the sentence contains Location entity person_list, org_list, loc_list, time_list, prof_list = NET.named_entity_tagging( sentence_list[i]) if loc_list != []: # If sentence contains location score = score + 6 candidate_sent_list.append(sentence_list[i]) master_loc_list.append((' '.join(loc_list), i)) sent_score_list.append(score) #print 'Master loc list is :',master_loc_list #print 'Candidate sentences based on Location entity are:',candidate_sent_list # For when and where questions the answer to the question could also be from the timeline of the story dateline_score = 0 for i in range(0, len(cleansedQuestion)): # 1. If question contains "happen", it is a good clue that timeline could be answer if cleansedQuestion[i].lower() == 'happen': dateline_score = dateline_score + 4 # 2. If question contains "take place", it is a good clue that timeline could be answer if i != len(cleansedQuestion) - 1 and cleansedQuestion[i].lower( ) == 'take' and cleansedQuestion[i + 1].lower() == 'place': dateline_score = dateline_score + 4 # 3. If question contains "this", it is slam_dunk that timeline could be answer if cleansedQuestion[i].lower() == 'this': dateline_score = dateline_score + 12 # 4. If question contains "story", it is slam_dunk that timeline could be answer if cleansedQuestion[i].lower() == 'story': dateline_score = dateline_score + 12 #print 'Date line score for the question is :',dateline_score # Selecting the sentence that has the maximum score. If the dateline score is greater than max of sent_score choose # dateline_score else choose the maximum score from sent_score_list max_score_value = max(sent_score_list) #print 'Max value is :', max_score_value # Checking which of the scores is greater. IF score from sent_Score_list is greater than dateline score, then we find # the corresponding sentences and choose the best among them. Else we return the dateline as the result. if max_score_value > dateline_score: # Now we have to choose the best sentence among the sentences in candidate list.Choosing sentences # which have both maximum value and present in candidate list for i in range(0, len(sent_score_list)): if sent_score_list[i] == max_score_value: final_sent_list.append((complete_sentence_list[i], i)) #print 'Final sent list is:',final_sent_list #TODO - check which works better #TODO - based on the verbs in the question select the appropriate sentence from sentence list '''for i in range(0, len(sent_score_list)): if sent_score_list[i] in candidate_sent_list: if sent_score_list[i]==max_score_value: final_sent_list.append(sentence_list[i]) else: final_temp_list.append(sentence_list[i])''' # Now from the sentences extracting out the years or the date /time values alone and representing them temp_solution = [] answer_loc = [] if len(final_sent_list) == 1: sent = final_sent_list[0][0] index = final_sent_list[0][1] #print index for i in range(0, len(master_loc_list)): answer_loc.append(master_loc_list[i][0]) print 'Answer: ', ' '.join(set(answer_loc)) + '\n' #print '\n' return ' '.join(set(answer_loc)) '''print master_loc_list[i] temp=master_loc_list[i][1] if temp==index: result=master_loc_list[i][0] print 'Result is :',master_loc_list[i][0] return result''' else: for i in range(0, len(final_sent_list)): temp = final_sent_list[i][0] temp_solution.append(temp) break result = ' '.join(temp_solution) print 'Answer: ', result + '\n' #print '\n' return result else: result = dateline print 'Answer: ', result + '\n' #print '\n' return result
""" ##init weights def weights_init(m): classname=m.__class__.__name__ if classname.find('Conv') != -1: xavier(m.weight.data) xavier(m.bias.data) """ #build cnn SVS = NET.SVS() SVS.cuda() #print(SVS) #SVS.apply(weights_init) #torch.save(SVS,'/home/lisa/SVSP/SVS.pkl') optimizer = torch.optim.Adam(SVS.parameters(), lr=args.LR) # optimize all paramenters loss_func = nn.L1Loss().cuda() lossM = nn.MSELoss().cuda() if __name__ == '__main__': for epoch in range(args.epochs): ##training------------------------------------------ for step, (batch_x, batch_y) in enumerate(trainloader):
def answering_when(cleansedQuestion,stop_words_free_question,sentence_list,dateline): # Declaring globals to be used in this function candidate_sent_list=[] sent_score_list={} final_sent_list=[] when_year_verbs=['play','fought'] #'win','lose','victorius'] when_time_values=['january','jan', 'february', 'feb', 'march', 'mar', 'april', 'apr', 'may','may', 'june', 'jun', 'july', 'jul','august','aug','september','sep','october','oct','november','nov','december','dec','1400', '1401', '1402', '1403', '1404', '1405', '1406', '1407', '1408', '1409', '1410', '1411', '1412', '1413', '1414', '1415', '1416', '1417', '1418', '1419', '1420', '1421', '1422', '1423', '1424', '1425', '1426', '1427', '1428', '1429', '1430', '1431', '1432', '1433', '1434', '1435', '1436', '1437', '1438', '1439', '1440', '1441', '1442', '1443', '1444', '1445', '1446', '1447', '1448', '1449', '1450', '1451', '1452', '1453', '1454', '1455', '1456', '1457', '1458', '1459', '1460', '1461', '1462', '1463', '1464', '1465', '1466', '1467', '1468', '1469', '1470', '1471', '1472', '1473', '1474', '1475', '1476', '1477', '1478', '1479', '1480', '1481', '1482', '1483', '1484', '1485', '1486', '1487', '1488', '1489', '1490', '1491', '1492', '1493', '1494', '1495', '1496', '1497', '1498', '1499', '1500', '1501', '1502', '1503', '1504', '1505', '1506', '1507', '1508', '1509', '1510', '1511', '1512', '1513', '1514', '1515', '1516', '1517', '1518', '1519', '1520', '1521', '1522', '1523', '1524', '1525', '1526', '1527', '1528', '1529', '1530', '1531', '1532', '1533', '1534', '1535', '1536', '1537', '1538', '1539', '1540', '1541', '1542', '1543', '1544', '1545', '1546', '1547', '1548', '1549', '1550', '1551', '1552', '1553', '1554', '1555', '1556', '1557', '1558', '1559', '1560', '1561', '1562', '1563', '1564', '1565', '1566', '1567', '1568', '1569', '1570', '1571', '1572', '1573', '1574', '1575', '1576', '1577', '1578', '1579', '1580', '1581', '1582', '1583', '1584', '1585', '1586', '1587', '1588', '1589', '1590', '1591', '1592', '1593', '1594', '1595', '1596', '1597', '1598', '1599', '1600', '1601', '1602', '1603', '1604', '1605', '1606', '1607', '1608', '1609', '1610', '1611', '1612', '1613', '1614', '1615', '1616', '1617', '1618', '1619', '1620', '1621', '1622', '1623', '1624', '1625', '1626', '1627', '1628', '1629', '1630', '1631', '1632', '1633', '1634', '1635', '1636', '1637', '1638', '1639', '1640', '1641', '1642', '1643', '1644', '1645', '1646', '1647', '1648', '1649', '1650', '1651', '1652', '1653', '1654', '1655', '1656', '1657', '1658', '1659', '1660', '1661', '1662', '1663', '1664', '1665', '1666', '1667', '1668', '1669', '1670', '1671', '1672', '1673', '1674', '1675', '1676', '1677', '1678', '1679', '1680', '1681', '1682', '1683', '1684', '1685', '1686', '1687', '1688', '1689', '1690', '1691', '1692', '1693', '1694', '1695', '1696', '1697', '1698', '1699', '1700', '1701', '1702', '1703', '1704', '1705', '1706', '1707', '1708', '1709', '1710', '1711', '1712', '1713', '1714', '1715', '1716', '1717', '1718', '1719', '1720', '1721', '1722', '1723', '1724', '1725', '1726', '1727', '1728', '1729', '1730', '1731', '1732', '1733', '1734', '1735', '1736', '1737', '1738', '1739', '1740', '1741', '1742', '1743', '1744', '1745', '1746', '1747', '1748', '1749', '1750', '1751', '1752', '1753', '1754', '1755', '1756', '1757', '1758', '1759', '1760', '1761', '1762', '1763', '1764', '1765', '1766', '1767', '1768', '1769', '1770', '1771', '1772', '1773', '1774', '1775', '1776', '1777', '1778', '1779', '1780', '1781', '1782', '1783', '1784', '1785', '1786', '1787', '1788', '1789', '1790', '1791', '1792', '1793', '1794', '1795', '1796', '1797', '1798', '1799', '1800', '1801', '1802', '1803', '1804', '1805', '1806', '1807', '1808', '1809', '1810', '1811', '1812', '1813', '1814', '1815', '1816', '1817', '1818', '1819', '1820', '1821', '1822', '1823', '1824', '1825', '1826', '1827', '1828', '1829', '1830', '1831', '1832', '1833', '1834', '1835', '1836', '1837', '1838', '1839', '1840', '1841', '1842', '1843', '1844', '1845', '1846', '1847', '1848', '1849', '1850', '1851', '1852', '1853', '1854', '1855', '1856', '1857', '1858', '1859', '1860', '1861', '1862', '1863', '1864', '1865', '1866', '1867', '1868', '1869', '1870', '1871', '1872', '1873', '1874', '1875', '1876', '1877', '1878', '1879', '1880', '1881', '1882', '1883', '1884', '1885', '1886', '1887', '1888', '1889', '1890', '1891', '1892', '1893', '1894', '1895', '1896', '1897', '1898', '1899', '1900', '1901', '1902', '1903', '1904', '1905', '1906', '1907', '1908', '1909', '1910', '1911', '1912', '1913', '1914', '1915', '1916', '1917', '1918', '1919', '1920', '1921', '1922', '1923', '1924', '1925', '1926', '1927', '1928', '1929', '1930', '1931', '1932', '1933', '1934', '1935', '1936', '1937', '1938', '1939', '1940', '1941', '1942', '1943', '1944', '1945', '1946', '1947', '1948', '1949', '1950', '1951', '1952', '1953', '1954', '1955', '1956', '1957', '1958', '1959', '1960', '1961', '1962', '1963', '1964', '1965', '1966', '1967', '1968', '1969', '1970', '1971', '1972', '1973', '1974', '1975', '1976', '1977', '1978', '1979', '1980', '1981', '1982', '1983', '1984', '1985', '1986', '1987', '1988', '1989', '1990', '1991', '1992', '1993', '1994', '1995', '1996', '1997', '1998', '1999'] #print 'Question is :',cleansedQuestion # 1. Check if the sentence contains "TIME" expression for i in range(0,len(sentence_list)): score=0 person_list,org_list,loc_list,time_list,prof_list = NET.named_entity_tagging(sentence_list[i]) if time_list != []: # Sentence contains a time expression candidate_sent_list.append(sentence_list[i]) # Now compute the wordmatch score score = score + 4 + WM.stemWordMatch(cleansedQuestion,sentence_list[i]) #sent_score_list.append((score,i)) # 2. Check if the Question contains "the last" and sentence contains any of "first,last,since,ago", then score+= slam_dunk for word in cleansedQuestion: if word.lower() in ['the','last']: for sent in sentence_list[i]: if sent in ['first','last','since','ago']: score = score +20 # 3. If the question contains {start,begin} and sentence contains {start,begin,since,year} if word.lower() in ['start','begin']: for sent in sentence_list[i]: if sent in ['start','begin','since','year']: score = score +20 sent_score_list[i]=score #print 'Candidate sentences list is :',candidate_sent_list #print 'Sent score list is :', sent_score_list # For when and where questions the answer to the question could also be from the timeline of the story dateline_score=0 for i in range(0,len(cleansedQuestion)): # 1. If question contains "happen", it is a good clue that timeline could be answer if cleansedQuestion[i].lower()=='happen': dateline_score= dateline_score+4 # 2. If question contains "take place", it is a good clue that timeline could be answer if i != len(cleansedQuestion)-1 and cleansedQuestion[i].lower()=='take' and cleansedQuestion[i+1].lower()=='place': dateline_score=dateline_score+4 # 3. If question contains "this", it is slam_dunk that timeline could be answer if cleansedQuestion[i].lower()=='this': dateline_score= dateline_score+12 # 4. If question contains "story", it is slam_dunk that timeline could be answer if cleansedQuestion[i].lower()=='story': dateline_score= dateline_score+12 #print 'Date line score for the question is :',dateline_score # Selecting the sentence that has the maximum score. If the dateline score is greater than max of sent_score choose # dateline_score else choose the maximum score from sent_score_list max_score_index=max(sent_score_list, key=lambda i: sent_score_list[i]) score_values=sent_score_list.values() max_score_value =max(score_values) #print 'Max value is :', max_score_value # Checking which of the scores is greater. IF score from sent_Score_list is greater than dateline score, then we find # the corresponding sentences and choose the best among them. Else we return the dateline as the result. if max_score_value > dateline_score: # Now we have to choose the best sentence among the sentences in candidate list # First step is to parse the stop-words free question and look for words in the question which might help us find #the answer #print 'Stopwords free question :', stop_words_free_question '''for i in stop_words_free_question: if i in when_year: final_sent_list.append(''' # Giving preference to sentences which contain a year value # for i in sent_score_list.keys(): '''temp=sentence_list[i].split() for j in range(0, len(temp)): if j in when_year: print 'Year is true' #final_sent_list.append(sentence_list[i]) final_sent_list.append(j)''' # If none of the sentences contain a year, then choose the one with maximum value if sent_score_list[i]==max_score_value: final_sent_list.append(sentence_list[i]) #print 'Final sentence list is:',final_sent_list # Now from the sentences extracting out the years or the date /time values alone and representing them final_temp_list=[] if len(final_sent_list) == 1: temp=nltk.word_tokenize(final_sent_list[0]) for j in range(0, len(temp)): if temp[j].lower() in when_time_values: #print 'year true' final_temp_list.append(temp[j]) if final_temp_list != []: result=' '.join(final_temp_list) print 'Answer: ', result+'\n' #print '\n' return result else: print 'Answer: ', final_sent_list[0]+'\n' #print '\n' return final_sent_list[0] else: for i in range(0,len(final_sent_list)): temp=nltk.word_tokenize(final_sent_list[i]) for j in range(0, len(temp)): if temp[j].lower() in when_time_values: #print 'year true' final_temp_list.append(temp[j]) if final_temp_list != []: result=' '.join(final_temp_list) print 'Answer: ', result+'\n' #print '\n' return result else: print 'Answer: ', ' '.join(final_sent_list)+'\n' #print '\n' return ' '.join(final_sent_list) else: result=dateline print 'Answer: ', result +'\n' #print '\n' return result
def answering_where(cleansedQuestion,stop_words_free_question,complete_sentence_list,sentence_list,dateline): # Declaring globals to be used in this function candidate_sent_list=[] sent_score_list=[] final_sent_list=[] master_loc_list=[] location_prepositions=['in','at','near','inside','on','behind','above','under','next to','below','between','around', 'outside','among','on the right', 'across','front','opposite','before','beneath','beside','against'] when_year_verbs=['play','fought'] #'win','lose','victorius'] when_year=['1400', '1401', '1402', '1403', '1404', '1405', '1406', '1407', '1408', '1409', '1410', '1411', '1412', '1413', '1414', '1415', '1416', '1417', '1418', '1419', '1420', '1421', '1422', '1423', '1424', '1425', '1426', '1427', '1428', '1429', '1430', '1431', '1432', '1433', '1434', '1435', '1436', '1437', '1438', '1439', '1440', '1441', '1442', '1443', '1444', '1445', '1446', '1447', '1448', '1449', '1450', '1451', '1452', '1453', '1454', '1455', '1456', '1457', '1458', '1459', '1460', '1461', '1462', '1463', '1464', '1465', '1466', '1467', '1468', '1469', '1470', '1471', '1472', '1473', '1474', '1475', '1476', '1477', '1478', '1479', '1480', '1481', '1482', '1483', '1484', '1485', '1486', '1487', '1488', '1489', '1490', '1491', '1492', '1493', '1494', '1495', '1496', '1497', '1498', '1499', '1500', '1501', '1502', '1503', '1504', '1505', '1506', '1507', '1508', '1509', '1510', '1511', '1512', '1513', '1514', '1515', '1516', '1517', '1518', '1519', '1520', '1521', '1522', '1523', '1524', '1525', '1526', '1527', '1528', '1529', '1530', '1531', '1532', '1533', '1534', '1535', '1536', '1537', '1538', '1539', '1540', '1541', '1542', '1543', '1544', '1545', '1546', '1547', '1548', '1549', '1550', '1551', '1552', '1553', '1554', '1555', '1556', '1557', '1558', '1559', '1560', '1561', '1562', '1563', '1564', '1565', '1566', '1567', '1568', '1569', '1570', '1571', '1572', '1573', '1574', '1575', '1576', '1577', '1578', '1579', '1580', '1581', '1582', '1583', '1584', '1585', '1586', '1587', '1588', '1589', '1590', '1591', '1592', '1593', '1594', '1595', '1596', '1597', '1598', '1599', '1600', '1601', '1602', '1603', '1604', '1605', '1606', '1607', '1608', '1609', '1610', '1611', '1612', '1613', '1614', '1615', '1616', '1617', '1618', '1619', '1620', '1621', '1622', '1623', '1624', '1625', '1626', '1627', '1628', '1629', '1630', '1631', '1632', '1633', '1634', '1635', '1636', '1637', '1638', '1639', '1640', '1641', '1642', '1643', '1644', '1645', '1646', '1647', '1648', '1649', '1650', '1651', '1652', '1653', '1654', '1655', '1656', '1657', '1658', '1659', '1660', '1661', '1662', '1663', '1664', '1665', '1666', '1667', '1668', '1669', '1670', '1671', '1672', '1673', '1674', '1675', '1676', '1677', '1678', '1679', '1680', '1681', '1682', '1683', '1684', '1685', '1686', '1687', '1688', '1689', '1690', '1691', '1692', '1693', '1694', '1695', '1696', '1697', '1698', '1699', '1700', '1701', '1702', '1703', '1704', '1705', '1706', '1707', '1708', '1709', '1710', '1711', '1712', '1713', '1714', '1715', '1716', '1717', '1718', '1719', '1720', '1721', '1722', '1723', '1724', '1725', '1726', '1727', '1728', '1729', '1730', '1731', '1732', '1733', '1734', '1735', '1736', '1737', '1738', '1739', '1740', '1741', '1742', '1743', '1744', '1745', '1746', '1747', '1748', '1749', '1750', '1751', '1752', '1753', '1754', '1755', '1756', '1757', '1758', '1759', '1760', '1761', '1762', '1763', '1764', '1765', '1766', '1767', '1768', '1769', '1770', '1771', '1772', '1773', '1774', '1775', '1776', '1777', '1778', '1779', '1780', '1781', '1782', '1783', '1784', '1785', '1786', '1787', '1788', '1789', '1790', '1791', '1792', '1793', '1794', '1795', '1796', '1797', '1798', '1799', '1800', '1801', '1802', '1803', '1804', '1805', '1806', '1807', '1808', '1809', '1810', '1811', '1812', '1813', '1814', '1815', '1816', '1817', '1818', '1819', '1820', '1821', '1822', '1823', '1824', '1825', '1826', '1827', '1828', '1829', '1830', '1831', '1832', '1833', '1834', '1835', '1836', '1837', '1838', '1839', '1840', '1841', '1842', '1843', '1844', '1845', '1846', '1847', '1848', '1849', '1850', '1851', '1852', '1853', '1854', '1855', '1856', '1857', '1858', '1859', '1860', '1861', '1862', '1863', '1864', '1865', '1866', '1867', '1868', '1869', '1870', '1871', '1872', '1873', '1874', '1875', '1876', '1877', '1878', '1879', '1880', '1881', '1882', '1883', '1884', '1885', '1886', '1887', '1888', '1889', '1890', '1891', '1892', '1893', '1894', '1895', '1896', '1897', '1898', '1899', '1900', '1901', '1902', '1903', '1904', '1905', '1906', '1907', '1908', '1909', '1910', '1911', '1912', '1913', '1914', '1915', '1916', '1917', '1918', '1919', '1920', '1921', '1922', '1923', '1924', '1925', '1926', '1927', '1928', '1929', '1930', '1931', '1932', '1933', '1934', '1935', '1936', '1937', '1938', '1939', '1940', '1941', '1942', '1943', '1944', '1945', '1946', '1947', '1948', '1949', '1950', '1951', '1952', '1953', '1954', '1955', '1956', '1957', '1958', '1959', '1960', '1961', '1962', '1963', '1964', '1965', '1966', '1967', '1968', '1969', '1970', '1971', '1972', '1973', '1974', '1975', '1976', '1977', '1978', '1979', '1980', '1981', '1982', '1983', '1984', '1985', '1986', '1987', '1988', '1989', '1990', '1991', '1992', '1993', '1994', '1995', '1996', '1997', '1998', '1999'] #print 'Question is :',cleansedQuestion # 1. Find score for each sentence using word march score first for i in range(0,len(sentence_list)): score=0 score= score + WM.stemWordMatch(cleansedQuestion,sentence_list[i]) #2. Check if the sentence contains location preposition, then it is a good clue temp=complete_sentence_list[i].split() flag=0 for j in range(0, len(temp)): if temp[j] in location_prepositions: flag=1 if flag == 1: score= score + 4 # 3. Check if the sentence contains Location entity person_list,org_list,loc_list,time_list,prof_list = NET.named_entity_tagging(sentence_list[i]) if loc_list != []: # If sentence contains location score=score + 6 candidate_sent_list.append(sentence_list[i]) master_loc_list.append((' '.join(loc_list),i)) sent_score_list.append(score) #print 'Master loc list is :',master_loc_list #print 'Candidate sentences based on Location entity are:',candidate_sent_list # For when and where questions the answer to the question could also be from the timeline of the story dateline_score=0 for i in range(0,len(cleansedQuestion)): # 1. If question contains "happen", it is a good clue that timeline could be answer if cleansedQuestion[i].lower()=='happen': dateline_score= dateline_score+4 # 2. If question contains "take place", it is a good clue that timeline could be answer if i != len(cleansedQuestion)-1 and cleansedQuestion[i].lower()=='take' and cleansedQuestion[i+1].lower()=='place': dateline_score=dateline_score+4 # 3. If question contains "this", it is slam_dunk that timeline could be answer if cleansedQuestion[i].lower()=='this': dateline_score= dateline_score+12 # 4. If question contains "story", it is slam_dunk that timeline could be answer if cleansedQuestion[i].lower()=='story': dateline_score= dateline_score+12 #print 'Date line score for the question is :',dateline_score # Selecting the sentence that has the maximum score. If the dateline score is greater than max of sent_score choose # dateline_score else choose the maximum score from sent_score_list max_score_value =max(sent_score_list) #print 'Max value is :', max_score_value # Checking which of the scores is greater. IF score from sent_Score_list is greater than dateline score, then we find # the corresponding sentences and choose the best among them. Else we return the dateline as the result. if max_score_value > dateline_score: # Now we have to choose the best sentence among the sentences in candidate list.Choosing sentences # which have both maximum value and present in candidate list for i in range(0, len(sent_score_list)): if sent_score_list[i]==max_score_value: final_sent_list.append((complete_sentence_list[i],i)) #print 'Final sent list is:',final_sent_list #TODO - check which works better #TODO - based on the verbs in the question select the appropriate sentence from sentence list '''for i in range(0, len(sent_score_list)): if sent_score_list[i] in candidate_sent_list: if sent_score_list[i]==max_score_value: final_sent_list.append(sentence_list[i]) else: final_temp_list.append(sentence_list[i])''' # Now from the sentences extracting out the years or the date /time values alone and representing them temp_solution=[] answer_loc=[] if len(final_sent_list) == 1: sent=final_sent_list[0][0] index=final_sent_list[0][1] #print index for i in range(0,len(master_loc_list)): answer_loc.append(master_loc_list[i][0]) print 'Answer: ',' '.join(set(answer_loc))+'\n' #print '\n' return ' '.join(set(answer_loc)) '''print master_loc_list[i] temp=master_loc_list[i][1] if temp==index: result=master_loc_list[i][0] print 'Result is :',master_loc_list[i][0] return result''' else: for i in range(0,len(final_sent_list)): temp=final_sent_list[i][0] temp_solution.append(temp) break result=' '.join(temp_solution) print 'Answer: ', result+'\n' #print '\n' return result else: result=dateline print 'Answer: ', result+'\n' #print '\n' return result
torch.cuda.set_device(args.gpus[0]) # Infer the dataset from the model name args.dataset = 'cifar10' if 'cifar' in args.arch else 'imagenet' args.num_classes = 10 if args.dataset == 'cifar10' else 1000 if args.earlyexit_thresholds: args.num_exits = len(args.earlyexit_thresholds) + 1 args.loss_exits = [0] * args.num_exits args.losses_exits = [] args.exiterrors = [] # Create the model #model = create_model(args.pretrained, args.dataset, args.arch, # parallel=not args.load_serialized, device_ids=args.gpus) model = NET.SVS() compression_scheduler = None # Create a couple of logging backends. TensorBoardLogger writes log files in a format # that can be read by Google's Tensor Board. PythonLogger writes to the Python logger. tflogger = TensorBoardLogger(msglogger.logdir) pylogger = PythonLogger(msglogger) # capture thresholds for early-exit training if args.earlyexit_thresholds: msglogger.info('=> using early-exit threshold values of %s', args.earlyexit_thresholds) # TODO(barrh): args.deprecated_resume is deprecated since v0.3.1 if args.deprecated_resume: msglogger.warning('The "--resume" flag is deprecated. Please use "--resume-from=YOUR_PATH" instead.') if not args.reset_optimizer: msglogger.warning('If you wish to also reset the optimizer, call with: --reset-optimizer')
def main(arg, path, Type, iter, learningrate, update = None, sharedmemory = None): global lendata, data lendata = 0 ITERATIONS = 10000000 #print (arg) f2 = None Net = NET(arg) if update == None: #path = os.getcwd() dirlist = os.listdir(path) l = [] for i in dirlist: if i[-3:] == "upd": l.append(os.path.basename(i)) if len (l)>0: l1 = [] for i in xrange(len(l)): l1.append(l[i][:-4].split(" ")) d = dict(l1[:]) l2 = d.keys() l2 = np.sort(np.array(l2, dtype = "float64")) for i in l: if i.find("%s"%l2[0]) != -1: #print iter, i, "FIND", "%s_%s_%s"%(arg[0],arg[1:-1] ,arg[-1]) if d["%s"%l2[0]] == "%s_%s_%s"%(arg[0],arg[1:-1] ,arg[-1]): #print "FINDED", iter f2 = i break dirlist = os.listdir(path) l = [] for i in dirlist: if i[-3:] == "xml": l.append(os.path.basename(i)) if len (l)>0: l1 = [] for i in xrange(len(l)): l1.append(l[i][:-4].split(" ")) d = dict(l1[:]) l2 = d.keys() l2 = np.sort(np.array(l2, dtype = "float64")) for i in l: if i.find("%s"%l2[0]) != -1: if f2 == None: Net.UpdateWeights(path+"/"+i) else: Net.UpdateWeights(path+"/"+i, path+"/"+f2) break if update != None: ITERATIONS = 0 for i in xrange(len(arg[1:-1])): ITERATIONS += arg[i+1] ITERATIONS *= 100 Net.Update(arg[1:-1], update) dirlist = os.listdir(path) l = [] for i in dirlist: if os.path.isfile(i): if i[-4:] == "xml": l.append(os.path.basename(i)) if len (l)>0: l1 = [] for i in xrange(len(l)): l1.append(l[i][:-5].split(" ")) d = dict(l1[:]) l2 = d.keys() l2 = np.sort(np.array(l2, dtype = "float64")) for i in l: if i.find("%s"%l2[0]) != -1: Net.UpdateWeights(path+"/"+i) break ReloadData(Net, path, learningrate) sock = socket.socket() sock.connect(("localhost", 8011)) lastfname = "" old_err = 0 count = 0 err_count = 0 first_step =0 fr = open(path+"/dMSE.err", "r") l = fr.read() fr.close() count_max = 0 for i in xrange(len(arg[1:-1])): count_max += arg[i+1] l = l[1:-1].split(",") #print ("len DMSE", len(l)) if len(l)>0 and l[0].isdigit() == True: for i in xrange(len(l)): l[i] = np.float64(l[i]) for i in xrange(ITERATIONS): #err= Net.TrainNet(5, 0.000001) err= Net.TrainNetOnce() err_count=err_count - err average = SpeedLearning(err_count, i+1) accuracy = check_error(data, Net) if i == 0: old_err = err if i >1: if np.abs(SpeedLearning(err_count, i+1)) <= np.abs(err): if Type == "Main": if sharedmemory != None: sharedmemory[0] = Type sharedmemory[1] = err sharedmemory[2] = iter sharedmemory[3] = i sharedmemory[4] = count sharedmemory[5] = average sharedmemory[6] = accuracy break if sharedmemory != None: sharedmemory[0] = Type sharedmemory[1] = err sharedmemory[2] = iter sharedmemory[3] = i sharedmemory[4] = count sharedmemory[5] = average sharedmemory[6] = accuracy if i < len(l): if l[i]<err: count += 1 if update ==None: if err >= old_err: count +=1 else: count = 0 else: if err == old_err: count +=1 else: count = 0 if count >= count_max: err = old_err if sharedmemory != None: sharedmemory[0] = Type sharedmemory[1] = err sharedmemory[2] = iter sharedmemory[3] = i sharedmemory[4] = count sharedmemory[5] = average sharedmemory[6] = accuracy break if i%(ITERATIONS/100) == 0: ReloadData(Net, path, learningrate) fname = '%s %s_%s_%s.work'%(Net.err, Net.inputsize, Net.hiden, Net.outputsize) print "process %s:%s"%(iter,i/(ITERATIONS/100)) if fname == lastfname: fnamecmp+=1 else: fnamecmp = 0 if fnamecmp>=2: if sharedmemory != None: sharedmemory[0].value = Type sharedmemory[1].value = err sharedmemory[2].value = iter sharedmemory[3].value = i sharedmemory[4].value = count sharedmemory[5].value = average sharedmemory[6].value = accuracy break lastfname = fname Net.SaveNet(path+"/"+fname) old_err = err if update != None: fname = '%s %s_%s_%s.upd'%(Net.err, Net.inputsize, Net.hiden, Net.outputsize) Net.SaveNet(path+"/"+fname) else: Net.SaveNet(path+'/%s %s_%s_%s.xml'%(Net.err, Net.inputsize, Net.hiden, Net.outputsize)) print ("end %s %s %s"%(arg, err, iter)) #sock.close() sys.exit()
MODEL_G_NET = './MODEL/G_MODEL.pt' DDN_SIZE = 256 #隐藏特征数量 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") if __name__ == '__main__': # 数据集 data_set = DataHandle.DATA_SET(r"G:\项目\20190830\faces") train_data = DataLoader(data_set, shuffle=True, batch_size=BATCH_SIZE, num_workers=2) # 网络 d_net = NET.D_NET().to(device) g_net = NET.G_NET().to(device) if os.path.exists(MODEL_D_NET): d_net.load_state_dict(torch.load(MODEL_D_NET), strict=False) if os.path.exists(MODEL_G_NET): g_net.load_state_dict(torch.load(MODEL_G_NET), strict=False) # 损失函数 loss_fn = nn.BCELoss() d_opt = torch.optim.Adam(d_net.parameters(), lr=0.0002, betas=(0.5, 0.999)) # ????? g_opt = torch.optim.Adam(g_net.parameters(), lr=0.0002, betas=(0.5, 0.999)) # tensorboard