def create_topic_by_subject_locator(self, subject_locator):
"""Returns a `Topic` instance with the specified subject locator.
This method returns either an existing `Topic` or creates a
new `Topic` instance with the specified subject locator.
:param subject_locator: the subject locator the topic should
contain
:type subject_locator: `Locator`
:rtype: `Topic`
"""
if subject_locator is None:
raise ModelConstraintException(
self, 'The subject locator may not be None')
reference = subject_locator.to_external_form()
try:
topic = self.topic_constructs.get(
subject_locators__address=reference)
except Topic.DoesNotExist:
topic = Topic(topic_map=self)
topic.save()
sl = SubjectLocator(topic=topic,
address=reference,
containing_topic_map=self)
sl.save()
topic.subject_locators.add(sl)
return topic
def main(argv=None):
if argv is None:
argv = sys.argv
try:
args = parse_args(argv)
config = PressgangConfig("%s/.pressgangcli.conf" %
os.path.expanduser("~"))
topic_server = TopicServer(config.get_location())
topic = Topic(topic_server, args.TOPIC)
output = None
if args.json:
output = topic.get_json(revision=int(args.revision))
elif args.html:
output = topic.get_html(revision=int(args.revision))
else:
output = topic.get_xml(revision=int(args.revision))
if output is None:
if args.revision != 0:
raise Error("No such topic/revision combination exists.")
else:
raise Error("No such topic exists.")
else:
print output
return 0
except Usage, err:
print >>sys.stderr, err.msg
print >>sys.stderr, ("For help and usage information use the --help " +
"argument.")
return 2
def create_topic_by_subject_locator (self, subject_locator):
"""Returns a `Topic` instance with the specified subject locator.
This method returns either an existing `Topic` or creates a
new `Topic` instance with the specified subject locator.
:param subject_locator: the subject locator the topic should
contain
:type subject_locator: `Locator`
:rtype: `Topic`
"""
if subject_locator is None:
raise ModelConstraintException(
self, 'The subject locator may not be None')
reference = subject_locator.to_external_form()
try:
topic = self.topic_constructs.get(
subject_locators__address=reference)
except Topic.DoesNotExist:
topic = Topic(topic_map=self)
topic.save()
sl = SubjectLocator(topic=topic, address=reference,
containing_topic_map=self)
sl.save()
topic.subject_locators.add(sl)
return topic
def handle_publisher(self, connection, message):
self.publishers[str(datetime.now())] = connection
new_topic = Topic(
' '.join(self.separate_str_by_space(message)[1:]).rstrip('\n'),
connection)
self.topics[str(datetime.now())] = new_topic
reply = "Server: topic \'" + new_topic.get_name(
) + "\' has been published."
print(reply)
try:
connection.sendall(str.encode(reply + '\n'))
self.receive_topic_messages_and_respond(connection)
except:
print('Connection to ' +
':'.join(map(str, connection.getpeername())) + ' lost.\n')
for topic in self.topics.values():
if topic.get_publisher() == connection:
topic.set_is_active(False)
self.send_inactive_topic_message_to_subs(topic)
self.remove_client_from_clients_list(connection)
self.remove_client_from_publishers_list(connection)
return
def test_predict_topic_when_doc_length_less_than_min_words(self):
min_words = 20
t = Topic(lang_code=self.lang_code, min_words=min_words)
pred = t.predict_topic(self.text)
self.assertTrue(isinstance(pred, str))
self.assertEqual(
"required at least {} words for prediction".format(min_words),
pred)
def create_empty_topic (self):
"""Returns a `Topic` instance with no other information.
:rtype: `Topic`
"""
topic = Topic(topic_map=self)
topic.save()
return topic
def export_entities():
Query.export_csv('all_queries.csv')
Action.export_csv('all_actions.csv')
Condition.export_csv('all_conditions.csv')
Document.export_csv('all_documents.csv')
Relevance.export_csv('all_relevances.csv')
Session.export_csv('all_sessions.csv')
Topic.export_csv('all_topics.csv')
User.export_csv('all_users.csv')
def create_empty_topic(self):
"""Returns a `Topic` instance with no other information.
:rtype: `Topic`
"""
topic = Topic(topic_map=self)
topic.save()
return topic
def test_predict_topic_for_invalid_language_model(self):
text = """ La Ligue des champions de l'UEFA (UEFA Champions League), parfois abrégée en C1 et anciennement
dénommée Coupe des clubs champions européens (de sa création en 1955 jusqu'en 1992), est une compétition
annuelle de football organisée par l'Union des associations européennes de football (UEFA) et regroupant
les meilleurs clubs du continent européen1. """
t = Topic(lang_code="fr")
pred = t.predict_topic(text)
self.assertTrue(isinstance(pred, str))
self.assertEqual(pred, 'language not supported')
def from_dict(self, dic):
if 'topics' not in dic:
print("Research: from_dict: 'topics' not in dic. Dic:", dic)
exit(-1)
self.topics = []
for dic_topic in dic['topics']:
topic = Topic()
topic.from_dict(dic_topic)
self.topics.append(topic)
def __tester_init():
topics = Topics('g:\\myq\\testers\\')
p0 = {t.name
for t in topics.topics
} == {'TA', 'TA -> TB', 'TA -> TB -> TC', 'TD', 'TD -> TE'}
ta = Topic(name='TA', priority='A', i_row=1)
tb = Topic(name='TA -> TB', priority='A', i_row=1)
tc = Topic(name='TA -> TB -> TC', priority='A', i_row=1)
td = Topic(name='TD', priority='A', i_row=1)
te = Topic(name='TD -> TE', priority='A', i_row=1)
p1 = topics.level == {0: [ta, td], 1: [tb, te], 2: [tc]}
p2 = topics.is_valid
u_tester.run(p0, p1, p2)
def monit(self):
try:
auth = Auth(self._session, self._user, self._config)
auth.confirm()
topic = Topic(self._session, self._user, self._config)
nt = Post(self._session, self._user, self._config,
topic.locate_post_url())
nt.reply()
except Exception as e:
Recorder.error(self, self._user, "init the watchdog", e)
return None
def getChildren(self):
# myRec = self.getRecord()
if self._children is None:
self._children = SortedValuesDict()
for tabData in self.topics:
try:
topicData = TopicData(tabData.data_path, tabData.num)
except NoTopPickDataError:
print "No Data Found in %s" % tabData.data_path
continue
topic = Topic(topicData)
topicId = topic.getId()
self._children[topicId] = topic
print 'made topic', topicId
return self._children
def prepocess(topics):
pre_list = []
for topic in topics:
topic = topic.strip()
tp = Topic(topic, comma=True, num=False)
pre_list.append(tp)
return pre_list
def __init__(self, file_name): DataFile.__init__(self, file_name) result_file_info = self._get_file_info() self.topic = Topic.create_or_update(result_file_info['topic_id']) self.user = User.create_or_update(result_file_info['user_id']) self.query = Query.create_or_update(result_file_info['query_id'], topic=self.topic, user=self.user) self.__parse()
def from_topic_list(topic_list):
"""Generates a Window from a list of "topics", in which each "topic" is a list of tuples of (message, reason))
Parameters
----------
topic_list : list(tuples(|message|, str))
List of lists of tuples of (message, reason)
Returns
-------
TYPE
:class:`nlp.text.window.Window`
"""
_window = Window()
for topic in topic_list[::-1]:
# Append topics from oldest to most recent
uninit = True
for m, r in topic:
if uninit:
# generate new topic and append to window
_window.activate_topic(Topic(start_message=m, reason=r))
uninit = False # no longer uninit
else:
_window.insert_message(message=m, reason=r)
return _window
def register_topic(self, type, name, period):
topic = self._get_topic(name)
if type == 'publish':
# 이미 있는 토픽에 publish.
if topic:
topic.append_publisher(self._address)
return self.send_data(
(1, f'Publisher register complete: {name}'))
# 없는 토픽 생성.
else:
new_topic = Topic(name, period, self._address)
self._topics.append(new_topic)
return self.send_data(
f'Topic publish register complete: {name}')
elif type == 'subscribe':
if topic:
topic.append_subscriber(self._address)
return self.send_data(f'Subscriber register complete: {name}')
else:
self.send_data(f'No such topic: {name}')
else:
self.send_data(f'Wrong topic type: {type}')
def topic_proc(self, topic, country, epoch):
"""
Every topic ingested needs to be added to the memory and to the expiration queue
"""
new_topic = Topic(topic)
self.memory.increase_counter(country, new_topic)
self.queue.push(epoch, country, new_topic)
def get_topic(self, key):
with dbapi2.connect(self.app.config['dsn']) as connection:
cursor = connection.cursor()
query = "SELECT TITLE, TEXT, CURTIME,CATEGORYID FROM TOPIC WHERE (ID = %s)"
cursor.execute(query, (key,))
title,text,time, categoryId = cursor.fetchone()
return Topic(title,text,time,categoryId)
def __tester_init():
t = Topic(name='A -> B -> C', priority='ABA', i_row=8)
p0 = t.name == 'A -> B -> C'
p1 = t.priority == 'ABA'
p2 = t.i_row == 8
p3 = t.level == 2
u_tester.run(p0, p1, p2, p3)
def topics_page():
if request.method == 'GET':
topics = app.store.get_topics()
categories = app.store.get_categories()
now = datetime.datetime.now()
return render_template('topics.html',
topics=topics,
categories=categories,
current_time=now.ctime())
elif 'topics_to_delete' in request.form or 'search' in request.form:
if request.form['submit'] == 'Delete':
keys = request.form.getlist('topics_to_delete')
for key in keys:
app.store.delete_topic(int(key))
return redirect(url_for('topics_page'))
elif request.form['submit'] == 'Search':
keyword = request.form['search']
topics = app.store.search_topic(keyword)
now = datetime.datetime.now()
return render_template('topics.html',
topics=topics,
current_time=now.ctime())
else:
title = request.form['title']
text = request.form['text']
time = str((datetime.datetime.now()))
time = time[:-7]
category_id = request.form.get('categoryId')
#time=request.form['time']
topic = Topic(title, text, time, category_id)
app.store.add_topic(topic)
return redirect(url_for('topic_page', key=app.store.topic_last_key))
def get_topics(self):
with dbapi2.connect(self.app.config['dsn']) as connection:
cursor = connection.cursor()
query = "SELECT * FROM TOPIC ORDER BY ID"
cursor.execute(query)
topics = [(key, Topic(title,text,time,categoryId))
for key, title,text,time,categoryId in cursor]
return topics
def extract_topics_from_url(db):
articles = db.articles.find()
topic_re = re.compile('\/\d+\/(?P<topic>[a-z]+)\/(?P<subtopic>[a-z]+)?\/')
topics = TopicsSet()
for article in articles:
match = topic_re.search(article['url'])
if match:
t = Topic(match.group('topic'))
if t not in topics:
topics.add(t)
else:
t = topics.getelement(t)
subtopic = match.group('subtopic')
if subtopic:
t.add_topic(subtopic)
return topics
def search_topic(self, key):
with dbapi2.connect(self.app.config['dsn']) as connection:
cursor = connection.cursor()
query = "SELECT * FROM TOPIC WHERE (TITLE ILIKE %s OR TEXT ILIKE %s)"
key = '%'+key+'%'
cursor.execute(query, (key, key))
topics = [(key, Topic(title, text, time, categoryId))
for key, title, text, time, categoryId in cursor]
return topics
def __parse( self ):
with open( self.file_name, 'r' ) as result_file:
result_reader = csv.DictReader( result_file, delimiter=',')
for row in result_reader:
topic = Topic.create_or_update( row['topic'] )
user = User.create_or_update( row['userid'] )
condition = Condition.create_or_update( row['condition'] )
autocomplete = row['autocomplete_used'] == 1
query = Query.create_or_update( row['queryid'], topic = topic, user = user, condition = condition, autocomplete = autocomplete, query_text = row['terms'], precision = self.__build_precision_dict( row ) )
def topic_add_page():
if request.method == 'GET':
return render_template('topic_add.html')
else:
content = str(request.form['content'])
writer = app.moderatorlist.get_moderator(current_user.nickname)
topic = Topic(content, writer)
current_app.topiclist.add_topic(topic)
return redirect(url_for('Topics.topics_page'))
def read_topic(self, input_file="topic_list/topic_0.txt", split_sign=":"):
"""
Read in a specific topic from a file named <input_file>
:param input_file: the name of the file that stores the topic
:return: a topic containing a list of word distribution tuples
"""
# read in all ines of input except the last line which is an empty new line
input_file = open(input_file, 'r')
word_dist = input_file.readlines()
# Create the topic object
# The topic object contains a list of word distribution tuples
topic = Topic()
for dist in word_dist:
dict = dist.strip().split(split_sign)
topic.add((dict[0], float(dict[1])))
return topic
def get_TopicList(self):
with dbapi2.connect(app.config['dsn']) as connection:
cursor = connection.cursor()
query = """SELECT TOP_ID, CONTENT, WRITER, NICKNAME FROM TOPICS JOIN
MODERATORS ON WRITER=ID
ORDER BY TOP_ID"""
cursor.execute(query)
topicTable = [(id, Topic(content, writer), modname)
for id, content, writer, modname in cursor]
return topicTable
def getTopics(self):
global conn
global cur
cur.execute("SELECT * FROM topics")
topicsData = cur.fetchall()
allTopicObjs = []
for topic in topicsData:
topicObj = Topic(topic['id'], topic['name'])
allTopicObjs.append(topicObj)
return allTopicObjs
def __init__(self, research_settings):
if 'topics' not in research_settings:
print("Research: provide topics")
exit(-1234234)
topics = research_settings['topics']
self.topics = []
for topic_settings in topics:
topic = Topic(topic_settings)
self.topics.append(topic)
for topic in self.topics:
print(
compare_noun_synsets.match(topic.getAllComments(),
topic.getPros(), topic.getCons()))
exit(0)
def childTopicsFormChildsUrlPage(html, currentTopic):
topics = []
soup = BeautifulSoup(html, "html.parser")
all = soup.find_all("a", {"class", "zm-item-tag"})
for i in all:
t = Topic(unicode(i.attrs['data-token']),
unicode(i.string).strip(), currentTopic.id)
if (currentTopic.id != t.id):
topics.append(t)
return topics
def get_topic(self, topic_name):
""" 获取Account的一个Topic对象
@type topic_name: string
@param topic_name: 主题名称
@rtype: Topic object
@return: 返回该Account的一个Topic对象
"""
return Topic(topic_name, self.mns_client, self.debug)
def topic():
# group = u"爱吃早餐群"
group = u"养老区交流协会V2.0"
a, b, c, d = Topic().start()
msg = "今日学位英语: (难度" + d + ")\n\n" + a + "\n\n" + b
send_group(group, msg)
time.sleep(60)
send_group(group, c)
timer = threading.Timer(86400, topic)
timer.start()
def main():
topic = Topic()
observer_one = TopicSubscriber("Topic subscriber 1")
observer_two = TopicSubscriber("Topic subscriber 2")
observer_three = TopicSubscriber("Topic subscriber 3")
topic.subscribe(observer_one)
topic.subscribe(observer_two)
topic.subscribe(observer_three)
topic.set_topic("Programming is fun!")
def pos_builder(self, current_pos):
'''This function begin to declare new POS's sub-objects
like publisher, topic and data.
:param current_pos: dictionary -- The dictionary tree of current POS.
'''
print 'begin'
for pub in current_pos["pos"]:
pub_obj = Publisher(pub["pub_id"], pub["pub_name"], pub["pub_topics"])
self.__add_publisher(pub_obj)
for topic in pub["pub_topics"]:
topic_obj = Topic(topic["id"], topic["name"], topic["version"], topic["metadata_path"], topic["data_list"])
for data in topic["data_list"]:
metadata = topic_obj.read_metadata(topic["metadata_path"])
acc_folder = os.path.join(acc_loc, "acc")
acc = self.acc_build(metadata, acc_folder)
data_obj = PosData(metadata["data_name"], metadata["data_id"], metadata["data_desc"],
acc)
print 'end'
def get_topic_node(self, topic_path, operation):
if topic_path is None:
return None
topic_node = self.trie.find_node(topic_path)
if topic_node is None and operation in (OperationType.SUBSCRIBE.value,
OperationType.PUBLISH.value):
new_topic = Topic(topic_path)
return self.trie.add(new_topic)
return topic_node
def create_topic_by_subject_identifier (self, subject_identifier):
"""Returns a `Topic` instance with the specified subject identifier.
This method returns either an existing `Topic` or creates a
new `Topic` instance with the specified subject identifier.
If a topic with the specified subject identifier exists in
this topic map, that topic is returned. If a topic with an
item identifier equal to the specified subject identifier
exists, the specified subject identifier is added to that
topic and the topic is returned. If neither a topic with the
specified subject identifier nor with an item identifier equal
to the subject identifier exists, a topic with the subject
identifier is created.
:param subject_identifier: the subject identifier the topic
should contain
:type subject_identifier: `Locator`
:rtype: `Topic`
"""
if subject_identifier is None:
raise ModelConstraintException(
self, 'The subject identifier may not be None')
reference = subject_identifier.to_external_form()
try:
topic = self.topic_constructs.get(
subject_identifiers__address=reference)
except Topic.DoesNotExist:
try:
topic = self.topic_constructs.get(
item_identifiers__address=reference)
except Topic.DoesNotExist:
topic = Topic(topic_map=self)
topic.save()
si = SubjectIdentifier(topic=topic, address=reference,
containing_topic_map=self)
si.save()
topic.subject_identifiers.add(si)
return topic
def __init__(self, file_name): DataFile.__init__(self, file_name) result_file_info = self._get_file_info() self.topic = Topic.create_or_update( result_file_info['topic_id'] ) self.user = User.create_or_update( result_file_info['user_id'] ) self.condition = Condition.create_or_update( result_file_info['condition'] ) self.__create_or_update_session() self.query = Query.create_or_update( result_file_info['query_id'], topic = self.topic, user = self.user, session = self.session ) self.actions = self.__parse() self.topic.add_actions( self.actions ) self.user.add_actions( self.actions ) self.query.add_actions( self.actions ) self.session.add_actions( self.actions ) self.session.add_query( self.query )
"""
Test of Topic class with related Question and Exercise
"""
from topic import Topic
from exercise import Exercise
from topicSet import TopicSet
import json
print "Test for Topic..."
mainTopic = Topic("Simple Topic A", 3600)
mainTopic.addTheory("Simple theory about the Topic A of the Subject.")
mainTopic.addTheory("Complex theory about the Topic A of the Subject.")
ex0 = Exercise("Notes for Exercise 0")
ex0.addQuestion("Question 0.1", ["1"], [1.0])
ex1 = Exercise("Notes for Exercise 1")
ex1.addQuestion("Question 1.1", ["one", "two", "three", "four"], [0.0, 0.2, 1, "0.5"])
ex1.addQuestion("Question 1.2", ["one", "two"], [1.0]).addQuestion(
"Question 1.3", [1, 2, 3, 4, 5, 6, 7, 8, 9], [1, 0.3, 0.4]
)
ex2 = Exercise("Notes for Exercise 2")
ex2.addQuestion("Question 2.1", ["1", "2", "3"], [0.0, 1.0])
ex3 = Exercise("Notes for Exercise 3")
ex4 = Exercise("Notes for Exercise 4")
ex4.addQuestion("Question 4.1", ["1"], [1.0, 0.0])
ex5 = Exercise("Notes for Exercise 5")
ex6 = Exercise("Notes for Exercise 6")
ex7 = Exercise("Notes for Exercise 7")
mainTopic.addExercise(ex0).addExercise(ex1).addExercise(ex2).addExercise(ex3)
def parse(address, learnmode=False):
pages = Set([address])
addr_site = re.findall('^.+/',address)[0]
addr_topic = re.findall('[^/]*$',address)[0]
site = get_site(address)
with open("wedt/forumstrings/"+site, 'r') as f:
strings = json.loads(f.read())
topic = Topic()
scores = []
classes = []
p = 0
while p < len(pages):
page = get_page(sorted(pages)[p])
soup = BeautifulSoup(page)
# find links to other pages
if strings.get('pagination',''): # for forums without pagination
for pagination in soup(True, attrs={"class": strings['pagination']}):
for link in pagination("a", href=re.compile(re.escape(addr_topic))):
t = re.search('[^/]*('+strings['page']+')[0-9]+', link['href'])
if t:
pages.add(addr_site+t.group(0))
# extract posts container if possible
postlist = (soup.find("div", id=strings['postlist']) if strings.get('postlist','') else soup)
author = []
title = []
text = []
source = []
link = []
for postbody in postlist(True, attrs={"class": strings['postbody']}):
txt = '\n'.join(postbody.stripped_strings)
if txt:
text.append(txt)
source.append(''.join(map(unicode,postbody.contents)))
if strings.get('username',''):
for username in postlist(True, attrs={"class": strings['username']}):
author.append(unicode(username.string))
if strings.get('optitle',""):
optitle = postlist.find(True, id=strings['optitle'])
title.append(unicode(optitle.string))
if strings.get('posttitle',""):
for posttitle in postlist(True, attrs={"class": strings['posttitle']}):
title.append(unicode(posttitle.string))
if strings.get('postlink',""):
for postlink in postlist('a', attrs={"class": strings['postlink']}):
link.append(addr_site+re.search('[^/]*$', postlink['href']).group(0))
if learnmode:
for postscore in postlist(True, attrs={"class": strings['postscore']}):
scores.append(unicode(postscore.span.string))
classes.append("acc" if strings['postaccepted'] in postscore.get_text() else "nope")
if len(text)-len(link)==1: #OP isn't permalinked
link = [address] + link
for (u,t,b,l,s) in izip_longest(author, title, text, link, source, fillvalue=""):
topic.append(Post(u,t,b,l,s))
# go to the next page
p=p+1
return (topic, scores, classes) if learnmode else topic
def topic_test():
start = time.time()
topic = Topic("http://www.zhihu.com/topic/19550429")
topic.parser()
print "Title of the topic is ", topic.get_title()
print "Introduction of the topic is ", topic.get_introduction()
print "Number of follower of the topic is ", topic.get_follower_num()
print "Parent topics are ", topic.get_parent_topic()
print "Sample child topics are ", topic.get_sample_child_topic()
print "Top answerers of this topic are ", topic.get_top_answerer()
topic.save_latest_feed()
topic.save_top_answers() # might be time-consuming
end = time.time()
print "Time used is", end - start
