from topics.models import Word, Topic, WordTopicRank from Tome.helpers.data.helpers.wordfix import getWordDataFromLine, WORDS_TITLE from Tome.helpers.data.helpers.general import BASE_PATH from Tome.helpers.debug import Printer from decimal import Decimal progress = Printer(True) out = Printer() def wipeTopics(): Topic.objects.all().delete() WordTopicRank.objects.all().delete() def buildTopic(key): return Topic(key=key) def buildWordTopicPair(word_id, topic_id, score): return WordTopicRank(word_id=word_id, topic_id=topic_id, score=score) def createTopics(): topics = [buildTopic(i) for i in range(100)] return Topic.objects.bulk_create(topics) def pairWords(): f = open(BASE_PATH + WORDS_TITLE + '.csv') counter = 0
from topics.models import Word from Tome.helpers.data.helpers.wordfix import getWordFromLine, WORDS_TITLE from Tome.helpers.data.helpers.general import BASE_PATH from Tome.helpers.debug import Printer progress = Printer(True) def wipeWords(): Word.objects.all().delete() def buildWord(text): return Word(text=text) def importWords(): f = open(BASE_PATH + WORDS_TITLE + '.csv') counter = 0 words = [] progress.reset() for line in f: words.append(buildWord(getWordFromLine(line, counter))) if (counter % 100000 == 0): progress.log(counter) Word.objects.bulk_create(words) words = [] counter += 1 if words: Word.objects.bulk_create(words)
from news.models import Newspaper, Location from Tome.helpers.data.helpers.metadatafix import getArticleFromLine,\ METADATA_TITLE from Tome.helpers.data.helpers.general import BASE_PATH import datetime from Tome.helpers.debug import Printer DATA_BASE_PATH = BASE_PATH DATA_TITLE = METADATA_TITLE defaultLoc = None out = Printer() def wipePapers(): ''' Deletes all the changes ''' Newspaper.objects.all().delete() def fixTitle(title): newTitle = '' for character in title: if character.isupper(): newTitle += ' ' newTitle += character return newTitle