def test_flush(self):
m1 = Markov(prefix="one", db=5)
m2 = Markov(prefix="two", db=5)
line = ['i', 'ate', 'a', 'peach']
line1 = ['i', 'ate', 'one', 'peach']
line2 = ['i', 'ate', 'a', 'sandwich']
m1.add_line_to_index(line)
m1.add_line_to_index(line1)
m1.add_line_to_index(line2)
important_line = ['we', 'all', 'have', 'phones']
m2.add_line_to_index(important_line)
r = redis.Redis(db=5)
assert len(r.keys("one:*")) == 6
assert len(r.keys("two:*")) == 3
m1.flush(prefix="one")
assert len(r.keys("one:*")) == 0
assert len(r.keys("two:*")) == 3
m2.flush(prefix="two")
assert len(r.keys("one:*")) == 0
assert len(r.keys("two:*")) == 0
def __init__(self):
# load the markov chain data
print('*** Loading markov data from {}...'.format(
MARKOV_DATA_SAVE_LOCATION))
try:
with open(MARKOV_DATA_SAVE_LOCATION, 'rb') as markov_file:
self._markov = Markov(pickle.load(markov_file))
markov_file.close()
except IOError:
print('*** Unable to load file!')
self._markov = Markov()
self._messages_since_last_autosave = 0
super().__init__()
def test_monus():
rules = {'1*1': '*', '*1': '*', '*': '.1'}
m = Markov(rules)
assert m.evaluate('11111*111') == '111'
assert m.evaluate('11*11111') == '1'
assert m.evaluate('11*11') == '1'
def test_single_sentence(self):
sentence = "i like curry chicken"
sentence_splitted = sentence.split(" ")
m = Markov()
m.add_sentence(sentence)
first_word = m.rand_first_word()
self.assertEqual(first_word, "i", \
"'i' should be the first word")
next_word = first_word
i = 0
while (next_word != None):
self.assertEqual(next_word, sentence_splitted[i]), \
"Wrong order in word output"
next_word = m.rand_followup_for(next_word)
i += 1
self.assertLessEqual(i, len(sentence_splitted), \
"Something gone wrong in the loop")
complete_sentence = m.gen_sentence()
self.assertNotEqual(complete_sentence[len(complete_sentence) - 1], \
"chicken", \
"The last word must be 'chicken'")
def get_city():
num_countries = 1
while random.randint(1, 30) == 1:
num_countries += 1
countries = []
corpus = ""
with open('countrycodes.csv', 'r', encoding='utf-8') as f:
lines = f.read().splitlines()
while len(countries) < num_countries:
try:
num_cities = 0
while num_cities < 100:
country_name, rand_country_code = random.choice(
lines).split(',')
cities = df[df['Country'] == rand_country_code.lower()]
num_cities = len(cities)
corpus += cities.iloc[:, 2].str.cat(sep='\n') + '\n'
countries.append(country_name)
except Exception:
pass
model = Markov(corpus)
city_name = None
while not city_name:
city_name = model.get_word()
mashup_edition = ''
if num_countries > 1:
mashup_edition = 'Special Mashup Edition:\n'
tweet_str = mashup_edition + city_name + ', ' + '-'.join(countries)
return tweet_str
def setUp(self):
self.test_text_one = "I have bought this bonnet.\r\nI am Mr. Darcy.\r\nI am.\r\n"
with open("in_one.txt",'w') as f:
f.write(self.test_text_one)
self.test_outfile_one = "out_one.txt"
ospl.process_data("in_one.txt", self.test_outfile_one, True)
self.test_markov_one = Markov(self.test_outfile_one)
self.test_text_two = "Stubb is whale.\r\nStarbuck is THE whale.\r\nI am a porpoise.\r\n"
with open("in_two.txt",'w') as f:
f.write(self.test_text_two)
self.test_outfile_two = "out_two.txt"
ospl.process_data("in_two.txt", self.test_outfile_two, True)
self.test_markov_two = Markov(self.test_outfile_two)
self.test_markov = DoubleMarkov(self.test_markov_one, self.test_markov_two)
def main():
global database
global markov
global updater
global chat_states
# This is safe as long as we only access the db within the dispatcher
# callbacks. If not then we need locks.
database = Database(sqlite3.connect(config.DBFILE,
check_same_thread=False))
database.initialize()
markov = Markov(database)
chat_states = ChatStates(database)
updater = Updater(config.TOKEN)
updater.last_update_id = database.get_parameter('last_update', -1) + 1
admin = Admin(database, markov, updater, chat_states, config.ADMIN_LIST)
dp = updater.dispatcher
admin.register_handlers(dp)
dp.add_handler(MessageHandler(Filters.sticker, on_sticker), 0)
dp.add_handler(MessageHandler(Filters.all, on_message), 0)
# Commit updates after being handled. This avoids messages being handled
# twice or updates being missed
dp.add_handler(MessageHandler(Filters.all, on_post_message), 1)
dp.add_handler(AllUpdateHandler(on_post_update), 1)
dp.add_error_handler(on_error)
updater.start_polling()
updater.idle()
os._exit(0)
def Recall(text, archivist):
lines = text.splitlines()
version = parse(lines[0]).strip()
version = version if len(version.strip()) > 1 else lines[4]
archivist.logger.info(
"Dictionary version: {} ({} lines)".format(version, len(lines))
)
if version == "v4":
chatlog = Chatlog.loadl(lines[0:9])
cache = "\n".join(lines[10:])
parrot = Markov.loads(cache)
elif version == "v3":
chatlog = Chatlog.loadl(lines[0:8])
cache = "\n".join(lines[9:])
parrot = Markov.loads(cache)
elif version == "v2":
chatlog = Chatlog.loadl(lines[0:7])
cache = "\n".join(lines[8:])
parrot = Markov.loads(cache)
elif version == "dict:":
chatlog = Chatlog.loadl(lines[0:6])
cache = "\n".join(lines[6:])
parrot = Markov.loads(cache)
else:
chatlog = Chatlog.loadl(lines[0:4])
cache = lines[4:]
parrot = Markov(load=cache, mode=Markov.ModeList)
# raise SyntaxError("Scribe: Chatlog format unrecognized.")
s = Scribe(chatlog, archivist)
s.parrot = parrot
return s
def setUp(self):
self.test_text = "I have bought this bonnet.\r\nI am Mr. Darcy.\r\nI am.\r\n"
with open("in.txt",'w') as f:
f.write(self.test_text)
self.test_outfile = "out.txt"
ospl.process_data("in.txt", self.test_outfile, True)
self.test_markov = Markov(self.test_outfile)
def main(argv):
update_data = False
generate_sentences = False
interactive_mode = False
if len(argv) < 2:
print("No params specified. Using default values.")
argv.extend(("-u", "-g"))
args_dict = { i : True for i in argv }
if "-u" in args_dict:
update_data = True
if "-g" in args_dict:
generate_sentences = True
if "-h" in args_dict:
print(get_help_info())
return
if "-a" in args_dict:
print(get_about_info())
return
if "-i" in args_dict:
interactive_mode = True
if update_data:
dirname = "rawdata"
shutil.rmtree(dirname, ignore_errors=True)
filename = "news.zip"
yesterday = datetime.date.today() - datetime.timedelta(days=1)
network.download_file(network.URL.format(yesterday), filename)
with zipfile.ZipFile(filename, 'r') as zip_ref:
zip_ref.extractall(dirname)
parse_files(dirname + "/day")
if generate_sentences:
markov = Markov(OUTFILE)
markov.get_sentences()
if interactive_mode:
markov = Markov(OUTFILE)
outfile = open(GOODFILE, 'a', encoding="utf-8") # TODO: post these news
while True:
sentence = markov.generate_sentence()
print(sentence)
user_input = getch()
if user_input == "\x1b":
break
elif user_input == "+":
outfile.write(sentence + "\n")
pass
outfile.close()
def main():
# Import an image and train a Markov model on its neighbors
im = Im("painting.jpg")
w, h = im.get_size()
dxys = [(-1, 0), (0, -1), (1, 0), (0, 1)]
# Get all colors
states = list(im.all_colors())
markov = Markov(states)
for x in range(w):
for y in range(h):
# Source pixel
src_color = im.get_pixel(x, y)
# Destination pixels
dest_colors = []
for (dx, dy) in dxys:
nx = x + dx
ny = y + dy
if nx < 0:
nx += w
elif nx >= w:
nx -= w
if ny < 0:
ny += h
elif ny >= h:
ny -= h
dest_color = im.get_pixel(nx, ny)
markov.update_transition(dest_color, src_color)
image_fnames = []
# Model trained, now generate new images!
for i in range(IMAGE_COUNT):
# Save old image
image_fname = "images/image{0}.png".format(i)
image_fnames.append(image_fname)
im.save(image_fname)
# Generate new image
for x in range(w):
for y in range(h):
# Get old color
color = im.get_pixel(x, y)
# Set state
markov.state = color
# Get new color
new_color = markov.step()
# Paint over image
im.set_pixel(x, y, new_color)
animation = mpy.ImageSequenceClip(image_fnames, fps=20)
animation.write_gif("images/animation.gif", fps=20)
def load_markov(filename):
"""markovオブジェクトを作成し、filenameから読み込みをおこなう"""
markov = Markov()
try:
markov.load(filename)
except IOError as e:
print(format_error(e))
return markov
def get_branches(self, order, nb_nodes, branching=None):
assert nb_nodes > 0
m = Markov(self.p0, self.p1)
b = [m() for i in xrange(nb_nodes)]
b[0] = 0
b[-1] = 0
return b
def test_markov_with_higher_level(size=100, file_name="sample1.txt"):
markov = Markov()
mark = markov.train_model_higher(
normalize(os.getcwd() + "\\samples\\" + file_name))
text = markov.generate_sentence(size, mark)
save_text(text)
save_long_file(text, file_name)
print(text)
def markov_test():
print("### Markov Test")
m = Markov(3, "word")
m.train("../data/texts/lotr_clean.txt")
generated = m.generate(10)
pp.pprint(generated)
print("\n")
def index():
words_list = read_file('seinfeld.txt')
num_words = int(request.args.get('num_words', 10))
chain = Markov(words_list)
sampled_sentence = chain.create_sentence(num_words)
return render_template('index.html',
random_sentence=sampled_sentence,
num_words=num_words)
def loadParrot(self, scribe):
newParrot = False
self.parrot = self.archivist.wakeParrot(scribe.cid())
if self.parrot is None:
newParrot = True
self.parrot = Markov()
scribe.teachParrot(self.parrot)
self.store(scribe)
return newParrot
def test_replace_identical_subs_with_single_letter():
rules = {
'aa': 'a',
'bb': 'b',
'cc': 'c',
}
m = Markov(rules)
assert m.evaluate('abbcacccaabb') == 'abcacab'
def markov_generate_text(filename):
'''
My method of markov chain text generation
'''
markov = Markov(filename)
result = None
while result is None:
result = markov.generate_text()
return result
def generate_using_data(file_path, sim_length, dest_path):
"""
file_path : location of midi file
sim_length : number of notes to generate
dest_path : save location
"""
reader = Midi_Reader(file_path)
note_seq = reader.get_note_sequence()
dur_seq = reader.get_duration_sequence()
note_markov, dur_markov = Markov(), Markov()
note_markov.set_sampled_matrix(note_seq)
dur_markov.set_sampled_matrix(dur_seq)
sim_notes = note_markov.simulate_markov_process(sim_length)
sim_durs = dur_markov.simulate_markov_process(sim_length)
creator = Midi_Creator()
creator.create_mid(sim_notes, sim_durs)
creator.save(dest_path)
def initialize_walkers_(self, population):
if self.memory_safe:
self.__dict__['population'] = population
for i in range(self.chain.no_of_nodes):
trans_prob = self.chain.get_node_transition_probability(self.chain.node_ids[i])
next_step_processor = Markov(trans_prob)
for j in range(population[i]):
wlk = Walker(next_step_processor = next_step_processor,\
initial_id = self.chain.node_ids[i])
self.walkers.append(wlk)
def lambda_handler(event, context):
"""Entrypoint for AWS Lambda. Event contains the payload from the AWS Lambda
call."""
user_text = event['userText']
mixin = event['mixin']
length = int(event['length'])
full_text = user_text + load_mixin(mixin)
markover = Markov(full_text)
return markover.generate(length)
def test_sort():
rules = {
'ba': 'ab',
'ca': 'ac',
'da': 'ad',
'cb': 'bc',
'db': 'bd',
'dc': 'cd'
}
m = Markov(rules)
assert m.evaluate('cbadb') == 'abbcd'
def test_iteration_count_gets_reset():
rules = {
'ba': 'ab',
'ca': 'ac',
'da': 'ad',
'cb': 'bc',
'db': 'bd',
'dc': 'cd'
}
m = Markov(rules, max_iter=6)
for i in range(2):
assert m.evaluate('cbadb') == 'abbcd'
def test_a_star_b_c_star():
rules = {'#a': '#', '#b': '$', '$c': '$', '$': '.1', '': '#'}
m = Markov(rules)
assert m.evaluate('aaabcc') == '1'
assert m.evaluate('b') == '1'
assert m.evaluate('abbc') == '1bc'
assert m.evaluate('bac') == '1ac'
# this one should be infinite
with pytest.raises(RuntimeError):
assert m.evaluate('aac') == '###c'
def send():
if request.method == 'POST':
username = request.form['username']
Mgen = Markov()
res = Mgen.markov(username)
error = None
return render_template('markovres.html',
res=res,
author=username,
error=error)
return render_template('index.html')
def caption(username, start, end):
scraper = Scraper(username, start, end)
text = scraper.text
try:
markov = Markov(text)
except:
if len(text) < 1:
return "Username not found or private account"
return "Not enough captions found to generate new text"
text = markov.generate()
if text == None or len(text) < 1:
text = "Not enough captions found to generate new text"
return text
def newTweet():
tweet_data = Markov(prefix="tweets")
tweetline = tweet_data.generate()
tweet = ""
for word in tweetline:
tweet += word + " "
tweet = tweet.rstrip()
if len(tweet) > 140:
return newTweet()
else:
return tweet
def main():
"""Entry point for the code execution."""
f = open('sample.txt', 'r')
text = f.read()
clean_text_list = prep_text(text)
m = Markov()
m.add_text(clean_text_list)
text = generate_paragraph(m, 5, 10)
print(text)
f.close()
def get_all_words(nightmare: Nightmare):
nightmare._cursor.execute('select comments.body from comments;')
data = nightmare._cursor.fetchall()
words = ''
for d in data:
words += d[0]
n = 10
l = words.split()
c = [l[i:i + n] for i in range(0, len(l), n)]
print(c)
m = Markov(c)
# print(words.split())
return m.generate_markov_text()
