def load_model_params(self):
"""This is standard tf_utils stuff."""
log.info("Loading Model Params")
class params(object): pass
params.list_all_ops = [n.name for n in tf.get_default_graph().as_graph_def().node]
log.debug("Num ops in model: {}".format(len(params.list_all_ops)))
params.final_layer = tf.get_collection_ref('final_layer')[0]
#log.debug("Found Final Layer: {}".format(params.final_layer))
params.input_word = tf.get_collection_ref('input_word')[0]
#log.debug("Found input tensor: {}".format(params.input_tensor))
params.input_label = tf.get_collection_ref('input_label')[0]
#log.debug("Found input label: {}".format(params.input_label))
params.global_step = tf.get_collection_ref('global_step')[0]
#log.debug("Found global_step: {}".format(params.global_step))
params.learn_rate = tf.get_collection_ref('learn_rate')[0]
#log.debug("Found learn_rate: {}".format(params.learn_rate))
params.correct_pred = tf.get_collection_ref('correct_pred')[0]
#log.debug("Found correct_pred op: {}".format(params.correct_pred))
params.accuracy = tf.get_collection_ref('accuracy')[0]
#log.debug("Found accuracy op: {}".format(params.accuracy))
params.cost = tf.get_collection_ref('cost')[0]
#log.debug("Found cost op: {}".format(params.cost))
params.optimizer = tf.get_collection_ref('optimizer')[0]
#log.debug("Found optimizer op: {}".format(params.optimizer))
params.init_op = tf.get_collection_ref('init_op')[0]
# log.debug("Found init_op op: {}".format(params.init_op))
# params.saver = tf.get_collection_ref('saver')[0]
# log.debug("Found saver op: {}".format(params.saver))
params.merged = tf.get_collection_ref('merged')[0]
# log.debug("Found merged op: {}".format(params.merged))
# params.config = tf.get_collection_ref('config')[0]
params.test = "okay"
self.params = params
return params
def read_data(fname):
"""Create numpy representation of text from path."""
log.info("Processing text at path: {}".format(fname))
if not os.path.isfile(fname):
log.warn("{} is an invalid path".format(fname))
return False
class sample_text:
pass
with open(fname) as f:
content = f.readlines()
content = [x.strip() for x in content]
content = [content[i].split() for i in range(len(content))]
content = np.array(content)
sample_text.content = np.reshape(content, [
-1,
])
sample_text.len = sample_text.content.shape[0]
sample_text.sample = sample_text.content[np.random.randint(
0, sample_text.len)]
# this should be red if lower than x and green if above y.
log.debug("Sample text is {} words long.".format(sample_text.len))
log.info("Sample word from text:\n\t{}".format(sample_text.sample))
log.info("File Loaded successfully.")
return sample_text
def sanity(self):
"""This kind of thing should be standardized."""
log.info("Starting Stanity Check")
key = "stuff"
value = "morestuff"
self.idx2word.write_data(key, value)
new_value = self.idx2word.read_data(key)
assert value == new_value
log.debug("Passed Stanity Check")
return True
def read_data(self, fname=None, normalize_digits=True):
"""Create numpy representation of text from path."""
if fname is None:
fname = "text/test.txt"
log.info("Processing text at path: {}".format(fname))
if not os.path.isfile(fname):
log.warn("{} is an invalid path".format(fname))
return False
class sample_text:
pass
# starting
vocab = {}
with open(fname) as f:
counter = 0
for line in f:
counter += 1
if counter % 100000 == 0:
print("Processing line #{}...".format(counter))
# print(line)
tokens = self.basic_tokenizer(line)
for w in tokens:
word = re.sub(self._DIGIT_RE, "0",
w) if normalize_digits else w
if word in vocab:
vocab[word] += 1
else:
vocab[word] = 1
# finishing
vocab_list = self._START_VOCAB + sorted(
vocab, key=vocab.get, reverse=True)
log.info('>> Full Vocabulary Size : {}'.format(len(vocab_list)))
# add words to database
for index, word in enumerate(vocab_list):
log.debug("adding word \"{}\" to database @ {}".format(
word, index))
self.idx2word.write_data(str(index), str(word))
self.word2idx.write_data(str(word), str(index))
# how much time does this add???
read_back = int(self.word2idx.read_data(str(word)))
assert index == read_back
# sanity check
if False:
encoded_sample = self.encode_line(line)
print("Sample Encoded Line:\n{} == {}".format(
line, encoded_sample))
decoded_sample = self.decode_line(encoded_sample)
print("Sample Decoded line: {}".format(decoded_sample))
# fin
log.info("File Loaded successfully.")
return True
def run(tags, min_payout=Decimal('0.50')):
log.info("Curate mode activated", tags=tags)
if tags is None or len(tags) < 1:
raise ValueError("You must specify at least one tag")
log.debug("initializing...")
steem = Steem(keys=[account.key])
chain = Blockchain(steem)
curation = Curation(chain, min_payout)
log.debug("ready", steem=steem, blockchain=chain, curator=curation)
curation.watch(tags)
def build_dataset(self, sample_set):
"""Gotta have docstrings."""
sample_set.count = collections.Counter(sample_set.content).most_common()
sample_set.dictionary = dict()
log.debug("adding word at pos. word[pos]")
for word, _ in sample_set.count:
cur_len = len(sample_set.dictionary)
#log.debug("{} [{}]".format(word, cur_len))
sample_set.dictionary[word] = cur_len
sample_set.reverse_dictionary = dict(zip(sample_set.dictionary.values(),
sample_set.dictionary.keys()))
sample_set.dict_len = len(sample_set.dictionary)
log.debug("len of dictionary {}".format(sample_set.dict_len))
return sample_set
def get_text_file(self, file_, trunk=True):
"""Gotta have docstrings."""
if not os.path.isfile(file_):
log.warn("{} is an invalid path".format(file_))
return False
class sample_text():
pass
msg = "Text Results:\n"
with open(file_) as f:
content = f.readlines()
sample_text.all_content = content
content = [x.strip() for x in content]
print(len(content))
content = [content[i].split() for i in range(len(content))]
content = np.array(content)
# print(content)
sample_text.content = np.reshape(content, [
-1,
])
print(content.shape[:])
sample_text.nwords = 0
sample_text.word_set = []
sample_text.token_to_vector = {}
for this_line in sample_text.all_content:
this_line = this_line.strip()
words_in_line = this_line.split(' ')
# TOKEN is the first word in the line
token = words_in_line[0]
# VECTOR is the line relitive to the token
vector = words_in_line[1:] # this line minus the token
# one hot encoded...
sample_text.token_to_vector[token] = vector
for word in words_in_line:
sample_text.nwords += 1
del sample_text.all_content # maybe ... save on some rams
msg += "Num Words: {}\n".format(sample_text.nwords)
sample_text.uwords = sorted(list(set(sample_text.word_set)))
msg += "Num Unique Words: {}\n".format(len(sample_text.uwords))
msg += "Num of Sentences or Unique Vectors: {}\n".format(
len(sample_text.token_to_vector))
log.debug(msg)
return sample_text
def run(tags):
log.info("Follow mode activated", tags=tags)
if tags is None or len(tags) < 1:
raise ValueError("You must specify at least one tag")
log.debug("initializing...")
steem = Steem(keys=[cred.key])
account = Account(cred.id, steem)
chain = Blockchain(steem)
log.debug("ready", steem=steem, account=account, blockchain=chain)
log.info("Gathering our following list...")
following = account.get_following()
pending = []
log.info("Following list retrieved", count=len(following))
log.info("Watching for new posts...")
while True:
stream = map(Post, chain.stream(filter_by=['comment']))
try:
for post in stream:
count = len(pending)
if count > 0:
copy = list(pending)
for i in range(count):
if have_bandwidth(steem, account):
user = copy[i]
log.info("following user", user=user)
steem.follow(user, account=cred.id)
del pending[0]
else:
log.warn("Waiting for more bandwidth before following another user")
break
if post.is_main_post():
log.debug("found a top-level post", author=post.author, tags=post.tags)
if post.author != cred.id:
for tag in tags:
if tag in post.tags:
if post.author not in following:
pending.append(post.author)
following.append(post.author)
break
except PostDoesNotExist as e:
log.debug("Post has vanished", exception=e)
except RPCError as e:
log.error("RPC problem while streaming posts", exception=e)
def __init__(self, options):
"""Gonna need a db, and some creds."""
log.info("Starting AG Chatter Bot.")
self.options = options
# Build Constructors
self.idx2word = Database(
host=options.redis_host, pass_=options.redis_pass, db=0
)
self.word2idx = Database(
host=options.redis_host, pass_=options.redis_pass, db=1
)
self.dataReader = DataReader(
self.options, self.idx2word, self.word2idx
)
self.model = Model(
self.options
)
log.debug(options)
log.info("Init complete.")
def talk():
text = request.args.get('text')
if text is None or text == '':
text = request.get_data().decode('us-ascii')
#log.debug("you said: ", text)
log.debug("talk", text=text, chatservice=chatservice)
if chatservice is not None:
try:
response = chatservice.talk(text)
log.info("server says: ", response)
return response
except:
log.error()
return "oops, malfunction"
else:
return "FlaskServer is working and your message received, but no chatbot service was provided"
def run(args):
log.info("Market summary mode activated", args=args)
if args is None or len(args) < 3:
raise ValueError(
"You must specify a currency pair, title, and one or more tags")
pair = args[0]
title = args[1]
tags = args[2:]
log.debug("initializing...")
steem = Steem(keys=[account.key])
commit = Commit(steem)
api = Poloniex()
market = Market(commit, api, pair)
log.debug("ready", steem=steem, commit=commit, api=api, market=market)
market.summarize(title, tags)
def process(self):
try_again = {}
local_max_payout = Decimal("0")
local_max_post = None
for key, post in self.posts.items():
try:
now = datetime.utcnow()
if now - post['created'] >= timedelta(minutes=27):
if now - post['created'] < timedelta(minutes=30):
post.refresh()
payout = Decimal(
post.get("pending_payout_value").amount)
if payout >= self.min_payout and payout > local_max_payout:
local_max_payout = payout
local_max_post = post
else: # post is not mature enough yet, check it again later
try_again[post.identifier] = post
except PostDoesNotExist as e:
log.debug("Post has vanished", exception=e)
except RPCError as e:
log.error("RPC problem while refreshing post", exception=e)
self.posts = try_again
if local_max_post is not None and local_max_payout > self.max_payout:
log.info("Upvoting post #{}".format(self.votes_today + 1),
post=local_max_post,
elapsed=local_max_post.time_elapsed(),
payout=local_max_payout)
local_max_post.upvote(voter=account.id)
self.max_payout = local_max_payout
self.votes_today += 1
if self.first_vote is None:
self.first_vote = now
def load_tf_model(self, folder=None):
"""This is standard tf_utils stuff."""
if folder is None: folder = self.logs_path
log.info("Loading Model: {}".format("Model_Name"))
if self.sess:
self.sess.close()
try:
self.sess = tf.InteractiveSession()
checkpoint_file = tf.train.latest_checkpoint(folder)
log.info("trying: {}".format(folder))
saver = tf.train.import_meta_graph(checkpoint_file + ".meta")
log.debug("loading modelfile {}".format(checkpoint_file))
self.sess.run(tf.global_variables_initializer())
saver.restore(self.sess, checkpoint_file)
log.info("model successfully Loaded: {}".format(checkpoint_file))
self.saver = saver
self.model_loaded = True
except Exception as e:
log.warn("This folder failed to produce a model {}\n{}".format(folder, e))
return False
return True
def have_bandwidth(steem, account):
'''
Determine if the given account has enough bandwidth to do stuff.
Note that this is just an estimate to return approximately >20% bandwidth remaining.
'''
# FIXME: rewrite this once we get a better calculation for determining available bandwidth
props = steem.get_dynamic_global_properties()
log.debug("got global properties", props=props)
total_vests = Amount(props['total_vesting_shares']).amount
max_bandwidth = int(props['max_virtual_bandwidth'])
vests = Amount(account['vesting_shares']).amount
bw_post = steem.get_account_bandwidth(account.name, "post")
bw_forum = steem.get_account_bandwidth(account.name, "forum")
bw_market = steem.get_account_bandwidth(account.name, "market")
log.debug("account bandwidth information",
vests=vests,
post=bw_post,
forum=bw_forum,
market=bw_market)
bw_allocated = vests / total_vests
bw_used = (int(bw_market['average_bandwidth']) +
int(bw_forum['average_bandwidth'])) / max_bandwidth
ratio = bw_used / bw_allocated
log.debug("bandwidth calculation",
allocated=bw_allocated,
used=bw_used,
ratio=ratio)
return ratio < 9
def main(self, args):
"""The Commandline exec of the chatbot network."""
log.info("Beginning commandline exec of the chatbot network.")
# get a text file... say lincoln.txt
try:
file_ = args[1]
log.info("Recieving Documents: {}".format(file_))
except:
file_ = None
if file_ is None:
# file_ = "../text/sample.txt"
file_ = "text/lincoln.txt"
log.debug("Going with sample: {}".format(file_))
# Get some data
log.info("Opening File: {}".format(file_))
sample_set = self.read_data(file_)
if not sample_set:
return False
# clean your data
log.info("Building Database Dictionary")
sample_set = self.build_redis_dataset(sample_set)
if not sample_set:
return False
"""
# build a tensorboard
log.info("build tensorflow network")
log.debug("Trying to Load Old Model")
if self.load_tf_model(self.logs_path):
network = self.load_model_params()
else:
log.debug("Creating a New Model")
network = self.build_network(sample_set)
log.debug("Working with Final Layer {}".format(network.final_layer))
# do some work
msg = "Train Iters: {}".format(self.train_iters)
log.info("Training Details:\n{}".format(msg))
final_loss, average_acc = self.process_network(sample_set, network)
"""
return True
def watch(self, tags):
log.info("Watching for new posts...")
while True:
stream = map(Post, self.chain.stream(filter_by=['comment']))
try:
for post in stream:
self.process()
if self.first_vote is not None:
elapsed = datetime.utcnow() - self.first_vote
if self.votes_today > 11 or elapsed >= timedelta(
hours=24):
wait = timedelta(hours=24) - elapsed
if wait.total_seconds() > 0:
log.info(
"Maximum votes reached for today, going to sleep now",
wait=wait)
sleep(wait.total_seconds())
log.info("New day!")
self.first_vote = None
self.votes_today = 0
self.max_payout = Decimal("0")
break
if post.is_main_post():
log.debug(
"found a top-level post",
post=post,
elapsed=post.time_elapsed(),
tags=post.tags,
total_payout=post.get("total_payout_value"),
pending_payout=post.get("pending_payout_value"))
for tag in tags:
if tag in post.tags:
log.debug(
"found a possible curation candidate",
post=post)
self.posts[post.identifier] = post
break
except PostDoesNotExist as e:
log.debug("Post has vanished", exception=e)
except RPCError as e:
log.error("RPC problem while streaming posts", exception=e)
def main(self):
if self.database:
log.debug("found that database")
if self.write_data('4', '20'):
log.debug("wrote that data")
pass
twenty = self.read_data('4')
log.debug("read data test: {}".format(twenty))
if int(twenty) is 20:
return True
log.warn("see here... see.")
return False
else:
log.warn("Not logging into the database.")
return False
def main(self):
"""Test of connection settings."""
if self.database:
log.debug("found that database")
if self.write_data('4', '20'):
log.debug("wrote that data")
twenty = self.read_data('4')
log.debug("read data test: {}, type: {}".format(
twenty, type(twenty)))
# Gotta stop comparing to literal.
if int(float(twenty)) is 20:
return True
log.warn("see here... see.")
return False
else:
log.warn("Not logging into the database.")
return False
def build_redis_dataset(self, sample_set):
"""Use redis for managing a dynamic words library."""
log.info("Accessing redis for text management.")
start_time = time.time()
sample_set.count = collections.Counter(
sample_set.content).most_common()
sample_set.dict = dict()
sample_set.rev_dict = dict()
sample_set.dict['UNK'] = 0
sample_set.rev_dict[0] = 'UNK'
sample_set.num_unk = 0
# unk replacer
unk_repacler = {
'{}'.format(y): '{}'.format(x)
for x, y in enumerate(sample_set.content)
}
sample_set.Unique_words = 0
for i, _ in sample_set.count:
sample_set.Unique_words += 1
for index, (word, word_instances) in enumerate(sample_set.count):
mesg = "Popularity Rank: {}, Word: {}: Num References: {}".format(
index + 1, word, word_instances)
# the plan! Stop after 10k words. After that,
# replace the words in the input text as UNK.
if index <= self.vocab_size:
# add each entry to the dict
sample_set.dict[word] = index
sample_set.rev_dict[index] = word
else:
# This takes time...
loop_start = time.time()
print("##################################")
print("- Looking for {} instances of {}".format(
word_instances, word))
for j in range(word_instances):
word_place = unk_repacler[word]
sample_set.num_unk += 1
print("- Place in data to replace a word: {}".format(
word_place))
print("-is {} this {}".format(
word, sample_set.content[int(word_place)]))
if word in sample_set.content[int(word_place)]:
print("-- Yes.")
sample_set.content[int(word_place)] = 'UNK'
print('-# Changed to UNK')
# update the unk_repacler
if j + 1 < word_instances:
print(
"-! Checking for other instances of word: {}.".
format(word))
unk_repacler = {
'{}'.format(y): '{}'.format(x)
for x, y in enumerate(sample_set.content)
}
else:
print("-! Bogus word. {}".format(word))
loop_end = time.time()
# if index % 100 == 0:
elap = loop_end - loop_start
left = sample_set.Unique_words - (sample_set.num_unk +
self.vocab_size)
print("Word took {} to fix.".format(elapsed(elap)))
print("######|| Have {} left to fix. should take {} ||######".
format(left, elapsed(elap * left)))
# lookup word in input and replace it with 'UNK'
# for i_content, content_word in enumerate(sample_set.content):
# if word in content_word:
# sample_set.content[i_content] = 'UNK'
# print("setting {} as UNK, num_unk: {}".format(word, sample_set.num_unk))
# sample_set.num_unk += 1
# do redis next...
end_time = time.time()
print("process took {}secs to complete.".format(
elapsed(end_time - start_time)))
print("sample_set.num_unk ", sample_set.num_unk)
log.debug("Recounting Words in dataset: {}".format(len(
sample_set.dict)))
log.info("Finished Creating Dictionaries from texts.")
"""
try:
word = float(word)
word_ = self.p.number_to_words(int(word))
sample_set.num_converted += 1
sample_set.converted.append((word, word_))
word = word_
except:
pass
# FIX ME... SEARCH FOR OLD REFERENCE FIRST!
# this is broken
if self.database.read_data(word) is cur_len:
pass
else:
self.database.write_data(str(word), int(cur_len))
self.rev_dict.write_data(int(cur_len), str(word))
sample_set.num_to_dict += 1
#self.database.set_wordposition(str(word), int(cur_len))
sample_set.dict_len += 1
"""
# log.debug("len of dictionary {}".format(sample_set.dict_len))
# log.debug("Num Converted words {}".format(sample_set.num_converted))
# log.debug("Num words added to database {}".format(sample_set.num_to_dict))
# print(sample_set.converted)
return sample_set
def read_data(self, key):
value = self.database.get(key).decode('UTF-8')
log.debug("read: {}, found: {}".format(key, value))
return value
def write_data(self, key, value):
self.database.set(key, value)
log.debug("write: [{}, {}]".format(key, value))
return True
# @%@~LICENSE~@%@
from datetime import datetime
from steem import Steem
from steem.account import Account
from steem.blockchain import Blockchain
from steem.commit import Commit
from steem.post import Post
from steem.utils import construct_identifier
from steembase.exceptions import PostDoesNotExist, RPCError
import ag.logging as log
from ag.boiler.__version__ import __version__
from ag.boiler.config import account as cred, mask
log.debug("Account credentials loaded", id=cred.id, key=mask(cred.key))
def run():
log.info("Timely post mode activated")
log.debug("initializing...")
steem = Steem(keys=[cred.key])
account = Account(cred.id, steem)
chain = Blockchain(steem)
commit = Commit(steem)
log.debug("ready", steem=steem, account=account, blockchain=chain, commit=commit)
# Because subsequent edits to a post show up as separate post entries in the blockchain,
# we'll keep a list of candidates keyed by the post identifier which the edits share.
candidates = {}
# @%@~LICENSE~@%@
from datetime import datetime, timedelta
from decimal import Decimal
from time import sleep
from steem import Steem
from steem.blockchain import Blockchain
from steem.post import Post
from steembase.exceptions import PostDoesNotExist, RPCError
import ag.logging as log
from ag.boiler.config import account, mask
log.debug("Account credentials loaded", id=account.id, key=mask(account.key))
class Curation():
def __init__(self, chain, min_payout):
self.chain = chain
self.min_payout = min_payout
self.posts = {} # queue of curation candidates, keyed by post id
self.first_vote = None # time of our first vote (today)
self.votes_today = 0 # number of times we have voted in the 24 hours since first_vote
self.max_payout = Decimal(
"0") # maximum pending payout ecountered today
def watch(self, tags):
log.info("Watching for new posts...")
while True:
def summarize(self, title, tags):
log.info("Summarizing market...",
symbol=self.symbol,
against=self.against)
if self.testing:
log.info("TESTING MODE ENABLED")
ticker = self.api.ticker()
try:
ticker = ticker[self.against + '_' + self.symbol]
except KeyError as e:
log.error("Currency pair not found in ticker data",
symbol=self.symbol,
against=self.against,
exception=e)
raise ValueError("Currency pair not found in ticker data")
tz = get_localzone()
now = datetime.now(tz)
nowstr = now.strftime('%Y-%m-%d %H:%M:%S %Z')
log.debug("got ticker data", now=nowstr, ticker=ticker)
last = Decimal(ticker['last'])
if self.against == 'USDT' or self.against == 'USD':
symbol = '$'
quant = Decimal('0.00')
else:
symbol = ''
quant = Decimal('0.00000000')
laststr = symbol + str(last.quantize(quant))
log.debug("last trade", value=laststr)
ath = None
newath = False
nowfile = path.join(
dir, 'market.' + self.symbol + '-' + self.against + '.time')
lastfile = path.join(
dir, 'market.' + self.symbol + '-' + self.against + '.last')
img_url = None
if path.exists(nowfile) and path.exists(lastfile):
prev = True
with open(nowfile, 'r') as infile:
prev_now = datetime.fromtimestamp(int(
infile.readline().strip()),
tz=tz)
with open(lastfile, 'r') as infile:
prev_last = Decimal(infile.readline().strip())
prev_permlink = self.make_permlink(prev_now)
prev_nowstr = prev_now.strftime('%Y-%m-%d %H:%M:%S %Z')
change_price = last - prev_last
if change_price < Decimal('0'):
change_pricestr = symbol + str(
change_price.copy_negate().quantize(quant))
else:
change_pricestr = symbol + str(change_price.quantize(quant))
change_pct = (Decimal('100') * change_price / prev_last).quantize(
Decimal('0.00'))
if change_pct < Decimal('0'):
change_pctstr = str(change_pct.copy_negate()) + '%'
else:
change_pctstr = str(change_pct) + '%'
highest = last
lowest = last
fig = plt.figure(figsize=(10, 7), facecolor='k')
ax = fig.add_subplot(1, 1, 1)
rect = ax.patch
rect.set_facecolor('k')
img_title = self.symbol + '-' + self.against + ' at ' + nowstr
plt.title(img_title)
ax.xaxis_date()
plt.xticks(rotation=25)
ax.xaxis.set_major_formatter(DateFormatter('%Y-%m-%d %H:%M'))
# first graph 30-minute candlesticks
log.info("Plotting 30-minute candlesticks...")
data = self.api.chartData(pair=self.against + '_' + self.symbol,
start=int(prev_now.strftime("%s")) + 1,
period=1800)
if len(data) < 0:
raise ValueError("No data returned")
elif len(data) == 1:
try:
error = data['error']
log.error("Received error from API", error=error)
raise ValueError(
"Received error from API: {}".format(error))
except KeyError:
if int(data[0]['date']) == 0:
raise ValueError(
"Too soon! You must wait at least 30 minutes between summaries for candlesticks."
)
for row in data:
high = Decimal(row['high'])
if high > highest:
highest = high
low = Decimal(row['low'])
if low < lowest:
lowest = low
time = datetime.fromtimestamp(int(row['date']))
popen = Decimal(row['open'])
close = Decimal(row['close'])
if close >= popen:
color = 'g'
else:
color = 'r'
vline = Line2D(xdata=(time, time),
ydata=(low, high),
linewidth=1.5,
color=color,
antialiased=False)
oline = Line2D(xdata=(time, time),
ydata=(popen, popen),
linewidth=1,
color=color,
antialiased=False,
marker=TICKLEFT,
markersize=7)
cline = Line2D(xdata=(time, time),
ydata=(close, close),
linewidth=1,
color=color,
antialiased=False,
marker=TICKRIGHT,
markersize=7)
ax.add_line(vline)
ax.add_line(oline)
ax.add_line(cline)
# then graph 5-minute lines
log.info("Plotting 5-minute lines...")
data = self.api.chartData(pair=self.against + '_' + self.symbol,
start=int(prev_now.strftime("%s")) + 1,
period=300)
if len(data) < 0:
raise ValueError("No data returned")
elif len(data) == 1:
try:
error = data['error']
log.error("Received error from API", error=error)
raise ValueError(
"Received error from API: {}".format(error))
except KeyError:
if int(data[0]['date']) == 0:
raise ValueError(
"Too soon! You must wait at least 5 minutes between summaries."
)
begin = None
for row in data:
high = Decimal(row['high'])
if high > highest:
highest = high
low = Decimal(row['low'])
if low < lowest:
lowest = low
time = int(row['date'])
popen = Decimal(row['open'])
close = Decimal(row['close'])
if begin is None:
begin = popen
line = Line2D(xdata=(datetime.fromtimestamp(time),
datetime.fromtimestamp(time + 300)),
ydata=(begin, close),
linewidth=0.7,
color='#FFFF00',
antialiased=True)
ax.add_line(line)
begin = close
higheststr = symbol + str(highest.quantize(quant))
loweststr = symbol + str(lowest.quantize(quant))
athfile = path.join(
dir, 'market.' + self.symbol + '-' + self.against + '.ath')
if path.exists(athfile):
with open(athfile, 'r') as infile:
ath = Decimal(infile.readline().strip())
if highest > ath:
ath = highest
newath = True
if not testing:
with open(athfile, 'w') as out:
out.write(str(ath))
ax.xaxis.grid(True, color='#555555', linestyle='dotted')
ax.yaxis.grid(True, color='#555555', linestyle='solid')
plt.tight_layout()
ax.autoscale_view()
# save image to file or memory buffer
if self.testing:
imgfile = '/tmp/' + self.symbol + '-' + self.against + '.png'
fig.savefig(imgfile)
log.info("Market graph PNG saved", file=imgfile)
else:
img = io.BytesIO()
fig.savefig(img, format='png')
img.seek(0)
plt.close(fig)
# now upload result to imgur
if not self.testing:
log.info("Uploading plot to imgur...")
img_b64 = base64.standard_b64encode(img.read())
client = 'bbe2ecf93d88915'
headers = {'Authorization': 'Client-ID ' + client}
imgur_data = {'image': img_b64, 'title': img_title}
req = Request(url='https://api.imgur.com/3/upload.json',
data=urlencode(imgur_data).encode('ASCII'),
headers=headers)
resp = urlopen(req).read()
resp = json.loads(resp)
log.debug("Got response from imgur", resp=resp)
if resp['success'] == True:
img_url = resp['data']['link']
log.info("Image uploaded successfully", url=img_url)
else:
log.error("Non-successful response from imgur", resp=resp)
raise ValueError("Non-successful response from imgur")
else:
prev = False
body = "Market Summary for " + self.symbol
body += "\n=="
body += "\n* All prices in *" + self.against + "*"
body += "\n---"
body += "\n"
if prev:
if change_pct > Decimal('0'):
body += "\nUp " + change_pctstr
title += ": Up " + change_pctstr
elif change_pct < Decimal('0'):
body += "\nDown " + change_pctstr
title += ": Down " + change_pctstr
else:
body += "\nFlat"
title += ": Flat"
if newath:
body += " (New All Time High Achieved)"
title += " -- New All Time High!"
body += "\n-"
body += "\n" + self.symbol + " **"
if change_price > Decimal('0'):
body += "gained " + change_pricestr
elif change_price < Decimal('0'):
body += "lost " + change_pricestr
else:
body += "had no change"
body += "** since the [last market summary]"
body += "(https://steemit.com/@" + account.id + "/" + prev_permlink + ")"
if change_pct > Decimal('0'):
body += ", a change of **" + change_pctstr + "**"
elif change_pct < Decimal('0'):
body += ", a change of **-" + change_pctstr + "**"
body += "."
else:
body += "\n*This is the first market summary, so no previous comparison data is available.*"
body += "\n"
body += "\n* Last trade: *" + laststr + "*"
if prev:
body += "\n* Highest trade: *" + higheststr + "*"
if newath:
body += " (new all time high)"
body += "\n* Lowest trade: *" + loweststr + "*"
if img_url is not None:
body += "\n"
body += "\n[![market activity plot](" + img_url + ")](" + img_url + ")"
body += "\n"
body += "\n---"
body += "\n"
body += "\n* Snapshot taken at *" + nowstr + "*"
if prev:
body += "\n* Previous snapshot: *[" + prev_nowstr + "]"
body += "(https://steemit.com/@" + account.id + "/" + prev_permlink + ")*"
body += "\n* Quote data from [Poloniex](http://poloniex.com)"
body += "\n"
body += "\n<center>Happy trading... stay tuned for the next summary!</center>"
body += "\n"
body += "\n---"
body += "\n<center>*This market summary produced automatically by:"
body += "\n["
body += "\nAlpha Griffin Boiler bot](https://github.com/AlphaGriffin/boiler)"
body += "\nv" + __version__ + "*</center>"
if self.testing:
print(body)
permlink = self.make_permlink(now)
tries = 0
post = None
while tries < self.max_tries:
try:
log.info("Posting summary...",
permlink=permlink,
title=title,
last=laststr,
tags=tags)
if self.testing:
log.warn("Not actually going to post (testing mode)")
break
post = self.commit.post(permlink=permlink,
title=title,
author=account.id,
body=body,
tags=tags,
self_vote=True)
break
except RPCError as e:
log.warn(
"Got RPC error while posting, trying again in 1 minute...",
exception=e)
tries += 1
sleep(60)
if post is not None:
log.info("Summary posted successfully", post=post)
with open(nowfile, 'w') as out:
out.write(now.strftime("%s"))
with open(lastfile, 'w') as out:
out.write(str(last))
return True
else:
if not self.testing:
log.error("Failed to post summary")
return False
def run():
log.info("Timely post mode activated")
log.debug("initializing...")
steem = Steem(keys=[cred.key])
account = Account(cred.id, steem)
chain = Blockchain(steem)
commit = Commit(steem)
log.debug("ready", steem=steem, account=account, blockchain=chain, commit=commit)
# Because subsequent edits to a post show up as separate post entries in the blockchain,
# we'll keep a list of candidates keyed by the post identifier which the edits share.
candidates = {}
log.info("Checking post history...")
history = map(Post, account.history(filter_by=['comment']))
# FIXME: use steem.get_posts() instead?
for post in history:
if post.is_main_post():
log.debug("found a top-level post", post=post, tags=post.tags)
if post.tags[0] == cred.id and 'boiled' not in post.tags:
candidates[post.identifier] = post
if len(candidates) > 0:
log.info("Found one or more historical posts to process", posts=candidates)
deleting = []
for key, post in candidates.items():
result = process(commit, post)
if result or result is None:
deleting.append(key)
for key in deleting:
del candidates[key]
log.info("Watching for new posts...")
while True:
stream = map(Post, chain.stream(filter_by=['comment']))
try:
for post in stream:
if post.is_main_post() and post.author == cred.id:
log.debug("found a top-level post", post=post, tags=post.tags)
if len(post.tags) == 2 and post.tags[0] == cred.id and post.tags[1] == cred.id:
candidates[post.identifier] = post
deleting = []
for key, post in candidates.items():
result = process(commit, post)
if result or result is None:
deleting.append(key)
for key in deleting:
del candidates[key]
except PostDoesNotExist as e:
log.debug("Post has vanished", exception=e)
except RPCError as e:
log.error("RPC problem while streaming posts", exception=e)
def process(commit, post):
log.debug("checking post", post=post.__dict__)
lines = post.body.splitlines()
if len(lines) < 2:
log.warn("this post appears to be empty or lacking timely data", post=post)
return None
timely = lines[-1].split(' ')
if len(timely) < 3:
log.warn("this post lacks timely data: <date> <time> <tag> ...", post=post)
return None
when = datetime.strptime('{} {}'.format(timely[0], timely[1]), '%Y-%m-%d %H:%M')
if datetime.now() >= when:
log.info("This post is boiling!", post=post)
tags = timely[2:]
meta = {'app' : 'boiler/{}'.format(__version__)}
link = '-' + post.permlink
if lines[-2] == '```':
body = "\n".join(lines[:-2])
else:
body = "\n".join(lines[:-1])
body += "\n---"
body += "\n<center>*This post made timely by:"
body += "\n["
body += "\nAlpha Griffin Boiler bot](https://github.com/AlphaGriffin/boiler)"
body += "\nv" + __version__ + "*</center>"
newpost = commit.post(
permlink = link,
title = post.title,
author = post.author,
body = body,
tags = tags,
json_metadata = meta,
self_vote = True
)
log.debug("new post committed!", result=newpost)
body = "This post has boiled! Find it now here:"
body += "\n* https://steemit.com/@"+post.author+"/"+link
body += "\n---"
body += "\n<center>*Timely posts made possible by:"
body += "\n["
body += "\nAlpha Griffin Boiler bot](https://github.com/AlphaGriffin/boiler)"
body += "\nv" + __version__ + "*</center>"
meta['tags'] = [post.category, 'boiled']
edited = commit.post(
permlink = post.permlink,
title = post.title,
author = post.author,
body = body,
tags = meta['tags'],
json_metadata = meta,
reply_identifier = construct_identifier(post["parent_author"], post["parent_permlink"])
)
log.debug("original post edited!", result=edited)
return True
else:
return False
# Copyright (C) 2017 Alpha Griffin
# @%@~LICENSE~@%@
import ag.logging as log
from os import mkdir, path
from appdirs import AppDirs
dirs = AppDirs("boiler", "Alpha Griffin")
dir = dirs.user_config_dir
log.debug("Starting up", configdir=dir)
if not path.exists(dir):
log.info("Running first-time setup for configuration...")
log.debug("Creating user config directory")
mkdir(dir)
if not path.isdir(dir):
log.fatal("Expected a directory for configdir", configdir=dir)
raise Exception("Not a directory: " + dir)
def mask(secret):
if secret is None:
return ''
elif len(secret) < 10:
return '*' * len(secret)
else:
return secret[:3] + ('*' * (len(secret) - 6)) + secret[-3:]
def process_network(
self,
sample_set,
network,
):
"""This is standard tf_utils stuff."""
# DEFINES!!
training_data = sample_set.content
# dictionary = sample_set.dictionary
# reverse_dictionary = sample_set.reverse_dictionary
n_input = self.n_input
vocab_size = sample_set.dict_len
# start here
start_time = time.time()
session = self.sess
#if self.sess:
# session = self.sess
#else:
# session = tf.Session()
#session.run(network.init_op)
writer = tf.summary.FileWriter(self.logs_path)
_step = 0
offset = random.randint(0, n_input + 1)
end_offset = n_input + 1
acc_total = 0
loss_total = 0
display_step = 10
pred_msg = ' "{}" *returns* "{}" *vs* "{}"\n'
msg = "step: {0:}, offset: {1:}, acc_total: {2:.2f}, loss_total: {3:.2f}"
log.debug("Starting the Train Session:")
# start by adding the whole graph to the Tboard
writer.add_graph(session.graph)
for i in range(self.train_iters):
# Generate a minibatch. Add some randomness on selection process.
if offset > (len(training_data) - end_offset):
offset = random.randint(0, self.n_input + 1)
symbols_in_keys = []
for i in range(offset, offset + self.n_input):
symbols_in_keys.append(
self.database.read_data(str(training_data[i])))
symbols_in_keys = np.reshape(np.array(symbols_in_keys),
[-1, n_input, 1])
symbols_out_onehot = np.zeros([vocab_size], dtype=float)
# symbols_out_onehot[dictionary[str(training_data[offset + n_input])]] = 1.0
one_hot = self.database.read_data(
str(training_data[offset + n_input]))
if one_hot is None:
one_hot = 0
symbols_out_onehot[int(one_hot)] = 1.0
symbols_out_onehot = np.reshape(symbols_out_onehot, [1, -1])
feed_dict = {
network.input_word: symbols_in_keys,
network.input_label: symbols_out_onehot
}
try:
_, acc, loss, onehot_pred, _step, summary = session.run(
[
network.optimizer, network.accuracy, network.cost,
network.final_layer, network.global_step,
network.merged
],
feed_dict=feed_dict)
log.debug("###WORKING {}!!####".format(_step))
# pool data results
loss_total += loss
acc_total += acc
if i % 25 == 0:
# acc pool
print("###WORKING2!!####")
acc_total = (acc_total * 100) / display_step
loss_total = loss_total / display_step
# gather datas
try:
symbols_in = [
training_data[i]
for i in range(offset, offset + n_input)
]
symbols_out = training_data[offset + n_input]
symbols_out_pred = self.rev_dict.read_data(
int(
tf.argmax(onehot_pred,
1).eval(session=session)))
# do save actions
log.info("Saving the Train Session:\n{}\n{}".format(
msg.format(_step, offset, acc_total, loss_total),
pred_msg.format(symbols_in, symbols_out,
symbols_out_pred)))
except Exception as e:
log.warn("Bad Things are happening here: {}\n\t{}\n{}".
format(elapsed(time.time() - start_time), e))
pass
# Save Functions
self.saver.save(session,
self.logs_path + self.filename,
global_step=network.global_step)
writer.add_summary(summary, global_step=_step)
# projector.visualize_embeddings(writer, network.config)
# reset the pooling counters
acc_total = 0
loss_total = 0
# end of loop increments
offset += (n_input + 1)
except Exception as e:
log.warn("BLowing it DUDE... {}\nError: {}".format(_step, e))
pass
# Save Functions
self.saver.save(session,
self.logs_path + self.filename,
global_step=network.global_step)
writer.add_summary(summary, global_step=_step)
# projector.visualize_embeddings(writer, network.config)
log.info("Optimization Finished!")
log.debug("Elapsed time: {}".format(elapsed(time.time() - start_time)))
return (loss_total, acc_total)
session.close()
