def load_model_params(self): """This is standard tf_utils stuff.""" log.info("Loading Model Params") class params(object): pass params.list_all_ops = [n.name for n in tf.get_default_graph().as_graph_def().node] log.debug("Num ops in model: {}".format(len(params.list_all_ops))) params.final_layer = tf.get_collection_ref('final_layer')[0] #log.debug("Found Final Layer: {}".format(params.final_layer)) params.input_word = tf.get_collection_ref('input_word')[0] #log.debug("Found input tensor: {}".format(params.input_tensor)) params.input_label = tf.get_collection_ref('input_label')[0] #log.debug("Found input label: {}".format(params.input_label)) params.global_step = tf.get_collection_ref('global_step')[0] #log.debug("Found global_step: {}".format(params.global_step)) params.learn_rate = tf.get_collection_ref('learn_rate')[0] #log.debug("Found learn_rate: {}".format(params.learn_rate)) params.correct_pred = tf.get_collection_ref('correct_pred')[0] #log.debug("Found correct_pred op: {}".format(params.correct_pred)) params.accuracy = tf.get_collection_ref('accuracy')[0] #log.debug("Found accuracy op: {}".format(params.accuracy)) params.cost = tf.get_collection_ref('cost')[0] #log.debug("Found cost op: {}".format(params.cost)) params.optimizer = tf.get_collection_ref('optimizer')[0] #log.debug("Found optimizer op: {}".format(params.optimizer)) params.init_op = tf.get_collection_ref('init_op')[0] # log.debug("Found init_op op: {}".format(params.init_op)) # params.saver = tf.get_collection_ref('saver')[0] # log.debug("Found saver op: {}".format(params.saver)) params.merged = tf.get_collection_ref('merged')[0] # log.debug("Found merged op: {}".format(params.merged)) # params.config = tf.get_collection_ref('config')[0] params.test = "okay" self.params = params return params
def decode_line(self, data): """Use redis for managing a dynamic words library.""" log.info("Accessing redis for text management.") decoded_line = "" for w in data: word_for_id = self.idx2word.read_data(str(w)) decoded_line += word_for_id + " " return decoded_line
def flaskytalkyrun(server=None): log.info("flaskytalkyrun", server=server) global chatservice chatservice = server if server is not None: flask.run(host=server.host, port=server.port, threaded=True) else: flask.run()
def encode_line(self, data): """Use redis for managing a dynamic words library.""" log.info("Accessing redis for text management.") words = self.basic_tokenizer(data) encoded_line = [] for w in words: id_for_word = self.word2idx.read_data(w) encoded_line.append(id_for_word) return encoded_line
def sanity(self): """This kind of thing should be standardized.""" log.info("Starting Stanity Check") key = "stuff" value = "morestuff" self.idx2word.write_data(key, value) new_value = self.idx2word.read_data(key) assert value == new_value log.debug("Passed Stanity Check") return True
def run(tags): log.info("Follow mode activated", tags=tags) if tags is None or len(tags) < 1: raise ValueError("You must specify at least one tag") log.debug("initializing...") steem = Steem(keys=[cred.key]) account = Account(cred.id, steem) chain = Blockchain(steem) log.debug("ready", steem=steem, account=account, blockchain=chain) log.info("Gathering our following list...") following = account.get_following() pending = [] log.info("Following list retrieved", count=len(following)) log.info("Watching for new posts...") while True: stream = map(Post, chain.stream(filter_by=['comment'])) try: for post in stream: count = len(pending) if count > 0: copy = list(pending) for i in range(count): if have_bandwidth(steem, account): user = copy[i] log.info("following user", user=user) steem.follow(user, account=cred.id) del pending[0] else: log.warn("Waiting for more bandwidth before following another user") break if post.is_main_post(): log.debug("found a top-level post", author=post.author, tags=post.tags) if post.author != cred.id: for tag in tags: if tag in post.tags: if post.author not in following: pending.append(post.author) following.append(post.author) break except PostDoesNotExist as e: log.debug("Post has vanished", exception=e) except RPCError as e: log.error("RPC problem while streaming posts", exception=e)
def run(tags, min_payout=Decimal('0.50')): log.info("Curate mode activated", tags=tags) if tags is None or len(tags) < 1: raise ValueError("You must specify at least one tag") log.debug("initializing...") steem = Steem(keys=[account.key]) chain = Blockchain(steem) curation = Curation(chain, min_payout) log.debug("ready", steem=steem, blockchain=chain, curator=curation) curation.watch(tags)
def __init__(self, options): """Gonna need a db, and some creds.""" log.info("Starting AG Chatter Bot.") self.options = options # Build Constructors self.idx2word = Database( host=options.redis_host, pass_=options.redis_pass, db=0 ) self.word2idx = Database( host=options.redis_host, pass_=options.redis_pass, db=1 ) self.dataReader = DataReader( self.options, self.idx2word, self.word2idx ) self.model = Model( self.options ) log.debug(options) log.info("Init complete.")
def process(self): try_again = {} local_max_payout = Decimal("0") local_max_post = None for key, post in self.posts.items(): try: now = datetime.utcnow() if now - post['created'] >= timedelta(minutes=27): if now - post['created'] < timedelta(minutes=30): post.refresh() payout = Decimal( post.get("pending_payout_value").amount) if payout >= self.min_payout and payout > local_max_payout: local_max_payout = payout local_max_post = post else: # post is not mature enough yet, check it again later try_again[post.identifier] = post except PostDoesNotExist as e: log.debug("Post has vanished", exception=e) except RPCError as e: log.error("RPC problem while refreshing post", exception=e) self.posts = try_again if local_max_post is not None and local_max_payout > self.max_payout: log.info("Upvoting post #{}".format(self.votes_today + 1), post=local_max_post, elapsed=local_max_post.time_elapsed(), payout=local_max_payout) local_max_post.upvote(voter=account.id) self.max_payout = local_max_payout self.votes_today += 1 if self.first_vote is None: self.first_vote = now
def talk(): text = request.args.get('text') if text is None or text == '': text = request.get_data().decode('us-ascii') #log.debug("you said: ", text) log.debug("talk", text=text, chatservice=chatservice) if chatservice is not None: try: response = chatservice.talk(text) log.info("server says: ", response) return response except: log.error() return "oops, malfunction" else: return "FlaskServer is working and your message received, but no chatbot service was provided"
def run(args): log.info("Market summary mode activated", args=args) if args is None or len(args) < 3: raise ValueError( "You must specify a currency pair, title, and one or more tags") pair = args[0] title = args[1] tags = args[2:] log.debug("initializing...") steem = Steem(keys=[account.key]) commit = Commit(steem) api = Poloniex() market = Market(commit, api, pair) log.debug("ready", steem=steem, commit=commit, api=api, market=market) market.summarize(title, tags)
def all_time_high(pair, price): pair = pair.split('-') if len(pair) != 2: raise ValueError("Currency pair must be of the form XXX-YYY") symbol = pair[0].upper() against = pair[1].upper() price = Decimal(price) file = path.join(dir, 'market.' + symbol + '-' + against + '.ath') log.info("Saving new all-time-high...", symbol=symbol, against=against, price=price, file=file) with open(file, 'w') as out: out.write(str(price)) print("All-time-high saved successfully.")
def read_data(fname): """Create numpy representation of text from path.""" log.info("Processing text at path: {}".format(fname)) if not os.path.isfile(fname): log.warn("{} is an invalid path".format(fname)) return False class sample_text: pass with open(fname) as f: content = f.readlines() content = [x.strip() for x in content] content = [content[i].split() for i in range(len(content))] content = np.array(content) sample_text.content = np.reshape(content, [ -1, ]) sample_text.len = sample_text.content.shape[0] sample_text.sample = sample_text.content[np.random.randint( 0, sample_text.len)] # this should be red if lower than x and green if above y. log.debug("Sample text is {} words long.".format(sample_text.len)) log.info("Sample word from text:\n\t{}".format(sample_text.sample)) log.info("File Loaded successfully.") return sample_text
def read_data(self, fname=None, normalize_digits=True): """Create numpy representation of text from path.""" if fname is None: fname = "text/test.txt" log.info("Processing text at path: {}".format(fname)) if not os.path.isfile(fname): log.warn("{} is an invalid path".format(fname)) return False class sample_text: pass # starting vocab = {} with open(fname) as f: counter = 0 for line in f: counter += 1 if counter % 100000 == 0: print("Processing line #{}...".format(counter)) # print(line) tokens = self.basic_tokenizer(line) for w in tokens: word = re.sub(self._DIGIT_RE, "0", w) if normalize_digits else w if word in vocab: vocab[word] += 1 else: vocab[word] = 1 # finishing vocab_list = self._START_VOCAB + sorted( vocab, key=vocab.get, reverse=True) log.info('>> Full Vocabulary Size : {}'.format(len(vocab_list))) # add words to database for index, word in enumerate(vocab_list): log.debug("adding word \"{}\" to database @ {}".format( word, index)) self.idx2word.write_data(str(index), str(word)) self.word2idx.write_data(str(word), str(index)) # how much time does this add??? read_back = int(self.word2idx.read_data(str(word))) assert index == read_back # sanity check if False: encoded_sample = self.encode_line(line) print("Sample Encoded Line:\n{} == {}".format( line, encoded_sample)) decoded_sample = self.decode_line(encoded_sample) print("Sample Decoded line: {}".format(decoded_sample)) # fin log.info("File Loaded successfully.") return True
def watch(self, tags): log.info("Watching for new posts...") while True: stream = map(Post, self.chain.stream(filter_by=['comment'])) try: for post in stream: self.process() if self.first_vote is not None: elapsed = datetime.utcnow() - self.first_vote if self.votes_today > 11 or elapsed >= timedelta( hours=24): wait = timedelta(hours=24) - elapsed if wait.total_seconds() > 0: log.info( "Maximum votes reached for today, going to sleep now", wait=wait) sleep(wait.total_seconds()) log.info("New day!") self.first_vote = None self.votes_today = 0 self.max_payout = Decimal("0") break if post.is_main_post(): log.debug( "found a top-level post", post=post, elapsed=post.time_elapsed(), tags=post.tags, total_payout=post.get("total_payout_value"), pending_payout=post.get("pending_payout_value")) for tag in tags: if tag in post.tags: log.debug( "found a possible curation candidate", post=post) self.posts[post.identifier] = post break except PostDoesNotExist as e: log.debug("Post has vanished", exception=e) except RPCError as e: log.error("RPC problem while streaming posts", exception=e)
def main(self, args): """The Commandline exec of the chatbot network.""" log.info("Beginning commandline exec of the chatbot network.") # get a text file... say lincoln.txt try: file_ = args[1] log.info("Recieving Documents: {}".format(file_)) except: file_ = None if file_ is None: # file_ = "../text/sample.txt" file_ = "text/lincoln.txt" log.debug("Going with sample: {}".format(file_)) # Get some data log.info("Opening File: {}".format(file_)) sample_set = self.read_data(file_) if not sample_set: return False # clean your data log.info("Building Database Dictionary") sample_set = self.build_redis_dataset(sample_set) if not sample_set: return False """ # build a tensorboard log.info("build tensorflow network") log.debug("Trying to Load Old Model") if self.load_tf_model(self.logs_path): network = self.load_model_params() else: log.debug("Creating a New Model") network = self.build_network(sample_set) log.debug("Working with Final Layer {}".format(network.final_layer)) # do some work msg = "Train Iters: {}".format(self.train_iters) log.info("Training Details:\n{}".format(msg)) final_loss, average_acc = self.process_network(sample_set, network) """ return True
def load_tf_model(self, folder=None): """This is standard tf_utils stuff.""" if folder is None: folder = self.logs_path log.info("Loading Model: {}".format("Model_Name")) if self.sess: self.sess.close() try: self.sess = tf.InteractiveSession() checkpoint_file = tf.train.latest_checkpoint(folder) log.info("trying: {}".format(folder)) saver = tf.train.import_meta_graph(checkpoint_file + ".meta") log.debug("loading modelfile {}".format(checkpoint_file)) self.sess.run(tf.global_variables_initializer()) saver.restore(self.sess, checkpoint_file) log.info("model successfully Loaded: {}".format(checkpoint_file)) self.saver = saver self.model_loaded = True except Exception as e: log.warn("This folder failed to produce a model {}\n{}".format(folder, e)) return False return True
def build_redis_dataset(self, sample_set): """Use redis for managing a dynamic words library.""" log.info("Accessing redis for text management.") start_time = time.time() sample_set.count = collections.Counter( sample_set.content).most_common() sample_set.dict = dict() sample_set.rev_dict = dict() sample_set.dict['UNK'] = 0 sample_set.rev_dict[0] = 'UNK' sample_set.num_unk = 0 # unk replacer unk_repacler = { '{}'.format(y): '{}'.format(x) for x, y in enumerate(sample_set.content) } sample_set.Unique_words = 0 for i, _ in sample_set.count: sample_set.Unique_words += 1 for index, (word, word_instances) in enumerate(sample_set.count): mesg = "Popularity Rank: {}, Word: {}: Num References: {}".format( index + 1, word, word_instances) # the plan! Stop after 10k words. After that, # replace the words in the input text as UNK. if index <= self.vocab_size: # add each entry to the dict sample_set.dict[word] = index sample_set.rev_dict[index] = word else: # This takes time... loop_start = time.time() print("##################################") print("- Looking for {} instances of {}".format( word_instances, word)) for j in range(word_instances): word_place = unk_repacler[word] sample_set.num_unk += 1 print("- Place in data to replace a word: {}".format( word_place)) print("-is {} this {}".format( word, sample_set.content[int(word_place)])) if word in sample_set.content[int(word_place)]: print("-- Yes.") sample_set.content[int(word_place)] = 'UNK' print('-# Changed to UNK') # update the unk_repacler if j + 1 < word_instances: print( "-! Checking for other instances of word: {}.". format(word)) unk_repacler = { '{}'.format(y): '{}'.format(x) for x, y in enumerate(sample_set.content) } else: print("-! Bogus word. {}".format(word)) loop_end = time.time() # if index % 100 == 0: elap = loop_end - loop_start left = sample_set.Unique_words - (sample_set.num_unk + self.vocab_size) print("Word took {} to fix.".format(elapsed(elap))) print("######|| Have {} left to fix. should take {} ||######". format(left, elapsed(elap * left))) # lookup word in input and replace it with 'UNK' # for i_content, content_word in enumerate(sample_set.content): # if word in content_word: # sample_set.content[i_content] = 'UNK' # print("setting {} as UNK, num_unk: {}".format(word, sample_set.num_unk)) # sample_set.num_unk += 1 # do redis next... end_time = time.time() print("process took {}secs to complete.".format( elapsed(end_time - start_time))) print("sample_set.num_unk ", sample_set.num_unk) log.debug("Recounting Words in dataset: {}".format(len( sample_set.dict))) log.info("Finished Creating Dictionaries from texts.") """ try: word = float(word) word_ = self.p.number_to_words(int(word)) sample_set.num_converted += 1 sample_set.converted.append((word, word_)) word = word_ except: pass # FIX ME... SEARCH FOR OLD REFERENCE FIRST! # this is broken if self.database.read_data(word) is cur_len: pass else: self.database.write_data(str(word), int(cur_len)) self.rev_dict.write_data(int(cur_len), str(word)) sample_set.num_to_dict += 1 #self.database.set_wordposition(str(word), int(cur_len)) sample_set.dict_len += 1 """ # log.debug("len of dictionary {}".format(sample_set.dict_len)) # log.debug("Num Converted words {}".format(sample_set.num_converted)) # log.debug("Num words added to database {}".format(sample_set.num_to_dict)) # print(sample_set.converted) return sample_set
def summarize(self, title, tags): log.info("Summarizing market...", symbol=self.symbol, against=self.against) if self.testing: log.info("TESTING MODE ENABLED") ticker = self.api.ticker() try: ticker = ticker[self.against + '_' + self.symbol] except KeyError as e: log.error("Currency pair not found in ticker data", symbol=self.symbol, against=self.against, exception=e) raise ValueError("Currency pair not found in ticker data") tz = get_localzone() now = datetime.now(tz) nowstr = now.strftime('%Y-%m-%d %H:%M:%S %Z') log.debug("got ticker data", now=nowstr, ticker=ticker) last = Decimal(ticker['last']) if self.against == 'USDT' or self.against == 'USD': symbol = '$' quant = Decimal('0.00') else: symbol = '' quant = Decimal('0.00000000') laststr = symbol + str(last.quantize(quant)) log.debug("last trade", value=laststr) ath = None newath = False nowfile = path.join( dir, 'market.' + self.symbol + '-' + self.against + '.time') lastfile = path.join( dir, 'market.' + self.symbol + '-' + self.against + '.last') img_url = None if path.exists(nowfile) and path.exists(lastfile): prev = True with open(nowfile, 'r') as infile: prev_now = datetime.fromtimestamp(int( infile.readline().strip()), tz=tz) with open(lastfile, 'r') as infile: prev_last = Decimal(infile.readline().strip()) prev_permlink = self.make_permlink(prev_now) prev_nowstr = prev_now.strftime('%Y-%m-%d %H:%M:%S %Z') change_price = last - prev_last if change_price < Decimal('0'): change_pricestr = symbol + str( change_price.copy_negate().quantize(quant)) else: change_pricestr = symbol + str(change_price.quantize(quant)) change_pct = (Decimal('100') * change_price / prev_last).quantize( Decimal('0.00')) if change_pct < Decimal('0'): change_pctstr = str(change_pct.copy_negate()) + '%' else: change_pctstr = str(change_pct) + '%' highest = last lowest = last fig = plt.figure(figsize=(10, 7), facecolor='k') ax = fig.add_subplot(1, 1, 1) rect = ax.patch rect.set_facecolor('k') img_title = self.symbol + '-' + self.against + ' at ' + nowstr plt.title(img_title) ax.xaxis_date() plt.xticks(rotation=25) ax.xaxis.set_major_formatter(DateFormatter('%Y-%m-%d %H:%M')) # first graph 30-minute candlesticks log.info("Plotting 30-minute candlesticks...") data = self.api.chartData(pair=self.against + '_' + self.symbol, start=int(prev_now.strftime("%s")) + 1, period=1800) if len(data) < 0: raise ValueError("No data returned") elif len(data) == 1: try: error = data['error'] log.error("Received error from API", error=error) raise ValueError( "Received error from API: {}".format(error)) except KeyError: if int(data[0]['date']) == 0: raise ValueError( "Too soon! You must wait at least 30 minutes between summaries for candlesticks." ) for row in data: high = Decimal(row['high']) if high > highest: highest = high low = Decimal(row['low']) if low < lowest: lowest = low time = datetime.fromtimestamp(int(row['date'])) popen = Decimal(row['open']) close = Decimal(row['close']) if close >= popen: color = 'g' else: color = 'r' vline = Line2D(xdata=(time, time), ydata=(low, high), linewidth=1.5, color=color, antialiased=False) oline = Line2D(xdata=(time, time), ydata=(popen, popen), linewidth=1, color=color, antialiased=False, marker=TICKLEFT, markersize=7) cline = Line2D(xdata=(time, time), ydata=(close, close), linewidth=1, color=color, antialiased=False, marker=TICKRIGHT, markersize=7) ax.add_line(vline) ax.add_line(oline) ax.add_line(cline) # then graph 5-minute lines log.info("Plotting 5-minute lines...") data = self.api.chartData(pair=self.against + '_' + self.symbol, start=int(prev_now.strftime("%s")) + 1, period=300) if len(data) < 0: raise ValueError("No data returned") elif len(data) == 1: try: error = data['error'] log.error("Received error from API", error=error) raise ValueError( "Received error from API: {}".format(error)) except KeyError: if int(data[0]['date']) == 0: raise ValueError( "Too soon! You must wait at least 5 minutes between summaries." ) begin = None for row in data: high = Decimal(row['high']) if high > highest: highest = high low = Decimal(row['low']) if low < lowest: lowest = low time = int(row['date']) popen = Decimal(row['open']) close = Decimal(row['close']) if begin is None: begin = popen line = Line2D(xdata=(datetime.fromtimestamp(time), datetime.fromtimestamp(time + 300)), ydata=(begin, close), linewidth=0.7, color='#FFFF00', antialiased=True) ax.add_line(line) begin = close higheststr = symbol + str(highest.quantize(quant)) loweststr = symbol + str(lowest.quantize(quant)) athfile = path.join( dir, 'market.' + self.symbol + '-' + self.against + '.ath') if path.exists(athfile): with open(athfile, 'r') as infile: ath = Decimal(infile.readline().strip()) if highest > ath: ath = highest newath = True if not testing: with open(athfile, 'w') as out: out.write(str(ath)) ax.xaxis.grid(True, color='#555555', linestyle='dotted') ax.yaxis.grid(True, color='#555555', linestyle='solid') plt.tight_layout() ax.autoscale_view() # save image to file or memory buffer if self.testing: imgfile = '/tmp/' + self.symbol + '-' + self.against + '.png' fig.savefig(imgfile) log.info("Market graph PNG saved", file=imgfile) else: img = io.BytesIO() fig.savefig(img, format='png') img.seek(0) plt.close(fig) # now upload result to imgur if not self.testing: log.info("Uploading plot to imgur...") img_b64 = base64.standard_b64encode(img.read()) client = 'bbe2ecf93d88915' headers = {'Authorization': 'Client-ID ' + client} imgur_data = {'image': img_b64, 'title': img_title} req = Request(url='https://api.imgur.com/3/upload.json', data=urlencode(imgur_data).encode('ASCII'), headers=headers) resp = urlopen(req).read() resp = json.loads(resp) log.debug("Got response from imgur", resp=resp) if resp['success'] == True: img_url = resp['data']['link'] log.info("Image uploaded successfully", url=img_url) else: log.error("Non-successful response from imgur", resp=resp) raise ValueError("Non-successful response from imgur") else: prev = False body = "Market Summary for " + self.symbol body += "\n==" body += "\n* All prices in *" + self.against + "*" body += "\n---" body += "\n" if prev: if change_pct > Decimal('0'): body += "\nUp " + change_pctstr title += ": Up " + change_pctstr elif change_pct < Decimal('0'): body += "\nDown " + change_pctstr title += ": Down " + change_pctstr else: body += "\nFlat" title += ": Flat" if newath: body += " (New All Time High Achieved)" title += " -- New All Time High!" body += "\n-" body += "\n" + self.symbol + " **" if change_price > Decimal('0'): body += "gained " + change_pricestr elif change_price < Decimal('0'): body += "lost " + change_pricestr else: body += "had no change" body += "** since the [last market summary]" body += "(https://steemit.com/@" + account.id + "/" + prev_permlink + ")" if change_pct > Decimal('0'): body += ", a change of **" + change_pctstr + "**" elif change_pct < Decimal('0'): body += ", a change of **-" + change_pctstr + "**" body += "." else: body += "\n*This is the first market summary, so no previous comparison data is available.*" body += "\n" body += "\n* Last trade: *" + laststr + "*" if prev: body += "\n* Highest trade: *" + higheststr + "*" if newath: body += " (new all time high)" body += "\n* Lowest trade: *" + loweststr + "*" if img_url is not None: body += "\n" body += "\n[![market activity plot](" + img_url + ")](" + img_url + ")" body += "\n" body += "\n---" body += "\n" body += "\n* Snapshot taken at *" + nowstr + "*" if prev: body += "\n* Previous snapshot: *[" + prev_nowstr + "]" body += "(https://steemit.com/@" + account.id + "/" + prev_permlink + ")*" body += "\n* Quote data from [Poloniex](http://poloniex.com)" body += "\n" body += "\n<center>Happy trading... stay tuned for the next summary!</center>" body += "\n" body += "\n---" body += "\n<center>*This market summary produced automatically by:" body += "\n[![Alpha Griffin logo](http://alphagriffin.com/usr/include/ag/favicon/favicon-128.png)" body += "\nAlpha Griffin Boiler bot](https://github.com/AlphaGriffin/boiler)" body += "\nv" + __version__ + "*</center>" if self.testing: print(body) permlink = self.make_permlink(now) tries = 0 post = None while tries < self.max_tries: try: log.info("Posting summary...", permlink=permlink, title=title, last=laststr, tags=tags) if self.testing: log.warn("Not actually going to post (testing mode)") break post = self.commit.post(permlink=permlink, title=title, author=account.id, body=body, tags=tags, self_vote=True) break except RPCError as e: log.warn( "Got RPC error while posting, trying again in 1 minute...", exception=e) tries += 1 sleep(60) if post is not None: log.info("Summary posted successfully", post=post) with open(nowfile, 'w') as out: out.write(now.strftime("%s")) with open(lastfile, 'w') as out: out.write(str(last)) return True else: if not self.testing: log.error("Failed to post summary") return False
def run(): log.info("Timely post mode activated") log.debug("initializing...") steem = Steem(keys=[cred.key]) account = Account(cred.id, steem) chain = Blockchain(steem) commit = Commit(steem) log.debug("ready", steem=steem, account=account, blockchain=chain, commit=commit) # Because subsequent edits to a post show up as separate post entries in the blockchain, # we'll keep a list of candidates keyed by the post identifier which the edits share. candidates = {} log.info("Checking post history...") history = map(Post, account.history(filter_by=['comment'])) # FIXME: use steem.get_posts() instead? for post in history: if post.is_main_post(): log.debug("found a top-level post", post=post, tags=post.tags) if post.tags[0] == cred.id and 'boiled' not in post.tags: candidates[post.identifier] = post if len(candidates) > 0: log.info("Found one or more historical posts to process", posts=candidates) deleting = [] for key, post in candidates.items(): result = process(commit, post) if result or result is None: deleting.append(key) for key in deleting: del candidates[key] log.info("Watching for new posts...") while True: stream = map(Post, chain.stream(filter_by=['comment'])) try: for post in stream: if post.is_main_post() and post.author == cred.id: log.debug("found a top-level post", post=post, tags=post.tags) if len(post.tags) == 2 and post.tags[0] == cred.id and post.tags[1] == cred.id: candidates[post.identifier] = post deleting = [] for key, post in candidates.items(): result = process(commit, post) if result or result is None: deleting.append(key) for key in deleting: del candidates[key] except PostDoesNotExist as e: log.debug("Post has vanished", exception=e) except RPCError as e: log.error("RPC problem while streaming posts", exception=e)
def process(commit, post): log.debug("checking post", post=post.__dict__) lines = post.body.splitlines() if len(lines) < 2: log.warn("this post appears to be empty or lacking timely data", post=post) return None timely = lines[-1].split(' ') if len(timely) < 3: log.warn("this post lacks timely data: <date> <time> <tag> ...", post=post) return None when = datetime.strptime('{} {}'.format(timely[0], timely[1]), '%Y-%m-%d %H:%M') if datetime.now() >= when: log.info("This post is boiling!", post=post) tags = timely[2:] meta = {'app' : 'boiler/{}'.format(__version__)} link = '-' + post.permlink if lines[-2] == '```': body = "\n".join(lines[:-2]) else: body = "\n".join(lines[:-1]) body += "\n---" body += "\n<center>*This post made timely by:" body += "\n[![Alpha Griffin logo](http://alphagriffin.com/usr/include/ag/favicon/favicon-128.png)" body += "\nAlpha Griffin Boiler bot](https://github.com/AlphaGriffin/boiler)" body += "\nv" + __version__ + "*</center>" newpost = commit.post( permlink = link, title = post.title, author = post.author, body = body, tags = tags, json_metadata = meta, self_vote = True ) log.debug("new post committed!", result=newpost) body = "This post has boiled! Find it now here:" body += "\n* https://steemit.com/@"+post.author+"/"+link body += "\n---" body += "\n<center>*Timely posts made possible by:" body += "\n[![Alpha Griffin logo](http://alphagriffin.com/usr/include/ag/favicon/favicon-128.png)" body += "\nAlpha Griffin Boiler bot](https://github.com/AlphaGriffin/boiler)" body += "\nv" + __version__ + "*</center>" meta['tags'] = [post.category, 'boiled'] edited = commit.post( permlink = post.permlink, title = post.title, author = post.author, body = body, tags = meta['tags'], json_metadata = meta, reply_identifier = construct_identifier(post["parent_author"], post["parent_permlink"]) ) log.debug("original post edited!", result=edited) return True else: return False
log.warn("Not logging into the database.") return False def write_data(self, key, value): """Should have an H value as well.""" self.database.set(key, value) return True def read_data(self, key): """A basic read of redis data with a utf check.""" try: value = self.database.get(key).decode('UTF-8') except: try: value = self.database.get(key) except Exception as e: log.fatal("Damnit on the reverse lookup!") value = 'unk' return value if __name__ == '__main__': log.info("Starting the Database Testing") try: app = Database() if app.main(): sys.exit("Everything checks out.") except Exception as e: log.error("and thats okay too.") sys.exit(e)
# Copyright (C) 2017 Alpha Griffin # @%@~LICENSE~@%@ import ag.logging as log from os import mkdir, path from appdirs import AppDirs dirs = AppDirs("boiler", "Alpha Griffin") dir = dirs.user_config_dir log.debug("Starting up", configdir=dir) if not path.exists(dir): log.info("Running first-time setup for configuration...") log.debug("Creating user config directory") mkdir(dir) if not path.isdir(dir): log.fatal("Expected a directory for configdir", configdir=dir) raise Exception("Not a directory: " + dir) def mask(secret): if secret is None: return '' elif len(secret) < 10: return '*' * len(secret) else: return secret[:3] + ('*' * (len(secret) - 6)) + secret[-3:]
def process_network( self, sample_set, network, ): """This is standard tf_utils stuff.""" # DEFINES!! training_data = sample_set.content # dictionary = sample_set.dictionary # reverse_dictionary = sample_set.reverse_dictionary n_input = self.n_input vocab_size = sample_set.dict_len # start here start_time = time.time() session = self.sess #if self.sess: # session = self.sess #else: # session = tf.Session() #session.run(network.init_op) writer = tf.summary.FileWriter(self.logs_path) _step = 0 offset = random.randint(0, n_input + 1) end_offset = n_input + 1 acc_total = 0 loss_total = 0 display_step = 10 pred_msg = ' "{}" *returns* "{}" *vs* "{}"\n' msg = "step: {0:}, offset: {1:}, acc_total: {2:.2f}, loss_total: {3:.2f}" log.debug("Starting the Train Session:") # start by adding the whole graph to the Tboard writer.add_graph(session.graph) for i in range(self.train_iters): # Generate a minibatch. Add some randomness on selection process. if offset > (len(training_data) - end_offset): offset = random.randint(0, self.n_input + 1) symbols_in_keys = [] for i in range(offset, offset + self.n_input): symbols_in_keys.append( self.database.read_data(str(training_data[i]))) symbols_in_keys = np.reshape(np.array(symbols_in_keys), [-1, n_input, 1]) symbols_out_onehot = np.zeros([vocab_size], dtype=float) # symbols_out_onehot[dictionary[str(training_data[offset + n_input])]] = 1.0 one_hot = self.database.read_data( str(training_data[offset + n_input])) if one_hot is None: one_hot = 0 symbols_out_onehot[int(one_hot)] = 1.0 symbols_out_onehot = np.reshape(symbols_out_onehot, [1, -1]) feed_dict = { network.input_word: symbols_in_keys, network.input_label: symbols_out_onehot } try: _, acc, loss, onehot_pred, _step, summary = session.run( [ network.optimizer, network.accuracy, network.cost, network.final_layer, network.global_step, network.merged ], feed_dict=feed_dict) log.debug("###WORKING {}!!####".format(_step)) # pool data results loss_total += loss acc_total += acc if i % 25 == 0: # acc pool print("###WORKING2!!####") acc_total = (acc_total * 100) / display_step loss_total = loss_total / display_step # gather datas try: symbols_in = [ training_data[i] for i in range(offset, offset + n_input) ] symbols_out = training_data[offset + n_input] symbols_out_pred = self.rev_dict.read_data( int( tf.argmax(onehot_pred, 1).eval(session=session))) # do save actions log.info("Saving the Train Session:\n{}\n{}".format( msg.format(_step, offset, acc_total, loss_total), pred_msg.format(symbols_in, symbols_out, symbols_out_pred))) except Exception as e: log.warn("Bad Things are happening here: {}\n\t{}\n{}". format(elapsed(time.time() - start_time), e)) pass # Save Functions self.saver.save(session, self.logs_path + self.filename, global_step=network.global_step) writer.add_summary(summary, global_step=_step) # projector.visualize_embeddings(writer, network.config) # reset the pooling counters acc_total = 0 loss_total = 0 # end of loop increments offset += (n_input + 1) except Exception as e: log.warn("BLowing it DUDE... {}\nError: {}".format(_step, e)) pass # Save Functions self.saver.save(session, self.logs_path + self.filename, global_step=network.global_step) writer.add_summary(summary, global_step=_step) # projector.visualize_embeddings(writer, network.config) log.info("Optimization Finished!") log.debug("Elapsed time: {}".format(elapsed(time.time() - start_time))) return (loss_total, acc_total) session.close()