def poll(self, timestamp, filters): logging.info('Polling feed "%s"' % self.url) result = [] self.last_poll = timestamp username = util.decode_password(self.username) password = util.decode_password(self.password) d = util.parse(self.url, username, password, self.etag, self.modified) self.etag = util.get(d, 'etag', None) self.modified = util.get(d, 'modified', None) feed = util.get(d, 'feed', None) if feed: self.title = self.title or util.get(feed, 'title', '') self.link = self.link or util.get(feed, 'link', self.url) entries = util.get(d, 'entries', []) for entry in reversed(entries): id = create_id(entry) if id in self.id_set: continue self.item_count += 1 self.id_list.append(id) self.id_set.add(id) item = Item(self, id) item.timestamp = calendar.timegm( util.get(entry, 'date_parsed', time.gmtime())) item.title = util.format(util.get(entry, 'title', ''), settings.POPUP_TITLE_LENGTH) item.description = util.format(util.get(entry, 'description', ''), settings.POPUP_BODY_LENGTH) item.link = util.get(entry, 'link', '') item.author = util.format(util.get(entry, 'author', '')) # TODO: max length if all(filter.filter(item) for filter in filters): result.append(item) self.clean_cache(settings.FEED_CACHE_SIZE) return result
def prep(type): global counts global classifiers global doclens global scores global docids global p_n global r_n counts = {} classifiers = {} # read classifiers for line in open(util.projdir + "/corpus/" + type + "-class", "r"): c = line.split(' ') classifiers[c[0].split('-')[1]] = c[1].split('\n')[0] for docid, c in classifiers.iteritems(): if c == "satire": r_n += 1 # read word counts docids = { "test": [util.format(i) for i in range(1, 1595 + 1)], "training": [util.format(i) for i in range(1, 2638 + 1)] } for docid in docids[type]: tmpcounts = util.read_counts_nopos("/data/bag/" + type + "/" + type + "-" + docid) counts[docid] = {} for w, c in tmpcounts.iteritems(): counts[docid][w.lower()] = c # these are floats because we need to divide something by them later doclens[docid] = float(sum([c for c in counts[docid].values()]))
def poll(self, timestamp, filters): logging.info('Polling feed "%s"' % self.url) result = [] self.last_poll = timestamp username = util.decode_password(self.username) password = util.decode_password(self.password) d = util.parse(self.url, username, password, self.etag, self.modified) self.etag = util.get(d, 'etag', None) self.modified = util.get(d, 'modified', None) feed = util.get(d, 'feed', None) if feed: self.title = self.title or util.get(feed, 'title', '') self.link = self.link or util.get(feed, 'link', self.url) entries = util.get(d, 'entries', []) for entry in reversed(entries): id = create_id(entry) if id in self.id_set: continue self.item_count += 1 self.id_list.append(id) self.id_set.add(id) item = Item(self, id) item.timestamp = calendar.timegm(util.get(entry, 'date_parsed', time.gmtime())) item.title = util.format(util.get(entry, 'title', ''), settings.POPUP_TITLE_LENGTH) item.description = util.format(util.get(entry, 'description', ''), settings.POPUP_BODY_LENGTH) item.link = util.get(entry, 'link', '') item.author = util.format(util.get(entry, 'author', '')) # TODO: max length if all(filter.filter(item) for filter in filters): result.append(item) self.clean_cache(settings.FEED_CACHE_SIZE) return result
def prep(type): global counts global classifiers global doclens global scores global docids global p_n global r_n counts = {} classifiers = {} # read classifiers for line in open(util.projdir + "/corpus/"+type+"-class", "r"): c = line.split(' ') classifiers[c[0].split('-')[1]] = c[1].split('\n')[0] for docid,c in classifiers.iteritems(): if c=="satire": r_n += 1 # read word counts docids = { "test": [util.format(i) for i in range(1,1595+1)], "training": [util.format(i) for i in range(1,2638+1)] } for docid in docids[type]: tmpcounts = util.read_counts_nopos("/data/bag/"+type+"/"+type+"-"+docid) counts[docid] = {} for w,c in tmpcounts.iteritems(): counts[docid][w.lower()] = c # these are floats because we need to divide something by them later doclens[docid] = float(sum([c for c in counts[docid].values()]))
def poll(self, timestamp, filters): try: logging.info('Polling feed "%s"' % self.url) result = [] self.last_poll = timestamp username = util.decode_password(self.username) password = util.decode_password(self.password) d = util.parse(self.url, username, password, self.etag, self.modified) self.etag = util.get(d, 'etag', None) self.modified = util.get(d, 'modified', None) feed = util.get(d, 'feed', None) if feed: self.title = self.title or util.get(feed, 'title', '') self.link = self.link or util.get(feed, 'link', self.url) entries = util.get(d, 'entries', []) for entry in reversed(entries): found = False id = create_id(entry) for i in self.id_set: # be cheap, compare the links - they should be unique. # Can't compare objects anymore because the time is present which will always be different if id[1] == i[1]: found = True break if found: continue # The entry is not new, move on to the next one. self.id_set.add(id) self.item_count += 1 item = Item(self, id) item.timestamp = calendar.timegm(util.get(entry, 'date_parsed', time.gmtime())) item.title = util.format(util.get(entry, 'title', ''), settings.POPUP_TITLE_LENGTH) item.description = util.format(util.get(entry, 'description', ''), settings.POPUP_BODY_LENGTH) item.link = util.get(entry, 'link', '') item.author = util.format(util.get(entry, 'author', '')) # TODO: max length if all(filter.filter(item) for filter in filters): result.append(item) # determine if there are any entries in self.id_set that are older than CACHE_AGE_LIMIT # if there are, remove them from the list (they aged out of the cache) idsToRemove = set() now = datetime.fromtimestamp(time.time()) for tempId in self.id_set: diff = now - datetime.fromtimestamp(tempId[3]) if abs(diff.days) > settings.CACHE_AGE_LIMIT: idsToRemove.add(tempId) # can't modify seld.id_set right here. for i in idsToRemove: logging.info('Removing %s because its too old' % i[1]) self.id_set.remove(i) return result except Exception, e: logging.info('Error durring poll: %s' % e) raise
def gen_cart_stack(init_commands, clock_commands, mode, loud=False): final_command_obj = None if clock_commands or init_commands: entities = [] entities.append(sands.normal_sand("activator_rail")) for command in init_commands: if hasattr(command, "cmd"): if loud: cprint(command.prettystr()) entities.append(cart(command.cmd)) offset = 1 for command in clock_commands: if offset == 1: command.block = "repeating_command_block" if loud: cprint(command.prettystr()) entities.append(cart_command_block(offset, command, 1, mode)) offset += 1 filloffset = offset+1 offset += 2 entities.append(cart_command_block(offset, Command(format("clone ~ ~-2 ~ ~ ~-{o1} ~ ~ ~-{o2} ~ replace move", o1=offset-1,o2=offset+2)))) offset += 1 entities.append(cart_command_block(offset, Command(format("fill ~ ~-{o1} ~ ~ ~-{o2} ~ air", o1=offset,o2=offset+2)))) offset += 1 activatesand = sands.normal_sand("command_block") activatesand["TileEntityData"] = {"auto": 1} entities.append(cart_block(offset, "air")) entities.append(cart(nbt.cmd(format("summon FallingSand ~ ~{o} ~ ", o=offset), activatesand, True))) entities.append(cart_command_block(filloffset, Command(format("fill ~ ~ ~ ~ ~{o} ~ air", o=offset-filloffset)))) entities.append(cart("kill @e[r=1,type=MinecartCommandBlock]")) stack = ride(entities) final_stack = sands.ride([ stack, sands.normal_sand("redstone_block"), sands.normal_sand("barrier") ], False) final_command_obj = nbt.cmd("summon FallingSand ~ ~1 ~ ", final_stack) final_command = nbt.JSON2Command(final_command_obj) return final_command
def send_batch(server_host, secret, batch): body = hashlib.sha256(secret + format(batch[0])).hexdigest() + "\n" + '\n'.join(map(format, batch)) while True: try: r = requests.post('http://{0}/logdata'.format(server_host), data=body, timeout=60) except requests.exceptions.RequestException, e: print >>sys.stderr, e time.sleep(1) continue else: if r.status_code >= 200 and r.status_code <= 204: print "Sent batch[{0}]: {1}...{2}".format(len(batch), batch[0], batch[-1]) return print "HTTP {0}".format(r.status_code) continue
def latest_entry(): keys = sorted([ x for x in redis.keys('blinks-*') if not x.startswith('blinks-x') ])[-1:] if not keys: return '0', 200 latest_ts = redis.zrevrangebyscore(keys[0], '+inf', '-inf', withscores=True, start=0, num=1)[0][1] return format(latest_ts), 200
def get_formatted(self, word, annotation): color = self.get_color(annotation) hl = self.get_highlight(annotation) return util.format(word, color, hl, self.latex)
def cart_block(offset, block, damage=0, data={}): cmd = format("setblock ~ ~{offset} ~ {block} {dmg} replace", block=block, dmg=damage) if offset: cmd = format(cmd, offset=offset) else: cmd = format(cmd, offset="") if data: cmd = nbt.cmd(cmd, data, True) return cart(cmd)
def main(assFile): lines = util.readFrom(assFile) origOffset = 0 updatedLines = [] ################## ### first pass ### ################## for line in lines: line = line.strip() if util.isComment(line) or line == '': continue elif util.isDirective(line): directive = line[:5] if directive == '.ORIG' or directive == '.orig': origOffset = 0 # in case .ORIG is used twice or more if '0x' in line: addrIndex = line.index('0x') origAddr = util.zext(line[addrIndex:], 8) else: addrIndex = '0x' + line.split()[-1] origAddr = util.zext(addrIndex, 8) updatedLines.append(line) # could update nameTable here... else: if util.isLabelDef(line): util.updateLabelTable( line[:-1], origOffset, origAddr) # just the label (without the :) continue elif util.isPseudoInstr(line): replacedLine, hasMoreLines = util.replacePseudoInstr(line) for l in replacedLine: updatedLines.append(l) if hasMoreLines: origOffset += 1 updatedLines.append(line) origOffset += 1 ################### ### second pass ### ################### mifComment = '-- @ ' origOffset = 0 hexLine = ''.join(header) + '\n' for line in updatedLines: if ';' in line: line = line[:line.index(';')] line = line.strip() if util.isComment(line) or line == '': continue elif util.isDirective(line): #util.parseDirective(line) directive = line[:5] if directive == '.ORIG' or directive == '.orig': origOffset = 0 if '0x' in line: addrIndex = line.index('0x') origAddr = util.format(line[addrIndex:], 8) else: addrIndex = '0x' + line.split()[-1] origAddr = util.format(addrIndex, 8) elif directive == '.NAME' or directive == '.name': util.updateNameTable(line) elif directive == '.WORD' or directive == '.word': pass else: raise Exception( 'Invalid directive!') #only three valid directives elif util.isLabelDef(line): continue else: opcode, regs, label = util.parseLine(line) opHex = util.transOpcode(opcode) if opcode == 'sw': # sw is the ONLY I-type instruction that has RS2 and RS1 backwards regs = regs[::-1] regsHex = util.transRegs(regs) if len(regs) == 1: # cases like mvhi, beqz... regsHex += '0' currAddr = '0x' + util.format( hex(int(origAddr, 16) + 4 * origOffset), 8) a32Addr = mifComment + ' ' + currAddr + ' : ' + line + '\n' mifAddr = util.format( hex((int(origAddr, 16) + 4 * origOffset) / 4), 8) if util.isImmType(opcode): # if I-type instruction if label.startswith('0x'): imm = util.zext(label, 4) elif util.isDecimalOffset(label): imm = util.zext(hex(int(label)), 4) else: if label in util.nameTable: imm = util.nameTable[label] if opcode == 'mvhi': # mvhi takes the 4 MSBs if len(imm[2:] ) < 4: # cases like .NAME STUFF=0x234 imm = (4 - len(imm[2:])) * '0' + imm else: imm = imm[ 2:6] # cases like .NAME STUFF=0xdf00000000 else: imm = util.format(imm, 4) elif label in util.labelTable: # we need to calculate address offset labelDefAddr = util.labelTable[ label] # this is the byte address #currAddr = hex(int(origAddr, 16) + 4 * origOffset) imm = util.calcLabelOffset(labelDefAddr, currAddr) imm = util.format(imm, 4) print('labelAddr: ', labelDefAddr) print('currAddr: ', currAddr) print('imm: ', imm) elif label in util.wordTable: # not sure if wordTable (and .WORD directive) is even going to be used... pass else: raise Exception('label not defined!') else: # if R-type instruction imm = '000' origOffset += 1 transLine = mifAddr + ' : ' + regsHex + imm + opHex + ';\n' hexLine += a32Addr + transLine print('labelTable: ', util.labelTable) print('nameTable: ', util.nameTable) #print(hexLine) filename, extension = os.path.splitext(assFile) util.writeTo(filename + '.mif', hexLine)
def test(nlp, src, gen, bert=False, print_annotations=False, print_latex=False, verbose=False): if print_annotations: print("source:", src_line[:50]) print("summary:", gen_line[:50]) src = nlp(src) gen = nlp(gen) if verbose: print("clusters:", src._.coref_clusters, gen._.coref_clusters) ce = CompoundEquivalency() spe = SpeakerPronounEquivalency() spe.register(src) spe.register(gen) kg = KnowledgeGraph(nlp, use_bert=bert, equivalencies=[ce, spe], verbose=verbose) if print_annotations: annotator = Annotator(src, gen, latex=print_latex) kg.add_document(src) contained = 0 contained_bert = 0 missing = 0 missing_verb = 0 missing_actors = 0 missing_acteds = 0 contradiction = 0 contradiction_bert = 0 invalid_simplification = 0 total = 0 for token in gen: if token.pos_ == "VERB": total += 1 relation = kg.get_relation(token) r = kg.query_relation(relation) if r[0] == KnowledgeGraph.entailment: if print_annotations: print(util.format("contained", "blue", latex=print_latex), "|", relation, "|", r[1]) contained += 1 if r[0] == KnowledgeGraph.entailment_bert: if print_annotations: print( util.format("contained (BERT)", "blue", latex=print_latex), "|", relation, "|", r[1]) contained_bert += 1 if r[0] == KnowledgeGraph.contradiction_bert: if print_annotations: print( util.format("contradiction (BERT)", "red", latex=print_latex), "|", relation, "|", r[1]) contradiction_bert += 1 elif r[0] == KnowledgeGraph.missing_dependencies: missing += 1 if print_annotations: print( util.format("generic missing dependency", "yellow", latex=print_latex), "|", relation, "|", r[1]) elif r[0] == KnowledgeGraph.missing_actors: missing_actors += 1 if print_annotations: print( util.format("missing actors", "magenta", latex=print_latex), "|", relation, "|", r[1]) elif r[0] == KnowledgeGraph.missing_acteds: missing_acteds += 1 if print_annotations: print( util.format("missing acteds", "magenta", latex=print_latex), "|", relation, "|", r[1]) elif r[0] == KnowledgeGraph.missing_verb: missing_verb += 1 if print_annotations: print( util.format("missing verb", "magenta", latex=print_latex), "|", relation, "|", r[1]) elif r[0] == KnowledgeGraph.invalid_simplification: invalid_simplification += 1 if print_annotations: print( util.format("invalid simplification", "magenta", latex=print_latex), "|", relation, "|", r[1]) elif r[0] == KnowledgeGraph.contradiction: contradiction += 1 if print_annotations: print( util.format("contradiction", "red", latex=print_latex), "|", relation, "|", r[1]) if print_annotations: annotator.annotate(relation, r) if print_annotations: annotated_document, annotated_summary = annotator.annotated() print("Document:", " ".join(annotated_document)) print("Summary:", " ".join(annotated_summary)) if total == 0: return 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 return 100.0 * contained / total, \ 100.0 * contained_bert / total, \ 100.0 * missing / total, \ 100.0 * missing_verb / total, \ 100.0 * missing_actors / total, \ 100.0 * missing_acteds / total, \ 100.0 * contradiction / total, \ 100.0 * contradiction_bert / total, \ 100.0 * invalid_simplification / total
def gen_stack(init_commands, clock_commands, mode, loud=False): final_command_obj = None if clock_commands or init_commands: command_sands = [] repeatoffsets = [] if mode == 'i': if clock_commands and isinstance(clock_commands[0], Command): repeatoffsets.append(len(clock_commands) + 2) for command in clock_commands: if command.block == "repeating_command_block" and not command.cond and command is not clock_commands[0]: repeatoffsets.append(len(clock_commands) - clock_commands.index(command) + 2 + len(repeatoffsets)) filloffset = len(init_commands) + len(repeatoffsets) if filloffset: filloffset += 1 if filloffset: if loud: cprint("minecraft:command_block:0\n - Initialization", color=bcolors.DARKGRAY, allow_repeat=True) sand = normal_sand("command_block") if mode == 'i': sand["TileEntityData"] = { "auto": 1 } command_sands.append(sand) for command in init_commands: if loud: cprint(command.prettystr(), allow_repeat=True) command_sands.append(generate_sand(command, 0)) for offset in repeatoffsets[::-1]: blockdata = Command(format("blockdata ~ ~-{offset} ~ {auto:1b}", offset = offset), init=True) if loud: cprint(blockdata.prettystr(), allow_repeat=True) sand = generate_sand(blockdata, 0) command_sands.append(sand) if filloffset: fill = Command(format("fill ~ ~-1 ~ ~ ~{offset} ~ air", offset = filloffset), init=True) if loud: cprint(fill.prettystr(), allow_repeat=True) cprint("minecraft:barrier\n - Initialization", color=bcolors.DARKGRAY, allow_repeat=True) command_sands.append(generate_sand(fill, 0)) command_sands.append(normal_sand("barrier")) for command in clock_commands[::-1]: if command is clock_commands[0] and isinstance(command, Command): command.block = "repeating_command_block" command_sands.append(generate_sand(command, 1)) else: sand = generate_sand(command, 1) if command.block == "repeating_command_block" and command.cond: sand["TileEntityData"]["auto"] = 1 command_sands.append(sand) if loud: cprint(command.prettystr(), allow_repeat=True) final_command_obj = nbt.cmd("summon FallingSand ~ ~1 ~ ", ride(command_sands, False)) final_command = nbt.JSON2Command(final_command_obj) return final_command
def prettystr(self): return format("{darkgray}{block}{init}", block = self, init = "\n - Initialization" if self.init else "")
def __str__(self): return format("{block} {data}", block=self.block, data=self.data)
def prettystr(self): return format("{cmd}{init}{cond}{repeat}", cmd = self.cmd, init = "\n - Initialization" if self.init else "", cond = "\n - Conditional" if self.cond else "", repeat="\n - Repeating" if self.block == "repeating_command_block" else "")