def poll(self, timestamp, filters):
logging.info('Polling feed "%s"' % self.url)
result = []
self.last_poll = timestamp
username = util.decode_password(self.username)
password = util.decode_password(self.password)
d = util.parse(self.url, username, password, self.etag, self.modified)
self.etag = util.get(d, 'etag', None)
self.modified = util.get(d, 'modified', None)
feed = util.get(d, 'feed', None)
if feed:
self.title = self.title or util.get(feed, 'title', '')
self.link = self.link or util.get(feed, 'link', self.url)
entries = util.get(d, 'entries', [])
for entry in reversed(entries):
id = create_id(entry)
if id in self.id_set:
continue
self.item_count += 1
self.id_list.append(id)
self.id_set.add(id)
item = Item(self, id)
item.timestamp = calendar.timegm(
util.get(entry, 'date_parsed', time.gmtime()))
item.title = util.format(util.get(entry, 'title', ''),
settings.POPUP_TITLE_LENGTH)
item.description = util.format(util.get(entry, 'description', ''),
settings.POPUP_BODY_LENGTH)
item.link = util.get(entry, 'link', '')
item.author = util.format(util.get(entry, 'author',
'')) # TODO: max length
if all(filter.filter(item) for filter in filters):
result.append(item)
self.clean_cache(settings.FEED_CACHE_SIZE)
return result
def prep(type):
global counts
global classifiers
global doclens
global scores
global docids
global p_n
global r_n
counts = {}
classifiers = {}
# read classifiers
for line in open(util.projdir + "/corpus/" + type + "-class", "r"):
c = line.split(' ')
classifiers[c[0].split('-')[1]] = c[1].split('\n')[0]
for docid, c in classifiers.iteritems():
if c == "satire":
r_n += 1
# read word counts
docids = {
"test": [util.format(i) for i in range(1, 1595 + 1)],
"training": [util.format(i) for i in range(1, 2638 + 1)]
}
for docid in docids[type]:
tmpcounts = util.read_counts_nopos("/data/bag/" + type + "/" + type +
"-" + docid)
counts[docid] = {}
for w, c in tmpcounts.iteritems():
counts[docid][w.lower()] = c
# these are floats because we need to divide something by them later
doclens[docid] = float(sum([c for c in counts[docid].values()]))
def poll(self, timestamp, filters):
logging.info('Polling feed "%s"' % self.url)
result = []
self.last_poll = timestamp
username = util.decode_password(self.username)
password = util.decode_password(self.password)
d = util.parse(self.url, username, password, self.etag, self.modified)
self.etag = util.get(d, 'etag', None)
self.modified = util.get(d, 'modified', None)
feed = util.get(d, 'feed', None)
if feed:
self.title = self.title or util.get(feed, 'title', '')
self.link = self.link or util.get(feed, 'link', self.url)
entries = util.get(d, 'entries', [])
for entry in reversed(entries):
id = create_id(entry)
if id in self.id_set:
continue
self.item_count += 1
self.id_list.append(id)
self.id_set.add(id)
item = Item(self, id)
item.timestamp = calendar.timegm(util.get(entry, 'date_parsed', time.gmtime()))
item.title = util.format(util.get(entry, 'title', ''), settings.POPUP_TITLE_LENGTH)
item.description = util.format(util.get(entry, 'description', ''), settings.POPUP_BODY_LENGTH)
item.link = util.get(entry, 'link', '')
item.author = util.format(util.get(entry, 'author', '')) # TODO: max length
if all(filter.filter(item) for filter in filters):
result.append(item)
self.clean_cache(settings.FEED_CACHE_SIZE)
return result
def prep(type):
global counts
global classifiers
global doclens
global scores
global docids
global p_n
global r_n
counts = {}
classifiers = {}
# read classifiers
for line in open(util.projdir + "/corpus/"+type+"-class", "r"):
c = line.split(' ')
classifiers[c[0].split('-')[1]] = c[1].split('\n')[0]
for docid,c in classifiers.iteritems():
if c=="satire":
r_n += 1
# read word counts
docids = {
"test": [util.format(i) for i in range(1,1595+1)],
"training": [util.format(i) for i in range(1,2638+1)]
}
for docid in docids[type]:
tmpcounts = util.read_counts_nopos("/data/bag/"+type+"/"+type+"-"+docid)
counts[docid] = {}
for w,c in tmpcounts.iteritems():
counts[docid][w.lower()] = c
# these are floats because we need to divide something by them later
doclens[docid] = float(sum([c for c in counts[docid].values()]))
def poll(self, timestamp, filters):
try:
logging.info('Polling feed "%s"' % self.url)
result = []
self.last_poll = timestamp
username = util.decode_password(self.username)
password = util.decode_password(self.password)
d = util.parse(self.url, username, password, self.etag, self.modified)
self.etag = util.get(d, 'etag', None)
self.modified = util.get(d, 'modified', None)
feed = util.get(d, 'feed', None)
if feed:
self.title = self.title or util.get(feed, 'title', '')
self.link = self.link or util.get(feed, 'link', self.url)
entries = util.get(d, 'entries', [])
for entry in reversed(entries):
found = False
id = create_id(entry)
for i in self.id_set:
# be cheap, compare the links - they should be unique.
# Can't compare objects anymore because the time is present which will always be different
if id[1] == i[1]:
found = True
break
if found:
continue # The entry is not new, move on to the next one.
self.id_set.add(id)
self.item_count += 1
item = Item(self, id)
item.timestamp = calendar.timegm(util.get(entry, 'date_parsed', time.gmtime()))
item.title = util.format(util.get(entry, 'title', ''), settings.POPUP_TITLE_LENGTH)
item.description = util.format(util.get(entry, 'description', ''), settings.POPUP_BODY_LENGTH)
item.link = util.get(entry, 'link', '')
item.author = util.format(util.get(entry, 'author', '')) # TODO: max length
if all(filter.filter(item) for filter in filters):
result.append(item)
# determine if there are any entries in self.id_set that are older than CACHE_AGE_LIMIT
# if there are, remove them from the list (they aged out of the cache)
idsToRemove = set()
now = datetime.fromtimestamp(time.time())
for tempId in self.id_set:
diff = now - datetime.fromtimestamp(tempId[3])
if abs(diff.days) > settings.CACHE_AGE_LIMIT:
idsToRemove.add(tempId) # can't modify seld.id_set right here.
for i in idsToRemove:
logging.info('Removing %s because its too old' % i[1])
self.id_set.remove(i)
return result
except Exception, e:
logging.info('Error durring poll: %s' % e)
raise
def gen_cart_stack(init_commands, clock_commands, mode, loud=False):
final_command_obj = None
if clock_commands or init_commands:
entities = []
entities.append(sands.normal_sand("activator_rail"))
for command in init_commands:
if hasattr(command, "cmd"):
if loud:
cprint(command.prettystr())
entities.append(cart(command.cmd))
offset = 1
for command in clock_commands:
if offset == 1:
command.block = "repeating_command_block"
if loud:
cprint(command.prettystr())
entities.append(cart_command_block(offset, command, 1, mode))
offset += 1
filloffset = offset+1
offset += 2
entities.append(cart_command_block(offset, Command(format("clone ~ ~-2 ~ ~ ~-{o1} ~ ~ ~-{o2} ~ replace move", o1=offset-1,o2=offset+2))))
offset += 1
entities.append(cart_command_block(offset, Command(format("fill ~ ~-{o1} ~ ~ ~-{o2} ~ air", o1=offset,o2=offset+2))))
offset += 1
activatesand = sands.normal_sand("command_block")
activatesand["TileEntityData"] = {"auto": 1}
entities.append(cart_block(offset, "air"))
entities.append(cart(nbt.cmd(format("summon FallingSand ~ ~{o} ~ ", o=offset), activatesand, True)))
entities.append(cart_command_block(filloffset, Command(format("fill ~ ~ ~ ~ ~{o} ~ air", o=offset-filloffset))))
entities.append(cart("kill @e[r=1,type=MinecartCommandBlock]"))
stack = ride(entities)
final_stack = sands.ride([
stack,
sands.normal_sand("redstone_block"),
sands.normal_sand("barrier")
], False)
final_command_obj = nbt.cmd("summon FallingSand ~ ~1 ~ ", final_stack)
final_command = nbt.JSON2Command(final_command_obj)
return final_command
def send_batch(server_host, secret, batch):
body = hashlib.sha256(secret + format(batch[0])).hexdigest() + "\n" + '\n'.join(map(format, batch))
while True:
try:
r = requests.post('http://{0}/logdata'.format(server_host), data=body, timeout=60)
except requests.exceptions.RequestException, e:
print >>sys.stderr, e
time.sleep(1)
continue
else:
if r.status_code >= 200 and r.status_code <= 204:
print "Sent batch[{0}]: {1}...{2}".format(len(batch), batch[0], batch[-1])
return
print "HTTP {0}".format(r.status_code)
continue
def latest_entry():
keys = sorted([ x for x in redis.keys('blinks-*') if not x.startswith('blinks-x') ])[-1:]
if not keys:
return '0', 200
latest_ts = redis.zrevrangebyscore(keys[0], '+inf', '-inf', withscores=True, start=0, num=1)[0][1]
return format(latest_ts), 200
def get_formatted(self, word, annotation):
color = self.get_color(annotation)
hl = self.get_highlight(annotation)
return util.format(word, color, hl, self.latex)
def cart_block(offset, block, damage=0, data={}):
cmd = format("setblock ~ ~{offset} ~ {block} {dmg} replace", block=block, dmg=damage)
if offset: cmd = format(cmd, offset=offset)
else: cmd = format(cmd, offset="")
if data: cmd = nbt.cmd(cmd, data, True)
return cart(cmd)
def main(assFile):
lines = util.readFrom(assFile)
origOffset = 0
updatedLines = []
##################
### first pass ###
##################
for line in lines:
line = line.strip()
if util.isComment(line) or line == '':
continue
elif util.isDirective(line):
directive = line[:5]
if directive == '.ORIG' or directive == '.orig':
origOffset = 0 # in case .ORIG is used twice or more
if '0x' in line:
addrIndex = line.index('0x')
origAddr = util.zext(line[addrIndex:], 8)
else:
addrIndex = '0x' + line.split()[-1]
origAddr = util.zext(addrIndex, 8)
updatedLines.append(line)
# could update nameTable here...
else:
if util.isLabelDef(line):
util.updateLabelTable(
line[:-1], origOffset,
origAddr) # just the label (without the :)
continue
elif util.isPseudoInstr(line):
replacedLine, hasMoreLines = util.replacePseudoInstr(line)
for l in replacedLine:
updatedLines.append(l)
if hasMoreLines:
origOffset += 1
updatedLines.append(line)
origOffset += 1
###################
### second pass ###
###################
mifComment = '-- @ '
origOffset = 0
hexLine = ''.join(header) + '\n'
for line in updatedLines:
if ';' in line:
line = line[:line.index(';')]
line = line.strip()
if util.isComment(line) or line == '':
continue
elif util.isDirective(line):
#util.parseDirective(line)
directive = line[:5]
if directive == '.ORIG' or directive == '.orig':
origOffset = 0
if '0x' in line:
addrIndex = line.index('0x')
origAddr = util.format(line[addrIndex:], 8)
else:
addrIndex = '0x' + line.split()[-1]
origAddr = util.format(addrIndex, 8)
elif directive == '.NAME' or directive == '.name':
util.updateNameTable(line)
elif directive == '.WORD' or directive == '.word':
pass
else:
raise Exception(
'Invalid directive!') #only three valid directives
elif util.isLabelDef(line):
continue
else:
opcode, regs, label = util.parseLine(line)
opHex = util.transOpcode(opcode)
if opcode == 'sw': # sw is the ONLY I-type instruction that has RS2 and RS1 backwards
regs = regs[::-1]
regsHex = util.transRegs(regs)
if len(regs) == 1: # cases like mvhi, beqz...
regsHex += '0'
currAddr = '0x' + util.format(
hex(int(origAddr, 16) + 4 * origOffset), 8)
a32Addr = mifComment + ' ' + currAddr + ' : ' + line + '\n'
mifAddr = util.format(
hex((int(origAddr, 16) + 4 * origOffset) / 4), 8)
if util.isImmType(opcode): # if I-type instruction
if label.startswith('0x'):
imm = util.zext(label, 4)
elif util.isDecimalOffset(label):
imm = util.zext(hex(int(label)), 4)
else:
if label in util.nameTable:
imm = util.nameTable[label]
if opcode == 'mvhi': # mvhi takes the 4 MSBs
if len(imm[2:]
) < 4: # cases like .NAME STUFF=0x234
imm = (4 - len(imm[2:])) * '0' + imm
else:
imm = imm[
2:6] # cases like .NAME STUFF=0xdf00000000
else:
imm = util.format(imm, 4)
elif label in util.labelTable:
# we need to calculate address offset
labelDefAddr = util.labelTable[
label] # this is the byte address
#currAddr = hex(int(origAddr, 16) + 4 * origOffset)
imm = util.calcLabelOffset(labelDefAddr, currAddr)
imm = util.format(imm, 4)
print('labelAddr: ', labelDefAddr)
print('currAddr: ', currAddr)
print('imm: ', imm)
elif label in util.wordTable:
# not sure if wordTable (and .WORD directive) is even going to be used...
pass
else:
raise Exception('label not defined!')
else: # if R-type instruction
imm = '000'
origOffset += 1
transLine = mifAddr + ' : ' + regsHex + imm + opHex + ';\n'
hexLine += a32Addr + transLine
print('labelTable: ', util.labelTable)
print('nameTable: ', util.nameTable)
#print(hexLine)
filename, extension = os.path.splitext(assFile)
util.writeTo(filename + '.mif', hexLine)
def test(nlp,
src,
gen,
bert=False,
print_annotations=False,
print_latex=False,
verbose=False):
if print_annotations:
print("source:", src_line[:50])
print("summary:", gen_line[:50])
src = nlp(src)
gen = nlp(gen)
if verbose:
print("clusters:", src._.coref_clusters, gen._.coref_clusters)
ce = CompoundEquivalency()
spe = SpeakerPronounEquivalency()
spe.register(src)
spe.register(gen)
kg = KnowledgeGraph(nlp,
use_bert=bert,
equivalencies=[ce, spe],
verbose=verbose)
if print_annotations:
annotator = Annotator(src, gen, latex=print_latex)
kg.add_document(src)
contained = 0
contained_bert = 0
missing = 0
missing_verb = 0
missing_actors = 0
missing_acteds = 0
contradiction = 0
contradiction_bert = 0
invalid_simplification = 0
total = 0
for token in gen:
if token.pos_ == "VERB":
total += 1
relation = kg.get_relation(token)
r = kg.query_relation(relation)
if r[0] == KnowledgeGraph.entailment:
if print_annotations:
print(util.format("contained", "blue", latex=print_latex),
"|", relation, "|", r[1])
contained += 1
if r[0] == KnowledgeGraph.entailment_bert:
if print_annotations:
print(
util.format("contained (BERT)",
"blue",
latex=print_latex), "|", relation, "|",
r[1])
contained_bert += 1
if r[0] == KnowledgeGraph.contradiction_bert:
if print_annotations:
print(
util.format("contradiction (BERT)",
"red",
latex=print_latex), "|", relation, "|",
r[1])
contradiction_bert += 1
elif r[0] == KnowledgeGraph.missing_dependencies:
missing += 1
if print_annotations:
print(
util.format("generic missing dependency",
"yellow",
latex=print_latex), "|", relation, "|",
r[1])
elif r[0] == KnowledgeGraph.missing_actors:
missing_actors += 1
if print_annotations:
print(
util.format("missing actors",
"magenta",
latex=print_latex), "|", relation, "|",
r[1])
elif r[0] == KnowledgeGraph.missing_acteds:
missing_acteds += 1
if print_annotations:
print(
util.format("missing acteds",
"magenta",
latex=print_latex), "|", relation, "|",
r[1])
elif r[0] == KnowledgeGraph.missing_verb:
missing_verb += 1
if print_annotations:
print(
util.format("missing verb",
"magenta",
latex=print_latex), "|", relation, "|",
r[1])
elif r[0] == KnowledgeGraph.invalid_simplification:
invalid_simplification += 1
if print_annotations:
print(
util.format("invalid simplification",
"magenta",
latex=print_latex), "|", relation, "|",
r[1])
elif r[0] == KnowledgeGraph.contradiction:
contradiction += 1
if print_annotations:
print(
util.format("contradiction", "red", latex=print_latex),
"|", relation, "|", r[1])
if print_annotations:
annotator.annotate(relation, r)
if print_annotations:
annotated_document, annotated_summary = annotator.annotated()
print("Document:", " ".join(annotated_document))
print("Summary:", " ".join(annotated_summary))
if total == 0:
return 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
return 100.0 * contained / total, \
100.0 * contained_bert / total, \
100.0 * missing / total, \
100.0 * missing_verb / total, \
100.0 * missing_actors / total, \
100.0 * missing_acteds / total, \
100.0 * contradiction / total, \
100.0 * contradiction_bert / total, \
100.0 * invalid_simplification / total
def gen_stack(init_commands, clock_commands, mode, loud=False):
final_command_obj = None
if clock_commands or init_commands:
command_sands = []
repeatoffsets = []
if mode == 'i':
if clock_commands and isinstance(clock_commands[0], Command):
repeatoffsets.append(len(clock_commands) + 2)
for command in clock_commands:
if command.block == "repeating_command_block" and not command.cond and command is not clock_commands[0]:
repeatoffsets.append(len(clock_commands) - clock_commands.index(command) + 2 + len(repeatoffsets))
filloffset = len(init_commands) + len(repeatoffsets)
if filloffset: filloffset += 1
if filloffset:
if loud:
cprint("minecraft:command_block:0\n - Initialization", color=bcolors.DARKGRAY, allow_repeat=True)
sand = normal_sand("command_block")
if mode == 'i':
sand["TileEntityData"] = {
"auto": 1
}
command_sands.append(sand)
for command in init_commands:
if loud:
cprint(command.prettystr(), allow_repeat=True)
command_sands.append(generate_sand(command, 0))
for offset in repeatoffsets[::-1]:
blockdata = Command(format("blockdata ~ ~-{offset} ~ {auto:1b}", offset = offset), init=True)
if loud:
cprint(blockdata.prettystr(), allow_repeat=True)
sand = generate_sand(blockdata, 0)
command_sands.append(sand)
if filloffset:
fill = Command(format("fill ~ ~-1 ~ ~ ~{offset} ~ air", offset = filloffset), init=True)
if loud:
cprint(fill.prettystr(), allow_repeat=True)
cprint("minecraft:barrier\n - Initialization", color=bcolors.DARKGRAY, allow_repeat=True)
command_sands.append(generate_sand(fill, 0))
command_sands.append(normal_sand("barrier"))
for command in clock_commands[::-1]:
if command is clock_commands[0] and isinstance(command, Command):
command.block = "repeating_command_block"
command_sands.append(generate_sand(command, 1))
else:
sand = generate_sand(command, 1)
if command.block == "repeating_command_block" and command.cond:
sand["TileEntityData"]["auto"] = 1
command_sands.append(sand)
if loud:
cprint(command.prettystr(), allow_repeat=True)
final_command_obj = nbt.cmd("summon FallingSand ~ ~1 ~ ", ride(command_sands, False))
final_command = nbt.JSON2Command(final_command_obj)
return final_command
def prettystr(self):
return format("{darkgray}{block}{init}",
block = self,
init = "\n - Initialization" if self.init else "")
def __str__(self):
return format("{block} {data}",
block=self.block,
data=self.data)
def prettystr(self):
return format("{cmd}{init}{cond}{repeat}",
cmd = self.cmd,
init = "\n - Initialization" if self.init else "",
cond = "\n - Conditional" if self.cond else "",
repeat="\n - Repeating" if self.block == "repeating_command_block" else "")
