def main():
global loopCount
while gamePlay == True:
clock.tick(45)
if loopCount % 90 == 0:
topPos = random.randint(0, height / 2) - 400
pipes.add(Pipe((width + 100, topPos + gapSize + 800)))
pipes.add(Pipe((width + 100, topPos), True))
for event in pygame.event.get():
if event.type == pygame.QUIT:
sys.exit()
if event.type == pygame.KEYDOWN:
if event.key == pygame.K_SPACE:
player.speed[1] = -10
screen.fill(color)
player.update()
pipes.update()
gets_hit = pygame.sprite.spritecollide(
player, pipes, False) or player.rect.center[1] > height
screen.blit(background, [0, 0])
pipes.draw(screen)
screen.blit(player.image, player.rect)
pygame.display.flip()
loopCount += 1
if gets_hit:
lose()
def __init__(self, x, y):
self.x = x
self.lower_pipe = Pipe(x, y, 'lower')
self.upper_pipe = Pipe(x, y - settings.PIPE_HEIGHT - settings.PIPE_GAP,
'upper')
self.remove = False
self.scored = False
def main():
global loopCount
while True:
clock.tick(60)
if loopCount % 60 == 0:
topPos = random.randint(0, 200)
pipes.add(Pipe((width + 100, topPos + gapSize + 380)))
pipes.add(Pipe((width + 100, topPos), True))
for event in pygame.event.get():
if event.type == pygame.QUIT:
sys.exit()
if event.type == pygame.KEYDOWN:
if event.key == pygame.K_SPACE:
player.speed[1] = -10
player.update()
pipes.update()
gets_hit = pygame.sprite.spritecollide(player,pipes,False) \
or player.rect.center[1] > height
window.blit(background, [0, 0])
pipes.draw(window)
window.blit(player.image, player.rect)
pygame.display.flip()
loopCount += 1
if gets_hit:
lose()
def add_items():
""" Initialize all items to be added to the model"""
# create all instances of goombas
model.items.append(Goomba(model, 700, 300, -0.1, 0))
model.items.append(Goomba(model, 800, 300, -0.1, 0))
model.items.append(Goomba(model, 1000, 300, -0.1, 0))
model.items.append(Goomba(model, 1300, 300, -0.1, 0))
model.items.append(Goomba(model, 1500, 300, -0.1, 0))
model.items.append(Goomba(model, 1700, 300, -0.1, 0))
model.items.append(Goomba(model, 2800, 300, -0.1, 0))
model.items.append(Goomba(model, 3000, 300, -0.1, 0))
# create all instances of pipes
model.items.append(Pipe(model, 800, 425, height=125))
model.items.append(Pipe(model, 2000, 425, height=125))
# create all instances of bricks in the air
model.items.append(Air_Bricks(model, 550, 450))
model.items.append(Air_Bricks(model, 1000, 450))
model.items.append(Air_Bricks(model, 1400, 450))
model.items.append(Air_Bricks(model, 2600, 450))
# create the flag and castle
model.items.append(Flag(model, length - 275, 250))
model.items.append(Castle(model, length - 200, 350))
# add clouds to display
for n in range(1, length, 400):
model.items.append(Cloud(x=n, y=random.randint(50, 250)))
def addPipes(self, numMiners, numFullNodes):
for identifier in range(numMiners):
self.pipes["M%d" % identifier] = Pipe(self.env, "M%d" % identifier,
self.nodes)
for identifier in range(numFullNodes):
self.pipes["F%d" % identifier] = Pipe(self.env, "F%d" % identifier,
self.nodes)
def create_pipe(ai_settings,pipes,number_pipe,uppipes_y):
pipe = Pipe(ai_settings)
pipe.x=ai_settings.screen_width+ai_settings.pipe_width/2\
+number_pipe*(ai_settings.screen_width /(ai_settings.number_pipes-2))
pipe.rect.x =pipe.x
pipe.rect.y=uppipes_y[number_pipe]+ai_settings.pipe_height+ai_settings.disty
pipes.add(pipe)
def run_game():
pygame.init()
gameDisplay = pygame.display.set_mode((DISPLAY_W,DISPLAY_H))
pygame.display.set_caption('Learn to fly')
running = True
bgImg = pygame.image.load(BG_FILENAME)
label_font = pygame.font.SysFont("monospace", DATA_FONT_SIZE)
clock = pygame.time.Clock()
dt = 0
game_time = 0
pi = Pipe(gameDisplay, DISPLAY_W, 300, PIPE_LOWER)
while running:
dt = clock.tick(FPS)
game_time += dt
gameDisplay.blit(bgImg, (0, 0))
for event in pygame.event.get():
if event.type == pygame.QUIT:
running = False
elif event.type == pygame.KEYDOWN:
running = False
update_data_labels(gameDisplay, dt, game_time, label_font)
pi.update(dt)
pygame.display.update()
def update_pipes(screen, settings, bird, pipes, sb, stats):
"""
Adds the first pipe pair.
Updates the pipes' pos.
Adds new pipes as the game progresses.
Removes pipes off the screen.
Increments score for when the bird passes through a pipe.
"""
#First pipe
if len(pipes) == 0:
pipe_pair = Pipe(screen, settings, bird)
pipes.append(pipe_pair)
#Move pipes
for pipe_pair in pipes:
pipe_pair.top_pipe['x'] -= settings.pipe_speed
pipe_pair.bot_pipe['x'] -= settings.pipe_speed
#Add new pipes, spaced by half the screen width
for pipe in pipes:
if pipe.top_pipe['x'] == settings.screen_width / 2:
new_pipe = Pipe(screen, settings, bird)
pipes.append(new_pipe)
#Remove pipes
if pipes[0].top_pipe['x'] < -pipe_pair.pipe_width:
pipes.pop(0)
#Scoring
if pipes[0].top_pipe['x'] == bird.x:
stats.score += 1
sb.prep_score()
check_hiscore(stats, sb)
def __init__(self):
self.pipes = []
self.pipes.append(Pipe((0, SIZE_GAME[1] - 100), (300, 100)))
self.pipes.append(Pipe((0, 0), (50, SIZE_GAME[1])))
self.pipes.append(Pipe((600, SIZE_GAME[1] - 200), (300, 100)))
self.pipes.append(Pipe((1000, SIZE_GAME[1] - 350), (300, 200)))
self.pipes.append(Pipe((1000, SIZE_GAME[1] - 700), (300, 100)))
def setup(self):
self.pipes = arcade.SpriteList()
# If the generation is the first we need to create the birds.
if self.demo_mode:
self.bird_list = arcade.SpriteList()
self.bird_list.append(Player(neural_net = self.demo_network))
self.player_count = 1
else:
if self.generation == 0:
self.bird_list = arcade.SpriteList()
for i in range(0, self.player_count):
self.bird_list.append(Player())
# Otherwise send to the genetic algorithm to modify them.
else:
self.bird_list = Genetic_Algorithm.next_gen(self.dead_birds_array)
# Create initial pipes
left, right = Pipe.generate_pipes(SCREEN_WIDTH + 20)
self.pipes.append(left)
self.pipes.append(right)
left, right = Pipe.generate_pipes(SCREEN_WIDTH + 20 + 150)
self.pipes.append(left)
self.pipes.append(right)
left, right = Pipe.generate_pipes(SCREEN_WIDTH + 20 + 300)
self.pipes.append(left)
self.pipes.append(right)
def animationFrame(self):
# Timer1 handler (Create new pipes)
if(self.timestamp > self.timer1 + self.settings["richness"] / self.settings["speed"]):
self.timer1 = self.timestamp
gap = randint(self.settings["minPipeHeight"], self.settings["height"] - self.settings["gapSize"] - self.settings["minPipeHeight"])
self.pipes.append(Pipe(self, 0, gap))
self.pipes.append(Pipe(self, gap + self.settings["gapSize"], self.settings["height"] - gap - self.settings["gapSize"]))
# Update and render pipes
for pipe in self.pipes:
if(self.player.isAlive):
pipe.update()
pipe.render()
# Remove pipes which are out of the view (Has to do it in separate loop to prevent flashing)
for pipe in self.pipes:
if(pipe.isRemoved):
self.pipes.remove(pipe)
# Render ground
self.renderGround()
# Update and render player
self.player.update()
self.player.render()
# Render text
self.renderScore()
self.canvas.create_text(15, 20, text=round(self.bestScore), anchor="w", font=("Consolas", 12), fill="white")
def main_play():
global Win, generation
win = Win
generation += 1
bird = Bird(200, 320)
BGround = Back_ground()
base = Base(FLOOR)
pipes = [Pipe(650)]
run = True
clock = pygame.time.Clock()
scores = 0
while run:
clock.tick(30)
for event in pygame.event.get():
if event.type == pygame.QUIT:
run = False
pygame.quit()
# bird.move()
if event.type == pygame.KEYDOWN and (event.key == pygame.K_SPACE
or event.key == pygame.K_UP):
bird.jump()
bird.y -= 15
bird.move()
BGround.move()
base.move()
disappear = []
addPipe = False
for pipe in pipes:
pipe.move()
if pipe.attack(bird):
run = False
if pipe.x + pipe.pipe_top.get_width() < 0:
disappear.append(pipe)
if not pipe.passed and pipe.x < bird.x:
pipe.passed = True
addPipe = True
if addPipe:
scores += 1
pipes.append(Pipe(500))
for pipe_remove in disappear:
pipes.remove(pipe_remove)
if bird.y + bird.image.get_height() - 10 >= FLOOR or bird.y < 0:
run = False
draw_win(Win, bird, pipes, base, BGround, scores, generation)
def reset(self):
# make game activate False until bird is operated
self.activate_game = False
self.bird = Bird()
self.pipe = Pipe()
self.score_count = 0
self.background = Background()
def generate(self, width, height):
for x in range(width):
for y in range(height):
pipe = Pipe(self, x, y)
#TODO birbirine bağlanabilen level algoritması yazılacak
#ya da hardcode şeklinde levellar tasarlanacak
random = randint(1, 4)
pipe.setPipeType(random)
self.pipeList.append(pipe)
def generate_pipes(self):
# generate the pipes with a delay of (param1) and (param for pipe 2)
p1 = Pipe(310)
p2 = Pipe(310 + 500)
p3 = Pipe(310 + 500 + 500)
self.pipe_array.append(p1)
self.pipe_array.append(p2)
self.pipe_array.append(p3)
def game(genome, config):
FPSCLOCK = pygame.time.Clock()
size = (SCREEN_WIDTH, SCREEN_HEIGHT)
brain = neat.nn.FeedForwardNetwork.create(genome, config)
bird = Bird(brain)
pipes = []
pipes.append(Pipe())
screen = pygame.display.set_mode(size)
while 1:
for event in pygame.event.get():
if event.type == QUIT or (event.type == KEYDOWN and event.key == K_ESCAPE):
pygame.quit()
sys.exit()
closest = closest_pipe([bird], pipes)
if closest != None:
brain_input = (bird.y / SCREEN_HEIGHT,
closest.x / SCREEN_WIDTH,
closest.upper_pipe_len / SCREEN_HEIGHT,
closest.bottom_pipe_y / SCREEN_HEIGHT)
output = bird.brain.activate(brain_input)
if output[0] >= 0.5:
bird.flap()
# Background
screen.fill(BLACK)
# Draw pipe(s)
for pipe in reversed(pipes):
pipe.update()
pipe.show(screen)
if bird.is_alive() and pipe.has_collided(bird):
bird.die()
print(bird.fitness_score())
if pipe.x < -PIPE_WIDTH:
pipes.remove(pipe)
# Draw bird(s)
if bird.is_alive():
bird.update()
bird.show(screen)
if pipes[-1].x == ADD_PIPE_POS:
pipes.append(Pipe())
# Change frames
pygame.display.flip()
FPSCLOCK.tick(FPS_PLAY)
def update(self):
if self.alive:
self.counter -= 60 / TICKRATE
if self.counter < 0:
self.gap = randint(PIPEGAP_MIN + PIPE_BUFFER,
400 - PIPE_BUFFER)
self.add(Pipe(self.gap, self.color, 0, self.bird))
self.add(Pipe(self.gap, self.color, 1, self.bird))
self.counter = 288 / PIPE_DENSITY
pygame.sprite.Group.update(self)
def main():
bird.position = [74, 100]
pipeTimer = 0
pipeDelay = 2.5
fall_speed = 0
global pipes
pipes = []
global score
score = 0
while True:
clock.tick(60)
bird.sprite = pygame.transform.rotate(bird.sprite_root,
bird.fall_speed * -2.5)
display()
for event in pygame.event.get():
if event.type == pygame.QUIT:
leave()
elif event.type == pygame.KEYDOWN:
# Figure out if it was an arrow key. If so adjust speed
if event.key == pygame.K_SPACE:
bird.flap()
if pipeTimer + pipeDelay < time.time():
offset = random.randint(-100, 350)
pipes.append((Pipe([500, offset - 390],
True), Pipe([500, offset + 390], False)))
pipeTimer = time.time()
for i in range(len(pipes)):
if bird.position[0] == pipes[i][0].position[0]:
score += 1
for j in range(2):
pipes[i][j].position[0] -= 2
pipes[i][j].rect = pipes[i][j].sprite.get_rect(
center=(pipes[i][j].position[0] +
pipes[i][j].dimensions[0] / 2,
pipes[i][j].position[1] +
pipes[i][j].dimensions[1] / 2))
if pygame.Rect.colliderect(bird.rect, pipes[i][j].rect):
display()
return
if len(pipes) > 0:
if pipes[0][0].position[0] - pipes[0][0].dimensions[0] < -200:
del pipes[0]
bird.fall_speed += 0.5
bird.position[1] += bird.fall_speed
if bird.fall_speed > 10:
bird.fall_speed = 10
bird.rect = bird.sprite_root.get_rect(
center=(bird.position[0] + bird.dimensions[0] / 2,
bird.position[1] + bird.dimensions[1] / 2 + 10))
if bird.position[1] + bird.dimensions[1] > 800 or bird.position[
1] < -100:
return
def start_flow(self):
self.flow_path = []
pipe = Pipe(kind=Pipe.Kind.STRAIGHT, direction=Pipe.Direction.RIGHT)
entry_dir = Pipe.Direction.LEFT
x, y = -1, 0
while True:
pipe, entry_dir, x, y = pipe.get_connected(self.board, x, y,
entry_dir)
self.flow_path.append((pipe, (x, y)))
if pipe is None:
break
def __init__(self):
self.player = Bird(100, 100, 30, 20)
self.speed = 0
self.score = 0
self.pipes = [
Pipe(START_PIPE_X, 120, SCREEN_HEIGHT, PIPE_HEIGHT, PIPE_WIDTH),
Pipe(START_PIPE_X + PIPE_DIST, 120, SCREEN_HEIGHT, PIPE_HEIGHT,
PIPE_WIDTH),
Pipe(START_PIPE_X + PIPE_DIST * 2, 120, SCREEN_HEIGHT, PIPE_HEIGHT,
PIPE_WIDTH)
]
def eval_genomes(pop):
global WIN
clock = pygame.time.Clock()
pipes = [Pipe(700)]
base = Floor(FLOOR)
win = WIN
score = 0
run = True
while run:
clock.tick(30)
for event in pygame.event.get():
if event.type == pygame.QUIT:
run = False
pygame.quit()
quit()
break
rem = []
add_pipe = False
for pipe in pipes:
pipe.move()
pop.check_colision_with_pipe(pipe, win)
if pipe.x + pipe.PIPE_TOP.get_width() < 0:
rem.append(pipe)
if not pipe.passed and pipe.x < pop.pop[0].x:
pipe.passed = True
add_pipe = True
if add_pipe:
score += 1
pop.increment_score_by_n(10)
pipes.append(Pipe(WIN_WIDTH))
for r in rem:
pipes.remove(r)
pipe_ind = 0
if pop.done() != True:
if len(pipes) > 1 and pop.pop[
0].x > pipes[0].x + pipes[0].PIPE_TOP.get_width():
pipe_ind = 1
pop.increment_score_by_n(0.03)
pop.update_alive(pipes, pipe_ind, FLOOR)
base.move()
else:
pop.natural_selection()
run = False
draw_window(WIN, pop.pop, pipes, base, score, gen, pipe_ind)
def create_pipe(settings, screen, pipes):
while settings.current_pipe < settings.number_of_pipes[
settings.current_level]:
pipe = Pipe(
settings.pipe_positions[settings.current_level][
settings.current_pipe], screen,
settings.pipe_sizes[settings.current_level][settings.current_pipe])
settings.current_pipe += 1
pipe.x = pipe.pos[0]
pipe.y = pipe.pos[1]
pipes.add(pipe)
def get_pipes(x_pos=SCREEN_WIDTH + 10):
'''return a pipes pair with random gap position'''
pipe_height = IMAGES['lpipe'].get_height()
# random position of the gap
gap_y_pos = random.randrange(int(Y_BASE * 0.2),
int(Y_BASE * 0.8 - GAP_SIZE))
return {
# upper pipe
'u': Pipe(IMAGES['upipe'], x_pos, gap_y_pos - pipe_height),
# lower pipe
'l': Pipe(IMAGES['lpipe'], x_pos, gap_y_pos + GAP_SIZE)
}
def __init__(self, bird):
pygame.sprite.Group.__init__(self)
self.counter = 288 / PIPE_DENSITY
if ZPIPES == 0:
self.color = randint(0, 1)
else:
self.color = ZPIPES - 1
self.gap = randint(PIPEGAP_MIN + PIPE_BUFFER, 400 - PIPE_BUFFER)
self.bird = bird
self.add(Pipe(self.gap, self.color, 0, self.bird))
self.add(Pipe(self.gap, self.color, 1, self.bird))
self.alive = True
def __init__(self, file_name, file_path, encoding, language):
super().__init__()
self.data = ConfigData(language)
self.file_name = file_name
self.file_path = file_path
self.encoding = encoding
self.pipe_read_clean = Pipe(Queue(), Condition())
self.pipe_clean_sound = Pipe(Queue(), Condition())
self.pipe_sound_syll = Pipe(Queue(), Condition())
self.pipe_syll_count = Pipe(Queue(), Condition())
self.pipe_count_txt = Pipe(Queue(), Condition())
def test2(self):
pin = Pipe(Queue(), Condition())
pipe_syllabify_count = Pipe(Queue(), Condition())
pout = Pipe(Queue(), Condition())
module = WriteModule([pipe_syllabify_count, pout], file_path)
module.start()
for word in words_to_do[0]:
run_through_module(word, module)
time.sleep(1)
self.assertEqual(os.path.exists(file_lengths), True)
self.assertEqual(os.path.exists(file_syllables), True)
self.assertEqual(text_of_file(file_syllables), syllables[1])
self.assertEqual(text_of_file(file_lengths), lengths[1])
def generate_pipes(self):
top_pipe_config = {
"x": SCREEN_WIDTH,
"y": random.randint(-150, -10),
"image": pygame.image.load(os.path.join("images", "pipeTop.png")),
}
bottom_pipe_config = {
"x": SCREEN_WIDTH,
"y": top_pipe_config["y"] + GAP,
"image": pygame.image.load(os.path.join("images",
"pipeBottom.png")),
}
top_pipe = Pipe(top_pipe_config)
bottom_pipe = Pipe(bottom_pipe_config)
self.pipes.append({"top": top_pipe, "bottom": bottom_pipe})
def __init__(self, geo_coder, sponsor_coder, conn, model, feature_space):
self.geo_coder = geo_coder
self.sponsor_coder = sponsor_coder
self.conn = conn
self.model = model
self.feature_space = feature_space
fgs = [
entity_text_bag_feature_generator.unigram_feature_generator(
force=True),
simple_entity_text_feature_generator.
simple_entity_text_feature_generator(force=True),
gen_geo_features.geo_feature_generator(force=True),
sponsor_feature_generator.SponsorFeatureGenerator(force=True),
]
self.pipe = Pipe(fgs, num_processes=1)
def reInit(self):
random.seed()
pygame.display.set_caption("Flippy Bird")
# window size & FPS
self.win_width = 288
self.win_height = 512
self.win = pygame.display.set_mode((self.win_width, self.win_height))
self.clock = pygame.time.Clock()
# game object
self.bird = Bird()
self.pipes = [Pipe(), Pipe()]
self.pipes[0].randInit()
self.pipes[1].randInit()
self.pipes[1].x = 288 + 170
self.score = 0
def on_execute(self): self.on_init() while(self.running): if len(self.bird)==0: self.nextGeneration() self.remaining = self.population if self.interval%100==0: self.pipes.append(Pipe()) keys = pygame.key.get_pressed() for event in pygame.event.get(): if event.type==pygame.QUIT: self.running = False closest_pipe = self.closest_pipe() self.remaining = len(self.bird) while self.j<self.remaining: self.bird[self.j].think(closest_pipe) if self.isHit(self.bird[self.j],closest_pipe): self.oldBird.append(self.bird[self.j]) self.bird.remove(self.bird[self.j]) self.remaining-=1 self.j+=1 self.j = 0 self.interval+=1 self.on_render() self.on_cleanup()
def __init__(self, locals = __builtin__.globals()):
InteractiveConsole.__init__(self, locals)
self.stdout = sys.stdout
self.stderr = sys.stderr
self.pipe = Pipe()
self.completer = Completer(locals)
def OnInit(self):
self.SetAppName(self.__APP_NAME__)
self._pipe=Pipe(self.GetAppName() + '-' + wx.GetUserId())
self._frames=[]
files=sys.argv[1:]
self.OpenFiles(files)
self.prefs_dialog=self.__DOC_FRAME__.__PREFS_DIALOG__(None)
self.prefs_dialog.Center()
return True
def __init__(self, geo_coder, sponsor_coder, conn, model, feature_space):
self.geo_coder = geo_coder
self.sponsor_coder = sponsor_coder
self.conn = conn
self.model = model
self.feature_space = feature_space
fgs = [
entity_text_bag_feature_generator.unigram_feature_generator(force=True),
simple_entity_text_feature_generator.simple_entity_text_feature_generator(force=True),
gen_geo_features.geo_feature_generator(force = True),
sponsor_feature_generator.SponsorFeatureGenerator(force = True),
]
self.pipe = Pipe(fgs, num_processes=1)
def parse_pipe(event, fct_array):
"""
read line in a pipe and parse it to call a function with given args
ex -> ["print", "un", "deux, "trois","soleil"] will be the same than
print("un","deux", "trois", "soleil")
"""
pipe = Pipe("/tmp/bibusPipe")
for line in pipe.readlines():
if not event.isSet():
return
if not line:
continue
try:
args = json.loads(line)
except ValueError:
continue
if not isinstance(args, list):
continue
try:
command = args.pop(0)
except IndexError:
continue
try:
fct = fct_array[command]
except KeyError:
print("Bad command", command)
continue
print(PIPE_START)
try:
fct(*args)
except Exception as e:
print("ERROR in pipe", e.args)
print(PIPE_END)
def main():
parser = argparse.ArgumentParser(description='get pickeled instances')
subparsers = parser.add_subparsers(dest='subparser_name' ,help='sub-command help')
parser_serialize = subparsers.add_parser('serialize', help='pickle instances')
parser_serialize.add_argument('--data_folder', required=True, help='path to output pickled files')
parser_serialize.add_argument('--threads', type=int, default = mp.cpu_count(), help='number of threads to run in parallel')
parser_serialize.add_argument('--positivefile', required=True, help='file containing entities identified as earmarks')
parser_serialize.add_argument('--negativefile', required=True, help='file containing negative example entities')
parser_add = subparsers.add_parser('add', help='add to pickled instances')
parser_add.add_argument('--data_folder', required=True, help='path to output pickled files')
parser_add.add_argument('--threads', type=int, default = 1, help='number of threads to run in parallel')
args = parser.parse_args()
logging.info("pid: " + str(os.getpid()))
if args.subparser_name == "serialize":
positive_entities = read_entities_file(args.positivefile)
negative_entities = read_entities_file(args.negativefile)
logging.info("Pulling entities from database")
positive_instance = get_instances_from_entities(get_entity_objects(positive_entities, args.threads), 1, args.threads )
negative_instance = get_instances_from_entities(get_entity_objects(negative_entities, args.threads), 0, args.threads )
instances = positive_instance + negative_instance
logging.info("Creating pipe")
feature_generators = [
#wikipedia_categories_feature_generator.wikipedia_categories_feature_generator(depth = 2, distinguish_levels=False, force=True ),
entity_text_bag_feature_generator.unigram_feature_generator(force=True),
#entity_text_bag_feature_generator.bigram_feature_generator(force=True),
simple_entity_text_feature_generator.simple_entity_text_feature_generator(force=True),
gen_geo_features.geo_feature_generator(force = True),
sponsor_feature_generator.SponsorFeatureGenerator(force = True),
#calais_feature_generator.CalaisFeatureGenerator(force=True)
]
elif args.subparser_name == "add":
logging.info("pid: " + str(os.getpid()))
instances = load_instances(args.data_folder)
logging.info("Creating pipe")
feature_generators = [
#wikipedia_categories_feature_generator.wikipedia_categories_feature_generator(depth = 2, distinguish_levels=False, force=True ),
entity_text_bag_feature_generator.unigram_feature_generator(force=True),
#entity_text_bag_feature_generator.bigram_feature_generator(force=True),
simple_entity_text_feature_generator.simple_entity_text_feature_generator(force=True),
gen_geo_features.geo_feature_generator(force = True),
#calais_feature_generator.CalaisFeatureGenerator(force=True),
#prefix_feature_generator.PrefixFeatureGenerator(force=True, prefixes = ['O&M', 'for'])
sponsor_feature_generator.SponsorFeatureGenerator(force = True),
]
pipe = Pipe(feature_generators, instances, num_processes=args.threads)
logging.info("Pushing into pipe")
pipe.push_all_parallel()
logging.info("Start Serializing")
serialize_instances(pipe.instances, args.data_folder)
logging.info("Done!")
class Console(InteractiveConsole):
def __init__(self, locals = __builtin__.globals()):
InteractiveConsole.__init__(self, locals)
self.stdout = sys.stdout
self.stderr = sys.stderr
self.pipe = Pipe()
self.completer = Completer(locals)
#-------------------------------------------------------------------------------
def redirectOutput(self, stdout, stderr):
sys.stdout = stdout
sys.stderr = stderr
#-------------------------------------------------------------------------------
def push(self, input):
self.redirectOutput(self.pipe, self.pipe)
InteractiveConsole.push(self, input)
self.redirectOutput(self.stdout, self.stderr)
output = self.pipe.flush()
return output
#-------------------------------------------------------------------------------
def complete(self, input):
completed = input
candidates = []
words = input.split(" ")
matches = self.completer.getMatches(words[-1])
for match in matches:
if callable(matches[match]):
candidates.append(match+"()")
else:
candidates.append(match)
if len(matches) == 1:
match = matches.iterkeys().next()
words = words[0:-1]
words.append(match)
completed = " ".join(words)
if callable(matches[match]):
completed += "("
return completed, candidates
#-------------------------------------------------------------------------------
def help(self, input):
text = None
doc = self.push("%s.__doc__" % input)
if ("Traceback" in doc) or ("SyntaxError" in doc):
doc = None
self.push("import inspect")
src = self.push("inspect.getsourcelines(%s)[0][0:6]" % input)
if ("Traceback" in src) or ("SyntaxError" in src):
src = None
if doc:
exec("text = ''.join(%s)" % doc)
elif src:
exec("text = ''.join(%s)" % src)
if text:
text = text.strip(os.linesep)
return text
class DocApp(wx.App):
__APP_NAME__='wxdocapp'
__DOC_FRAME__=DocumentFrame
def OnInit(self):
self.SetAppName(self.__APP_NAME__)
self._pipe=Pipe(self.GetAppName() + '-' + wx.GetUserId())
self._frames=[]
files=sys.argv[1:]
self.OpenFiles(files)
self.prefs_dialog=self.__DOC_FRAME__.__PREFS_DIALOG__(None)
self.prefs_dialog.Center()
return True
def OnExit(self):
del self._singleInstanceChecker
def ShowPreferences(self):
if not self.prefs_dialog.IsShown():
self.prefs_dialog.Show()
def OpenFiles(self, files):
self._singleInstanceChecker = wx.SingleInstanceChecker(self.GetAppName() + '-' + wx.GetUserId(), tempfile.gettempdir())
if self._singleInstanceChecker.IsAnotherRunning():
self._pipe.write(files)
else:
if not files:
files=[None]
for filename in files:
self.OpenFile(filename)
self.timer=wx.Timer(self,-1)
self.Bind(wx.EVT_TIMER, self.CheckForOpenFiles, self.timer)
self.timer.Start(1000)
def CheckForOpenFiles(self, event):
files=self._pipe.read()
if files:
for filename in files:
self.OpenFile(filename)
def OpenFile(self, filename):
# see if we already have a window
# open for the filename passed in
empty_frame=None
if filename:
for frame in self._frames:
if frame.GetDocFilename() == filename:
frame.Raise()
return
# otherwise see if there's an empty frame we can use
empty_frames=[frame for frame in self._frames if frame.IsEmpty()]
if empty_frames:
empty_frame=empty_frames[0]
if not empty_frame:
frame=self.__DOC_FRAME__(None)
self._frames.append(frame)
else:
frame=empty_frame
frame.Show()
if filename:
frame.Load(filename)
frame.Raise()
def MacOpenFile(self, filename):
self.OpenFile(filename)
def FrameClosed(self, frame):
if frame in self._frames:
self._frames.remove(frame)
# see if this is the last
# frame being closed and if so
# destroy the prefs dialog
if not self._frames:
self.prefs_dialog.Destroy()
def FrameRaised(self, frame):
# add frame to front of list
if frame in self._frames:
self._frames.remove(frame)
self._frames.insert(0, frame)
def Quit(self):
# try to close all windows
# copy list of frames, so as to avoid
# problems as windows get raised etc
frames=list(self._frames)
for frame in frames:
res=frame.OnClose(None)
if res == wx.ID_CANCEL:
break # cancel pressed
class EarmarkDetector:
def __init__(self, geo_coder, sponsor_coder, conn, model, feature_space):
self.geo_coder = geo_coder
self.sponsor_coder = sponsor_coder
self.conn = conn
self.model = model
self.feature_space = feature_space
fgs = [
entity_text_bag_feature_generator.unigram_feature_generator(force=True),
simple_entity_text_feature_generator.simple_entity_text_feature_generator(force=True),
gen_geo_features.geo_feature_generator(force = True),
sponsor_feature_generator.SponsorFeatureGenerator(force = True),
]
self.pipe = Pipe(fgs, num_processes=1)
def get_instance_from_row(self, row, column_indices):
instance = Instance()
row_offset = row.offset
(entity_text, entity_inferred_name) = get_row_entity_text_and_entity_inferred_name(row, column_indices)
instance.attributes["entity_inferred_name"] = entity_inferred_name[:2048]
instance.attributes['entity_text'] = entity_text[:2048]
instance.attributes["id"] = 0
return self.pipe.push_single(instance)
def label_row(self, row, column_indices, table_offset, congress, chamber, document_type, number, sponsor_indices):
instance = self.get_instance_from_row(row, column_indices)
X, y, space = pipe.instances_to_matrix([instance,], feature_space = self.feature_space, dense = False)
scores = self.model.decision_function(X)
fields = ['congress', 'chamber','document_type','number', 'row', 'row_offset', 'row_length', 'score', 'state', 'sponsors']
cmd = "insert into candidate_earmarks (" + ", ".join(fields) + ") values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) returning id"
attributes = instance.attributes
state = self.geo_coder.get_state(attributes['entity_text'])
cur = self.conn.cursor()
if sponsor_indices:
print sponsor_indices
sponsors = []
for index in sponsor_indices:
try:
sponsor_cell = attributes['entity_text'].split("|")[index]
sponsors_in_cell = string_functions.tokenize(string_functions.normalize_no_lower(sponsor_cell))
for sic in sponsors_in_cell:
if sic in self.sponsor_coder.sponsors[congress]:
sponsors.append(sic)
except Exception as e:
print "Index: %d" % index
print len(attributes['entity_text'].split("|"))
print attributes['entity_text']
logging.exception("SCREW UP")
sponsors_string = "|".join(sponsors)[:1024]
cur.execute(cmd, (congress, chamber, document_type, number, attributes['entity_text'], row.offset+table_offset, row.length, scores[0], state, sponsors_string))
curr_id = cur.fetchone()[0]
for sponsor in sponsors:
cur.execute('insert into sponsors (candidate_earmark_id, sponsor) values (%s, %s)', (curr_id,sponsor ))
self.conn.commit()
def label_doc(self, doc_path, congress, chamber, document_type, number):
print doc_path
paragraphs_list = text_table_tools.get_paragraphs(open(doc_path,'r'))
tables = text_table_tools.identify_tables(paragraphs_list)
for table in tables:
table_offset = table.offset
column_indices = sorted(text_table_tools.get_candidate_columns(table))
sponsor_indices = self.sponsor_coder.find_sponsor_index(table, congress)
for row in table.rows:
self.label_row(row, column_indices, table_offset, congress, chamber, document_type, number, sponsor_indices)
