def main(genomes, config):
global GEN
GEN += 1
nets = []
ge = []
copters = []
for _, g in genomes:
net = neat.nn.FeedForwardNetwork.create(g, config)
nets.append(net)
copters.append(Copter())
g.fitness = 0
ge.append(g)
clock = pygame.time.Clock()
run = True
ghosts = []
score = 0
pygame.time.set_timer(SPAWN_EVENT, SPAWN_TIME)
i = 0
pressed = {}
global COUNTER
while run:
clock.tick(FPS)
COUNTER += 1
if len(copters) == 0:
run = False
break
for event in pygame.event.get():
if event.type == pygame.QUIT:
run = False
pygame.quit()
if event.type == SPAWN_EVENT:
ghosts.append(Ghost())
for a, ghost in enumerate(ghosts):
ghost.move()
if ghost.rect.x < 0 - GHOST_WIDTH:
ghosts.remove(ghost)
score += 1
for g in ge:
g.fitness += 3
continue
for x, copter in enumerate(copters):
if copter.collision(ghost):
if len(ghosts) != 0:
ghosts.pop()
ge[x].fitness -= 1
copters.pop(x)
ge.pop(x)
nets.pop(x)
if copter.collison_with_boundary():
ge[x].fitness -= 2
copters.pop(x)
ge.pop(x)
nets.pop(x)
# if COUNTER % 100 == 0:
# COUNTER = 0
for x, copter in enumerate(copters):
if ghost is not None:
output = nets[x].activate(
(copter.rect.y, abs(copter.rect.x - ghost.rect.x),
abs(copter.rect.y - ghost.rect.y)))
copter.update(output)
if score > 60:
break
draw_window(ghosts, copters, score)
def game(game_state, q_net, gamma, sample_epsilon, replay_buffer_size):
global constants
constants.score = 0
constants.wall_collide_number = 0
generateLevel.createLevel()
pacmanMain = Pacman()
blinky = Ghost()
walls = generateLevel.walls
#initialize some feature variables
pacmanCurrentTile = featureExtraction.on_current_tile(dynamicPositions.pacman, pacmanMain)
blinkyCurrentTile = featureExtraction.on_current_tile((blinky.rect.x, blinky.rect.y), blinky)
#closest_food = featureExtraction.bfs([featureExtraction.on_current_tile(dynamicPositions.pacman, pacmanMain)], [featureExtraction.on_current_tile(dynamicPositions.pacman, pacmanMain)], 0, generateLevel.coins)
crashCount = 1
frightenModeCount = 0
time = 0
scatterModeCount = 0
while not game_state:
for event in pygame.event.get():
if event.type == pygame.QUIT:
constants.added_previous_t = False
changeGameState()
game_state = True
if event.type == pygame.KEYDOWN:
if event.key == pygame.K_q:
constants.added_previous_t = False
changeGameState()
game_state = True
reward = 0
score_before_script = constants.score
wall_collide_number_before_script = constants.wall_collide_number
constants.screen.fill(black)
generateLevel.drawWalls()
generateLevel.drawCoins()
label = font.render("Score: " + str(constants.score), 1, (255, 255, 255))
constants.screen.blit(label, (constants.display_width * 0.02, constants.display_height * 0.9))
pacmanMain.checkCollision()
blinky.checkCollision()
pacmanMain.update()
#Update (x, y) position value of pacman in the global variables file dynamicPositions.py
dynamicPositions.pacman = (pacmanMain.x, pacmanMain.y)
if blinky.reviveMode == False:
blinky.update()
elif blinky.reviveMode == True:
if blinky.noMovementTime % 50 == 0: #idle time for 5/6th of a second.
blinky.noMovementTime = 1
blinky.reviveMode = False
blinky.noMovementTime += 1
if blinky.noMovementTime % 5 == 0: #shutter respawn effect.
constants.screen.blit(blinky.image, (blinky.rect.x, blinky.rect.y))
if constants.frightenMode == True:
if constants.scatterMode == True or constants.chaseMode == True:
constants.scatterMode = False
constants.chaseMode = False
frightenModeCount += 1
if frightenModeCount % 150 == 0: #2.5 second frighten mode
constants.frightenMode = False
frightenModeCount = 0
if constants.scatterMode == True:
if constants.chaseMode or constants.frightenMode:
constants.frightenMode = False
constants.chaseMode = False
scatterModeCount += 1
if scatterModeCount % 180 == 0: #scatter lasts for 3 seconds
constants.scatterMode = False
scatterModeCount = 0
if constants.chaseMode == False and constants.frightenMode == False and constants.scatterMode == False:
constants.chaseMode = True
if time % 600 == 0: #every 10 seconds (including when game starts) it has 70% chance to scatter.
if random.randint(1,100) >= 70:
constants.scatterMode = True
constants.chaseMode = False
time = 0
time += 1
if blinky.rect.colliderect(pacmanMain.rect):
if constants.frightenMode == True:
blinky.rect.x = 960
blinky.rect.y = 320
constants.frightenMode = False
blinky.reviveMode = True
constants.score += 5
'''#FITNESS: update fitness - for eating ghost'''
reward += 2
else: #lose the game
constants.scores.append(constants.score)
changeGameState()
'''#FITNESS: update fitness - losing the game'''
reward -= 2
constants.added_previous_t = False
game_state = True
pacmanMain.kill()
blinky.kill()
break
if featureExtraction.on_current_tile((blinky.rect.x, blinky.rect.y), blinky) != blinkyCurrentTile: #blinky change tile
blinkyCurrentTile = featureExtraction.on_current_tile((blinky.rect.x, blinky.rect.y), blinky)
#only do bfs when pacman changes tiles
if featureExtraction.on_current_tile(dynamicPositions.pacman, pacmanMain) != pacmanCurrentTile or constants.closest_food == None:
pacmanCurrentTile = featureExtraction.on_current_tile(dynamicPositions.pacman, pacmanMain)
constants.closest_food = featureExtraction.bfs([featureExtraction.on_current_tile(dynamicPositions.pacman, pacmanMain)], [featureExtraction.on_current_tile(dynamicPositions.pacman, pacmanMain)], 0, generateLevel.coins)
#features
food_pos = featureExtraction.check_tile(generateLevel.coins, pacmanCurrentTile, 1, "food", blinkyCurrentTile)
enemy_pos = featureExtraction.check_tile(generateLevel.coins, pacmanCurrentTile, 1, "ghost", blinkyCurrentTile)
wall_pos = featureExtraction.check_tile(generateLevel.wallPositions, pacmanCurrentTile, 1, "wall", blinkyCurrentTile)
food_pos_2 = featureExtraction.check_tile(generateLevel.coins, pacmanCurrentTile, 2, "food", blinkyCurrentTile)
enemy_pos_2 = featureExtraction.check_tile(generateLevel.coins, pacmanCurrentTile, 2, "ghost", blinkyCurrentTile)
distance_between = featureExtraction.distance_between((pacmanMain.rect.x, pacmanMain.rect.y), (blinky.rect.x, blinky.rect.y))
inputVector = featureExtraction.extract(food_pos, enemy_pos, wall_pos, food_pos_2, enemy_pos_2, constants.closest_food, distance_between, constants.frightenMode)
##print(inputVector)
if constants.randoming == False:
random_movement_epsilon = random.uniform(0, 1)
if random_movement_epsilon < 0:
constants.randoming = True
action = random.randint(0, 3)
constants.movement = action
constants.max_movement_t = int(np.random.normal(32, 10))
#print("Randoming with movement time: " + str(constants.max_movement_t))
else:
if constants.target_network.apply_softmax:
probability = constants.target_network.process(inputVector)
action = np.random.choice(np.array([0,1,2,3]), p=probability) # Sample from softmax probability distribution.
constants.randoming = True
constants.movement = action
constants.max_movement_t = int(np.random.normal(15, 4))
#print(probability)
else:
action = constants.target_network.process(inputVector)
else:
action = constants.movement
constants.random_movement_t += 1
if constants.max_movement_t != 0:
if(constants.random_movement_t % constants.max_movement_t == 0):
constants.random_movement_t = 0
constants.randoming = False
#Adding state on time step "t+1" into the transition at time step t
if constants.added_previous_t:
rb.replay_buffer[rb.count-1].extend([reward, inputVector])
##print(rb.replay_buffer[rb.count-1])
constants.added_previous_t = False
elif constants.added_previous_t_two:
rb.replay_buffer_two[rb.count_two-1].extend([reward, inputVector])
constants.added_previous_t_two = False
pacmanMain.automate(action)
##print(q_net.process(inputVector))
##print(q_net.show(inputVector))
score_after_script = constants.score
wall_collide_number_after_script = constants.wall_collide_number
if not game_state: #don't update score if game is over
if score_after_script - score_before_script == 1: #reward for eating a food/coin
reward += 1
''' ---- Saving memory into experience buffer ---- '''
if(reward > 0): #remember states where reward is earned
if rb.count<=replay_buffer_size:
rb.replay_buffer.append([inputVector, action])
rb.count += 1
constants.added_previous_t = True
print("Added to replay_buffer. Transition count: "+ str(rb.count))
p = random.uniform(0, 1)
if p < 0.04 and reward <= 0: #~4 percent probability of remembering the current state.
if rb.count_two <= replay_buffer_size:
rb.replay_buffer_two.append([inputVector, action])
rb.count_two += 1
constants.added_previous_t_two = True
print("Added to replay_buffer_two. Transition count: "+ str(rb.count_two))
''' ---- End of saving memory into experience buffer ---- '''
blinky.shortest_distance = []
blinky.tileToMove = []
blinky.futureMovementNumber = []
constants.t += 1 #increment a time-step
#Sample from replay buffer every 20 timesteps
if constants.t % 5 == 0:
if random.randint(0, 100) < 60: #sample from first replay buffer
if rb.count >= replay_buffer_size:
print("Sampling from replay buffer one.")
e = random.uniform(0, 1)
i = 0
if e > sample_epsilon: #sample stochastically rather than greedily.
i = random.randint(0, replay_buffer_size-21)
end = i+20
while i < end:
#calculate target q(s,a)
if i > len(rb.replay_buffer)-1:
break
if len(rb.replay_buffer[i]) == 4: #s(t+1) may not have been added to the transition yet, so check if it has then proceed.
q_t = constants.q_network.forward(rb.replay_buffer[i][0])
q_t_plus_1 = constants.target_network.forward(rb.replay_buffer[i][3])
t_index = list(q_t_plus_1).index(max(q_t_plus_1))
q_value_target = rb.replay_buffer[i][2] + gamma*max(q_t_plus_1)
for x in range(4):
if x == t_index:
q_t_plus_1[x] = q_value_target
else:
q_t_plus_1[x] = q_t[x]
print("Q TARGET IS: " + str(q_value_target))
constants.q_network.backpropagate(q_t_plus_1, q_t)
rb.count -= 1
rb.pop_experience(i, 1)
i += 1
else:
break
else: #sample from second replay buffer
if rb.count_two >= replay_buffer_size:
print("Sampling from replay buffer two.")
e = random.uniform(0, 1)
i = 0
if e > sample_epsilon: #sample stochastically rather than greedily.
i = random.randint(0, replay_buffer_size-21)
end = i+20
while i < end:
if i > len(rb.replay_buffer_two)-1:
break
if len(rb.replay_buffer_two[i]) == 4: #s(t+1) may not have been added to the transition yet, so check if it has then proceed.
q_t = constants.q_network.forward(rb.replay_buffer_two[i][0])
q_t_plus_1 = constants.target_network.forward(rb.replay_buffer_two[i][3])
t_index = list(q_t_plus_1).index(max(q_t_plus_1))
q_value_target = rb.replay_buffer_two[i][2] + gamma*max(q_t_plus_1)
for x in range(4):
if x == t_index:
q_t_plus_1[x] = q_value_target
else:
q_t_plus_1[x] = q_t[x]
constants.q_network.backpropagate(q_t_plus_1, q_t)
rb.count_two -= 1
rb.pop_experience(i, 2)
i+=1
else:
break
#Freeze interval of 100000 time steps, update the target_network with the weights of the q_network.
if constants.t % 2000 == 0:
constants.target_network = constants.q_network
constants.to_save = True
constants.t = 0
#print("Target network is now up-to-date with Q network.")
#if constants.t % 10 == 0:
# rgbArray = pygame.surfarray.array_colorkey(constants.screen)
# #print("hallo")
pygame.display.update()
clock.tick(60)
#!/usr/bin/python
# -*- coding: utf-8 -*-
from ghost import Ghost
from pyquery import PyQuery as pq
from mongoconnect import *
import hashlib
KEY_WORD = 'news'
exec('database=db_' + KEY_WORD)
g = Ghost()
s = g.start(display=True)
checkset = set()
def crawl_page(s):
#select data
ct = s.content
d = pq(ct)
ul = d('div.listInfo ul.titMode')
print('crawl page ing')
cell = []
for li in ul('li').items():
href = li('a').attr('href')
title = li('a span.txt').text()
time = li('a span.time').text()
print(time)
hashid = hashlib.md5((title + time).encode()).hexdigest()
tem_len = len(checkset)
checkset.add(hashid)
def broadcast(bot,update):
ghost = Ghost()
return_array = []
cat_list = ["spock","beard","mauveover","cymric","gold","otaku","saycheese","googly","mainecoon","whixtensions","wingtips","chestnut","jaguar"]
counter = 0
trigger = True
web3 = Web3(HTTPProvider('http://*****:*****@nthwin @iczac @jaynertwx"
filename = "/home/elanor/ftp/files/cryptokitties/images/"+str(kitten['kitty']['id'])+".png"
with ghost.start() as session:
session.open(cattribute['image_url'])
session.capture_to(filename)
"""
svg_filename = "/home/elanor/ftp/files/cryptokitties/images/"+str(kitten['kitty']['id'])+".svg"
f = open(svg_filename,'wb')
f.write(urllib.request.urlopen(cattribute['image_url']).read())
f.close()
cairosvg.svg2png(url=svg_filename, write_to=filename)"""
bot.sendMessage(chat_id=Tokens.channel('livechannel'),text=message,parse_mode='HTML')
bot.sendPhoto(chat_id=Tokens.channel('livechannel'),photo=open(filename,'rb'))
os.remove(filename)
break
scrapednumber = "Nthwin "+str(counter)+" scraped"
counter += 100
bot.sendMessage(chat_id=Tokens.channel('errorchannel'),text=scrapednumber,parse_mode='HTML')
print (str(counter)+" has been scraped")
else:
trigger = False
except:
#API will time out after x amount of requests. need to let it sleep then resume.
catcherror = traceback.format_exc()
bot.sendMessage(chat_id=Tokens.channel('errorchannel'),text=catcherror,parse_mode='HTML')
message = """Timed out API on page number """+str(counter)+"""
Sleeping for 15 seconds"""
bot.sendMessage(chat_id=Tokens.channel('errorchannel'),text=message,parse_mode='HTML')
time.sleep(15)
def __init__(self):
self.nodes = NodeGroup(gridUnit, gridUnit)
self.nodes.createNodeListFromFile("map.txt")
# initialize agents
self.agentRed = Ghost(self.nodes.nodeList[randint(0, 2499)])
self.agentRed.color = (255, 0, 0)
#print self.clyde1.id
self.agentGreen = Ghost(self.nodes.nodeList[randint(0, 2499)])
self.agentGreen.color = (0, 255, 0)
#print self.clyde2.id
self.agentBlue = Ghost(self.nodes.nodeList[randint(0, 2499)])
self.agentBlue.color = (0, 0, 255)
#print self.clyde3.id
self.agentYellow = Ghost(self.nodes.nodeList[randint(0, 2499)])
self.agentYellow.color = (255, 255, 0)
#print self.clyde4.id
self.agentPurple = Ghost(self.nodes.nodeList[randint(0, 2499)])
self.agentPurple.color = (128, 0, 128)
#print self.clyde5.id
# initialize targets
self.targetRed1 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentRed.id)
self.targetRed1.color = (240, 128, 128)
self.targetRed2 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentRed.id)
self.targetRed2.color = (240, 128, 128)
self.targetRed3 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentRed.id)
self.targetRed3.color = (240, 128, 128)
self.targetRed4 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentRed.id)
self.targetRed4.color = (240, 128, 128)
self.targetRed5 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentRed.id)
self.targetRed5.color = (240, 128, 128)
self.targetGreen1 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentGreen.id)
self.targetGreen1.color = (128, 240, 128)
self.targetGreen2 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentGreen.id)
self.targetGreen2.color = (128, 240, 128)
self.targetGreen3 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentGreen.id)
self.targetGreen3.color = (128, 240, 128)
self.targetGreen4 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentGreen.id)
self.targetGreen4.color = (128, 240, 128)
self.targetGreen5 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentGreen.id)
self.targetGreen5.color = (128, 240, 128)
self.targetBlue1 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentBlue.id)
self.targetBlue1.color = (128, 128, 240)
self.targetBlue2 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentBlue.id)
self.targetBlue2.color = (128, 128, 240)
self.targetBlue3 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentBlue.id)
self.targetBlue3.color = (128, 128, 240)
self.targetBlue4 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentBlue.id)
self.targetBlue4.color = (128, 128, 240)
self.targetBlue5 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentBlue.id)
self.targetBlue5.color = (128, 128, 240)
self.targetYellow1 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentYellow.id)
self.targetYellow1.color = (240, 240, 128)
self.targetYellow2 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentYellow.id)
self.targetYellow2.color = (240, 240, 128)
self.targetYellow3 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentYellow.id)
self.targetYellow3.color = (240, 240, 128)
self.targetYellow4 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentYellow.id)
self.targetYellow4.color = (240, 240, 128)
self.targetYellow5 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentYellow.id)
self.targetYellow5.color = (240, 240, 128)
self.targetPurple1 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentPurple.id)
self.targetPurple1.color = (240, 128, 240)
self.targetPurple2 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentPurple.id)
self.targetPurple2.color = (240, 128, 240)
self.targetPurple3 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentPurple.id)
self.targetPurple3.color = (240, 128, 240)
self.targetPurple4 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentPurple.id)
self.targetPurple4.color = (240, 128, 240)
self.targetPurple5 = PacMan(self.nodes.nodeList[randint(0, 2499)],
self.agentPurple.id)
self.targetPurple5.color = (240, 128, 240)
self.Agents = [
self.agentRed, self.agentGreen, self.agentBlue, self.agentYellow,
self.agentPurple
]
self.Targets = [
self.targetRed1, self.targetRed2, self.targetRed3, self.targetRed4,
self.targetRed5, self.targetGreen1, self.targetGreen2,
self.targetGreen3, self.targetGreen4, self.targetGreen5,
self.targetBlue1, self.targetBlue2, self.targetBlue3,
self.targetBlue4, self.targetBlue5, self.targetYellow1,
self.targetYellow2, self.targetYellow3, self.targetYellow4,
self.targetYellow5, self.targetPurple1, self.targetPurple2,
self.targetPurple3, self.targetPurple4, self.targetPurple5
]
#for target in self.Targets:
# print target.owner
self.checkList = []
from ghost import Ghost
url = "http://macrochan.org/view.php?u=SFOKRTC3YTTYBNS4YSMZKBFPQUAAFU7Q"
# save bandwidth by not loading images through ghost
ghost = Ghost(download_images=False)
# open the webpage
page = ghost.open(url)
img_urls = ghost.evaluate("""
var listRet = []; // empty list
// grab all `<a href=>` tags with `tags`
var links = document.querySelectorAll("a[href*=tags]");
// loop to check every link
for (var i=0; i<links.length; i++){
// return href= links
listRet.push(links[i].href);
}
listRet; // return list
""")
# Print the links
for l in img_urls[0]:
print(l)
def fetch_content(url, queue):
# Ghost.py stuff
ghost = Ghost(
user_agent=ua_string,
#viewport_size=(2000,2000),
wait_timeout=10)
#with ghost.start(java_enabled=False) as session:#user_agent=ua_string) as session:
# set to use privoxy proxy defaults
ghost.set_proxy(type_="http", host="127.0.0.1", port=8118)
# make the Ghost request
# page is resources[0]
for i in range(5):
try:
#ghost.open(url, wait=False)
#time.sleep(5)
#page, resources = ghost.wait_for_page_loaded()
page, resources = ghost.open(url)
except Exception as e:
print("Error with ", url)
print(e)
#traceback.print_tb(sys.exc_info()[2])
continue
if page != None:
break
if page == None:
print("Unrecoverable error with Ghost.py", url)
ghost.exit()
# create the list of dependencies
'''
url_l = []
for r in resources:
url_l.append(r.url)
'''
# list of [url, data] lists
data_l = []
# list of fetched urls
fetched = []
p_r = [page]
p_r.extend(resources)
#print(page.content)
# fill data_l
for r in p_r:
url = r.url.split("?")[0].strip()
# if we haven't seen this before
if not url in fetched:
#print(url)
fetched.append(url)
# if it's ASCII
if type(r.content) == str:
data = bytes(r.content, encoding='utf-8')
else:
data = r.content.data()
data_l.append(Resource(url, r.http_status, data))
#queue.put((url_l, data_l))
queue.put(data_l)
CounterSpell, HealingSpell)
if __name__ == '__main__':
hagrid = HogwartsMember('Hagrid', 1952, 'male')
dumby = HogwartsMember.dumbledore()
harry = Pupil.harry()
ron = Pupil.ron()
hermione = Pupil.hermione()
malfoy = Pupil.malfoy()
snape = Professor.snape()
mcgonagall = Professor.mcgonagall()
nick = Ghost.nearly_headless_nick()
bloody_baron = Ghost("Bloody Baron's", 1409, 'male', 1468, 'Slytherin')
lumos = Charm.lumos()
wingardium_leviosa = Charm.wingardium_leviosa()
ron.add_trait('kind')
ron.add_trait('tidy-minded')
ron.add_trait('impatient', value=False)
bellatrix = DeathEater('Bellatrix Lestrange', 1951)
wing_lev = Charm.wingardium_leviosa()
rictum = Charm('tickling_charm',
'Rictumsempra',
'Causes victim to buckle with laughter',
min_year=5)
def __init__(
self,
url,
debug=False,
):
self.debug = debug
self.url = url
hr_dir = os.environ.get("HR_DIR")
if not hr_dir:
raise Exception(
"Error, set HR_DIR to your hacker_rank directory as an environment variable"
)
if not os.path.isdir(hr_dir):
raise Exception(
"Error, cannot find hackerRank directory: {}".format(hr_dir))
self.save_file = "{}/debug.json".format(hr_dir)
self.ghost = Ghost(log_level=logging.ERROR, )
if self.debug:
if not os.path.isfile(self.save_file):
raise Exception(
"Error, debug enabled but save file is not populated")
self.doc = json.load(open(self.save_file, 'r'))
else:
self.doc = self.__getDoc()
open(self.save_file, 'w').write(json.dumps(self.doc, indent=5))
sys.stderr.write("Save file created: {}\n".format(self.save_file))
if not self.doc:
raise Exception("Error fetching JSON Doc!")
self.html = self.__getHtml()
self.root = ET.fromstring(self.html)
slug = self.doc.get('slug')
self.lang = self.doc.get('track', {}).get('track_slug')
if self.lang not in __lang_switches__.keys():
sys.stderr.write(
"hrm... seems track/slug_name '{}'' isn't in our languages. Defaulting to: '{}'"
.format(self.lang, __default_lang__))
self.lang = __default_lang__
if not slug:
raise Exception(
"Error detecting key ['slug'] see {} for details".format(
self.save_file))
if not self.lang:
raise Exception(
"Error detecting key ['track']['track_slug'] see {} for details"
.format(self.save_file))
hr_name = slug
template_fn = '{}/templates/{}'.format(hr_dir, self.lang)
if not os.path.isfile(template_fn):
raise Exception(
"Error, template for language: {} does not exist".format(
template_fn))
self.template_doc = open(template_fn, 'r').read()
self.dirname = "{}/{}/{}".format(hr_dir, self.lang, hr_name)
self.__makeDir()
def __init__(self):
pygame.init()
pygame.font.init()
self.pacman = Pacman()
self.map_start = time.monotonic()
self.Highscore = highscore.Highscore()
self.positionD = 380, 306
self.positionG = 400, 306
self.positionS = 360, 306
self.position = 380, 290
self.Ghost = Ghost(self.position)
self.Gannibal = Gannibal(self.positionG)
self.Stalker = Stalker(self.positionS)
self.Debil = Debil(self.positionD)
self.Output = Output()
self.Seed = Seed()
self.Berry = Berry()
self.bigseed = BigSeed()
self.gameover = False
self.map = [[
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2
],
[
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2
],
[
2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 2
],
[
2, 2, 2, 2, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,
0, 0, 0, 0, 0, 0, 1, 2
],
[
2, 2, 2, 2, 2, 2, 1, 3, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1,
1, 1, 0, 1, 1, 3, 1, 2
],
[
2, 2, 2, 2, 2, 2, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1,
1, 1, 0, 1, 1, 0, 1, 2
],
[
2, 2, 2, 2, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 1, 2
],
[
2, 2, 2, 2, 2, 2, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1,
0, 1, 0, 1, 1, 0, 1, 2
],
[
2, 2, 2, 2, 2, 2, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0,
0, 1, 0, 0, 0, 0, 1, 2
],
[
2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 0, 1, 1, 1, 2, 1, 2, 1,
1, 1, 0, 1, 1, 1, 1, 2
],
[
2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 0, 1, 2, 2, 2, 2, 2, 2,
2, 1, 0, 1, 2, 2, 2, 2
],
[
2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 0, 1, 2, 1, 1, 1, 1, 1,
2, 1, 0, 1, 1, 1, 1, 2
],
[
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 2, 2, 1, 2, 2, 2, 1,
2, 2, 0, 2, 2, 2, 2, 2
],
[
2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 0, 1, 2, 1, 1, 1, 1, 1,
2, 1, 0, 1, 1, 1, 1, 2
],
[
2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 0, 1, 2, 2, 2, 2, 2, 2,
2, 1, 0, 1, 2, 2, 2, 2
],
[
2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 0, 1, 2, 1, 1, 1, 1, 1,
2, 1, 0, 1, 1, 1, 1, 2
],
[
2, 2, 2, 2, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,
0, 0, 0, 0, 0, 0, 1, 2
],
[
2, 2, 2, 2, 2, 2, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1,
1, 1, 0, 1, 1, 0, 1, 2
],
[
2, 2, 2, 2, 2, 2, 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0,
0, 0, 0, 1, 0, 3, 1, 2
],
[
2, 2, 2, 2, 2, 2, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1,
0, 1, 0, 1, 0, 1, 1, 2
],
[
2, 2, 2, 2, 2, 2, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0,
0, 1, 0, 0, 0, 0, 1, 2
],
[
2, 2, 2, 2, 2, 2, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1,
1, 1, 1, 1, 1, 0, 1, 2
],
[
2, 2, 2, 2, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 1, 2
],
[
2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 2
]]
def ghost(self):
global ghost
ghost = Ghost(self.x, self.y, self.dir*3)
game_world.add_object(ghost, 1)
#print('%s加入列表！' % ip)
ip_list.append((type, ip, port))
##########################################
##########################################
UA = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'
header = {'User-Agent':UA,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
'Accept-Encoding': 'gzip, deflate'
}
s = requests.session()
s.keep_alive = False
se = Session(Ghost(), user_agent=UA, wait_timeout=30, wait_callback=None, display=False, viewport_size=(800, 680), download_images=False)
#################################
# hidemy
def get_hidemy():
url = 'https://hidemy.name/en/proxy-list/?country=US&type=h&anon=4#list'
se.open(url)
se.wait_for_selector('table.proxy__t')
html = se.content
soup = BeautifulSoup(html, "html.parser")
sources = soup.select('tbody > tr')
for i in sources:
ip_info = hidemy_info(i)
add_task(ip_info[0], ip_info[1], ip_info[2])
def hidemy_info(source):
def main(url, output, option={}):
result = {
"error": [],
"page": {},
"resources": [],
"capture": None,
}
#savedir = appdir + "/artifacts/ghost/" + output
#dump = savedir + "/ghost.pkl"
savedir = os.path.join(appdir, "artifacts/ghost")
dump = savedir + "/" + output
try:
#if os.path.exists(savedir):
# shutil.rmtree(savedir)
#os.makedirs(savedir)
with open(dump, 'wb') as d:
#umsgpack.dump(result, d)
json.dump(result, d)
except Exception as e:
logger.error(str(e))
result["error"].append(str(e))
defaults = {
"wait_timeout": 60,
"display": False,
"viewport_size": (800, 600),
"plugins_enabled": True,
"java_enabled": True,
}
proxy_url = None
http_method = "GET"
req_headers = None
body = None
if option:
if "user_agent" in option:
defaults["user_agent"] = str(option["user_agent"])
if "timeout" in option:
defaults["wait_timeout"] = int(option["timeout"])
if "proxy" in option:
proxy_url = option["proxy"]
if "method" in option:
http_method = option["method"]
if "headers" in option:
req_headers = option["headers"]
if "post_data" in option:
body = str(option["post_data"])
logger.info(defaults)
ghost = None
try:
ghost = Ghost(
#log_level=logging.DEBUG,
log_level=logging.INFO,
plugin_path=[
appdir + '/plugins',
'/usr/lib/mozilla/plugins',
],
defaults=defaults,
)
except Exception as e:
logger.error("ghost init failed. " + str(e))
result["error"].append(str(e))
with ghost.start() as session:
if proxy_url:
try:
type = proxy_url.split(":")[0]
server = proxy_url.split("/")[2]
host = server.split(":")[0]
port = server.split(":")[1]
session.set_proxy(
str(type),
host=str(host),
port=int(port),
)
except Exception as e:
logger.debug(e)
headers = {}
if req_headers:
logger.debug(req_headers)
for h in req_headers:
headers[str(h)] = str(req_headers[h])
logger.debug(headers)
if hasattr(ghost, "xvfb"):
logger.info(ghost.xvfb)
page = None
resources = None
try:
page, resources = session.open(url.decode("utf-8"),
method=http_method,
headers=headers,
body=body)
except Exception as e:
logger.error(str(e))
result["error"].append(str(e))
#if error:
# result["error"] = error.spilt(".")[-1]
if page:
result["page"] = {
"url": page.url,
"http_status": page.http_status,
"headers": page.headers,
#"content":session.content.encode("utf-8"),
"content": base64.b64encode(session.content.encode("utf-8")),
"seq": 0,
#"error":page.error.encode("utf-8").split(".")[-1],
"error": page.error.split(".")[-1],
}
try:
image = session.capture()
ba = QByteArray()
buffer = QBuffer(ba)
buffer.open(QIODevice.WriteOnly)
image.save(buffer, "PNG")
bio = BytesIO(ba)
bio.seek(0)
#result["capture"] = bio.read()
result["capture"] = base64.b64encode(bio.read())
bio.flush()
bio.close()
ba.clear()
buffer.close()
except Exception as e:
logger.error(str(e))
result["error"].append(str(e))
if resources:
seq = 0
for r in resources:
seq += 1
#logger.debug(r.url)
dict = {
"url": r.url,
"http_status": r.http_status,
"headers": r.headers,
#"content":r.content.encode("utf-8"),
"content": base64.b64encode(r.content),
"seq": seq,
#"error":r.error.encode("utf-8").split(".")[-1],
"error": r.error.split(".")[-1],
}
result["resources"].append(dict)
with open(dump, 'wb') as d:
#umsgpack.dump(result, d)
json.dump(result, d)
logger.debug(dump)
ghost.exit()
return dump
from ghost import Ghost, Session
import time
wrapping_div = ''
next_button = ''
initial_url = ''
series_name = ''
chunks = 10
core_session = Ghost()
session = Session(core_session, display=False)
lower_bound = 0
upper_bound = chunks + lower_bound
def save_story(series_name, lower_bound, upper_bound, content):
file_name = './chap/{0}_{1}-{2}.txt'.format(series_name, lower_bound,
upper_bound)
content = ''.join([i if ord(i) < 128 else ' ' for i in content])
print(file_name)
with open(file_name, 'wt', encoding='utf-8') as file:
file.write(content)
searching = True
story_buffer = ''
next_url = initial_url
"type": a[2]
} for a in zip(names, urls, types)]
dic = [findOptionType(op) for op in dic]
return dic
def bulkDownloadAvailable(res_sections):
types = [dic["type"] for dic in res_sections]
if "menu" in types:
return False
else:
return True
#Open INEGI massive download website
ghost = Ghost().start()
url = 'http://www3.inegi.org.mx/sistemas/descarga/'
ghost.open(url)
soup = BeautifulSoup(ghost.content)
#Parse sections
res_sections = getNamesAndUrlsFromSoup(soup)
#Print sections
printOptions(res_sections)
#Ask for a section number
section = int(raw_input('Selecciona una opción: '))
#Check if value is valid
while not section in range(len(res_sections)):
section = int(raw_input('Opción incorrecta. Selecciona una sección: '))
def poltergeist(self):
global ghost
ghost = Ghost(self.x, self.y, self.dir)
game_world.add_object(ghost, 1)
win.blit(profile_pic, (62, 60))
pygame.draw.rect(win, (255, 0, 0), (140, 80, 200, 10))
pygame.draw.rect(win, (0, 128, 0),
(140, 80, 200 - (5 * (50 - player.health)), 10))
win.blit(health_label, (140, 60))
def quit():
pass
#main loop
run = True
clock = pygame.time.Clock()
player = Player(400, 490, 129, 129)
ghost = Ghost(250, 520, 130, 105)
player_graphics = CharacterGraphics(player, win)
ghost_graphics = GhostGraphics(ghost, win)
projectiles = []
font = pygame.font.SysFont('Comic Sans Ms', 15)
while run:
clock.tick(15)
for event in pygame.event.get():
if event.type == pygame.QUIT:
run = False
pygame.quit()
sys.exit()
keys = pygame.key.get_pressed()
if keys[pygame.K_UP]:
def enter(boy, event):
boy.frame = 0
ghost = Ghost(boy.x - 25, boy.y - 25)
game_world.add_object(ghost, 1)
def __init__(self, fb_email=None, fb_password=None):
self.session = Ghost().start()
self.facebook = Facebook(self.session, fb_email, fb_password)
self.login_successful = False
self.fat = None # Facebook access token
def loadpage(url,
outformat='JPG',
w=1024,
h=10,
selector=None,
waitsecond=None,
waitforselector=None,
waittext=None,
recalc=False,
agent=None):
""" Load page and capture output """
status = "404 NotFound"
response_headers = []
response_body = ["404 NotFound"]
content_type="text/html"
if url is None or url == "":
return status, response_headers, response_body
ghost = Ghost()
try:
# append suffix if any
agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.121 Safari/535.2%s' % (' '+agent if agent is not None else '')
with ghost.start(java_enabled=False, display=False, user_agent=agent) as session:
# manage proxy
if "http_proxy" in os.environ:
host, port = os.environ["http_proxy"].replace("http://","").split(":")
session.set_proxy("http", host=host, port=int(port))
# set viewport size to a minimal height
session.set_viewport_size(w, h)
# load page
open_url(session, url, waitforselector, waittext, recalc)
if recalc:
# set the view port to stick to the real height before to reload
session.set_viewport_size(w, session.main_frame.contentsSize().height())
# if recalc is true, we can now
# reload page with the correct viewport size
open_url(session, url, waitforselector, waittext)
if waitsecond is not None:
waitsecond = float(waitsecond) if waitsecond <= MAXSLEEP else MAXSLEEP
while waitsecond > 0.0:
waitsecond -= SLEEPSTEP
session.sleep(SLEEPSTEP)
if outformat.upper() in ('PNG', 'JPG', 'JPEG'):
buffer, content_type = capture_image(session, outformat=outformat, selector=selector)
buffer = buffer.data()
else:
buffer = session.content.encode("utf-8")
# write image
response_body = [bytes(buffer)]
status = "200 OK"
response_headers = [
('Content-Type', content_type),
]
except Exception, e:
logging.exception(e)
response_body = [
"There were an error...",
"\n",
str(e)
]
status = "500 InternalServerError"
response_headers = [
('Content-Type', 'text/plain'),
]
import numpy as np
import matplotlib.pyplot as plt
UP = 0
DOWN = 2
RIGHT = 1
LEFT = 3
PACMAN = 12
BLINKY = -2
Grid = Map()
Omap = np.array(Grid.map)
np.copyto(Grid.map, Omap)
#tpos = Pos((1, 2, 1, 2), 1)
pac = Ghost(Pos((1, 2, 1, 2), RIGHT))
print "Pacman located at: "
pac.pos.printD()
Grid.colorPos(pac.pos, PACMAN)
pacT = Pos((26, 27, 14, 15), LEFT)
#tpos.printD()
#Grid.colorPos(tpos, PACMAN)
g = Ghost(Pos((13, 14, 14, 15), UP)) #den
#g = Ghost(Pos((26, 27, 29, 30), LEFT)) #corner
Grid.colorPos(g.pos, BLINKY)
plt.pcolor(Grid.map)
plt.colorbar()
plt.axis('equal')
i = 0
from ghost import Ghost
import time
# Creates a Ghost instance telling to share cache and don't share cookies
gh = Ghost(share_cache=True, share_cookies=False)
page1, name = gh.create_page()
page2, name = gh.create_page()
# Open google
page1.open("http://www.google.com", wait_for_loading=False)
# Open Hacker News
page2.open("http://news.ycombinator.com/", wait_for_loading=False)
# At this point ghost it's rendering the two pages at the same time. Now we
# need to check if the pages are loaded.
while page1.get_loaded_page() is None:
page_resource = page1.get_loaded_page()
gh.process_events()
time.sleep(0.01)
# Saves an image of the screen 1
page1.capture_to("/tmp/tab1.png")
print "open /tmp/tab1.png"
#Or we can use wait_for_page_loaded() instead
page2.wait_for_page_loaded()
# Saves an image of the screen 2
page2.capture_to("/tmp/tab2.png")
def ev_get_title():
title = self.page.evaluate("document.title")
# Every time that one event it's executed successfully (it returns True)
# The result_data will be saved into a private variable called (_last_result).
# That result information can be retrived with "get_result", It has to be a
# dictionary that can be converted to a json.
return True, dict(title=unicode(title))
self.add_event(ev_get_title)
def start(self, data):
# We don't do anything with the data in this example
self._add_events()
def get_result(self):
return self._last_result
# We define the main intance of ghost that we want to run
gh = Ghost(display=True,
cache_size=10000,
wait_timeout=30,
prevent_download=["jpg", "png", "gif"],
download_images=False,
individual_cookies=True)
# Run Black Pearl Server
blackPerl = BlackPearl(gh, GhostWork)
blackPerl.start()
from ghost import Ghost
from pacman import Pacman
canvasX = 600
canvasY = 600
canvasC = 255 / 2
nGhosts = 20
initialX = []
initialY = []
ghosts = []
for i in range(nGhosts):
initialX.append(random(0, canvasX))
initialY.append(random(0, canvasY))
ghosts.append(Ghost(initialX[i], initialY[i], 50))
pacman = Pacman(250, 250, 50)
def setup(): # happens once
size(canvasX, canvasY) # always first line of setup
ellipseMode(CENTER)
rectMode(CENTER)
background(canvasC)
frameRate(10)
startButton()
def draw(): # happens every frame
global canvasC
background(canvasC)
import sqlite3
import threading
from bs4 import BeautifulSoup
from ghost import Ghost, Session
#################################
UA = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'
ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14'
header = {'User-Agent':UA,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
'Accept-Encoding': 'gzip, deflate'
}
s = requests.session()
s.keep_alive = False
gh = Ghost()
se = Session(gh, user_agent=ua, wait_timeout=30, wait_callback=None, display=False, viewport_size=(800, 680), download_images=False)
#####################################
def get_guobanjia():
for i in range(1, 9):
url = 'http://www.goubanjia.com/free/isp/%E7%A7%BB%E5%8A%A8/index' + str(i) + '.shtml'
r = s.get(url, headers=header)
if r.status_code == 200:
print('正在载入第%s页' %i)
html = r.content
soup = BeautifulSoup(html, "html.parser")
hidden_tags = soup.select('p')
for tag in hidden_tags:
tag.extract()
source = soup.select('tbody > tr')
html = '''
<html>
<body>
<iframe src="''' + URL + '''" height='600px' width='800px'></iframe>
</body>
</html>'''
html_filename = 'clickjack.html'
log_filename = 'test.log'
f = open(html_filename, 'w+')
f.write(html)
f.close()
fh = logging.FileHandler(log_filename)
ghost = Ghost(log_level=logging.INFO, log_handler=fh)
page, resources = ghost.open(html_filename)
l = open(log_filename, 'r')
if 'forbidden by X-Frame-Options.' in l.read():
print 'Clickjacking mitigated via X-FRAME-OPTIONS'
else:
href = ghost.evaluate('document.location.href')[0]
if html_filename not in href:
print 'Frame busting detected'
else:
print 'Frame busting not detected, page is likely vulnerable to clickjacking'
l.close()
logging.getLogger('ghost').handlers[0].close()
os.unlink(log_filename)
from ghost import Ghost
import time
class Tools:
def showExists(self, session, selector):
print selector + " is exists: " + str(session.exists(selector))
ghost = Ghost()
tools = Tools()
with ghost.start(user_agent="Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:39.0) Gecko/20100101 Firefox/39.0") as session:
page, extra_resources = session.open("http://qzone.qq.com/",user_agent="Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:39.0) Gecko/20100101 Firefox/39.0")
#print page.content
print session.exists("#login")
print session.evaluate("document.getElementById('login_frame').src")
print session.wait_for_selector("#login_frame")
print session.frame("login_frame") # Frame by name
print session.exists("#login")
#Switch to login_frame
print "--------switch to the login_frame"
session.click("#switcher_plogin") # not necessary?
tools.showExists(session, "#loginform")
#Commit the login form
#print session.evaluate("document.getElementById('u').outerHTML")
session.click("#u")
session.click("#p")
#print session.fill("#loginform",{"u":"490089331","p":"490089331"})
#print session.evaluate("document.getElementById('u').value='490089331'")
def __init__(self):
self._ghost = Ghost()
def create_ghost(self):
self.ghost = Ghost(self)
game_world.add_object(self.ghost, 1)
powerPellet_list.append(PowerPellet(30, 490, screen))
powerPellet_list.append(PowerPellet(570, 490, screen))
def doesPowerPelletExistHere(x, y):
for powerPellet in powerPellet_list:
if powerPellet.x == x and powerPellet.y == y:
return True
return False
placePowerPellets()
placePellets()
createGhostPath()
# create ghosts
ghostBlue = Ghost(14, 14, ghostNodes)
ghostPink = Ghost(14, 14, ghostNodes)
ghost = Ghost(14, 14, ghostNodes)
def isColliding(obj1, obj2):
distSquared = (obj1.x - obj2.x)**2 + (obj1.y - obj2.y)**2
if distSquared <= (obj1.rad + obj2.rad)**2:
return True
else:
return False
# game loop
running = True
pygame.event.set_allowed([pygame.QUIT, pygame.KEYDOWN, pygame.KEYUP])
