def __init__(self):
self._links = []
self._domains = []
self._grab = Grab()
self._ok_codes = [200, 301, 302, 401]
self._max_depth = 0
self._clog = ConsoleLogger()
self._flog = FileLogger()
def log_episode(run_name, output_directory, run_directory, conf_directory,
conf_file, max_frames, total_episodes, total_frames,
total_duration, total_original_rewards,
total_shaped_rewards, episode_frames,
episode_original_reward, episode_shaped_reward,
episode_loss, episode_duration):
ConsoleLogger.log_episode(
output_directory=output_directory,
run_directory=run_directory,
max_frames=max_frames,
total_episodes=total_episodes + 1,
total_frames=total_frames,
total_duration=total_duration,
total_original_rewards=total_original_rewards,
total_shaped_rewards=total_shaped_rewards,
episode_frames=episode_frames + 1,
episode_original_reward=episode_original_reward,
episode_shaped_reward=episode_shaped_reward,
episode_loss=episode_loss,
episode_duration=episode_duration)
FileLogger.log_episode(output_directory=output_directory,
run_directory=run_directory,
max_frames=max_frames,
total_episodes=total_episodes + 1,
total_frames=total_frames,
total_duration=total_duration,
total_original_rewards=total_original_rewards,
total_shaped_rewards=total_shaped_rewards,
episode_frames=episode_frames + 1,
episode_original_reward=episode_original_reward,
episode_shaped_reward=episode_shaped_reward,
episode_loss=episode_loss,
episode_duration=episode_duration)
TelegramLogger.log_episode(
run_name=run_name,
output_directory=output_directory,
run_directory=run_directory,
conf_directory=conf_directory,
conf_file=conf_file,
max_frames=max_frames,
total_episodes=total_episodes + 1,
total_frames=total_frames,
total_duration=total_duration,
total_original_rewards=total_original_rewards,
total_shaped_rewards=total_shaped_rewards,
episode_frames=episode_frames + 1,
episode_original_reward=episode_original_reward,
episode_shaped_reward=episode_shaped_reward,
episode_loss=episode_loss,
episode_duration=episode_duration)
def __init__(self, configurations):
www_path = configurations['WWWPath']
self.factory = StaticFactory.initialise_factory()
self.factory.register('DateTimeWrapper', DateTimeWrapper())
self.factory.register('FsWrapper', FsWrapper())
self.set_up_database(configurations['DatabasePath'], configurations['MigrationsPath'])
self.factory.register('ToDoTasks', ToDoTasks())
self.taskdataaccess = TaskDataAccess(self.factory)
self.factory.register('TaskDataAccess', self.taskdataaccess)
self.factory.register('GoDaddyTaskRetriever', GoDaddyTaskRetriever())
self.restcommandparser = RestCommandParser()
self.filepathhandler = FilePathHandler(www_path)
daily_log_writer = DailyFileWriter(self.factory, DAILY_LOG_NAME, DAILY_LOG_EXT)
daily_err_writer = DailyFileWriter(self.factory, DAILY_LOG_NAME, DAILY_ERR_EXT)
daily_logger = Logger(self.factory, daily_log_writer, daily_err_writer)
self.logger = LogChainer(daily_logger)
self.logger.chain(ConsoleLogger(True))
self.factory.register('Logger', self.logger)
self.factory.register('TaskConverter', TaskConverter())
self.factory.register('GoDaddyTaskMerger', GoDaddyTaskMerger(self.factory))
self.factory.register('GoDaddyTasksMerger', GoDaddyTasksMerger(self.factory))
self.tesk_reloader = TaskReloader(self.factory)
def __init__(self):
self._links = []
self._domains = []
self._grab = Grab()
self._ok_codes = [200, 301, 302, 401]
self._max_depth = 0
self._clog = ConsoleLogger()
self._flog = FileLogger()
def setUp(self):
self.logger = ConsoleLogger()
self.mock_disk_util = mock.create_autospec(DiskUtil)
self.mock_passhprase_filename = "mock_passphrase_filename"
mock_public_settings = {}
self.resource_disk = ResourceDiskUtil(self.logger, self.mock_disk_util,
self.mock_passhprase_filename,
mock_public_settings,
["ubuntu", "16"])
def __init__(self, www_path):
self.factory = Factory()
self.factory.register('DateTimeWrapper', DateTimeWrapper())
self.factory.register('FsWrapper', FsWrapper())
self.filepathhandler = FilePathHandler(www_path)
daily_log_writer = DailyFileWriter(self.factory, DAILY_LOG_NAME,
DAILY_LOG_EXT)
daily_err_writer = DailyFileWriter(self.factory, DAILY_LOG_NAME,
DAILY_ERR_EXT)
daily_logger = Logger(self.factory, daily_log_writer, daily_err_writer)
self.logger = LogChainer(daily_logger)
self.logger.chain(ConsoleLogger(True))
def __init__(self, repo_url, testcasedir, runscript):
try:
self.repo_url = repo_url
logger = ConsoleLogger()
self.context = Context(logger, testcasedir, runscript)
self.context.repo_dir = self.git_clone()
self.check()
self.test()
except Exception as e:
print traceback.format_exc()
raise
finally:
self.cleanup()
self.print_output()
exit_code = 0 if self.context.error_count == 0 else 1
exit(exit_code)
class Test_GeoQueryRequestProcessor(unittest.TestCase):
config = {"databaseFile": "truck.db"}
logger = ConsoleLogger()
gqrp = GQRP(config, logger)
def test_coit_tower_foodtrucks(self):
req = {
"Latitude": 37.80241,
"Longitude": -122.4059
} # this is actually Coit Tower!
self.logger.logTrace(
"Test starting", str(req),
"Test_GeoQueryRequestProcessor::test_coit_tower_foodtrucks")
response = self.gqrp.requestHandler(req)
self.assertEqual("Mike's Catering", response["Foodtrucks"][0]["Name"])
self.assertEqual("Philz Coffee Truck",
response["Foodtrucks"][1]["Name"])
self.assertEqual("Akuranvyka USA Inc",
response["Foodtrucks"][2]["Name"])
self.assertEqual("M M Catering", response["Foodtrucks"][3]["Name"])
self.assertEqual("Wu Wei LLC dba MoBowl",
response["Foodtrucks"][4]["Name"])
class Spider:
def __init__(self):
self._links = []
self._domains = []
self._grab = Grab()
self._ok_codes = [200, 301, 302, 401]
self._max_depth = 0
self._clog = ConsoleLogger()
self._flog = FileLogger()
def add_url(self, url):
self._links += [SpiderLink(url, 0)]
def add_domain(self, domain):
self._domains += [domain]
def set_max_depth(self, depth):
self._max_depth = depth
def start(self):
# Уже обработанные url
visited = set([])
while len(self._links) > 0:
spider_link = self._links.pop(0)
self._clog.show_progress(spider_link.url, len(visited), len(self._links) + 1)
if spider_link.url in visited:
continue
visited = visited | {spider_link.url}
# Пытаемся загрузить страницу
is_loaded = self.try_load_url(spider_link.url)
if not is_loaded:
self._clog.add_error_message(self._grab.response.code, spider_link)
self._flog.add_error_message(self._grab.response.code, spider_link)
continue
# Проверка на глубину
if 0 < self._max_depth == spider_link.depth:
continue
# Если страница за пределами доменов, парсить не нужно
in_domain = False
for d in self._domains:
if d in spider_link.url:
in_domain = True
break
if not in_domain:
continue
# Парсим страницу на новые ссылки
new_spider_links = self.get_spider_links(spider_link)
if len(new_spider_links) == 0:
continue
new_spider_links = self.filter_links(new_spider_links)
new_spider_links = self.remove_visited_links(new_spider_links, visited)
for l in new_spider_links:
l.parent = spider_link
# А теперь можно и добавить в очередь
self._links += new_spider_links
def try_load_url(self, url):
try:
self._grab.go(url)
except:
return False
if self._grab.response.code not in self._ok_codes:
return False
return True
def get_spider_links(self, spider_link):
objects = self._grab.doc.select("//body//a")
spider_links = []
for o in objects:
try:
url = o.attr("href")
except DataNotFound:
continue
if url.startswith("/"):
url = "http://" + self._grab.response.url_details().hostname + url
if not url.startswith("http"):
ind = spider_link.url.rfind("/")
if ind == -1:
url = spider_link.url + "/" + url
else:
url = spider_link.url[:(ind+1)] + url
spider_links += [SpiderLink(url, spider_link.depth + 1)]
return spider_links
def filter_links(self, spider_links):
filtered = []
for l in spider_links:
if l.url is not "" and l.url is not "#":
filtered += [l]
return filtered
def remove_visited_links(self, spider_links, visited_urls):
res = []
for s in spider_links:
if s.url in visited_urls:
continue
res += [s]
return res
class Spider:
def __init__(self):
self._links = []
self._domains = []
self._grab = Grab()
self._ok_codes = [200, 301, 302, 401]
self._max_depth = 0
self._clog = ConsoleLogger()
self._flog = FileLogger()
def add_url(self, url):
self._links += [SpiderLink(url, 0)]
def add_domain(self, domain):
self._domains += [domain]
def set_max_depth(self, depth):
self._max_depth = depth
def start(self):
# Уже обработанные url
visited = set([])
while len(self._links) > 0:
spider_link = self._links.pop(0)
self._clog.show_progress(spider_link.url, len(visited),
len(self._links) + 1)
if spider_link.url in visited:
continue
visited = visited | {spider_link.url}
# Пытаемся загрузить страницу
is_loaded = self.try_load_url(spider_link.url)
if not is_loaded:
self._clog.add_error_message(self._grab.response.code,
spider_link)
self._flog.add_error_message(self._grab.response.code,
spider_link)
continue
# Проверка на глубину
if 0 < self._max_depth == spider_link.depth:
continue
# Если страница за пределами доменов, парсить не нужно
in_domain = False
for d in self._domains:
if d in spider_link.url:
in_domain = True
break
if not in_domain:
continue
# Парсим страницу на новые ссылки
new_spider_links = self.get_spider_links(spider_link)
if len(new_spider_links) == 0:
continue
new_spider_links = self.filter_links(new_spider_links)
new_spider_links = self.remove_visited_links(
new_spider_links, visited)
for l in new_spider_links:
l.parent = spider_link
# А теперь можно и добавить в очередь
self._links += new_spider_links
def try_load_url(self, url):
try:
self._grab.go(url)
except:
return False
if self._grab.response.code not in self._ok_codes:
return False
return True
def get_spider_links(self, spider_link):
objects = self._grab.doc.select("//body//a")
spider_links = []
for o in objects:
try:
url = o.attr("href")
except DataNotFound:
continue
if url.startswith("/"):
url = "http://" + self._grab.response.url_details(
).hostname + url
if not url.startswith("http"):
ind = spider_link.url.rfind("/")
if ind == -1:
url = spider_link.url + "/" + url
else:
url = spider_link.url[:(ind + 1)] + url
spider_links += [SpiderLink(url, spider_link.depth + 1)]
return spider_links
def filter_links(self, spider_links):
filtered = []
for l in spider_links:
if l.url is not "" and l.url is not "#":
filtered += [l]
return filtered
def remove_visited_links(self, spider_links, visited_urls):
res = []
for s in spider_links:
if s.url in visited_urls:
continue
res += [s]
return res
def fetch_configurations(config_path):
config = open(config_path, "r").read()
return ConfigurationParser().parse_configuration(config)
try:
factory = Factory()
factory.register('DateTimeWrapper', DateTimeWrapper())
factory.register('FsWrapper', FsWrapper())
daily_log_writer = DailyFileWriter(factory, DAILY_LOG_NAME, DAILY_LOG_EXT)
daily_err_writer = DailyFileWriter(factory, DAILY_LOG_NAME, DAILY_ERR_EXT)
daily_logger = Logger(factory, daily_log_writer, daily_err_writer)
logger = LogChainer(daily_logger)
logger.chain(ConsoleLogger(True))
factory.register('Logger', logger)
factory.register('UrlValidator', UrlValidator())
factory.register('UrlPagegroupExtractor', UrlPagegroupExtractor())
factory.register(
'UrlRouter',
UrlRouter(DefaultDestination(factory,
'/home/dev/Documents/todo20/www')))
factory.register('UrlRequestHandler', UrlRequestHandler(factory))
configurations = fetch_configurations('config.cfg')
factory.fetch('UrlRouter').register_destination(
'test',
TestPagegroup(
auth_service = AuthService()
game_service = GameService()
app.install(JsonPlugin())
app.install(ErrorFilterPlugin())
app.install(
BottleSQLAlchemySessionPlugin(engine=ENGINE,
commit=False,
create_session_by_default=True))
app.install(AuthPlugin(auth_service))
app.install(EnableCors())
converter = PyJsonConverter()
app.install(BottlePyJsonPlugin(converter))
app.install(ControllerPlugin())
logger = ConsoleLogger()
auth_controller = AuthController(app, logger, auth_service)
game_controller = GameController(app, game_service, logger)
@app.route('/<:re:.*>', method='OPTIONS')
def cors():
print('After request hook.')
response.headers['Access-Control-Allow-Origin'] = '*'
if __name__ == "__main__":
run(
app,
host='localhost',
def log_parameters(
run_name, output_directory, run_directory, conf_directory,
conf_file, environment_id, batch_size, learning_rate, gamma,
eps_start, eps_end, eps_decay, num_atoms, vmin, vmax, eta, beta,
lambda1, normalize_shaped_reward, reward_shaping_dropout_rate,
target_update_rate, model_save_rate, episode_log_rate,
replay_memory_size, num_frames,
reward_pong_player_racket_hits_ball,
reward_pong_player_racket_covers_ball,
reward_pong_player_racket_close_to_ball_linear,
reward_pong_player_racket_close_to_ball_quadratic,
reward_pong_opponent_racket_hits_ball,
reward_pong_opponent_racket_covers_ball,
reward_pong_opponent_racket_close_to_ball_linear,
reward_pong_opponent_racket_close_to_ball_quadratic,
reward_breakout_player_racket_hits_ball,
reward_breakout_player_racket_covers_ball,
reward_breakout_player_racket_close_to_ball_linear,
reward_breakout_player_racket_close_to_ball_quadratic,
reward_breakout_ball_hitting_upper_block,
reward_space_invaders_player_avoids_line_of_fire,
reward_freeway_distance_walked, reward_freeway_distance_to_car,
reward_ms_pacman_far_from_enemy, reward_potential_based):
ConsoleLogger.log_parameters(
run_name=run_name,
output_directory=output_directory,
run_directory=run_directory,
environment_id=environment_id,
batch_size=batch_size,
learning_rate=learning_rate,
gamma=gamma,
eps_start=eps_start,
eps_end=eps_end,
eps_decay=eps_decay,
num_atoms=num_atoms,
vmin=vmin,
vmax=vmax,
eta=eta,
beta=beta,
lambda1=lambda1,
normalize_shaped_reward=normalize_shaped_reward,
reward_shaping_dropout_rate=reward_shaping_dropout_rate,
target_update_rate=target_update_rate,
model_save_rate=model_save_rate,
episode_log_rate=episode_log_rate,
replay_memory_size=replay_memory_size,
num_frames=num_frames,
reward_pong_player_racket_hits_ball=
reward_pong_player_racket_hits_ball,
reward_pong_player_racket_covers_ball=
reward_pong_player_racket_covers_ball,
reward_pong_player_racket_close_to_ball_linear=
reward_pong_player_racket_close_to_ball_linear,
reward_pong_player_racket_close_to_ball_quadratic=
reward_pong_player_racket_close_to_ball_quadratic,
reward_pong_opponent_racket_hits_ball=
reward_pong_opponent_racket_hits_ball,
reward_pong_opponent_racket_covers_ball=
reward_pong_opponent_racket_covers_ball,
reward_pong_opponent_racket_close_to_ball_linear=
reward_pong_opponent_racket_close_to_ball_linear,
reward_pong_opponent_racket_close_to_ball_quadratic=
reward_pong_opponent_racket_close_to_ball_quadratic,
reward_breakout_player_racket_hits_ball=
reward_breakout_player_racket_hits_ball,
reward_breakout_player_racket_covers_ball=
reward_breakout_player_racket_covers_ball,
reward_breakout_player_racket_close_to_ball_linear=
reward_breakout_player_racket_close_to_ball_linear,
reward_breakout_player_racket_close_to_ball_quadratic=
reward_breakout_player_racket_close_to_ball_quadratic,
reward_breakout_ball_hitting_upper_block=
reward_breakout_ball_hitting_upper_block,
reward_space_invaders_player_avoids_line_of_fire=
reward_space_invaders_player_avoids_line_of_fire,
reward_freeway_distance_walked=reward_freeway_distance_walked,
reward_freeway_distance_to_car=reward_freeway_distance_to_car,
reward_ms_pacman_far_from_enemy=reward_ms_pacman_far_from_enemy,
reward_potential_based=reward_potential_based)
FileLogger.log_parameters(
run_name=run_name,
output_directory=output_directory,
run_directory=run_directory,
environment_id=environment_id,
batch_size=batch_size,
learning_rate=learning_rate,
gamma=gamma,
eps_start=eps_start,
eps_end=eps_end,
eps_decay=eps_decay,
num_atoms=num_atoms,
vmin=vmin,
vmax=vmax,
eta=eta,
beta=beta,
lambda1=lambda1,
normalize_shaped_reward=normalize_shaped_reward,
reward_shaping_dropout_rate=reward_shaping_dropout_rate,
target_update_rate=target_update_rate,
model_save_rate=model_save_rate,
episode_log_rate=episode_log_rate,
replay_memory_size=replay_memory_size,
num_frames=num_frames,
reward_pong_player_racket_hits_ball=
reward_pong_player_racket_hits_ball,
reward_pong_player_racket_covers_ball=
reward_pong_player_racket_covers_ball,
reward_pong_player_racket_close_to_ball_linear=
reward_pong_player_racket_close_to_ball_linear,
reward_pong_player_racket_close_to_ball_quadratic=
reward_pong_player_racket_close_to_ball_quadratic,
reward_pong_opponent_racket_hits_ball=
reward_pong_opponent_racket_hits_ball,
reward_pong_opponent_racket_covers_ball=
reward_pong_opponent_racket_covers_ball,
reward_pong_opponent_racket_close_to_ball_linear=
reward_pong_opponent_racket_close_to_ball_linear,
reward_pong_opponent_racket_close_to_ball_quadratic=
reward_pong_opponent_racket_close_to_ball_quadratic,
reward_breakout_player_racket_hits_ball=
reward_breakout_player_racket_hits_ball,
reward_breakout_player_racket_covers_ball=
reward_breakout_player_racket_covers_ball,
reward_breakout_player_racket_close_to_ball_linear=
reward_breakout_player_racket_close_to_ball_linear,
reward_breakout_player_racket_close_to_ball_quadratic=
reward_breakout_player_racket_close_to_ball_quadratic,
reward_breakout_ball_hitting_upper_block=
reward_breakout_ball_hitting_upper_block,
reward_space_invaders_player_avoids_line_of_fire=
reward_space_invaders_player_avoids_line_of_fire,
reward_freeway_distance_walked=reward_freeway_distance_walked,
reward_freeway_distance_to_car=reward_freeway_distance_to_car,
reward_ms_pacman_far_from_enemy=reward_ms_pacman_far_from_enemy,
reward_potential_based=reward_potential_based)
TelegramLogger.log_parameters(
run_name=run_name,
output_directory=output_directory,
run_directory=run_directory,
conf_directory=conf_directory,
conf_file=conf_file,
environment_id=environment_id,
batch_size=batch_size,
learning_rate=learning_rate,
gamma=gamma,
eps_start=eps_start,
eps_end=eps_end,
eps_decay=eps_decay,
num_atoms=num_atoms,
vmin=vmin,
vmax=vmax,
eta=eta,
beta=beta,
lambda1=lambda1,
normalize_shaped_reward=normalize_shaped_reward,
reward_shaping_dropout_rate=reward_shaping_dropout_rate,
target_update_rate=target_update_rate,
model_save_rate=model_save_rate,
episode_log_rate=episode_log_rate,
replay_memory_size=replay_memory_size,
num_frames=num_frames,
reward_pong_player_racket_hits_ball=
reward_pong_player_racket_hits_ball,
reward_pong_player_racket_covers_ball=
reward_pong_player_racket_covers_ball,
reward_pong_player_racket_close_to_ball_linear=
reward_pong_player_racket_close_to_ball_linear,
reward_pong_player_racket_close_to_ball_quadratic=
reward_pong_player_racket_close_to_ball_quadratic,
reward_pong_opponent_racket_hits_ball=
reward_pong_opponent_racket_hits_ball,
reward_pong_opponent_racket_covers_ball=
reward_pong_opponent_racket_covers_ball,
reward_pong_opponent_racket_close_to_ball_linear=
reward_pong_opponent_racket_close_to_ball_linear,
reward_pong_opponent_racket_close_to_ball_quadratic=
reward_pong_opponent_racket_close_to_ball_quadratic,
reward_breakout_player_racket_hits_ball=
reward_breakout_player_racket_hits_ball,
reward_breakout_player_racket_covers_ball=
reward_breakout_player_racket_covers_ball,
reward_breakout_player_racket_close_to_ball_linear=
reward_breakout_player_racket_close_to_ball_linear,
reward_breakout_player_racket_close_to_ball_quadratic=
reward_breakout_player_racket_close_to_ball_quadratic,
reward_breakout_ball_hitting_upper_block=
reward_breakout_ball_hitting_upper_block,
reward_space_invaders_player_avoids_line_of_fire=
reward_space_invaders_player_avoids_line_of_fire,
reward_freeway_distance_walked=reward_freeway_distance_walked,
reward_freeway_distance_to_car=reward_freeway_distance_to_car,
reward_ms_pacman_far_from_enemy=reward_ms_pacman_far_from_enemy,
reward_potential_based=reward_potential_based)
def setUp(self):
self.logger = ConsoleLogger()
self.cmd_executor = CommandExecutor(self.logger)
def setUp(self):
self.logger = ConsoleLogger()
self.cutil = CheckUtil(self.logger)
def _launch_fired(self):
"""Called when the 'launch' button is clicked."""
context = self._new_xp_context()
context.console_logger = ConsoleLogger()
self.curworker = _WorkerThread(context, self)
self.curworker.start()
def _set_up_logging(self, base_dir):
self.log = Log()
self.log.add_logger(ConsoleLogger())
self.log.add_logger(TFBoardLogger(base_dir))
def setUp(self):
self.logger = ConsoleLogger()
self.disk_util = DiskUtil(None,
MockDistroPatcher('Ubuntu', '14.04', '4.15'),
self.logger,
EncryptionEnvironment(None, self.logger))