def run_multiple():
epochs = [10, 50]
for e in epochs:
config.epochs = e
print(config.epochs)
player = StateManager()
player.play_game()
def play_with_agents(self, agt1, agt2):
player_turn = 1
player1_wins = 0
for i in range(self.G):
print("[{}>{}]".format("-" * i, "." * (self.G - i - 1)), end="\r")
sm = StateManager(5)
agent = MCTS(exploration_rate=1, anet=agt1)
game = sm.create_game()
tree = Tree(game, chanceOfRandom=0.0)
state = tree.root
while not sm.is_finished():
if player_turn == 1:
agent.anet = agt1
best_child = agent.uct_search(tree, state,
num_search_games)
else:
agent.anet = agt2
best_child = agent.uct_search(tree, state,
num_search_games)
game.execute_move(best_child.move)
state = best_child
if sm.get_winner() == 1:
player1_wins += 1
print("{} won {}/{} against {}.".format(agt1.name, player1_wins,
self.G, agt2.name))
print(np.reshape(sm.game.board, (boardsize, boardsize)))
def test_pressing_number_after_result_calculated_should_display_number(self):
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '1'))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '+'))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '3'))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '='))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '8'))
self.assertEqual('8', self.s.display)
def test_divide_2_numbers_should_return_float(self):
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '1'))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '+'))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '3'))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '='))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '8'))
self.assertEqual('8', self.s.display)
def start():
"""
Start the Backend service
"""
state = StateManager(settings.STATE_FILE)
state.start()
listner = RequestListner(state.repository)
listner.start()
def _init_state(self, opt):
self.state = StateManager(
opt.experiment_class, opt.model_name, self.device, opt.model_type,
opt.model_num_layers, opt.model_num_layers_vgg,
opt.train_weights_init, opt.train_learning_rate,
opt.train_weight_decay, opt.train_scheduler_step_size)
if opt.model_load is not None:
self.state.load(opt.model_load, opt.model_disable_lr_loading)
def _init_state(self, opt):
self.state = StateManager(
opt.experiment_class, opt.model_name, self.device,
opt.model_split_pos, opt.model_num_layers,
opt.train_depth_grad_scale, opt.train_segmentation_grad_scale,
opt.train_weights_init, opt.model_depth_resolutions,
opt.model_num_layers_pose, opt.train_learning_rate,
opt.train_weight_decay, opt.train_scheduler_step_size)
if opt.model_load is not None:
self.state.load(opt.model_load, opt.model_disable_lr_loading)
def vmc_login(sessionId, info):
global g_id_state_manager
g_id_state_manager[sessionId] = StateManager()
g_id_state_manager[sessionId].vmc_util = VMCUtil()
g_id_state_manager[sessionId].vmc_util.set_info(info)
g_id_state_manager[sessionId].vmc_util.login()
print("vmc_login: Logged in successfully")
def __init__(self, feature_name, entity_field, sum_value_field, incl_count,
incl_sum, incl_mean, decay_rate):
"""Instance initiated with state_id and sample_decay_rate"""
self.feature_name = feature_name
self.entity_field = entity_field
self.sum_value_field = sum_value_field
self.incl_count = incl_count
self.incl_sum = incl_sum
self.incl_mean = incl_mean
self.decay_rate = decay_rate
self.state_manager = StateManager()
class Algorithm: def __init__(self, point_array): self.point_array = point_array self.stack = list() self.state_manager = StateManager() def increment_state(self): self.state_manager.increment_state() def segments(self): if not self.stack: return [] tuple_list = zip(self.stack, self.stack[1:]) segment_list = map(lambda x: Segment(x[0], x[1]), tuple_list) return segment_list def min_point(self): #return leftmost min point by y coord min_point = Point(sys.maxsize, sys.maxsize) for i in self.point_array: if i < min_point: min_point=i return min_point def polar_angle_sort(self, min_point): sort_list = copy.deepcopy(self.point_array) sort_list.remove(min_point) sort_list.sort(key = lambda point: Point.calculate_polar_angle(min_point, point)) sort_list.insert(0, min_point) return sort_list def sort_point_array(self): self.point_array = self.polar_angle_sort(self.min_point()) def init_stack(self): self.stack = list() self.stack.append(self.point_array[0]) self.stack.append(self.point_array[1]) self.stack.append(self.point_array[2]) def next_step(self): if self.state_manager.current_state == State.not_run: self.increment_state() elif self.state_manager.current_state == State.initial_point: self.sort_point_array() self.increment_state() elif self.state_manager.current_state == State.init_stack: self.init_stack()
def main():
# initialize tk window for embedding pygame
root = tk.Tk()
embed = tk.Frame(root, width=WINDOW_WIDTH, height=WINDOW_HEIGHT)
embed.pack(side=tk.LEFT)
os.environ['SDL_WINDOWID'] = str(embed.winfo_id())
root.update()
# initialize pygame
pygame.init()
# These are the only events we care about handling
pygame.event.set_allowed([KEYUP, KEYDOWN, MOUSEBUTTONDOWN, ACTIVEEVENT])
pygame.mouse.set_visible(True)
manager = StateManager(root)
root.protocol("WM_DELETE_WINDOW", manager.ask_quit)
# Main game loop
# We put root.update() in here so that pygame.display.update()
# doesn't interrupt tkinter's event handling
while not manager.quit:
t = time.clock()
manager.update()
manager.draw()
pygame.display.update()
root.update()
dt = 1000 * (time.clock() - t)
REAL_FPS = round(1000.0 / dt)
root.title(f"FPS: {min(REAL_FPS, FPS)}/{FPS}")
# The FPS is actually slightly more. This shouldn't matter.
root.after(max(1, round(FRAME_TIME - dt) - 1))
manager.terminate()
def __init__(self, master = None, pathToImage = None, name=None, image=None):
super().__init__(master = master)
if pathToImage is not None and image is None:
self.pathToImage = pathToImage
self.image = ImageSaved(pathToImage)
elif image is not None:
self.image = image
self.manager = StateManager(self.image.cv2Image)
self.set_images()
self.set_geometry()
self.set_basic(master, name)
self.place_menu()
self.manage_line_profile()
self.bind_functions()
def __init__(self, point_array): self.point_array = point_array self.stack = list() self.state_manager = StateManager() self.index = 0 self.minimum_point = self.min_point()
def main():
config = Config()
config_dict = config.get_config()
board_size = config_dict['board_size']
episodes = config_dict['episodes']
num_simulations = config_dict['simulations']
la = config_dict['la']
optimizer = config_dict['optimizer']
M = config_dict['M']
G = config_dict['G']
conv_layer_filters = config_dict['conv_layer_filters']
conv_layer_kernels = config_dict['conv_layer_kernels']
max_num_moves = int(board_size**2)
state_space_size = int(board_size**2 + 1)
conv_layers = (conv_layer_filters, conv_layer_kernels)
state_space_size = 128
hidden_layers = [state_space_size, max_num_moves]
state_manager = StateManager(board_size)
player = NeuralActor(conv_layers, hidden_layers, max_num_moves, la,
optimizer)
mct = MCT(player, num_simulations)
# Train progressive policies
for i in range(episodes):
mct.play_game(copy.deepcopy(state_manager))
training_data = mct.get_training_data()
loss = player.update_Q(training_data)
print(str(i) + " " + str(loss))
if i % (episodes // (M - 1)) == 0:
player.store_model('models/iteration' + str(i))
player.store_model('models/iteration' + str(episodes))
def __init__(self):
self.state_manager = StateManager()
self.level = None
self.max_steps = 0
self.number_steps = 0
self.max_time = 120
self.date_begin = 0
self.verbose = False
def main():
num_simulations = 20
max_depth = 5
mct1 = MCT(None, num_simulations, max_depth)
state_manager = StateManager(4)
for i in range(0, 1):
mct1.play_game(copy.deepcopy(state_manager))
def play_series(self, agt1, agt2):
player1_wins = 0
for i in range(self.G):
print("[{}>{}]".format("-" * i, "." * (self.G - i - 1)), end="\r")
sm = StateManager(5)
while not sm.is_finished():
player = sm.game.get_current_player()
state = np.array(
[np.concatenate((player, sm.game.board), axis=None)])
if player == 1:
predictions = agt1.predict(state)[0]
else:
predictions = agt2.predict(state)[0]
legal_moves = sm.get_legal_moves()
if len(sm.game.executedMoves) <= 1:
best_move = random.choice(legal_moves)
else:
best_move = self.choose_best_move(predictions, legal_moves)
sm.execute_move(best_move)
if sm.get_winner() == 1:
player1_wins += 1
print("{} won {}/{} against {}.".format(agt1.name, player1_wins,
self.G, agt2.name))
print(np.reshape(sm.game.board, (boardsize, boardsize)))
print(sm.game.executedMoves)
return player1_wins
def calc():
try:
print('Server received ' + str(request.json))
resp = StateManager.process_request(request.json)
print('Server returned ' + str(resp))
return resp
except Exception as e:
print(e)
abort(500)
def get_action(self, state_str):
state = np.fromstring(state_str, np.int8) - 48
state = torch.from_numpy(state).float()
nn_output = self.nn(state) # Forward pass
move_index = torch.argmax(nn_output.data)
while not StateManager.is_legal(move_index, state_str):
nn_output.data[0, move_index] = -1.0
move_index = torch.argmax(nn_output.data)
return move_index.item()
def expand(self):
size = int((len(self.state) - 1)**0.5)
state_manager = StateManager(size, self.state)
possible_moves = state_manager.get_moves()
self.children = []
for move in possible_moves:
state_manager.make_move(move)
self.children.append((Node(self,
state_manager.string_representation()),
0)) # Visited 0 times
state_manager.undo_move(move)
def test_display_when_2_valid_calculation_are_preformed(self):
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '1'))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '+'))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '3'))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '='))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '-'))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '7'))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '='))
self.assertEqual('-3', self.s.display)
def rollout(self, leaf):
size = int((len(leaf.state) - 1) ** 0.5)
leaf_state, first_iteration = StateManager(size, leaf.state), True
while True:
if leaf_state.player1_won():
score = 1.0
return score
elif leaf_state.player2_won():
score = -1.0
return score
if leaf_state.is_finished():
print("No winner error")
quit()
if first_iteration:
possible_moves = leaf_state.get_moves()
move = possible_moves[random.randint(0, len(possible_moves) - 1)]
first_iteration = False
else:
move = leaf_state.convert_to_move(self.nn.get_action(leaf_state.string_representation()))
leaf_state.make_move(move)
class TestClustering(unittest.TestCase):
def setUp(self):
self.state = StateManager("database/unittest.pickle")
def test_clustering_pref(self):
ts = time.time()
self.state.run_for_unittest()
te = time.time()
logging.info("Clustering perf test took: %2.2f seconds", te - ts)
self.assertLess(te - ts, 30.0, "Clustering is running slowly")
def test_tf_idf_perf(self):
ts = time.time()
for i in range(100000):
self.state.repository.index.tf_idf("portugal")
self.state.repository.index.tf_idf("london")
self.state.repository.index.tf_idf("ukraine")
self.state.repository.index.tf_idf("russia")
te = time.time()
logging.info("tf_idf perf test took: %2.2f seconds", te - ts)
self.assertLess(te - ts, 1.0, "tf_idf is running slowly")
class ValueLastMatchFeature(object):
"""Extract transient feature from previous event"""
def __init__(self, field_name, entity_field, feature_name):
self.field_name = field_name
self.entity_field = entity_field
self.feature_name = feature_name
self.state_manager = StateManager()
def initialise_state(self, entity_id):
"""Sum and count state initiated as 0. and 0."""
self.state_manager.write(entity_id, None)
def get_feature_names(self):
"""Return feature name"""
return [self.feature_name]
def pre_extraction_update(self, event):
pass
def extract_feature_dict(self, event):
# if state missing, initialise state
entity_id = event[self.entity_field]
if self.state_manager.entity_missing(entity_id):
self.initialise_state(entity_id)
return {self.feature_name: self.state_manager.read(entity_id)}
def post_extraction_update(self, event):
self.state_manager.write(event[self.entity_field],
event[self.field_name])
def test_display_when_number_operation_numer_operation_are_pressed_should_evaluate(self):
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '1'))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '+'))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '3'))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '-'))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '2'))
self.s = json_2_obj(StateManager.calculate_next_state(self.s, '='))
self.assertEqual('2', self.s.display)
def play_series(self, agt1, agt2):
player1_wins = 0
for i in range(self.G):
print("[{}>{}]".format("-" * i, "." * (self.G - i - 1)), end="\r")
game = StateManager(size=self.get_size())
if i == self.G - 1:
print_game = input("Print the last game? (Y/n): ")
else:
print_game = "n"
move_counter = 0
while not game.is_game_over():
game_state = game.get_nn_state()
player_turn = game_state[0]
if player_turn == 1:
predictions = agt1.predict(game_state)
else:
predictions = agt2.predict(game_state)
legal_moves = game.get_legal_moves()
if move_counter < topp_random_moves * 2:
move = random.choice(legal_moves)
else:
move = legal_moves[np.argmax(predictions)]
game.move(move)
move_counter += 1
if print_game.lower() == "y":
game.print_board()
if game.get_winner() == 1:
player1_wins += 1
print("{} won {}/{} against {}.".format(agt1.name, player1_wins,
self.G, agt2.name))
def execute(self):
self.state_manager = StateManager((REMOTE_HOST, STATE_MANAGER_PORT))
self.hardware_monitor = HardwareMonitor()
self.transfer_manager = TransferManager(
"HTTP://%s:%s/" % (REMOTE_HOST, TRANSFER_MANAGER_PORT),
self.job_queue, self.completed_queue)
self.adaptor = Adaptor(self.state_manager, self.hardware_monitor,
self.transfer_manager, TRANSFER_POLICY)
self._bootstrap()
self._process()
self._aggregate()
self.tock = time.time()
logging.info("Run time: %d" % (self.tock - self.tick))
class DecayFeature(object):
def __init__(self, feature_name, entity_field, sum_value_field, incl_count,
incl_sum, incl_mean, decay_rate):
"""Instance initiated with state_id and sample_decay_rate"""
self.feature_name = feature_name
self.entity_field = entity_field
self.sum_value_field = sum_value_field
self.incl_count = incl_count
self.incl_sum = incl_sum
self.incl_mean = incl_mean
self.decay_rate = decay_rate
self.state_manager = StateManager()
def initialise_state(self, entity_id):
raise Exception("Implementation missing")
def get_feature_names(self):
feature_names = []
if self.incl_count:
feature_names.append(self.feature_name + "_count")
if self.incl_sum:
feature_names.append(self.feature_name + "_sum")
if self.incl_mean:
feature_names.append(self.feature_name + "_mean")
return feature_names
def pre_extraction_update(self, event):
raise Exception("Implementation missing")
def extract_feature_dict(self, event):
"""Return feature dict state"""
# read state using state manager
state = self.state_manager.read(event[self.entity_field])
feature_dict = {}
if self.incl_count:
feature_dict[self.feature_name + "_count"] = state['count']
if self.incl_sum:
feature_dict[self.feature_name + "_sum"] = state['sum']
if self.incl_mean:
feature_dict[self.feature_name + "_mean"] = safe_division(
state['sum'], state['count'], 1.)
return feature_dict
def run(self):
self.state_manager = StateManager((LOCAL_HOST, STATE_MANAGER_PORT))
self.hardware_monitor = HardwareMonitor()
self.transfer_manager = TransferManager("HTTP://%s:%s/" % (LOCAL_HOST, TRANSFER_MANAGER_PORT),
self.job_queue, self.completed_queue)
self.adaptor = Adaptor(self.state_manager, self.hardware_monitor,
self.transfer_manager, TRANSFER_POLICY)
# remote node won't start processing phase until bootstrap phase finishes
self.transfer_manager.bootstrap_finished.wait()
worker_thread = threading.Thread(target=worker,
args=(self.job_queue, self.adaptor, self.completed_queue))
worker_thread.daemon = True
worker_thread.start()
self.adaptor.adapt()
while True:
time.sleep(1)
def __init__(self, is_master, remote_ip, gui=None):
self.is_master = is_master
self.gui = gui
self.finished_jobs = []
if is_master:
self.vector = [111.1111] * 1024 * 1024 * 32
else:
self.vector = None
self.job_queue = Queue()
self.work_threads = []
for i in range(NUM_THREADS):
self.work_threads.append(WorkerThread(self.job_queue, self, i))
self.hardware_monitor = HardwareMonitor(self.work_threads)
self.transfer_manager = TransferManager(self.job_queue, remote_ip, self)
self.state_manager = StateManager(remote_ip)
self.adaptor = Adaptor(self.work_threads[0], self.job_queue, self.transfer_manager,
self.state_manager, self.hardware_monitor, self.gui)
def tree_policy_select(self, parent, children, is_first_iteration):
if is_first_iteration:
selected_child = children[random.randrange(0, len(children))]
return selected_child
size = int((len(parent.state) - 1) ** 0.5)
parent_state = StateManager(size, parent.state)
if parent_state.player1_to_move:
max_Q = 0.0
selected_child = None
for child in children:
Q = child.get_Q(parent_state.player1_to_move) + np.sqrt(np.log(parent.num_traversed) / (1.0 + parent.num_traversed_edge(child.state)))
if Q > max_Q or selected_child == None:
max_Q = Q
selected_child = child
else:
max_Q = 0.0
selected_child = None
for child in children:
Q = child.get_Q(parent_state.player1_to_move) + np.sqrt(np.log(parent.num_traversed) / (1.0 + parent.num_traversed_edge(child.state)))
if Q < max_Q or selected_child == None:
max_Q = Q
selected_child = child
return selected_child
anchor=kytten.ANCHOR_CENTER,
theme=gTheme,
on_escape=on_escape)
if __name__ == '__main__':
window = pyglet.window.Window(800,
600,
caption='Kytten Theme Editor',
resizable=True,
vsync=True)
batch = pyglet.graphics.Batch()
bg_group = pyglet.graphics.OrderedGroup(0)
fg_group = pyglet.graphics.OrderedGroup(1)
# Update as often as possible (limited by vsync, if not disabled)
window.register_event_type('on_update')
def update(dt):
window.dispatch_event('on_update', dt)
pyglet.clock.schedule(update)
# StateManager keeps track of what we're doing
manager = StateManager(window)
# Start off by picking the theme directory
manager.push(ThemeDirSelectState())
pyglet.app.run()
on_escape=on_escape)
if __name__ == '__main__':
window = pyglet.window.Window( 800, 600, caption='Kytten Theme Editor', resizable=True, vsync=True)
batch = pyglet.graphics.Batch()
bg_group = pyglet.graphics.OrderedGroup(0)
fg_group = pyglet.graphics.OrderedGroup(1)
# Update as often as possible (limited by vsync, if not disabled)
window.register_event_type('on_update')
def update(dt):
window.dispatch_event('on_update', dt)
pyglet.clock.schedule(update)
# StateManager keeps track of what we're doing
manager = StateManager(window)
parser = argparse.ArgumentParser()
parser.add_argument('-p', "--theme-path", help='path to theme json')
parser.add_argument('-c', '--charsets', help='Specify number of character sets in file vertival & horizontal.')
parser.add_argument("-f", '--frames', help="display list of backup folder")
args = parser.parse_args()
if args.theme_path:
manager.push(ThemeFileSelectState(args.theme_path))
else:# Start off by picking the theme directory
manager.push(ThemeDirSelectState())
pyglet.app.run()
import RPi.GPIO as GPIO
import elemental_api_class as liveapi
from s_av_ctrl import StreamAvailController as StreamAvailCtrl
from reloader import Reloader
from state_manager import StateManager
# from helpers import setup_logger
# sys.path.append('/home/pi/config')
import loggers as lg
import config as cf
try:
# main_log = lg.setup_logger('main_logger','main')
main_log = lg.get_main_logger()
main_log.info('Starting Ad Avail Controller ver. 0.7')
stater = StateManager(cf.LAST_EXIT_FILE, cf.STATE_FILE)
def check_elemental_connection():
global main_log
live_api = liveapi.Elemental_api(cf.elemental_ip)
try:
response = live_api.list_live_events()
if response.status_code != 200:
raise Exception("Elemental server error: {}".format(
response.status_code))
except Exception as e:
main_log.error('Error: {}'.format(e))
# Set-up state manager
# Make a new dict with GPIs as Keys and (class)StreamAvailCtrl as values
class Algorithm: def __init__(self, point_array): self.point_array = point_array self.stack = list() self.state_manager = StateManager() self.index = 0 self.minimum_point = self.min_point() def increment_state(self, is_right_turn = False, out_of_points = False): self.state_manager.increment_state(is_right_turn = is_right_turn, out_of_points = out_of_points) def segments(self): if not self.stack: return [] tuple_list = zip(self.stack, self.stack[1:]) segment_list = map(lambda x: Segment(x[0], x[1]), tuple_list) if self.state_manager.current_state == State.complete: segment_list.append(Segment(self.stack[-1], self.stack[0])) return segment_list def min_point(self): #return leftmost min point by y coord min_point = Point(sys.maxsize, sys.maxsize) for i in self.point_array: if i < min_point: min_point=i return min_point def polar_angle_sort(self, min_point): sort_list = copy.deepcopy(self.point_array) sort_list.remove(min_point) sort_list.sort(key = lambda point: Point.calculate_polar_angle(min_point, point)) sort_list.insert(0, min_point) return sort_list def sort_point_array(self): self.point_array = self.polar_angle_sort(self.min_point()) def init_stack(self): self.stack = list() self.stack.append(self.point_array[0]) self.stack.append(self.point_array[1]) self.stack.append(self.point_array[2]) self.index = 2 def is_checking_point(self): return self.state_manager.current_state == State.check_angle def cross_product(self, v1, v2): return (v1.x * v2.y - v1.y * v2.x) def is_nonleft_turn(self, p1, p2, p3): v1 = Point((p1.x - p2.x), (p1.y - p2.y)) v2 = Point((p3.x - p2.x), (p3.y - p2.y)) determinant = self.cross_product(v1, v2) return True if determinant > 0 else False def select_point(self): self.index += 1 def check_angle(self): return self.is_nonleft_turn(self.stack[-2], self.stack[-1], self.point_array[self.index]) def pop(self): self.stack.pop() def is_out_of_points(self): return self.index >= len(self.point_array) - 1 def push(self): self.stack.append(self.point_array[self.index]) def next_step(self): if self.state_manager.current_state == State.not_run: self.increment_state() elif self.state_manager.current_state == State.initial_point: self.sort_point_array() self.increment_state() elif self.state_manager.current_state == State.init_stack: self.init_stack() self.increment_state() elif self.state_manager.current_state == State.select_point: self.select_point() self.increment_state() elif self.state_manager.current_state == State.check_angle: self.increment_state(is_right_turn = self.check_angle()) elif self.state_manager.current_state == State.pop: self.pop() self.increment_state() elif self.state_manager.current_state == State.push: self.push() self.increment_state(out_of_points = self.is_out_of_points()) elif self.state_manager.current_state == State.complete: pass
# if not found, create a new one if not spec: spec = Species (state) r = RewardInfo () self.species [s] = r return spec, r # return reward for inputted state return spec, self.species [spec] g = Game () sm = StateManager () scores = [] random_moves = [] informed_moves = [] dif_states = [] x_data = [] running_length = 500 averages = [] num_games = int (4 * 1e3) print 'Playing %s games.' % num_games
def test_increment_state(self): test_state = StateManager() self.assertEqual(test_state.current_state, State.not_run) test_state.increment_state() self.assertEqual(test_state.current_state, State.initial_point) test_state.increment_state() self.assertEqual(test_state.current_state, State.init_stack) test_state.increment_state() self.assertEqual(test_state.current_state, State.select_point) test_state.increment_state() self.assertEqual(test_state.current_state, State.check_angle) test_state.increment_state(is_right_turn = True) self.assertEqual(test_state.current_state, State.pop) test_state.increment_state() self.assertEqual(test_state.current_state, State.check_angle) test_state.increment_state(is_right_turn = False) self.assertEqual(test_state.current_state, State.push) test_state.increment_state() self.assertEqual(test_state.current_state, State.select_point) test_state.increment_state() self.assertEqual(test_state.current_state, State.check_angle) test_state.increment_state(is_right_turn = True) self.assertEqual(test_state.current_state, State.pop) test_state.increment_state() self.assertEqual(test_state.current_state, State.check_angle) test_state.increment_state(is_right_turn = False) self.assertEqual(test_state.current_state, State.push) test_state.increment_state(out_of_points = True) self.assertEqual(test_state.current_state, State.complete)
class Launcher:
"""
The initiator of the whole program
"""
def __init__(self, is_master, remote_ip, gui=None):
self.is_master = is_master
self.gui = gui
self.finished_jobs = []
if is_master:
self.vector = [111.1111] * 1024 * 1024 * 32
else:
self.vector = None
self.job_queue = Queue()
self.work_threads = []
for i in range(NUM_THREADS):
self.work_threads.append(WorkerThread(self.job_queue, self, i))
self.hardware_monitor = HardwareMonitor(self.work_threads)
self.transfer_manager = TransferManager(self.job_queue, remote_ip, self)
self.state_manager = StateManager(remote_ip)
self.adaptor = Adaptor(self.work_threads[0], self.job_queue, self.transfer_manager,
self.state_manager, self.hardware_monitor, self.gui)
def bootstrap(self):
"""
Define the behavior for bootstrap phase
"""
if self.is_master:
self.allocate_jobs()
self.transfer_jobs()
self.transfer_manager.receive_job()
# wait until receiving half of jobs
while self.job_queue.qsize() != NUM_JOB/2:
sleep(0.1)
# receive all jobs and exit the bootstrap stage
self.hardware_monitor.start()
self.state_manager.receive_state()
self.state_manager.start()
for i in range(NUM_THREADS):
self.work_threads[i].start()
def allocate_jobs(self):
"""
Divide the vector into NUM_JOB jobs, and put jobs into job
queue
"""
job_size = len(self.vector) / NUM_JOB
for i in range(NUM_JOB):
pos = i * job_size
job = Job(i, pos, self.vector[pos: pos + job_size])
self.job_queue.put(job)
def transfer_jobs(self):
"""
Send half of jobs to slave node through transfer manager
"""
self.transfer_manager.send_jobs(NUM_JOB / 2)
def on_job_finish(self, job):
"""
Callback function when a job finishes
:param job: Job Object
"""
# put the job into finished job if in master node
# send the job to master node if in slave node
if self.is_master:
self.finished_jobs.append(job)
else:
self.transfer_manager.send_job(job)
# inform gui
if self.gui:
self.gui.on_job_finish()
# start to aggregate jobs when all jobs finished
if len(self.finished_jobs) == NUM_JOB:
self.aggregate_jobs()
self.check_data()
def aggregate_jobs(self):
"""
Map the data back to vector
"""
print "Aggregating..."
for job in self.finished_jobs:
pos = job.pos
for data in job.work_data:
self.vector[pos] = data
pos += 1
def print_data(self):
"""
Print value stored in the vector
:param vector:
"""
for i, v in enumerate(self.vector):
print "A[%d]= %d" % (i, v)
def check_data(self):
"""
Print value stored in the vector
:param vector:
"""
print "Checking"
if len(set(self.vector)) == 1:
print "All elements in the vector are equal"
else:
print "There is error in the vector"
def __init__ (self, logger, product, sysVarDir, sysRunDir, dataVarDir, rpmsDir,
appDir, appDirReal, bootDevPath, grubConf, prevRfsDir=None, vitalDir=None, systemRoot="/", useFakeChrootMode=False):
"""
product : Name of product we handle, e.g. 'qb'.
sysVarDir : path to sys dir relative to root, e.g. '/opt/qb/sys/oscar/install/0/var'.
sysRunDir : path to sys dir relative to root, e.g. '/opt/qb/sys/oscar/install/0/run'.
dataVarDir : path to data dir relative to root, e.g. '/opt/qb/data/oscar/install/0/var'.
rpmsDir : path to rpms dir relative to root, e.g. '/opt/qb/rpms'.
appDir : path to app dir relative to root, e.g. '/opt/qb/app'.
appDirReal : real path to app dir relative to root, e.g. '/opt/qb/sys/rfs/sys/0/var/app'.
bootDevPath : Path to boot device, e.g. '/boot'
grubConf : name of grub.conf file relative to boot device, e.g. 'grub/grub.conf'.
systemRoot : For unit tests, specifies location of the file-system root.
useFakeChrootMode : For unit tests, True to use fakechroot instead of chroot (via rpm --root or yum --installroot)
prevRfsDir : Location to copy some RFS data to during OS install
vitalDir : Path to vital dir.
"""
self._raiseIfNotAbsPath(systemRoot, "systemRoot")
self._log = logger.createLogger("sys-install", "install-manager")
self._log.setInstance(product)
self._utils = Utils(self._log)
self._product = product
self._bootDevPath = bootDevPath
self._grubConf = grubConf
self._systemRoot = systemRoot
self._useFakeChrootMode = useFakeChrootMode
self._vitalDir = vitalDir
self._rpmsDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(rpmsDir))
self._sysVarDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(sysVarDir))
self._sysRunDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(sysRunDir))
self._dataVarDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(dataVarDir))
self._logDir=os.path.join(self._dataVarDir, 'log')
self._stateDir = os.path.join(self._sysVarDir, "state")
self._yumConfActiveRfsDir = os.path.join(self._sysVarDir, "yum-conf-active-rfs")
self._yumConfNewVersionDir = os.path.join(self._sysVarDir, "yum-conf-new-version")
self._appDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(appDir))
self._appDirReal = os.path.join(systemRoot, self._chopLeadingSlashIfFound(appDirReal))
self._installDir = os.path.join(self._sysVarDir,"sub-pkgs")
self._workDir=os.path.join(self._sysVarDir, 'work')
self._apiDir=os.path.join(self._sysVarDir, 'api')
self._startupScriptsDir=os.path.join(self._sysVarDir, 'startup')
self._startupScriptsPrevDir=os.path.join(self._startupScriptsDir, 'prev')
self._rpmKeysDir = os.path.join(self._sysVarDir, "rpm/keys")
self._tempDir = os.path.join(self._sysRunDir, "tmp")
self._installTempDir = os.path.join(self._tempDir, "sub-pkgs") # install-temp can be cleared between runs
self._subPkgManager = SubPkgManager(self._log)
self._subPkgManager.setDirs(self._systemRoot, self._installDir, self._installTempDir, self._rpmKeysDir, self._useFakeChrootMode)
self._stateFile = os.path.join(self._stateDir, product+"-state")
self._stateManager = StateManager(self._log)
self._stateManager.setFile(self._stateFile)
self._pilotHints=""
self._prevRfsDir = prevRfsDir
self._preparedPackageFilePath = None
self._log("init").info("InstallManager::__init__() called, self._product=%s, self._sysVarDir=%s, self._sysRunDir=%s, "\
"self._dataVarDir=%s, self._rpmsDir=%s, self._appDir=%s, self._bootDevPath=%s, "\
"self._grubConf=%s, self._prevRfsDir=%s, self._vitalDir=%s",
self._product, self._sysVarDir, self._sysRunDir, self._dataVarDir, self._rpmsDir,
self._appDir, self._bootDevPath, self._grubConf, self._prevRfsDir, self._vitalDir)
# If true, we will not actually install RPMs. This helps modify this module in-box and testing the
# changes, to avoid having to create a package file each time
self._debugNoInstallMode=False
class InstallManager(object):
"""
This is the main install manager
"""
def __init__ (self, logger, product, sysVarDir, sysRunDir, dataVarDir, rpmsDir,
appDir, appDirReal, bootDevPath, grubConf, prevRfsDir=None, vitalDir=None, systemRoot="/", useFakeChrootMode=False):
"""
product : Name of product we handle, e.g. 'qb'.
sysVarDir : path to sys dir relative to root, e.g. '/opt/qb/sys/oscar/install/0/var'.
sysRunDir : path to sys dir relative to root, e.g. '/opt/qb/sys/oscar/install/0/run'.
dataVarDir : path to data dir relative to root, e.g. '/opt/qb/data/oscar/install/0/var'.
rpmsDir : path to rpms dir relative to root, e.g. '/opt/qb/rpms'.
appDir : path to app dir relative to root, e.g. '/opt/qb/app'.
appDirReal : real path to app dir relative to root, e.g. '/opt/qb/sys/rfs/sys/0/var/app'.
bootDevPath : Path to boot device, e.g. '/boot'
grubConf : name of grub.conf file relative to boot device, e.g. 'grub/grub.conf'.
systemRoot : For unit tests, specifies location of the file-system root.
useFakeChrootMode : For unit tests, True to use fakechroot instead of chroot (via rpm --root or yum --installroot)
prevRfsDir : Location to copy some RFS data to during OS install
vitalDir : Path to vital dir.
"""
self._raiseIfNotAbsPath(systemRoot, "systemRoot")
self._log = logger.createLogger("sys-install", "install-manager")
self._log.setInstance(product)
self._utils = Utils(self._log)
self._product = product
self._bootDevPath = bootDevPath
self._grubConf = grubConf
self._systemRoot = systemRoot
self._useFakeChrootMode = useFakeChrootMode
self._vitalDir = vitalDir
self._rpmsDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(rpmsDir))
self._sysVarDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(sysVarDir))
self._sysRunDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(sysRunDir))
self._dataVarDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(dataVarDir))
self._logDir=os.path.join(self._dataVarDir, 'log')
self._stateDir = os.path.join(self._sysVarDir, "state")
self._yumConfActiveRfsDir = os.path.join(self._sysVarDir, "yum-conf-active-rfs")
self._yumConfNewVersionDir = os.path.join(self._sysVarDir, "yum-conf-new-version")
self._appDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(appDir))
self._appDirReal = os.path.join(systemRoot, self._chopLeadingSlashIfFound(appDirReal))
self._installDir = os.path.join(self._sysVarDir,"sub-pkgs")
self._workDir=os.path.join(self._sysVarDir, 'work')
self._apiDir=os.path.join(self._sysVarDir, 'api')
self._startupScriptsDir=os.path.join(self._sysVarDir, 'startup')
self._startupScriptsPrevDir=os.path.join(self._startupScriptsDir, 'prev')
self._rpmKeysDir = os.path.join(self._sysVarDir, "rpm/keys")
self._tempDir = os.path.join(self._sysRunDir, "tmp")
self._installTempDir = os.path.join(self._tempDir, "sub-pkgs") # install-temp can be cleared between runs
self._subPkgManager = SubPkgManager(self._log)
self._subPkgManager.setDirs(self._systemRoot, self._installDir, self._installTempDir, self._rpmKeysDir, self._useFakeChrootMode)
self._stateFile = os.path.join(self._stateDir, product+"-state")
self._stateManager = StateManager(self._log)
self._stateManager.setFile(self._stateFile)
self._pilotHints=""
self._prevRfsDir = prevRfsDir
self._preparedPackageFilePath = None
self._log("init").info("InstallManager::__init__() called, self._product=%s, self._sysVarDir=%s, self._sysRunDir=%s, "\
"self._dataVarDir=%s, self._rpmsDir=%s, self._appDir=%s, self._bootDevPath=%s, "\
"self._grubConf=%s, self._prevRfsDir=%s, self._vitalDir=%s",
self._product, self._sysVarDir, self._sysRunDir, self._dataVarDir, self._rpmsDir,
self._appDir, self._bootDevPath, self._grubConf, self._prevRfsDir, self._vitalDir)
# If true, we will not actually install RPMs. This helps modify this module in-box and testing the
# changes, to avoid having to create a package file each time
self._debugNoInstallMode=False
def unit_test_initializeSystemFirstInstall (self, curVersion, curBuild, keys):
"""
Should be called once after first install to initialize the state file and keys.
Used only by unit tests. In real life, the extract_pkg script does this.
curVersion : Name of the current version
curBuild : Name of the current build
keys : listof paths to ascii armoured files with gpg keys for rpm validation
"""
self.initializeSystemFirstInstall(curVersion, curBuild)
# Import rpm keys
rpm=self._createRpmForActiveRfs()
for key in keys:
rpm.doImport(key)
def initializeSystemFirstInstall (self, curVersion, curBuild):
"""
Should be called once after first install to initialize the state file and keys.
curVersion : Name of the current version
curBuild : Name of the current build
"""
self._log("initial-set-cur-version").info("initialSetCurVersion() called, curVersion=%s", curVersion)
# Create initial dirs
self._utils.runCommandRaiseIfFail("mkdir -p "+self._stateDir)
# Set initial state
self._stateManager.initialSetCurVersion(self.combineVersion(curVersion, curBuild))
def init (self):
"""
Must be called after creating this object, to initialize it, on a system that has been initialized
(i.e. has a state file on disk)
"""
self._log("init").info("init() called")
# Make sure these dirs exist. If not, create them.
self._utils.runCommandRaiseIfFail("mkdir -p "+self._sysRunDir)
self._utils.runCommandRaiseIfFail("mkdir -p "+self._logDir)
self._utils.runCommandRaiseIfFail("mkdir -p "+self._workDir)
self._utils.runCommandRaiseIfFail("mkdir -p "+self._apiDir)
self._utils.runCommandRaiseIfFail("mkdir -p "+self._startupScriptsDir)
self._utils.runCommandRaiseIfFail("mkdir -p "+self._startupScriptsPrevDir)
#TODO(orens): catch InstallException that might be raised during load, reset state and restart oscar
self._log("init").info("init() loading state")
self._stateManager.load()
self._initPlatformBasic()
self._initSdUtils()
self._initBootUtils()
def isPendingRestart (self):
"""
Returns true if 'switch' was executed, we are not waiting for system to restart.
"""
return self._stateManager.isLocked()
def setPilotHints (self, hints):
self._log("setPilotHints").info("setPilotHints(hints=%s) called", hints)
self._pilotHints = hints
def startup (self, logDir = None):
"""
Must be called during startup of an installed system, i,e, upon every Oscar start.
Call this after callint init()
When logDir is passed, the start-up script writes its logs into that directory, otherwise default directory is used.
"""
self._log("startup").info("startup() called")
#TODO(orens): In the future, when we do not need to support upgrading from EFT versions,
# remove this call, and all the setVersionToActivateOnStartup() support from StateManager
self._stateManager.startup()
# Run all scripts from the startup dir, move them to 'prev' later
self._log("startup-running").info("startup(): Scanning directory %s for startup scripts", self._startupScriptsDir)
files=os.listdir(self._startupScriptsDir)
for file_ in files:
fullPath=os.path.join(self._startupScriptsDir, file_)
if logDir is not None:
startupCommand = fullPath + " --log-dir " + logDir
else:
startupCommand = fullPath
# Do not run .tmp files or directories :-)
if (not fullPath.endswith(".tmp")) and os.path.isfile(fullPath):
self._log("startup-running").info("startup(): running startup script '%s'", startupCommand)
# If script fails, raise and get out of here. This should fail oscar start
self._utils.runCommandRaiseIfFail(startupCommand)
# Move used script to 'prev' dir, to lay there forever shamed.
self._utils.runCommandRaiseIfFail("mv -f %s %s" % (fullPath, self._startupScriptsPrevDir))
def getActiveVersionAndBuild (self):
"""
Returns the current active (version, build)
"""
self._log("get-active-version").info("getActiveVersionAndBuid() called")
versionAndBuild=self._stateManager.getActiveVersion()
(version_, build) = self.splitVersion(versionAndBuild)
self._log("get-active-version").info("getActiveVersionAndBuid() got %s, returning %s, %s",
versionAndBuild, version_, build)
return (version_, build)
def getReadyVersion (self):
"""
Returns (version, build) of currently ready version.
If no version is ready, returns None
"""
versionCombined = self._stateManager.getReadyVersion()
if versionCombined==None:
return None
(version_, build) = self.splitVersion(versionCombined, raiseIfInvalid=True)
return (version_, build)
def getReadyVersion2 (self):
"""
Returns (version, build, switch type, install direction) of currently ready version.
If no version is ready, returns None
"""
versionCombined = self._stateManager.getReadyVersion()
if versionCombined==None:
return None
switchType = self._stateManager.getSwitchType(versionCombined)
(version_, build) = self.splitVersion(versionCombined, raiseIfInvalid=True)
return (version_, build, switchType)
def getStateManager (self):
return self._stateManager
def getPreparedPackageFilePath (self):
return self._preparedPackageFilePath
def isPackageValid (self, fileName):
"""
Checks that an RPM file is a signed, and contains a proper version for our product.
Returns None if invalid, or (version, build) if valid
"""
self._log("is-valid").info("isPackageValid(fileName=%s) called", fileName)
if not self._isPackageSigned(fileName):
self._log("is-valid").warning("isPackageValid() returning false, fileName=%s not signed", fileName)
return None
versionCombined = self._subPkgManager.findVersionInFile(self._product, fileName)
if versionCombined is None:
self._log("is-valid").warning("isPackageValid() returning false, no version of product %s found in it", self._product)
return None
(version_, build) = self.splitVersion(versionCombined)
if version_ is None:
self._log("is-valid").warning("isPackageValid() returning false, no version of product %s found in it", self._product)
return None
return (version_, build)
def addSubPkgs (self, fileName):
"""
Adds a package file to the repos. Raises InvalidPackageFileException if not a valid package file for this product
Returns (version, build)
"""
self._log("add").info("addSubPkgs(fileName=%s) called", fileName)
self._raiseIfNotAbsPath(fileName, "fileName")
valid=self._isPackageSigned(fileName)
if not valid:
self._log("add-bad-rpm").error("addSubPkgs() got bad package file, not properly signed", self._product)
raise InvalidPackageFileException()
versionCombined = self._subPkgManager.findVersionInFile(self._product, fileName)
if versionCombined is None:
self._log("add-bad-version").error("addSubPkgs() got bad package file, no version of product %s found in it", self._product)
raise InvalidPackageFileException()
(version_, build) = self.splitVersion(versionCombined)
if version_ is None:
self._log("add-bad-version").error("addSubPkgs() got versionInFile '%s', cannot split.", versionCombined)
raise InvalidPackageFileException()
# Check if version is active or ready. If yes, raise exception
self._raiseIfVersionActiveOrReady(versionCombined)
# Extract package file into sub-pkgs dir
(subPkgs, newSubPkgs)=self._subPkgManager.add(fileName)
self._log("add-got-subpkgs").info("addSubPkgs() returning newSubPkgs=%s", newSubPkgs)
self._stateManager.addVersion(versionCombined, subPkgs=subPkgs, subPkgsToRemove=newSubPkgs)
# Remember the file path
self._preparedPackageFilePath = fileName
self._log("add").info("addSubPkgs() returning version=%s, build=%s", version_, build)
return (version_, build)
def addSubPkgsFromHttp (self, repoHttpAddr, version_, build):
"""
Adds a package file to the repos. Raises InstallException if package not found.
"""
self._log("add-from-http").info("addSubPkgsFromHttp(repoHttpAddr=%s, version_=%s, build=%s) called", repoHttpAddr, version_, build)
versionCombined = self.combineVersion(version_, build)
if versionCombined is None:
self._log("add-from-http-bad-version").error("addSubPkgsFromHttp() got bad version(=%s) or build(=%s) parameters", version_, build)
raise InstallException("Invalid version '%s', build '%s' specificaion" % (version_, build))
# Check if version is active or ready. If yes, raise exception
self._raiseIfVersionActiveOrReady(versionCombined)
packageName = "%s-%s" % (self._product, versionCombined)
self._log("add-from-http-package").info("addSubPkgsFromHttp(): installing package %s", packageName)
# Extract package file into sub-pkgs dir
(subPkgs, newSubPkgs)=self._subPkgManager.add(httpRepo=repoHttpAddr, packageName=packageName)
self._log("add-from-http-got-subpkgs").info("addSubPkgsFromHttp() got newSubPkgs=%s", newSubPkgs)
self._stateManager.addVersion(versionCombined, subPkgs=subPkgs, subPkgsToRemove=newSubPkgs)
self._log("add-from-http-done").info("addSubPkgsFromHttp() done")
def removeVersionAndSubPkgs (self, version_, build, removeSubPackages=True):
"""
Removes sub-pkgs added by a version during 'prepare', and the version itself.
Version must have state 'on-disk'
"""
self._log("remove-subpkgs").info("removeVersionAndSubPkgs(version=%s, build=%s) called", version_, build)
versionCombined = self.combineVersion(version_, build)
versionState = self._stateManager.getVersionState(versionCombined)
if (versionState != StateManager.kStateOnDisk):
msg = "removeVersionAndSubPkgs(): Version %s is not on-disk, it is in state '%s'" % (versionCombined, versionState)
self._log("remove-subpkgs").error(msg)
raise InstallException(msg)
if removeSubPackages:
subPkgsToRemove = self._stateManager.getSubPkgsToRemoveOnRemoval(versionCombined)
self._log("remove-subpkgs").info("Removing sub-packages: %s", subPkgsToRemove)
if len(subPkgsToRemove):
self._subPkgManager.removeSubPackages(subPkgsToRemove)
self._log("remove-subpkgs").info("Removing version %s from state", versionCombined)
self._stateManager.removeVersion(versionCombined)
def removeUnneededVersionsFromDisk (self):
"""
Removes old versions from disk.
Scans all versions maintained in the state, and for each version which is 'on-disk', if it is not needed (i.e.
for a possible rollback command), removes sub-packages and RPMs installed by it and not required by the active versions
"""
self._log("remove-undeeded-vers-called").info("removeUnneededVersionsFromDisk() called")
# Find things needed by the current active and ready versions
# Currently no rollback support, hence nothing else is needed.
versions = self._stateManager.getVersions()
versionsToRemove=[]
versionsToKeep=[]
for ver in versions:
state=self._stateManager.getVersionState(ver)
# TODO(orens): When we add rollback support, keep also the 'on-disk' version we may rollback into.
if state==StateManager.kStateOnDisk:
versionsToRemove.append(ver)
else:
versionsToKeep.append(ver)
# Find list of RPMs and list of subpackages we must keep
(rpmsToKeep, subPkgsToKeep)=self._getRpmsAndSubPkgsOfVersions(versionsToKeep)
# Find list of RPMs used by versions we remove
(rpmsToRemove, subPkgsToRemove)=self._getRpmsAndSubPkgsOfVersions(versionsToRemove)
# The crucial moment
rpmsToRemove -= rpmsToKeep
subPkgsToRemove -= subPkgsToKeep
# Filter rpms in order to check which ones are not installed- only installed ones should be removed
self._log("ruv-filtering1").info("removeUnneededVersionsFromDisk(): Filtering RPMs %s for not installed packages", rpmsToRemove)
rpm=self._createRpmForActiveRfs()
rpmsToRemove = filter(rpm.isInstalled, rpmsToRemove)
# Check if pilot wants to modify RPM list
activeVersion = self._stateManager.getActiveVersion()
self._log("ruv-filtering3").info("removeUnneededVersionsFromDisk(): activeVersion is %s", activeVersion)
(pilot, _) = self._createPilot(activeVersion, None)
if hasattr(pilot, "hookModifyRpmsForRemovalOld"):
rpmsToRemove=pilot.hookModifyRpmsForRemovalOld(rpmsToRemove)
self._log("ruv-filtering5").info("removeUnneededVersionsFromDisk(): pilot hookModifyRpmsForRemovalOld() returned %s", rpmsToRemove)
else:
self._log("ruv-filtering6").info("removeUnneededVersionsFromDisk(): pilot hookModifyRpmsForRemovalOld() not implemented")
# We do not need to filter sub-pkgs for existance - only existing ones would be removed
self._log("ruv-removing-ver").info("removeUnneededVersionsFromDisk(): Removing RPMs %s", rpmsToRemove)
self._log("ruv-removing-ver").info("removeUnneededVersionsFromDisk(): Removing sub-pkgs %s", subPkgsToRemove)
if len(rpmsToRemove)>0:
yum=self._createYumWithAllRepos()
yum.doRemove(" ".join(rpmsToRemove))
self._subPkgManager.removeSubPackages(subPkgsToRemove)
self._log("ruv-removing-ver").info("removeUnneededVersionsFromDisk(): Removing versions %s from state", versionsToRemove)
for ver in versionsToRemove:
self._stateManager.removeVersion(ver)
def _getRpmsAndSubPkgsOfVersions(self, versions):
"""
Returns a tuple with two sets: (rpmsOfVersions, subPkgsOfVersions)
"""
rpmsOfVersions=set()
subPkgsOfVersions=set()
for ver in versions:
self._log("get-rpms-and-subpkgs").info("_getRpmsAndSubPkgsOfVersions() Handling version %s", ver)
leaders=self._stateManager.getLeaders(ver)
self._log("get-rpms-and-subpkgs").info("_getRpmsAndSubPkgsOfVersions(): leaders=%s", leaders)
for leader in leaders.values():
rpmsOfVersions.add(leader)
subPkgs=self._stateManager.getSubPkgs(ver)
self._log("get-rpms-and-subpkgs").info("_getRpmsAndSubPkgsOfVersions(): subPkgs=%s", subPkgs)
for subPkg in subPkgs:
subPkgsOfVersions.add(subPkg)
# The pilot of a version is always called like this:
rpmsOfVersions.add(self.calcPilotName(ver))
# Expand set by adding all required packages
rpmsOfVersions=self._expandRpmsUsingReqires(rpmsOfVersions)
self._log("get-rpms-and-subpkgs").info("_getRpmsAndSubPkgsOfVersions(): Returning rpmsOfVersions=%s", rpmsOfVersions)
self._log("get-rpms-and-subpkgs").info("_getRpmsAndSubPkgsOfVersions(): Returning subPkgsOfVersions=%s", subPkgsOfVersions)
return (rpmsOfVersions, subPkgsOfVersions)
def _expandRpmsUsingReqires (self, packages):
"""
Gets an iterable with a list of packages (Some may not be installed it is OK).
Returns a set with the same packages, plus all installed packages required by these installed packages
"""
self._log("expand-requires").info("_expandRpmsUsingReqires(packages=%s) called, ", packages)
rpm=self._createRpmForActiveRfs()
doIt=True
ret=set(packages)
while doIt:
self._log("expand-requires").info("_expandRpmsUsingReqires() ret=%s", ret)
doIt=False
addedPackages=set()
for package in packages:
requires=rpm.doListRequires(package)
for (capName, capCondition, capVersion) in requires:
# Ignore 'rpmxxx' capabilities
if capName.startswith('rpm'):
continue
# Ignore capabilities which are not '='.
if capCondition != '=':
continue
addedPackages.add(capName+'-'+capVersion)
# Found something new ? Add to result and continue iterating
self._log("expand-requires").info("_expandRpmsUsingReqires() addedPackages=%s, ret=%s", addedPackages, ret)
newPackages = addedPackages-ret
self._log("expand-requires").info("_expandRpmsUsingReqires() newPackages=%s", newPackages)
if len(newPackages) > 0:
doIt=True
ret |= newPackages
packages = newPackages
self._log("expand-requires").info("_expandRpmsUsingReqires() returning %s", ret)
return ret
def prepare (self, version_, build, usePilotVersion=None):
"""
Prepares a version for installation.
If a version is already prepared, raises InstallException().
usePilotVersion: If != None, sets pilot version to be used instead of auto-selecting.
"""
self._log("prepare").info("prepare(version=%s, build=%s) called", version_, build)
if self._stateManager.getReadyVersion() != None:
msg="prepare(): A version is ready"
self._log("prepare").error(msg)
raise InstallException(msg)
if build == kFakeBuildNumber:
msg="prepare(): Invalid build number. Build '%s' is reserved for fake builds" % kFakeBuildNumber
self._log("prepare").error(msg)
raise InstallException(msg)
newVersion=self.combineVersion(version_, build)
activeVersion=self._stateManager.getActiveVersion()
self._installPilot(newVersion)
self._log("prepare").info("Creating pilot for new version %s", newVersion)
(newVersionPilot, newVersionPs) = self._createPilot(newVersion, newVersion)
self._log("prepare").info("Creating pilot for active version %s", activeVersion)
(activeVersionPilot, activeVersionPs) = self._createPilot(activeVersion, newVersion)
# Teach each pilot services about the other pilot
newVersionPs.setOtherPilot(activeVersionPilot)
activeVersionPs.setOtherPilot(newVersionPilot)
if usePilotVersion != None:
self._log("prepare").info("Creating pilot for version %s, manually selected", usePilotVersion)
if (usePilotVersion != newVersion) and (usePilotVersion != activeVersion):
msg="Manually selected pilot should be one of %s or %s, not %s" % (newVersion, activeVersion, usePilotVersion)
self._log("prepare").error("prepare(): %s", msg)
raise InstallException(msg)
selectedVersion=usePilotVersion
else:
self._log("prepare").info("Running pilot selection")
# old pilot selection logic uses 'selectPilot'
# new pilot selection logic uses 'getPilotGeneration'
# If new version pilot supports new logic, use it.
# If new version pilot does not support new logic, then this is downgrade, we choose our pilot.
if hasattr(newVersionPilot, "getPilotGeneration"):
newPilotGeneration=newVersionPilot.getPilotGeneration()
activePilotGeneration=activeVersionPilot.getPilotGeneration()
self._log("prepare").info("prepare(): pilot selection: newVersion=%s, activeVersion=%s, newPilotGeneration=%s, activePilotGeneration=%s",
newVersion, activeVersion, newPilotGeneration, activePilotGeneration)
if newPilotGeneration > activePilotGeneration:
selectedVersion = newVersion
elif newPilotGeneration < activePilotGeneration:
selectedVersion = activeVersion
else: # Pilot generations are equal, choose according to version numbers
selectedVersion=self._selectLatestVersion(newVersion, activeVersion)
else: # Downgrading, choose our pilot
selectedVersion = activeVersion
if selectedVersion == newVersion:
pilot = newVersionPilot
elif selectedVersion == activeVersion:
pilot = activeVersionPilot
else:
msg="Bad pilot: newVersion=%s, activeVersion=%s, selectedVersion=%s" % (newVersion, activeVersion, selectedVersion)
self._log("prepare").error("prepare(): pilot selection: %s", msg)
raise InstallException(msg)
# Version is selected, We have a pilot, let's take off
self._log("prepare").info("prepare(): Selected pilot version is %s", selectedVersion)
self._stateManager.setPilotName(newVersion, selectedVersion)
rpm = self._createRpmForActiveRfs()
installedRpmsBefore=rpm.doGetInstalledPackages()
self._log("prepare").info("prepare(): Installed RPMs before prepare() are: %s", installedRpmsBefore)
# As last, Run pilot 'prepare' function !
# Pilot is executed in try block. In case of failure, we clear pilot name.
self._log("prepare").info("prepare(): running pilot...")
try:
# Old pilots do not have doPrepare2(), but we should never call old pilots from new code,
# because the active version pilot should win.
pilot.doPrepare2(newVersion, newVersionPilot, activeVersionPilot)
except:
self._stateManager.setPilotName(newVersion, None)
raise
installedRpmsAfter=rpm.doGetInstalledPackages()
self._log("prepare").info("prepare(): Installed RPMs after prepare() are: %s", installedRpmsBefore)
newRpms = sorted(list(set(installedRpmsAfter)-set(installedRpmsBefore)))
self._log("prepare").info("prepare(): New RPMs: %s", newRpms)
self._stateManager.setRpmsToRemoveOnRemoval(newVersion, newRpms)
# Pilot prepare went ok - time to mark this version as 'ready' !
self._log("prepare").info("prepare(): pilot did good, marking version as ready")
self._stateManager.setReadyVersion(newVersion)
def cancel (self, version_, build):
self._log("cancel").info("cancel(version=%s, build=%s) called", version_, build)
versionCombined = self.combineVersion(version_, build)
versionState = self._stateManager.getVersionState(versionCombined)
if (versionState != StateManager.kStateReady):
msg = "cancel(): Version %s is not ready, it is in state '%s'" % (versionCombined, versionState)
self._log("cancel").error(msg)
raise InstallException(msg)
self._log("cancel").info("Creating pilot for version %s", versionCombined)
(pilot,_)=self._createPilot(self._stateManager.getPilotName(versionCombined), versionCombined)
if hasattr(pilot, "hookCancelBeforeRemove"):
self._log("cancel").info("Calling pilot hookCancelBeforeRemove()")
pilot.hookCancelBeforeRemove(versionCombined)
rpmsToRemove = self._stateManager.getRpmsToRemoveOnRemoval(versionCombined)
rpm = self._createRpmForActiveRfs()
self._log("cancel").info("Removing rpms: %s", rpmsToRemove)
if len(rpmsToRemove):
rpm.doRemove(" ".join(rpmsToRemove))
if hasattr(pilot, "hookCancelAfterRemove"):
self._log("cancel").info("Calling pilot hookCancelAfterRemove()")
pilot.hookCancelAfterRemove(versionCombined)
self._log("cancel").info("Changing state of version %s to 'on-disk'", versionCombined)
self._stateManager.setVersionStateOnDisk(versionCombined)
def switch (self, version_, build):
"""
version_, build are parameters of the ready version
On success, returns pilot doSwitch() return code.
On failure, raises something
"""
self._log("switch-called").info("switch(version=%s, build=%s) called", version_, build)
versionCombined = self.combineVersion(version_, build)
versionState = self._stateManager.getVersionState(versionCombined)
if (versionState != StateManager.kStateReady):
msg = "cancel(): Version %s is not ready, it is in state '%s'" % (versionCombined, versionState)
self._log("cancel").error(msg)
raise InstallException(msg)
self._log("switch").info("switch() creating pilot for version %s", versionCombined)
pilotName=self._stateManager.getPilotName(versionCombined)
(pilot,_)=self._createPilot(pilotName, versionCombined)
self._log("switch").info("switch() forgetting things to remove on removal for version %s", versionCombined)
self._stateManager.setSubPkgsToRemoveOnRemoval(versionCombined, None)
self._stateManager.setRpmsToRemoveOnRemoval(versionCombined, None)
# Get params needed to determine which user-log message to emit, and emit it
activeVersion=self._stateManager.getActiveVersion()
switchType = self._stateManager.getSwitchType(versionCombined)
installDirection = self._stateManager.getInstallDirection(versionCombined)
self._log("switch").info("switch() got switchType='%s', installDirection='%s'", switchType, installDirection)
if (switchType == "OS Restart") and (installDirection == "upgrade"):
a.infra.process.logUserMessage(a.api.user_log.msg.install.OsUpgrade(activeVersion, versionCombined))
elif (switchType == "OS Restart") and (installDirection == "downgrade"):
a.infra.process.logUserMessage(a.api.user_log.msg.install.OsDowngrade(activeVersion, versionCombined))
elif (switchType == "Application Restart") and (installDirection == "upgrade"):
a.infra.process.logUserMessage(a.api.user_log.msg.install.AppUpgrade(activeVersion, versionCombined))
elif (switchType == "Application Restart") and (installDirection == "downgrade"):
a.infra.process.logUserMessage(a.api.user_log.msg.install.AppDowngrade(activeVersion, versionCombined))
self._log("switch").info("switch() calling pilot doSwitch(%s)", versionCombined)
# Returned value from pilot should signal caller something (Probably exit code).
ret = pilot.doSwitch(versionCombined)
# State is now locked. Let's check it, in case pilot was sloppy.
if not self._stateManager.isLocked():
msg="ERROR: switch() expecting state to be locked after pilot doSwitch()"
self._log("switch").error(msg)
raise InstallException(msg)
return ret
def verify (self):
"""
Verifies all installed RPMs.
Returns list of problems, empty list if OK
"""
self._log("verify-called").info("verify() called")
rpm=self._createRpmForActiveRfs()
problems = rpm.doVerifyAllInstalled()
self._log("verify-returning").info("verify() returning %s", problems)
# We skip problems about some files which normally change after being installed
filteredProblems = []
for problem in problems:
# /etc and /var files onbviously should not be checked
if " /etc/" in problem:
continue
if " /var/" in problem:
continue
# .ini and .cfg files in /opt/dell seem to be changing
if (" /opt/dell/" in problem) and (problem.endswith(".ini") or problem.endswith(".cfg")):
continue
# .pyc files in Oscar seem to be changing
if (" /opt/qb/rpms/oscar-" in problem) and problem.endswith(".pyc"):
continue
# We over-ride firmware files when upgrading the kernel
if " /lib/firmware/" in problem:
continue
# Handle a few specific files
if (" /opt/MegaRAID/MegaCli/MegaCli" in problem) or (" /root/.bash_profile" in problem):
continue
filteredProblems.append(problem)
return filteredProblems
def getRpmsDir (self):
return self._rpmsDir
def getAppDir (self):
return self._appDir
def getAppNextDir (self):
return self._appDir+".next"
def getAppPrevDir (self):
return self._appDir+".prev"
def getAppDirReal (self):
return self._appDirReal
def getBootDevPath (self):
return self._bootDevPath
def getGrubConf (self):
return self._grubConf
def getVitalDir (self):
return self._vitalDir
def getSubPkgManager (self):
return self._subPkgManager
def debugSetNoInstallMode (self):
self._log("debug-no-install-mode").notice("debugSetNoInstallMode() called, NO RPMS WILL BE INSTALLED")
self._debugNoInstallMode=True
def unit_test_ExtractPackageFile (self, fileName):
self._subPkgManager.add(r2pmFile=fileName)
def unit_test_InstallPilot (self, version_, build):
versionCombined=self.combineVersion(version_, build)
self._installPilot(versionCombined)
def unit_test_InstallRpm (self, name):
yum=self._createYumWithAllRepos()
self._log("debug-install-rpm").info("debugInstallRpm(): installing %s", name)
if not self._debugNoInstallMode:
yum.doInstall(name)
def unit_test_GetInstallTempDir (self):
"""Helper for unit tests"""
return self._installTempDir
def unit_test_GetStateFile (self):
"""Helper for unit tests"""
self._log("get-state-file").info("unit_test_GetStateFile(): returning %s", self._stateFile)
return self._stateFile
def _initPlatformBasic (self):
self._log("init-platform-basic").info("initializing the platform-basic")
self._platformBasic = a.infra.process.getPlatformBasicDataOrCrash()
def _initSdUtils (self):
self._log("init-sd-utils").info("initializing the sd-utils")
self._sdUtils = SdUtils(self._log)
def _initBootUtils (self):
self._log("init-boot-utils").info("initializing the boot-utils")
self._bootUtils = BootUtils.s_getFromOsefOrCrash(a.infra.process.getOsef())
def _installPilot(self, versionCombined):
yum=self._createYumWithAllRepos()
pilotPackageName = self.calcPilotName(versionCombined)
self._log("install-pilot").info("_installPilot(): pilotPackageName=%s", pilotPackageName)
if not self._debugNoInstallMode:
yum.doInstall(pilotPackageName)
def _createPilot (self, versionCombined, newVersionCombined):
"""
Returns (pilot, pilotServices)
versionCombined - name of pilot to create
newVersionCombined - name of version being installed ('v-b')
"""
self._log("create-pilot").info("_createPilot() called, versionCombined = %s", versionCombined)
pilotRpmDirName = self.calcPilotName(versionCombined)
pilotDirName = os.path.join(self._rpmsDir, pilotRpmDirName)
pilotFileName = os.path.join(pilotDirName, "pilot.py")
self._log("create-pilot").info("_createPilot(): pilotFileName = %s", pilotFileName)
if not os.path.exists(pilotFileName):
self._log("create-pilot").error("_createPilot() Could not find pilot file %s", pilotFileName)
raise InstallException("Could not find pilot file %s" % pilotFileName)
# Keep a copy of sys.path
oldSysPath=list(sys.path)
try:
sys.path.append(pilotDirName)
self._log("create-pilot").info("_createPilot(): about to import, sys.path=%s", sys.path)
import pilot
reload(pilot)
p=pilot.Pilot()
except:
self._log("create-pilot").error("_createPilot(): Got exception when importing and creating pilot")
raise
finally:
# Restore old sys.path
sys.path=oldSysPath
self._log("create-pilot").info("_createPilot(): Restored sys.path to %s", sys.path)
self._log("create-pilot").info("_createPilot(): creating pilot services, pilotDirName=%s", pilotDirName)
pilotServices=self._createPilotServices(versionCombined, pilotDirName, newVersionCombined)
p.setPilotServices(pilotServices)
self._log("create-pilot-returning").info("_createPilot() returning pilot")
return (p, pilotServices)
def calcPilotName (self, versionCombined):
pilotName="pilot-%s-%s" % (self._product, versionCombined)
return pilotName
def _isPackageSigned (self, fileName):
"""
Checks that an RPM file is a signed
"""
rpm=self._createRpmForActiveRfs()
return rpm.doValidateRpm(fileName)
def combineVersion (self, version_, build):
"""Combines version and build into a single string "v-b". version and buil cannot contain '-' chars. """
if version_.find("-") != -1:
self._log("combine-version-bad").error("combineVersion() got version='%s', invalid", version_)
raise InstallException("combineVersion(): bad version '%s'" % version_)
if build.find("-") != -1:
self._log("combine-version-bad").error("combineVersion() got build='%s', invalid", build)
raise InstallException("combineVersion(): bad build '%s'" % build)
return version_+"-"+build
def splitVersion (self, versionCombined, raiseIfInvalid=False):
"""
Splits versionCombined into a version and a build. versionCombined must contain exactly 1 '-' char.
Returns (version, build), or (None, None) if versionCombined is invalid
if raiseIfInvalid is True, then instead of returning (None, None), raises InstallException
"""
if versionCombined.count("-") != 1:
self._log("version-split-bad").error("splitVersion() got '%s', cannot split", versionCombined)
if raiseIfInvalid:
raise InstallException("splitVersion(): bad versionCombined '%s'" % versionCombined)
return (None, None)
(version_,build) = versionCombined.split("-")
return (version_,build)
def _createPilotServices (self, pilotVersion, pilotDir, newVersionCombined):
self._log("create-pilot-services").info("_createPilotServices(): Called, pilotVersion = %s, newVersionCombined = %s",
pilotVersion, newVersionCombined)
rpmActiveRfs=self._createRpmForActiveRfs()
yumActiveRfs=self._createYumWithAllRepos()
yumNewVersionOnly=self._createYumWithAllRepos(repoFilter=newVersionCombined)
self._log("create-pilot-services").info("_createPilotServices(): creating yum.conf for active RFS and yum.conf "\
"for new version %s", newVersionCombined)
yumConfActiveRfs=yumActiveRfs.createYumConfAt(self._yumConfActiveRfsDir)
yumConfNewVersionOnly=yumNewVersionOnly.createYumConfAt(self._yumConfNewVersionDir)
self._log("create-pilot-services").info("_createPilotServices(): got yum.conf at %s, yum.conf for new versiom only at %s",
yumConfActiveRfs, yumConfNewVersionOnly)
pilotServices=PilotServices(self._log, self._stateManager, self, pilotVersion, pilotDir, self._logDir, self._workDir, self._apiDir,
self._startupScriptsDir, rpmActiveRfs, yumActiveRfs, yumConfActiveRfs, yumConfNewVersionOnly,
self._prevRfsDir, self._sdUtils, self._bootUtils)
pilotServices.setPilotHints(self._pilotHints)
return pilotServices
def _createRpmForActiveRfs (self):
rpm=Rpm(self._log)
rpm.setRoot(self._systemRoot, useFakeChrootMode=self._useFakeChrootMode)
return rpm
def _createYumWithAllRepos (self, repoFilter=None):
"""
Returns a yum object configured to see all repos
if repoFilter != None, yum is configured to see only repos which contain the subs-string repoFilter:
"""
self._log("create-yum-with-all-repos").info("createYumWithAllRepos() called, repoFilter=%s", repoFilter)
if self._useFakeChrootMode:
yum=Yum(self._log, self._tempDir.replace(self._systemRoot, ""))
yum.setRoot(self._systemRoot, useFakeChrootMode=True)
else:
yum=Yum(self._log, os.path.join(self._tempDir, "yum"))
self._subPkgManager.addReposToYum(yum, repoFilter)
return yum
def _chopLeadingSlashIfFound (self, path_):
if path_.startswith('/'):
return path_[1:]
return path_
def _selectLatestVersion(self, version1, version2):
# Split versions to version name and build number
self._log("select-latest-version").info("_selectLatestVersion() called, version1=%s, version2=%s", version1, version2)
# Versions must look like this: "v-b"
vb1=version1.split('-')
if len(vb1) != 2:
raise InstallException("Invalid version number: %s is invalid" % version1)
vb2=version2.split('-')
if len(vb2) != 2:
raise InstallException("Invalid version number: %s is invalid" % version2)
ver1=vb1[0]
b1=vb1[1]
ver2=vb2[0]
b2=vb2[1]
# A version is official if it is made from numbers and dots only
onlyDigitsAndDots="^[\d\.]+$"
ver1IsOfficial=bool(re.match(onlyDigitsAndDots, version1))
ver2IsOfficial=bool(re.match(onlyDigitsAndDots, version2))
# Official versions beat private versions
if ver1IsOfficial != ver2IsOfficial:
if ver1IsOfficial:
return version1
return version2
# Now we have either two official versions, or two private versions.
if ver1 != ver2:
# Versions are not identical. Split version into elements
v1=ver1.split(".")
v2=ver2.split(".")
len1=len(v1)
len2=len(v2)
# Numerically compare the numbers, as long as both versions have them
minLen=min(len1, len2)
for i in range(minLen):
# Both version elements are the same ? no need to compare
if v1[i] == v2[i]:
continue
# Need to compare version elements. Convert to integers.
try:
ve1=int(v1[i])
ve2=int(v2[i])
except:
# Not an integer ? Cannot compare. Choose first version arbitrarily
return version1
# Different numbers ? We have a winner !
if ve1<ve2:
return version2
if ve1>ve2:
return version1
# Note that we may get here despite the 'if' that prevented comparing the same strings.
# For example, 04 and 4 are the same.
# Out of the loop with no decision ? Let's see if the version lengths want to talk
#
# Longer length wins: 2.1.0.0 is 'later' than 2.1.0
if len1<len2:
return version2
if len1>len2:
return version1
# If we are here, then versions are not identical, but all their components are the same
# AND they are the same length ? This could happen with versions like 2.04 vs 2.4.
# Choose arbitrarily
return version1
# Versions are the same. We need to decide based on build numbers
# Make sure build number is an integer
try:
b1n=int(b1)
b2n=int(b2)
except:
raise InstallException("Invalid build number: Both '%s' and '%s' should be numeric" % (b1, b2))
# Decide according to the build number
if b1n<b2n:
return version2
return version1
def _raiseIfVersionActiveOrReady(self, versionCombined):
# Do not allow adding an existing version
curState=self._stateManager.getVersionState(versionCombined)
if curState == StateManager.kStateActive:
self._log("add-existing-version").error("_raiseIfVersionActiveOrReady() Found existing active version '%s'", versionCombined)
raise InstallException("Version %s is the currently active version" % versionCombined)
if curState == StateManager.kStateReady:
self._log("add-existing-version").error("_raiseIfVersionActiveOrReady() Found existing ready version '%s'", versionCombined)
raise InstallException("Version %s is already ready" % versionCombined)
def _raiseIfAbsPath (self, pathToCheck, name):
if pathToCheck.startswith(os.path.sep):
raise ValueError("Parameter '%s' must not start with '/', value given is '%s'" % (name, pathToCheck))
def _raiseIfNotAbsPath (self, pathToCheck, name):
if not pathToCheck.startswith(os.path.sep):
raise ValueError("Parameter '%s' must start with '/', value given is '%s'" % (name, pathToCheck))
execute_parser = sub_parsers.add_parser(action_execute)
execute_parser.add_argument("task_id",
help="The task-id to be executed",
type=int)
parser.add_argument(
"-d",
"--dev",
help="Run API in development mode (when executing from a CLI)",
action="store_true",
)
args = parser.parse_args()
task_manager = None
state_manager = StateManager()
if args.dev:
logging.basicConfig(level=logging.DEBUG)
# noinspection PyBroadException
try:
if args.action == action_get:
if args.state_id:
state = state_manager.get_state(args.state_id)
else:
state = state_manager.get_latest_state(args.file_path)
context = json.loads(args.context) if args.context else None
task_manager = TaskManager()
tasks = task_manager.get_suggested_tasks(state, context)
class Game(object):
"""
A Game is a general abstraction for local search based games
Known children are MinConflictsGame and SimulatedAnnealingGame
These children should be also abstract
"""
def __init__(self):
self.state_manager = StateManager()
self.level = None
self.max_steps = 0
self.number_steps = 0
self.max_time = 120
self.date_begin = 0
self.verbose = False
def generate(self, level):
"""
Generates a new game. Level determines how hard the game will be
"""
levels = {
"easy": GameLevel.EASY,
"medium": GameLevel.MEDIUM,
"hard": GameLevel.HARD
}
assert(level in levels)
self.level = levels[level]
self.max_steps = 10000
self.state_manager.build_new_game(self.level)
self.state_manager.indexing_constraints()
self.state_manager.generate_game()
def run(self):
"""
Runs the current game.
It's just an execution of algorithms in the case of MinConflictsGame
and SimulatedAnnealingGame classes
"""
pass
EVAL_MAX = 1000.0
def evaluate(self, state):
"""
Evaluates a state (objective function)
Min value is 1 and max value is EVAL_MAX
"""
# The idea is to subtract the number of violated constraints (NVC) to a
# fix value (EVAL_MAX). But since the NVC can be greater than EVAL_MAX
# we must keep 0 <= NVC <= EVAL_MAX with a rule of three.
# There is still a problem: the number of total constraints is not
# perfect because a constraint can modify the count with a coeff (see
# Magic Square game with SumEqualsConstraint).
# So here we increase this number by a coeff of 1000. It works fine
# for this exercise but this solution is far from perfect.
total_constraints = len(self.state_manager.constraints) * 1000
nb_constraints = self.state_manager.count_constraint_violated(state)
_max = Game.EVAL_MAX
_eval = _max - (nb_constraints * _max) / total_constraints
return max(1, _eval)
def is_terminated(self):
"""
Returns True if the current state is an optimal one, False else
"""
return self.state_manager.is_optimal() or \
self.outofsteps() or self.outoftime()
def outofsteps(self):
return self.number_steps >= self.max_steps
def outoftime(self):
return self.date_begin > 0 and \
time() > (self.date_begin + self.max_time)
def show_state_manager(self):
if self.verbose:
print self.state_manager
"kytten Image Editor", kytten.Scrollable(kytten.VerticalLayout(content), width=750, height=500)
),
window=window,
anchor=kytten.ANCHOR_CENTER,
theme=gTheme,
on_escape=on_escape,
)
if __name__ == "__main__":
window = pyglet.window.Window(800, 600, caption="Kytten Theme Editor", resizable=True, vsync=True)
batch = pyglet.graphics.Batch()
bg_group = pyglet.graphics.OrderedGroup(0)
fg_group = pyglet.graphics.OrderedGroup(1)
# Update as often as possible (limited by vsync, if not disabled)
window.register_event_type("on_update")
def update(dt):
window.dispatch_event("on_update", dt)
pyglet.clock.schedule(update)
# StateManager keeps track of what we're doing
manager = StateManager(window)
# Start off by picking the theme directory
manager.push(ThemeDirSelectState())
pyglet.app.run()
from kitchen_sink_logger import KitchenSinkLogger
from handler.firehose import KinesisFirehoseHandler
from json_formatter.json import SimpleJsonFormatter
from backpack import Backpack
from state_manager import StateManager
import time
import logging
logger = KitchenSinkLogger()
logger.with_level(logging.INFO)
handler = KinesisFirehoseHandler(stream_name='STREAM_NAME')
handler.setFormatter(SimpleJsonFormatter())
logger.with_handler(handler)
logger.with_timer("TestTimer")
time.sleep(2)
logger.with_timer("SecondTimer")
logger2 = logger.clone()
logger.with_item('test', 'value')
j = logger.backpack.to_json()
sm = StateManager(TableName='BackpackState')
sm.upsert(logger.backpack)
logger.backpack = sm.get(logger.backpack.id)
logger.info('starting')
logger.log_metric("Getting data", "ReadsPerSecond", 100)
def __init__(self, point_array): self.point_array = point_array self.stack = list() self.state_manager = StateManager()
def setUp(self):
self.state = StateManager("database/unittest.pickle")
from database import Database
from state_manager import StateManager
from states import *
import shared
import sys
if __name__ == "__main__":
# allow database input
if len(sys.argv) == 1:
db_file = "db/database.db"
else:
db_file = sys.argv[1]
shared.db = Database(db_file)
# initialize state manager and insert all used states
sm = StateManager()
sm.add_state(LoginState(), "login")
sm.add_state(PostState(), "post")
sm.add_state(MenuState(), "menu")
sm.add_state(QuestionState(), "question")
sm.add_state(SearchState(), "search")
sm.add_state(AnswerState(), "answer")
sm.add_state(EditState(), "edit")
sm.start("login")
