def extract_game_data(gcs_path, root_node):
props = root_node.properties
komi = float(sgf_wrapper.sgf_prop(props.get('KM')))
result = sgf_wrapper.sgf_prop(props.get('RE', ''))
board_size = int(sgf_wrapper.sgf_prop(props.get('SZ')))
value = utils.parse_game_result(result)
was_resign = '+R' in result
filename = os.path.basename(gcs_path)
filename_no_ext, _ = os.path.splitext(filename)
# BigQuery's TIMESTAMP() takes in unix millis.
completion_millis = 1000 * int(filename_no_ext.split('-')[0])
worker_id = filename_no_ext.split('-')[-1]
model_num = shipname.detect_model_num(props.get('PW')[0])
sgf_url = gcs_path
first_comment_node_lines = root_node.next.properties['C'][0].split('\n')
# in-place edit to comment node so that first move's comment looks
# the same as all the other moves.
root_node.next.properties['C'][0] = '\n'.join(first_comment_node_lines[1:])
resign_threshold = float(first_comment_node_lines[0].split()[-1])
return {
'worker_id': worker_id,
'completed_time': completion_millis
'board_size': board_size,
'model_num': model_num,
'result_str': result,
'value': value,
'was_resign': was_resign,
'sgf_url': sgf_url,
'resign_threshold': resign_threshold,
}
def get_model_paths(model_dir):
'''Returns all model paths in the model_dir.'''
all_models = gfile.Glob(os.path.join(model_dir, '*.meta'))
model_filenames = [os.path.basename(m) for m in all_models]
model_numbers_names = [
(shipname.detect_model_num(m), shipname.detect_model_name(m))
for m in model_filenames]
model_names = sorted(model_numbers_names)
return [os.path.join(model_dir, name[1]) for name in model_names]
def get_latest_model():
"""Finds the latest model, returning its model number and name
Returns: (17, 000017-modelname)
"""
all_models = gfile.Glob(os.path.join(MODELS_DIR, '*.meta'))
model_filenames = [os.path.basename(m) for m in all_models]
model_numbers_names = [
(shipname.detect_model_num(m), shipname.detect_model_name(m))
for m in model_filenames]
latest_model = sorted(model_numbers_names, reverse=True)[0]
return latest_model
def get_models():
"""Finds all models, returning a list of model number and names
sorted increasing.
Returns: [(13, 000013-modelname), (17, 000017-modelname), ...etc]
"""
# all_models = glob.glob(os.path.join(MODELS_DIR, '*.meta'))
all_models = glob.glob(os.path.join(MODELS_DIR, '*'))
model_filenames = [os.path.basename(m) for m in all_models]
model_numbers_names = sorted([(shipname.detect_model_num(m),
shipname.detect_model_name(m))
for m in model_filenames])
return model_numbers_names
def get_models():
"""Finds all models, returning a list of model number and names
sorted increasing.
Returns: [(13, 000013-modelname), (17, 000017-modelname), ...etc]
"""
all_models = gfile.Glob(os.path.join(MODELS_DIR, '*.meta'))
model_filenames = [os.path.basename(m) for m in all_models]
model_numbers_names = sorted([
(shipname.detect_model_num(m), shipname.detect_model_name(m))
for m in model_filenames])
return model_numbers_names
def get_latest_model():
"""Finds the latest model, returning its model number and name
Returns: (17, 000017-modelname)
"""
all_models = gfile.Glob(os.path.join(MODELS_DIR, '*.meta'))
model_filenames = [os.path.basename(m) for m in all_models]
model_numbers_names = [
(shipname.detect_model_num(m), shipname.detect_model_name(m))
for m in model_filenames]
latest_model = sorted(model_numbers_names, reverse=True)[0]
return latest_model
def test_detect_num(self):
string = '000017-model.index'
detected_name = shipname.detect_model_num(string)
self.assertEqual(17, detected_name)
def test_detect_num(self):
string = '000017-model.index'
detected_name = shipname.detect_model_num(string)
self.assertEqual(detected_name, 17)
def get_latest_pb():
pb = os.path.basename(get_pbs()[-1])
return shipname.detect_model_num(pb), pb
