def main():
if PROJECT_PATH is None:
raise Exception("No PROJECT_ROOT_PATH")
if GITHUB_ACCESS_TOKEN is None:
raise Exception("No GITHUB_ACCESS_TOKEN")
issues = utils.read_csv_ignore_headers(
ISSUES_TO_COLLECT_COMMENTS_FOR_CSV_FILE_PATH,
ISSUES_CSV_FILE_FIELD_NAMES)
print("Getting all comments for issues...")
total = len(issues)
count = 0
print('Skipping thru read ones...')
for issue in issues:
try:
count += 1
issue_id = issue['id']
repo_name = issue['repo_name']
comments_url = issue['comments_url']
output_file_name = "{}/comments@issue@{}@{}.json".format(
OUTPUT_FOLDER_PATH, issue_id, repo_name.replace('/', '@'))
if utils.file_or_read_file_already_exists(output_file_name):
continue
print("\t{}/{} repo={}".format(count, total, repo_name))
issue_comments = get_comments(comments_url)
utils.write_to_json_file(output_file_name, issue_comments)
except Exception as e:
print("ERROR: Failed getting comments for issue={}".format(
issue['id']))
print("Done")
def main():
if PROJECT_PATH is None:
raise Exception("No PROJECT_ROOT_PATH")
if GITHUB_ACCESS_TOKEN is None:
raise Exception("No GITHUB_ACCESS_TOKEN")
if LIBRARIES_IO_ACCESS_TOKEN is None:
raise Exception("No LIBRARIES_IO_ACCESS_TOKEN")
entity_to_collect_for = utils.read_csv_ignore_headers(
INPUT_CSV_FILE_PATH, INPUT_CSV_FILE_FIELD_NAMES)
total = len(entity_to_collect_for)
count = 0
for x in entity_to_collect_for:
try:
repo_half_name = x['repo_name']
count += 1
print("\t{}/{} repo={}".format(count, total, repo_half_name))
repo_full_name = get_repo_full_name_from_libraries_io(
repo_half_name)
output_file_name = "{}/{}.json".format(
OUTPUT_FOLDER_PATH, repo_full_name.replace('/', '@'))
if utils.file_or_read_file_already_exists(output_file_name):
print("File Exists - Continuing")
continue
dependents = get_dependents(repo_full_name)
utils.write_to_json_file(output_file_name, dependents)
except Exception as e:
print("ERROR: Failed for repo={}".format(x['repo_name']))
print("Done")
def download(source):
res_factory = utils.BlueprintResourceFactory()
resource_base_dir = res_factory._get_resources_dir(TEST_SERVICE_NAME)
resource_path = os.path.join(resource_base_dir, 'tmp-res-name')
utils.mkdir(resource_base_dir)
utils.write_to_json_file('port: 8080', resource_path)
return resource_path
def main():
if PROJECT_PATH is None:
raise Exception("No PROJECT_ROOT_PATH")
if GITHUB_ACCESS_TOKEN is None:
raise Exception("No GITHUB_ACCESS_TOKEN")
commits_to_collect = utils.read_csv_ignore_headers(
INPUT_CSV_FILE_PATH, INPUT_CSV_FILE_FIELD_NAMES)
print("Getting all comments for issues...")
total = len(commits_to_collect)
count = 0
for c in commits_to_collect:
try:
issue_id = c['issue_id']
repo_name = c['repo_name']
sha = c['commit_id']
count += 1
print("\t{}/{} sha={}".format(count, total, sha))
output_file_name = "{}/commit@{}@{}.json".format(
OUTPUT_FOLDER_PATH, sha, repo_name.replace('/', '@'))
if utils.file_or_read_file_already_exists(output_file_name):
print("File Exists - Continuing")
continue
commit_url = f'https://api.github.com/repos/{repo_name}/commits/{sha}'
commit_json = get_commit_json(commit_url)
utils.write_to_json_file(output_file_name, commit_json)
except Exception as e:
print("ERROR: Failed getting comments for issue={}".format(
c['issue_id']))
print("Done")
def download(source):
resource_base_dir = utils.resource_factory.get_resources_dir(
TEST_SERVICE_NAME)
resource_path = os.path.join(resource_base_dir, 'tmp-res-name')
utils.mkdir(resource_base_dir)
utils.write_to_json_file('port: 8080', resource_path)
return resource_path
def resample_train_test(data_path: str, lengths: [int, int]) -> [str, str]:
with open(data_path, mode='r', encoding='utf-8') as fp:
ppdb_pairs = json.load(fp)
ppdb_size = len(ppdb_pairs)
print(f"load {ppdb_size} ppdb pairs")
train_size, test_size = lengths[0], lengths[1]
train_pairs = list(np.random.choice(ppdb_pairs, train_size, replace=False))
test_pairs = list(np.random.choice(ppdb_pairs, test_size, replace=True))
P = Path(data_path)
train_path = os.path.join(P.parent, "ppdb_train")
test_path = os.path.join(P.parent, "ppdb_test")
write_to_json_file(train_path, train_pairs)
write_to_json_file(test_path, test_pairs)
return train_path, test_path
def main():
parser = argparse.ArgumentParser(description="Program to build docker images")
parser.add_argument("--buildspec", type=str)
parser.add_argument("--framework", type=str)
parser.add_argument("--device_types", type=str, default=constants.ALL)
parser.add_argument("--image_types", type=str, default=constants.ALL)
parser.add_argument("--py_versions", type=str, default=constants.ALL)
args = parser.parse_args()
device_types = args.device_types.split(",") if not args.device_types == constants.ALL else args.device_types
image_types = args.image_types.split(",") if not args.image_types == constants.ALL else args.image_types
py_versions = args.py_versions.split(",") if not args.py_versions == constants.ALL else args.py_versions
# create the empty json file for images
build_context = os.getenv("BUILD_CONTEXT")
ei_dedicated = os.getenv("EIA_DEDICATED") == "True"
neuron_dedicated = os.getenv("NEURON_DEDICATED") == "True"
# Get config value options
frameworks_to_skip = parse_dlc_developer_configs("build", "skip_frameworks")
ei_build_mode = parse_dlc_developer_configs("dev", "ei_mode")
neuron_build_mode = parse_dlc_developer_configs("dev", "neuron_mode")
# A general will work if in non-EI and non-NEURON mode and its framework not been disabled
general_builder_enabled = (
not ei_dedicated
and not neuron_dedicated
and not ei_build_mode
and not neuron_build_mode
and args.framework not in frameworks_to_skip
)
# An EI dedicated builder will work if in EI mode and its framework not been disabled
ei_builder_enabled = (
ei_dedicated and ei_build_mode and args.framework not in frameworks_to_skip
)
# A NEURON dedicated builder will work if in NEURON mode and its framework has not been disabled
neuron_builder_enabled = (
neuron_dedicated
and neuron_build_mode
and args.framework not in frameworks_to_skip
)
utils.write_to_json_file(constants.TEST_TYPE_IMAGES_PATH, {})
# A builder will always work if it is in non-PR context
if general_builder_enabled or ei_builder_enabled or neuron_builder_enabled or build_context != "PR":
utils.build_setup(
args.framework, device_types=device_types, image_types=image_types, py_versions=py_versions,
)
image_builder(args.buildspec)
def parse_blog(path):
if is_file_exist(path + '/linked_papers.json'):
return
if not is_file_exist(path + '/urls.json'):
return
with open(path + '/urls.json') as f:
urls = json.load(f)
output = []
dir_path = path + '/linked_papers.json'
for url in urls:
output.append(crawl_paper_links(url))
write_to_json_file(dir_path, output)
def main():
gk_issues = utils.read_csv_ignore_headers(GREENKEEPER_ISSUES_FILE_PATH, GREENKEEPER_ISSUES_FIELD_NAMES)
library_names = list()
for issue in gk_issues:
if issue['issue_dependency_name'] not in library_names:
library_names.append(issue['issue_dependency_name'])
total = len(library_names)
count = 0
for lib_name in library_names:
count += 1
try:
print("\t{}/{} pr_url={}".format(count, total, lib_name))
if not lib_name:
continue
lib_name_for_url = lib_name.replace('/', '%2F')
url = 'https://libraries.io/api/NPM/{}?api_key={}'.format(lib_name_for_url, LIBRARIES_IO_ACCESS_TOKEN)
lib_info = utils.send_request(url, None, ignore_token=True, sleep_time=1.2)
lib_name_for_file = lib_name.replace('/', '%2F')
file_name = "{}/{}.json".format(OUTPUT_FOLDER_PATH, lib_name_for_file)
utils.write_to_json_file(file_name, lib_info)
except Exception as e:
print("Error on {}".format(lib_name))
def main():
if PROJECT_PATH is None:
raise Exception("No PROJECT_ROOT_PATH")
if GITHUB_ACCESS_TOKEN is None:
raise Exception("No GITHUB_ACCESS_TOKEN")
repos = utils.read_csv_ignore_headers(REPOS_FILE_PATH, REPOs_FILED_NAMES)
print("Getting all Issues for repos...")
total = len(repos)
count = 0
for repo in repos:
try:
repo_name = repo['dependent']
count += 1
print("\t{}/{} repo={}".format(count, total, repo_name))
output_file_name = "{}/issues@{}.json".format(OUTPUT_FOLDER_PATH, repo_name.replace('/', '@'))
if utils.file_or_read_file_already_exists(output_file_name):
print("File Exists - Continuing -- repo={}".format(repo_name))
continue
repo_issues = get_issues(repo_name)
utils.write_to_json_file(output_file_name, repo_issues)
except Exception as e:
print("ERROR: Failed getting issues for repo={}".format(repo['repo']))
print("Done")
task_name: create_task(
task_name, args, datasets[task_name]["nclasses"], emb_layer
)
for task_name in args.task
}
model = EmmentalModel(name="TC_task")
if Meta.config["model_config"]["model_path"]:
model.load(Meta.config["model_config"]["model_path"])
else:
for task_name, task in tasks.items():
model.add_task(task)
emmental_learner = EmmentalLearner()
emmental_learner.learn(model, dataloaders)
scores = model.score(dataloaders)
logger.info(f"Metrics: {scores}")
write_to_json_file(f"{Meta.log_path}/metrics.txt", scores)
if args.checkpointing:
logger.info(
f"Best metrics: "
f"{emmental_learner.logging_manager.checkpointer.best_metric_dict}"
)
write_to_file(
f"{Meta.log_path}/best_metrics.txt",
emmental_learner.logging_manager.checkpointer.best_metric_dict,
)
config.read('config.ini')
researchers_to_bows = []
papers = []
for author in os.listdir(papers_DIR):
papers_sub_dir = os.path.join(papers_DIR, author, "")
if os.path.isdir(papers_sub_dir) and os.listdir(papers_sub_dir):
bow_sub_DIR = os.path.join(bow_DIR, author, "")
all_texts = prepare_bow_content(papers_sub_dir, bow_sub_DIR)
# Persist to intermediate file under subfolder
outfile_location = os.path.join(bow_sub_DIR, "_.json")
content = {author: ' '.join(all_texts)}
utils.write_to_json_file(file_location=outfile_location,
data=content)
# Add to master list
researchers_to_bows.append({
'researcher': author,
'bow_content': ' '.join(all_texts)
})
papers.extend(all_texts)
RESET_BOW = config['PREPROCESSING'].getboolean('RESET_BOW')
try:
if RESET_BOW:
data = []
else:
data = utils.read_json_file(master_output_file)
def save_abilities(file_name="abilities.json"):
new_abilities_dict = {
'values': ability_values,
'variable_values': variable_ability_values.keys()
}
write_to_json_file(new_abilities_dict, 'abilities.json')
def train(dataset,
classifier,
encoder,
featurizer,
path = "./model_checkpoint",
epochs=100,
lr=0.01,
batch_size=1024,
):
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(f"Using {device} for training")
# Construct PyTorch DataLoader
train_test_split = [int(len(dataset) * 0.8), len(dataset) - int(len(dataset) * 0.8)]
train_set, val_set = torch.utils.data.random_split(dataset, train_test_split)
train_dataloader = DataLoader(train_set,
sampler=RandomSampler(train_set),
batch_size=batch_size,
num_workers = 4)
eval_dataloader = DataLoader(val_set,
sampler=RandomSampler(val_set),
batch_size=batch_size,
num_workers = 4)
loss_function = nn.NLLLoss()
optimizer = optim.Adam(model.parameters(), lr=lr)
total_step = len(train_dataloader)
classifier.to(device)
encoder.to(device)
classifier.train()
# Setup train loss, eval loss tracking every epoch
train_loss = []
file_name_head = f"{type(encoder).__name__}-{featurizer.__name__}-{type(classifier).__name__}"
# eval_loss = []
for epoch in trange(epochs, desc='Epochs'):
tr_loss = 0.
nb_tr_examples, nb_tr_steps = 0, 0
print(f"Start Training Epoch {epoch}")
for step, (X, X_mask, labels) in enumerate(tqdm(train_dataloader, desc="Iteration")):
X = X.to(device)
X_mask = X_mask.to(device)
# BERT Encoder
with torch.no_grad():
final_hidden_states, cls_output = encoder(X, attention_mask = X_mask)
inputs = featurizer(final_hidden_states) # cls_token
inputs = inputs.to(device)
labels = labels.to(device)
classifier.zero_grad()
outputs = classifier(inputs)
loss = loss_function(outputs, labels)
loss.backward()
optimizer.step()
tr_loss += loss.item()
nb_tr_examples += inputs.size(0)
nb_tr_steps += 1
train_loss.append((loss.item(), nb_tr_steps))
logger.info('Total loss at epoch %d: %.5f' % (epoch+1, tr_loss))
logger.info('Avrg loss at epoch %d: %.5f' % (epoch+1, tr_loss / nb_tr_examples))
# Evaluate the model f-1
start = time.time()
print("Testing {}".format(epoch))
f1_test, acc_test = test(eval_dataloader, classifier, featurizer, encoder, device)
f1_train, acc_train = test(train_dataloader, classifier, featurizer, encoder, device)
logger.info('[F1, Accuracy] score at epoch %d | train: (%.5f, %.5f) | test: (%.5f, %.5f)' \
% (epoch+1, f1_test, f1_train, acc_test, acc_train))
end = time.time()
if epoch == 0: print(f"Test cost {end-start}")
if epoch % 1 == 0:
# Save Model Checkpoint
create_directory(path)
torch.save(model.state_dict(), os.path.join(
path, f"{file_name_head}-{epoch+1}"))
# Write train loss per step
write_to_json_file(os.path.join(path,
f"{file_name_head}_train_loss_per_epoch"), train_loss)
def main():
parser = argparse.ArgumentParser(
description="Program to build docker images")
parser.add_argument("--buildspec", type=str)
parser.add_argument("--framework", type=str)
parser.add_argument("--device_types", type=str, default=constants.ALL)
parser.add_argument("--image_types", type=str, default=constants.ALL)
parser.add_argument("--py_versions", type=str, default=constants.ALL)
args = parser.parse_args()
device_types = args.device_types.split(
",") if not args.device_types == constants.ALL else args.device_types
image_types = args.image_types.split(
",") if not args.image_types == constants.ALL else args.image_types
py_versions = args.py_versions.split(
",") if not args.py_versions == constants.ALL else args.py_versions
# create the empty json file for images
build_context = os.getenv("BUILD_CONTEXT")
ei_dedicated = os.getenv("EIA_DEDICATED", "false").lower() == "true"
neuron_dedicated = os.getenv("NEURON_DEDICATED", "false").lower() == "true"
graviton_dedicated = os.getenv("GRAVITON_DEDICATED",
"false").lower() == "true"
habana_dedicated = os.getenv("HABANA_DEDICATED", "false").lower() == "true"
# Get config value options
frameworks_to_skip = parse_dlc_developer_configs("build",
"skip_frameworks")
ei_build_mode = parse_dlc_developer_configs("dev", "ei_mode")
neuron_build_mode = parse_dlc_developer_configs("dev", "neuron_mode")
graviton_build_mode = parse_dlc_developer_configs("dev", "graviton_mode")
habana_build_mode = parse_dlc_developer_configs("dev", "habana_mode")
# Write empty dict to JSON file, so subsequent buildspec steps do not fail in case we skip this build
utils.write_to_json_file(constants.TEST_TYPE_IMAGES_PATH, {})
# Skip tensorflow-1 PR jobs, as there are no longer patch releases being added for TF1
# Purposefully not including this in developer config to make this difficult to enable
# TODO: Remove when we remove these jobs completely
build_arn = utils.get_codebuild_build_arn()
if build_context == "PR":
tf_1_build_regex = re.compile(r"dlc-pr-tensorflow-1:")
if tf_1_build_regex.search(build_arn):
return
# A general will work if in non-EI, non-NEURON and non-GRAVITON mode and its framework not been disabled
general_builder_enabled = (not ei_dedicated and not neuron_dedicated
and not graviton_dedicated
and not habana_dedicated and not ei_build_mode
and not neuron_build_mode
and not graviton_build_mode
and not habana_build_mode
and args.framework not in frameworks_to_skip)
# An EI dedicated builder will work if in EI mode and its framework not been disabled
ei_builder_enabled = ei_dedicated and ei_build_mode and args.framework not in frameworks_to_skip
# A NEURON dedicated builder will work if in NEURON mode and its framework has not been disabled
neuron_builder_enabled = neuron_dedicated and neuron_build_mode and args.framework not in frameworks_to_skip
# A GRAVITON dedicated builder will work if in GRAVITON mode and its framework has not been disabled
graviton_builder_enabled = graviton_dedicated and graviton_build_mode and args.framework not in frameworks_to_skip
# A HABANA dedicated builder will work if in HABANA mode and its framework has not been disabled
habana_builder_enabled = habana_dedicated and habana_build_mode and args.framework not in frameworks_to_skip
# A builder will always work if it is in non-PR context
if (general_builder_enabled or ei_builder_enabled or neuron_builder_enabled
or graviton_builder_enabled or habana_builder_enabled
or build_context != "PR"):
utils.build_setup(
args.framework,
device_types=device_types,
image_types=image_types,
py_versions=py_versions,
)
image_builder(args.buildspec)
def save_abilities(file_name="abilities.json"):
new_abilities_dict = {
'values': ability_values,
'variable_values': variable_ability_values.keys()
}
write_to_json_file(new_abilities_dict, 'abilities.json')
device_types = (args.device_types.split(",")
if not args.device_types == constants.ALL else
args.device_types)
image_types = (args.image_types.split(",") if
not args.image_types == constants.ALL else args.image_types)
py_versions = (args.py_versions.split(",") if
not args.py_versions == constants.ALL else args.py_versions)
# create the empty json file for images
build_context = os.getenv("BUILD_CONTEXT")
ei_dedicated = os.getenv("EIA_DEDICATED") == "True"
# A general/non-EI builder will work if in non-EI mode and its framework not been disabled
non_ei_builder_enabled = not ei_dedicated and \
not build_config.ENABLE_EI_MODE and \
args.framework not in build_config.DISABLE_FRAMEWORK_TESTS
# An EI dedicated builder will work if in EI mode and its framework not been disabled
ei_builder_enabled = ei_dedicated and \
build_config.ENABLE_EI_MODE and \
args.framework not in build_config.DISABLE_FRAMEWORK_TESTS
utils.write_to_json_file(constants.TEST_TYPE_IMAGES_PATH, {})
# A builder will always work if it is in non-PR context
if non_ei_builder_enabled or ei_builder_enabled or build_context != "PR":
utils.build_setup(
args.framework,
device_types=device_types,
image_types=image_types,
py_versions=py_versions,
)
image_builder(args.buildspec)
stream_of_docs = [
paper for x in researchers_to_bow for paper in x["bow_content"]
] # Flatten a list of list
tokenized_docs, bigram_model = bow_phrases.text_preprocess_with_phrases(
stream_of_docs)
pickle.dump(bigram_model, open("bigram_model.p", "wb"))
papers = [' '.join(x) for x in tokenized_docs]
unflattened_docs = split_list_by_indices(tokenized_docs, num_publications)
authors = [x['researcher'] for x in researchers_to_bow]
researchers_to_bow_with_phrases = []
for author, text_tokens in zip(authors, unflattened_docs):
aggregated_texts = ' '.join(
[word for paper in text_tokens for word in paper])
researchers_to_bow_with_phrases.append({
'researcher': author,
'bow_content': aggregated_texts
})
utils.write_to_file(
os.path.join(data_directory, author, "_with_phrases.all"),
aggregated_texts)
utils.write_to_json_file(file_location="data_with_phrases.json",
data=researchers_to_bow_with_phrases)
utils.write_to_json_file(file_location="papers_with_phrases.json",
data=papers)
def serialized_train(dataset,
classifier,
encoder,
featurizer,
path = "./model_checkpoint",
epochs=100,
lr=0.01,
batch_size=8,
):
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(f"Using {device} for training on serialized data")
# Construct PyTorch DataLoader
train_test_split = [int(len(dataset) * 0.8), len(dataset) - int(len(dataset) * 0.8)]
train_set, val_set = torch.utils.data.random_split(dataset, train_test_split)
train_dataloader = DataLoader(train_set,
sampler=RandomSampler(train_set),
batch_size=batch_size,
num_workers = 1)
eval_dataloader = DataLoader(val_set,
sampler=RandomSampler(val_set),
batch_size=batch_size,
num_workers = 1)
loss_function = nn.NLLLoss()
optimizer = optim.Adam(model.parameters(), lr=lr)
total_step = len(train_dataloader)
classifier.to(device)
classifier.train()
# Setup train loss, eval loss tracking every epoch
train_loss = []
file_name_head = f"{encoder}-{featurizer}-{type(classifier).__name__}"
# eval_loss = []
train_f1_monitor, test_f1_monitor = [], []
for epoch in trange(epochs, desc='Epochs'):
tr_loss = 0.
nb_tr_examples, nb_tr_steps = 0, 0
print(f"Start Training Epoch {epoch}")
for step, (inputs, labels) in enumerate(tqdm(train_dataloader, desc="Iteration")):
inputs = inputs.to(device)
labels = labels.to(device)
classifier.zero_grad()
outputs = classifier(inputs)
loss = loss_function(outputs, labels)
loss.backward()
optimizer.step()
tr_loss += loss.item()
nb_tr_examples += inputs.size(0)
nb_tr_steps += 1
train_loss.append((loss.item(), nb_tr_steps))
logger.info('Total loss at epoch %d: %.5f' % (epoch+1, tr_loss))
logger.info('Avrg loss at epoch %d: %.5f' % (epoch+1, tr_loss / nb_tr_examples))
# Evaluate the model f-1
start = time.time()
f1_test, acc_test = test(eval_dataloader, classifier, device)
f1_train, acc_train = test(train_dataloader, classifier, device)
train_f1_monitor.append(f1_train)
test_f1_monitor.append(f1_test)
logger.info('[F1, Accuracy] score at epoch %d | train: (%.5f, %.5f) | test: (%.5f, %.5f)' \
% (epoch+1, f1_test, f1_train, acc_test, acc_train))
end = time.time()
if epoch == 0: print(f"Test cost {end-start}")
if epoch % 1 == 0:
# Save Model Checkpoint
create_directory(path)
torch.save(model.state_dict(), os.path.join(
path, f"{file_name_head}-{epoch+1}"))
if (epoch+1) % 10 == 0: adjust_learning_rate(optimizer, lr*0.1) # every 10 epochs reduce lr by factor of 10
early_stop = early_stopping_monitor({'val_f1': test_f1_monitor, 'train_f1': train_f1_monitor})
if early_stop:
break
# Write train loss per step
write_to_json_file(os.path.join(path,
f"{file_name_head}_train_loss_per_epoch"), train_loss)
from card import Card, save_abilities
from utils import write_to_json_file, read_from_json_file
if __name__ == '__main__':
spec_list = None
spec_list = read_from_json_file('cards.json')
cards = [Card(**card_spec) for card_spec in spec_list]
for card in cards:
print str(card)
save_abilities()
card_list = [card.serialize() for card in cards]
write_to_json_file(card_list, 'cards.json')
finale_population_size=number_of_birds,
percentage_for_parenting=percentage_for_parenting)
pipe_surface = pygame.image.load("assets/pipe-green.png").convert()
pipe_surface = pygame.transform.scale2x(pipe_surface)
pipe_list = []
pipe_height = [i for i in range(400, 850, 50)]
while True:
if stop_evolution_flag:
for index in range(len(bird_chromosomes)):
if active_birds[index]:
bird_chromosomes[index].complete_training(score)
bird_chromosomes.sort(reverse=True)
write_to_json_file([bird_chromosomes[i].to_dict() for i in range(number_of_birds)])
break
for event in pygame.event.get():
if event.type == pygame.QUIT:
pygame.quit()
sys.exit()
if event.type == SPAWN_PIPE:
pipe_list.extend(create_pipe())
if len(pipe_list) > 8:
del pipe_list[0]
del pipe_list[0]
if event.type in FLY:
ind = event.type - FLY[0]
bird_movement[ind] = 0
bird_movement[ind] -= up_velocity
def eval_genomes(genomes, config):
global current_generation, high_score, game_active, bird_movement, dt, floor_x, get_ticks_last_frame, pipe_list, score
current_generation += 1
# start by creating lists holding the genome itself, the
# neural network associated with the genome and the
# bird object that uses that network to play
nets = []
bird_cromoshomes = []
ge = []
for genome_id, genome in genomes:
genome.fitness = 0 # start with fitness level of 0
net = neat.nn.FeedForwardNetwork.create(genome, config)
nets.append(net)
bird_cromoshomes.append(Chromosome(NeuralBird()))
ge.append(genome)
print("CURRENT GENERATION: {}".format(current_generation))
bird_rects = [
bird_surface.get_rect(center=(100, 512))
for _ in range(number_of_birds)
]
active_birds = [True] * number_of_birds
pipe_list = []
score = 0
# pygame.time.set_timer(FLY, 850)
# bird_cromoshomes = [Chromosome(NeuralBird()) for _ in range(number_of_birds)]
# bird_cromoshomes = Chromosome.read_from_file("training.json", population_size=number_of_birds)
# bird_cromoshomes = one_generation_evolution(bird_cromoshomes,
# crossover_probability=crossover_probability,
# mutation_probability=mutation_probability,
# finale_population_size=number_of_birds,
# percentage_for_parenting=percentage_for_parenting)
while True:
for event in pygame.event.get():
if event.type == pygame.QUIT:
pygame.quit()
sys.exit()
if event.type == SPAWNPIPE:
pipe_list.extend(create_pipe())
if len(pipe_list) > 8:
del pipe_list[0]
del pipe_list[0]
if event.type in FLY:
ind = event.type - FLY[0]
bird_movement[ind] = 0
bird_movement[ind] -= up_velocity * dt
screen.blit(background_sf, (0, 0))
if game_active:
# Bird
bird_movement = list(map(lambda x: x + gravity * dt,
bird_movement))
rotated_bird = [
rotate_bird(bird_surface, i) for i in range(number_of_birds)
]
for index in range(number_of_birds):
if active_birds[index]:
bird_rects[index].centery += bird_movement[index]
screen.blit(rotated_bird[index], bird_rects[index])
# te uiti la coliziuni
if check_collision(pipe_list, bird_rects[index]):
active_birds[index] = False
ge[index].fitness -= 1
# nets.pop(index)
# ge.pop(index)
# bird_cromoshomes[index].complete_training(score)
distance = 100
pipe_up = 100
pipe_down = 100
# dai update la neuronii de input
for i in range(0, len(pipe_list), 2):
distance = pipe_list[i].bottomleft[0] - bird_rects[
index].bottomright[0]
pipe_down = abs(pipe_list[i].topright[1] -
bird_rects[index].bottomleft[1])
pipe_up = abs(pipe_list[i + 1].bottomright[1] -
bird_rects[index].topleft[1])
if distance > 0:
break
output = nets[index].activate(
(distance, pipe_down, pipe_up))
if output[
0] > 0.5: # we use a tanh activation function so result will be between -1 and 1. if over 0.5 jump
pygame.event.post(fly_events[index])
# dai compute la noua valoare. Daca e True generezi event
# print("Bird {}: {}".format(index, bird_cromoshomes[index].bird.compute_output()))
if bird_cromoshomes[index].bird.compute_output():
pygame.event.post(fly_events[index])
# cand mor toti faci gameActive = false
game_active = any(active_birds)
# Pipes
pipe_list = move_pipes(pipe_list)
draw_pipes(pipe_list)
score += 0.01
score_display(score, high_score)
else:
# alg genetic
# salvare parametrii cel mai bun fitness
# resetare populatie
game_active = True
bird_cromoshomes.sort(reverse=True)
write_to_json_file([
bird_cromoshomes[i].to_dict() for i in range(number_of_birds)
])
if score > high_score:
high_score = score
print("game over")
score = 0
break
pygame.quit()
sys.exit()
# Floor
floor_x -= 1
draw_floor()
if floor_x <= -576:
floor_x = 0
pygame.display.update()
t = pygame.time.get_ticks()
dt = (t - get_ticks_last_frame) / 1000
get_ticks_last_frame = t
if score > 100:
pickle.dump(nets[0], open("best.pickle", "wb"))
break
from card import Card, save_abilities
from utils import write_to_json_file, read_from_json_file
if __name__ == '__main__':
spec_list = None
spec_list = read_from_json_file('cards.json')
cards = [Card(**card_spec) for card_spec in spec_list]
for card in cards:
print str(card)
save_abilities()
card_list = [card.serialize() for card in cards]
write_to_json_file(card_list, 'cards.json')
for paper_filename in glob.glob(author_dir + '*.pdf'):
paper_title = os.path.basename(paper_filename).replace(".pdf", "")
if not is_paper_seen(papers_collection, paper_title):
papers_collection.append(paper_title)
pdfPath = paper_filename
print "pdfPath = " + pdfPath
fileNameOut = dataset_DIR + "p" + str(index) + ".txt"
print 'converting (%s %s -> %s)' % ("pdftotext", pdfPath,
fileNameOut)
try:
subprocess.check_call(["pdftotext", pdfPath, fileNameOut])
papers_id.append(index)
index += 1
# os.system(""" %s "%s" "%s" """ % ("pdftotext", pdfPath, fileNameOut))
except subprocess.CalledProcessError:
logging.error("Syntax Error: Couldn't parse pdf file. PDF: %s",
paper_title)
# papers_id.remove(index)
# index -= 1
else:
paper_id = papers_collection.index(paper_title)
papers_id.append(paper_id)
print "***** Paper " + paper_title + " already exists. Paperid=" + str(
paper_id)
author_papers_dict[author] = papers_id
utils.write_to_json_file("dataset/authors.json", author_papers_dict)
