def read_config(self):
config = configparser.ConfigParser(allow_no_value=True)
config.read(CONFIG_FILE)
processes = config["processes"]
num_proc = int(processes["parsing"])
self.values[
"parser_processes"] = num_proc if num_proc > 0 else multiprocessing.cpu_count(
)
curseforge = config["curseforge"]
self.values["game_version"] = curseforge["game version"]
db = config["db"]
self.values["db_location"] = relative_path(db["location"])
self.values["cache_timeout"] = int(db["timeout"])
log = config["logging"]
self.values["log_location"] = relative_path(log["location"])
self.values["log_use_stdout"] = log["print to stdout"]
self.values["log_level"] = log["log level"]
rate_limiting = config["rate_limiting"]
max_search = int(rate_limiting["max search"])
self.values[
"max_search"] = max_search if max_search > 0 else sys.maxsize
def test_mod_parse_specific(self):
jei_record = "(238222, '/minecraft/mc-mods/jei', 'https://github.com/mezz/JustEnoughItems', 'https://github.com/mezz/JustEnoughItems/issues?q=is%3Aissue', None, 'MIT License')"
with open(relative_path("test/resources/jei_page_20190715.html"),
"rb") as f:
modrecord = parse.fetch_and_scrape("/minecraft/mc-mods/jei")
self.assertEqual(jei_record, modrecord.test_form().__repr__())
def test_number_pages_specific(self):
with open(
relative_path(
"test/resources/mod_listing_page_1_20190715.html"),
"rb") as f:
maxpage = parse.get_number_pages(f.read())
self.assertEqual(maxpage, 279)
def test_project_links_specific(self):
links = {
'/minecraft/mc-mods/mouse-tweaks',
'/minecraft/mc-mods/crafttweaker',
'/minecraft/mc-mods/iron-chests',
'/minecraft/mc-mods/inventory-tweaks',
'/minecraft/mc-mods/resource-loader', '/minecraft/mc-mods/chisel',
'/minecraft/mc-mods/codechicken-lib-1-8',
'/minecraft/mc-mods/applied-energistics-2',
'/minecraft/mc-mods/shadowfacts-forgelin',
'/minecraft/mc-mods/baubles', '/minecraft/mc-mods/ctm',
'/minecraft/mc-mods/appleskin', '/minecraft/mc-mods/jei',
'/minecraft/mc-mods/modtweaker',
'/minecraft/mc-mods/thermal-foundation',
'/minecraft/mc-mods/tinkers-construct',
'/minecraft/mc-mods/mantle', '/minecraft/mc-mods/cofh-core',
'/minecraft/mc-mods/journeymap',
'/minecraft/mc-mods/custom-main-menu'
}
with open(
relative_path(
"test/resources/mod_listing_page_1_20190715.html"),
"rb") as f:
self.assertEqual(links, parse.get_project_links(f.read()))
import cv2
import numpy as np
from utils import relative_path
HAARCASCADE_CONFIG = relative_path(
'../resources/haarcascade_frontalface_default.xml')
face_detector = cv2.CascadeClassifier(HAARCASCADE_CONFIG)
def detect_faces(frame):
'''
Recognize faces in frame and check if there is face looking at screen
Args:
frame (numpy): Image frame from webcam (or whatever)
Returns:
boolean indicating whether face is present
'''
return face_detector.detectMultiScale(frame)
"geometry": {
"type": "Point",
"coordinates": [float(place['lon']),
float(place['lat'])]
},
"properties": {
"fips": place['fips']
}
}
if __name__ == "__main__":
if len(sys.argv) > 1:
fn = sys.argv[1]
else:
fn = relative_path("../raw/Gaz_places_national.txt")
try:
gazetteer = load_csv_columns(fn,
GAZETTEER_COLUMNS,
delimiter='\t',
quoting=csv.QUOTE_NONE)
except IOError:
print "unable to load", fn
sys.exit(-1)
for (abbr, data) in split_dict_by(gazetteer, 'state').items():
state_name = STATE_ABBR[abbr].replace(' ', '_')
geojson_collection = {"type": "FeatureCollection", "features": list()}
for place in data:
model.fit_generator(
train_seq,
epochs=args.epochs,
validation_data=test_seq,
use_multiprocessing=True,
workers=8,
callbacks=callbacks)
if __name__ == '__main__':
# Command line arguments parsing
parser = argparse.ArgumentParser(description='Train a colorization deep learning model')
parser.add_argument('--train-dataset',
type=str,
default=relative_path('../data/imagenet-sample/train/'),
help='Train dataset base path. Folder should contain subfolder for each class.')
parser.add_argument('--test-dataset',
type=str,
default=relative_path('../data/imagenet-sample/test/'),
help='Test dataset base path. Folder should contain images directly.')
parser.add_argument('--validation-path',
type=str,
default=relative_path('../data/imagenet-sample/val/'),
help='Path to directory with validation images that will be uploaded to comet after each epoch')
parser.add_argument('--batch-size',
type=int,
default=128,
help='Batch size used during training')
parser.add_argument('--img-w',
type=int,
from utils import relative_path
TIMESTEPS = 100 # Number of timesteps feeded to model
BATCH_SIZE = 128
TEXT_CORPUS_PATH = relative_path('../data/javascript_codes.js')
MODELS_DIR = relative_path('../model/') # Model save dir
FINAL_MODEL_PATH = relative_path('../model/jscode-final-model.h5') # Last learned model - arch+weight
SAMPLE_ITERS = 2000 # Number of chars generated when sampling from model
CATEGORY_BASE = getattr(settings, "DPUBLISHING_CATEGORY_BASE_URL", 'category')
MPTT_ADMIN_LEVEL_INDENT = getattr(settings, "DPUBLISHING_CATEGORY_ADMIN_INDENT_LEVEL", 30)
#Global settings which needs to be available to other applications
settings.GRAPPELLI_ADMIN_TITLE = getattr(settings, "DPUBLISHING_SITE_NAME", 'YourSite.com')
settings.URL_FILEBROWSER_MEDIA = settings.MEDIA_ROOT
settings.FILEBROWSER_DEBUG = False
settings.FILEBROWSER_DIRECTORY = "uploads/"
settings.FILEBROWSER_URL_FILEBROWSER_MEDIA = "/static/filebrowser/"
settings.TEMPLATE_DIRS += (
relative_path("templates/themes/%s/") % (THEME_NAME),
)
settings.STATICFILES_DIRS = (
relative_path("static"),
)
if ENABLE_SEARCH:
HAYSTACK_SITECONF = getattr(settings, "DPUBLISHING_HAYSTACK_SITECONF", 'dpublishing.site_search')
settings.HAYSTACK_SEARCH_ENGINE = getattr(settings, "DPUBLISHING_HAYSTACK_SEARCH_ENGINE", 'whoosh')
settings.HAYSTACK_WHOOSH_PATH = getattr(settings, "DPUBLISHING_HAYSTACK_WHOOSH_PATH", "%s/var" % settings.PROJECT_ROOT)
settings.INSTALLED_APPS += (
'haystack',
)
def build_native_dir(directory, output_dir, cache_dir, abis, std_includes_path, rules: BaseConfig):
executables = {}
for abi in abis:
executable = prepare_compiler_executable(abi)
if executable is None:
print("failed to acquire GCC executable from NDK for abi " + abi)
return CODE_FAILED_NO_GCC
executables[abi] = executable
try:
manifest = get_manifest(directory)
targets = {}
soname = "lib" + manifest["shared"]["name"] + ".so"
for abi in abis:
targets[abi] = os.path.join(output_dir, "so/" + abi + "/" + soname)
except Exception as err:
print("failed to read manifest for directory " + directory + " error: " + str(err))
return CODE_FAILED_INVALID_MANIFEST
keep_sources = rules.get_value("keepSources", fallback=False)
if keep_sources:
# copy everything and clear build files
copy_directory(directory, output_dir, clear_dst=True)
clear_directory(os.path.join(output_dir, "so"))
os.remove(os.path.join(output_dir, soname))
else:
clear_directory(output_dir)
# copy manifest
copy_file(os.path.join(directory, "manifest"), os.path.join(output_dir, "manifest"))
# copy includes
keep_includes = rules.get_value("keepIncludes", fallback=True)
for include_path in manifest["shared"]["include"]:
src_include_path = os.path.join(directory, include_path)
output_include_path = os.path.join(output_dir, include_path)
if keep_includes:
copy_directory(src_include_path, output_include_path, clear_dst=True)
else:
clear_directory(output_include_path)
std_includes = []
for std_includes_dir in os.listdir(std_includes_path):
std_includes.append(os.path.abspath(os.path.join(std_includes_path, std_includes_dir)))
# compile for every abi
overall_result = CODE_OK
for abi in abis:
printed_compilation_title = f"compiling {os.path.basename(directory)} for {abi}"
print("\n")
print(f"{'=' * (48 - len(printed_compilation_title) // 2)} {printed_compilation_title} {'=' * (48 - (1 + len(printed_compilation_title)) // 2)}")
executable = executables[abi]
gcc = [executable, "-std=c++11"]
includes = []
for std_includes_dir in std_includes:
includes.append(f'-I{std_includes_dir}')
dependencies = [f'-L{get_fake_so_dir(abi)}', "-landroid", "-lm", "-llog"]
for link in rules.get_value("link", fallback=[]) + make_config.get_value("make.linkNative", fallback=[]) + ["horizon"]:
add_fake_so(executable, abi, link)
dependencies.append(f'-l{link}')
if "depends" in manifest:
search_dir = os.path.abspath(os.path.join(directory, "..")) # always search for dependencies in current dir
for dependency in manifest["depends"]:
if dependency is not None:
add_fake_so(executable, abi, dependency)
dependencies.append("-l" + dependency)
dependency_dir = search_directory(search_dir, dependency)
if dependency_dir is not None:
try:
for include_dir in get_manifest(dependency_dir)["shared"]["include"]:
includes.append("-I" + os.path.join(dependency_dir, include_dir))
except KeyError:
pass
else:
print(f"ERROR: dependency directory {dependency} is not found, it will be skipped")
# prepare directories
source_files = get_all_files(directory, extensions=(".cpp", ".c"))
preprocessed_dir = os.path.abspath(os.path.join(cache_dir, "preprocessed", abi))
ensure_directory(preprocessed_dir)
object_dir = os.path.abspath(os.path.join(cache_dir, "object", abi))
ensure_directory(object_dir)
# pre-process and compile changes
import filecmp
object_files = []
recompiled_count = 0
for file in source_files:
relative_file = relative_path(directory, file)
sys.stdout.write("preprocessing " + relative_file + " " * 64 + "\r")
object_file = os.path.join(object_dir, relative_file) + ".o"
preprocessed_file = os.path.join(preprocessed_dir, relative_file)
tmp_preprocessed_file = preprocessed_file + ".tmp"
ensure_file_dir(preprocessed_file)
ensure_file_dir(object_file)
object_files.append(object_file)
result = subprocess.call(gcc + ["-E", file, "-o", tmp_preprocessed_file] + includes)
if result == CODE_OK:
if not os.path.isfile(preprocessed_file) or not os.path.isfile(object_file) or \
not filecmp.cmp(preprocessed_file, tmp_preprocessed_file):
if os.path.isfile(preprocessed_file):
os.remove(preprocessed_file)
os.rename(tmp_preprocessed_file, preprocessed_file)
if os.path.isfile(object_file):
os.remove(object_file)
sys.stdout.write("compiling " + relative_file + " " * 64 + "\n")
result = max(result, subprocess.call(gcc + ["-c", preprocessed_file, "-shared", "-o", object_file]))
if result != CODE_OK:
if os.path.isfile(object_file):
os.remove(object_file)
overall_result = result
else:
recompiled_count += 1
else:
if os.path.isfile(object_file):
os.remove(object_file)
overall_result = result
print(" " * 128)
if overall_result != CODE_OK:
print("failed to compile", overall_result)
return overall_result
else:
print(f"recompiled {recompiled_count}/{len(object_files)} files with result {overall_result}")
ensure_file_dir(targets[abi])
command = []
command += gcc
command += object_files
command.append("-shared")
command.append("-Wl,-soname=" + soname)
command.append("-o")
command.append(targets[abi])
command += includes
command += dependencies
print("linking object files...")
result = subprocess.call(command)
if result == CODE_OK:
print("build successful")
else:
print("linker failed with result code", result)
overall_result = result
return overall_result
return overall_result
import numpy as np
import pickle
from scipy.io import wavfile
from feature_extraction import extract_features, OUTPUT_DIM
from utils import file_listing, dir_listing, last_component, relative_path, file_exists
DATASET_TRAIN_PATH = relative_path('../data/train/')
TRAIN_PERSIST_PATH = relative_path('../data/train.npy')
def read_wav(filepath):
freq, audio = wavfile.read(filepath)
return extract_features(audio, freq)
def read_dataset_dir(base_dir):
dirs = dir_listing(base_dir)
X = {last_component(dir): np.empty(OUTPUT_DIM) for dir in dirs}
for dir_path in dirs:
for file in file_listing(dir_path, 'wav'):
speaker = last_component(dir_path)
audio_np = read_wav(file)
X[speaker] = np.vstack((X[speaker], audio_np))
return X
def get_speakers(base_dir=DATASET_TRAIN_PATH):
return set(
map(lambda dir_path: last_component(dir_path), dir_listing(base_dir)))
# Save model
if batch_idx % LOG_PERIOD == 0:
path_suffix = f'{args.img_w}x{args.img_h}_epoch-{epoch + 1}_{int(batch_idx / 1000)}K.h5'
gan.save_weights(f'{args.model_save_path}/gan_{path_suffix}')
generator.save_weights(
f'{args.model_save_path}/generator_{path_suffix}')
if __name__ == '__main__':
# Command line arguments parsing
parser = argparse.ArgumentParser(
description='Train a colorization deep learning model')
parser.add_argument(
'--train-dataset',
type=str,
default=relative_path('../data/train/'),
help=
'Train dataset base path. Folder should contain subfolder for each class.'
)
parser.add_argument(
'--validation-path',
type=str,
default=relative_path('../data/val/'),
help=
'Path to directory with validation images that will be uploaded to comet after each epoch'
)
parser.add_argument('--batch-size',
type=int,
default=128,
help='Batch size used during training')
parser.add_argument('--img-w', type=int, default=256, help='Image width')
def __get_explicits_path(cls):
""" Return path of encrypted explicits file """
path_to_enc_file = relative_path('../data/explicits-list')
return path_to_enc_file
# Hyperparams
ARCFACE_M = 0.5
ARCFACE_S = 10.
CENTERLOSS_ALPHA = 0.008
CENTERLOSS_LAMBDA = 0.5
EMBEDDING_SIZE = 256
MIN_FACES_PER_PERSON = 5 # Min num of samples per class - or class is removed
MAX_FACES_PER_PERSON = 200 # Max num of samples per class - additional samples are removed
MIN_FACES_UNSAMPLE = 5 # All classes with lower num of samples are upscaled to this num of samples
DEV_FACES_PER_PERSON = 2 # Number of images per person in dev data
BATCH_SIZE = 256
EPOCHS = 50
TARGET_IMG_WIDTH = 96
TARGET_IMG_HEIGHT = 112
MIN_IMG_WIDTH = TARGET_IMG_WIDTH # no image upscale allowed
MIN_IMG_HEIGHT = TARGET_IMG_HEIGHT # no image upscale allowed
INPUT_SHAPE = (TARGET_IMG_HEIGHT, TARGET_IMG_WIDTH, 3)
# Paths
MODEL_SAVE_PATH = os.environ.get('MODEL_SAVE_PATH', relative_path('../model/'))
VGG_TRAIN_PATH = os.environ.get('VGG_DATASET', relative_path('../data/VGGFace2/')) + '/train/'
VGG_TEST_PATH = os.environ.get('VGG_DATASET', relative_path('../data/VGGFace2/')) + '/test/'
VGG_BB_TRAIN_MAP = os.environ.get('BB_TRAIN', relative_path('../data/vggface_bb_landmark/loose_bb_train.csv'))
VGG_BB_TEST_MAP = os.environ.get('BB_TEST', relative_path('../data/vggface_bb_landmark/loose_bb_test.csv'))
CASIA_PATH = os.environ.get('CASIA_DATASET', relative_path('../data/CASIA-WebFace/'))
CASIA_BB_MAP = os.environ.get('CASIA_BB', relative_path('../data/casia_landmark.csv'))
LFW_PATH = os.environ.get('LFW_DATASET', relative_path('../data/lfw/'))
LFW_BB_MAP = os.environ.get('LFW_BB', relative_path('../data/lfw_landmark.csv'))
LFW_PAIRS_PATH = os.environ.get('LFW_PAIRS', relative_path('../data/lfw_pairs.txt'))
import os
import pickle
from numpy import inf
from sklearn.mixture import GaussianMixture as GMM
from dataset import load_local_dataset, read_wav
from utils import file_listing, dir_listing, last_component, relative_path
# Hyperparameters
MIXTURE_COMPONENTS = 24
MAX_ITERS = 300
NUM_INITS = 10
MODEL_PERSIST_PATH = relative_path('../model/gmm/')
def get_gmm_path(speaker):
format_tuple = (speaker, MIXTURE_COMPONENTS, MAX_ITERS, NUM_INITS)
return MODEL_PERSIST_PATH + 'speaker%s_%scomps_%smaxiter_%sninit.gmm' % format_tuple
def load_gmm(speaker):
with open(get_gmm_path(speaker), 'rb') as f:
gmm = pickle.load(f)
return gmm
def load_models(speakers):
if len(file_listing(MODEL_PERSIST_PATH, 'gmm')) > 0:
print('Loading saved GMM models from file')
return {speaker: load_gmm(speaker) for speaker in speakers}
else:
print('GMMs need to be trained first')
"coordinates": [float(d['lon']), float(d['lat'])]
},
"properties": {
"city": d['city'],
"other_cities": d['other_cities'],
"state": d['state'],
"county": d['county'],
}
}
if __name__ == "__main__":
if len(sys.argv) > 1:
fn = sys.argv[1]
else:
fn = relative_path("../raw/zip_code_database.csv")
try:
zipcode_db = load_csv_columns(fn, ZIPCODE_COLUMNS)
except IOError:
print "unable to load", fn
sys.exit(-1)
print "loaded %s zipcodes" % len(zipcode_db)
for (abbr, data) in split_dict_by(zipcode_db, 'state').items():
state_name = STATE_ABBR.get(abbr, '').replace(' ', '_')
geojson_collection = {"type": "FeatureCollection", "features": list()}
for place in data:
geojson_collection['features'].append(geojson_feature(place))
import os
from gmm_train import load_models, get_gmm_path, predict
from dataset import get_speakers
from utils import file_listing, dir_listing, relative_path, get_file_name
EVAL_DIR_PATH = relative_path('../data/eval/')
if __name__ == '__main__':
speakers = get_speakers()
gmm_models = load_models(speakers)
for file in file_listing(EVAL_DIR_PATH, 'wav'):
speaker = predict(gmm_models, file)
print('%s -> %s' % (get_file_name(file), speaker))
validation_steps=64,
callbacks=[tb_callback, cp_callback])
print('TRAINING COMPLETE')
model.save(MODEL_STRUCT_PATH)
for dp in glob(os.path.join(TEST_DATA_PATH, '*')):
for fp in glob(os.path.join(TEST_DATA_PATH, dp, '*')):
(fn, _) = os.path.splitext(fp)
arr = numpy.array(
load_img(fp,
target_size=(SIGN_IMG_HEIGHT, SIGN_IMG_WIDTH),
grayscale=False,
color_mode='rgb',
interpolation='nearest'))
a = get_activations(model, [[arr]], auto_compile=True)
rp = os.path.join(KERACT_PATH, relative_path(fn, TEST_DATA_PATH))
display_activations(a, directory=rp, save=True)
print(f'VISUALIZATION SAVED: {rp}')
break
print('DONE')
yp = model.predict_generator(test_generator)
yp = numpy.argmax(yp, axis=1)
print('CONFUSION MATRIX:')
print(confusion_matrix(test_generator.classes, yp))
print('Classification Report')
print(classification_report(test_generator.classes, yp, target_names=TYPES))
import configparser
import multiprocessing
from pathlib import Path
import sys
from utils import relative_path
CONFIG_FILE = relative_path("config.ini")
class Config:
# In theory this could be a dict but setting/accessing things globally in python is rather messy
# better to follow OOP and encapsulate as much as I can
def __init__(self):
self.values = {}
if Path(CONFIG_FILE).is_file():
self.read_config()
return
config = configparser.ConfigParser(allow_no_value=True)
# TODO - now that we use asynchronous downloads default these to the number of cpus available on the system
# and only have a parsing thread count, but include a semaphore for the async stuff to avoid overloading
# the server
config["processes"] = {
"# 0 or less defaults to number of cpus on the system": None,
"parsing": -1
}
config[
"curseforge"] = { # TODO - support for games other than minecraft?
"# can be found in curseforge url for mods listing": None,