def read_config(self): config = configparser.ConfigParser(allow_no_value=True) config.read(CONFIG_FILE) processes = config["processes"] num_proc = int(processes["parsing"]) self.values[ "parser_processes"] = num_proc if num_proc > 0 else multiprocessing.cpu_count( ) curseforge = config["curseforge"] self.values["game_version"] = curseforge["game version"] db = config["db"] self.values["db_location"] = relative_path(db["location"]) self.values["cache_timeout"] = int(db["timeout"]) log = config["logging"] self.values["log_location"] = relative_path(log["location"]) self.values["log_use_stdout"] = log["print to stdout"] self.values["log_level"] = log["log level"] rate_limiting = config["rate_limiting"] max_search = int(rate_limiting["max search"]) self.values[ "max_search"] = max_search if max_search > 0 else sys.maxsize
def test_mod_parse_specific(self): jei_record = "(238222, '/minecraft/mc-mods/jei', 'https://github.com/mezz/JustEnoughItems', 'https://github.com/mezz/JustEnoughItems/issues?q=is%3Aissue', None, 'MIT License')" with open(relative_path("test/resources/jei_page_20190715.html"), "rb") as f: modrecord = parse.fetch_and_scrape("/minecraft/mc-mods/jei") self.assertEqual(jei_record, modrecord.test_form().__repr__())
def test_number_pages_specific(self): with open( relative_path( "test/resources/mod_listing_page_1_20190715.html"), "rb") as f: maxpage = parse.get_number_pages(f.read()) self.assertEqual(maxpage, 279)
def test_project_links_specific(self): links = { '/minecraft/mc-mods/mouse-tweaks', '/minecraft/mc-mods/crafttweaker', '/minecraft/mc-mods/iron-chests', '/minecraft/mc-mods/inventory-tweaks', '/minecraft/mc-mods/resource-loader', '/minecraft/mc-mods/chisel', '/minecraft/mc-mods/codechicken-lib-1-8', '/minecraft/mc-mods/applied-energistics-2', '/minecraft/mc-mods/shadowfacts-forgelin', '/minecraft/mc-mods/baubles', '/minecraft/mc-mods/ctm', '/minecraft/mc-mods/appleskin', '/minecraft/mc-mods/jei', '/minecraft/mc-mods/modtweaker', '/minecraft/mc-mods/thermal-foundation', '/minecraft/mc-mods/tinkers-construct', '/minecraft/mc-mods/mantle', '/minecraft/mc-mods/cofh-core', '/minecraft/mc-mods/journeymap', '/minecraft/mc-mods/custom-main-menu' } with open( relative_path( "test/resources/mod_listing_page_1_20190715.html"), "rb") as f: self.assertEqual(links, parse.get_project_links(f.read()))
import cv2 import numpy as np from utils import relative_path HAARCASCADE_CONFIG = relative_path( '../resources/haarcascade_frontalface_default.xml') face_detector = cv2.CascadeClassifier(HAARCASCADE_CONFIG) def detect_faces(frame): ''' Recognize faces in frame and check if there is face looking at screen Args: frame (numpy): Image frame from webcam (or whatever) Returns: boolean indicating whether face is present ''' return face_detector.detectMultiScale(frame)
"geometry": { "type": "Point", "coordinates": [float(place['lon']), float(place['lat'])] }, "properties": { "fips": place['fips'] } } if __name__ == "__main__": if len(sys.argv) > 1: fn = sys.argv[1] else: fn = relative_path("../raw/Gaz_places_national.txt") try: gazetteer = load_csv_columns(fn, GAZETTEER_COLUMNS, delimiter='\t', quoting=csv.QUOTE_NONE) except IOError: print "unable to load", fn sys.exit(-1) for (abbr, data) in split_dict_by(gazetteer, 'state').items(): state_name = STATE_ABBR[abbr].replace(' ', '_') geojson_collection = {"type": "FeatureCollection", "features": list()} for place in data:
model.fit_generator( train_seq, epochs=args.epochs, validation_data=test_seq, use_multiprocessing=True, workers=8, callbacks=callbacks) if __name__ == '__main__': # Command line arguments parsing parser = argparse.ArgumentParser(description='Train a colorization deep learning model') parser.add_argument('--train-dataset', type=str, default=relative_path('../data/imagenet-sample/train/'), help='Train dataset base path. Folder should contain subfolder for each class.') parser.add_argument('--test-dataset', type=str, default=relative_path('../data/imagenet-sample/test/'), help='Test dataset base path. Folder should contain images directly.') parser.add_argument('--validation-path', type=str, default=relative_path('../data/imagenet-sample/val/'), help='Path to directory with validation images that will be uploaded to comet after each epoch') parser.add_argument('--batch-size', type=int, default=128, help='Batch size used during training') parser.add_argument('--img-w', type=int,
from utils import relative_path TIMESTEPS = 100 # Number of timesteps feeded to model BATCH_SIZE = 128 TEXT_CORPUS_PATH = relative_path('../data/javascript_codes.js') MODELS_DIR = relative_path('../model/') # Model save dir FINAL_MODEL_PATH = relative_path('../model/jscode-final-model.h5') # Last learned model - arch+weight SAMPLE_ITERS = 2000 # Number of chars generated when sampling from model
CATEGORY_BASE = getattr(settings, "DPUBLISHING_CATEGORY_BASE_URL", 'category') MPTT_ADMIN_LEVEL_INDENT = getattr(settings, "DPUBLISHING_CATEGORY_ADMIN_INDENT_LEVEL", 30) #Global settings which needs to be available to other applications settings.GRAPPELLI_ADMIN_TITLE = getattr(settings, "DPUBLISHING_SITE_NAME", 'YourSite.com') settings.URL_FILEBROWSER_MEDIA = settings.MEDIA_ROOT settings.FILEBROWSER_DEBUG = False settings.FILEBROWSER_DIRECTORY = "uploads/" settings.FILEBROWSER_URL_FILEBROWSER_MEDIA = "/static/filebrowser/" settings.TEMPLATE_DIRS += ( relative_path("templates/themes/%s/") % (THEME_NAME), ) settings.STATICFILES_DIRS = ( relative_path("static"), ) if ENABLE_SEARCH: HAYSTACK_SITECONF = getattr(settings, "DPUBLISHING_HAYSTACK_SITECONF", 'dpublishing.site_search') settings.HAYSTACK_SEARCH_ENGINE = getattr(settings, "DPUBLISHING_HAYSTACK_SEARCH_ENGINE", 'whoosh') settings.HAYSTACK_WHOOSH_PATH = getattr(settings, "DPUBLISHING_HAYSTACK_WHOOSH_PATH", "%s/var" % settings.PROJECT_ROOT) settings.INSTALLED_APPS += ( 'haystack', )
def build_native_dir(directory, output_dir, cache_dir, abis, std_includes_path, rules: BaseConfig): executables = {} for abi in abis: executable = prepare_compiler_executable(abi) if executable is None: print("failed to acquire GCC executable from NDK for abi " + abi) return CODE_FAILED_NO_GCC executables[abi] = executable try: manifest = get_manifest(directory) targets = {} soname = "lib" + manifest["shared"]["name"] + ".so" for abi in abis: targets[abi] = os.path.join(output_dir, "so/" + abi + "/" + soname) except Exception as err: print("failed to read manifest for directory " + directory + " error: " + str(err)) return CODE_FAILED_INVALID_MANIFEST keep_sources = rules.get_value("keepSources", fallback=False) if keep_sources: # copy everything and clear build files copy_directory(directory, output_dir, clear_dst=True) clear_directory(os.path.join(output_dir, "so")) os.remove(os.path.join(output_dir, soname)) else: clear_directory(output_dir) # copy manifest copy_file(os.path.join(directory, "manifest"), os.path.join(output_dir, "manifest")) # copy includes keep_includes = rules.get_value("keepIncludes", fallback=True) for include_path in manifest["shared"]["include"]: src_include_path = os.path.join(directory, include_path) output_include_path = os.path.join(output_dir, include_path) if keep_includes: copy_directory(src_include_path, output_include_path, clear_dst=True) else: clear_directory(output_include_path) std_includes = [] for std_includes_dir in os.listdir(std_includes_path): std_includes.append(os.path.abspath(os.path.join(std_includes_path, std_includes_dir))) # compile for every abi overall_result = CODE_OK for abi in abis: printed_compilation_title = f"compiling {os.path.basename(directory)} for {abi}" print("\n") print(f"{'=' * (48 - len(printed_compilation_title) // 2)} {printed_compilation_title} {'=' * (48 - (1 + len(printed_compilation_title)) // 2)}") executable = executables[abi] gcc = [executable, "-std=c++11"] includes = [] for std_includes_dir in std_includes: includes.append(f'-I{std_includes_dir}') dependencies = [f'-L{get_fake_so_dir(abi)}', "-landroid", "-lm", "-llog"] for link in rules.get_value("link", fallback=[]) + make_config.get_value("make.linkNative", fallback=[]) + ["horizon"]: add_fake_so(executable, abi, link) dependencies.append(f'-l{link}') if "depends" in manifest: search_dir = os.path.abspath(os.path.join(directory, "..")) # always search for dependencies in current dir for dependency in manifest["depends"]: if dependency is not None: add_fake_so(executable, abi, dependency) dependencies.append("-l" + dependency) dependency_dir = search_directory(search_dir, dependency) if dependency_dir is not None: try: for include_dir in get_manifest(dependency_dir)["shared"]["include"]: includes.append("-I" + os.path.join(dependency_dir, include_dir)) except KeyError: pass else: print(f"ERROR: dependency directory {dependency} is not found, it will be skipped") # prepare directories source_files = get_all_files(directory, extensions=(".cpp", ".c")) preprocessed_dir = os.path.abspath(os.path.join(cache_dir, "preprocessed", abi)) ensure_directory(preprocessed_dir) object_dir = os.path.abspath(os.path.join(cache_dir, "object", abi)) ensure_directory(object_dir) # pre-process and compile changes import filecmp object_files = [] recompiled_count = 0 for file in source_files: relative_file = relative_path(directory, file) sys.stdout.write("preprocessing " + relative_file + " " * 64 + "\r") object_file = os.path.join(object_dir, relative_file) + ".o" preprocessed_file = os.path.join(preprocessed_dir, relative_file) tmp_preprocessed_file = preprocessed_file + ".tmp" ensure_file_dir(preprocessed_file) ensure_file_dir(object_file) object_files.append(object_file) result = subprocess.call(gcc + ["-E", file, "-o", tmp_preprocessed_file] + includes) if result == CODE_OK: if not os.path.isfile(preprocessed_file) or not os.path.isfile(object_file) or \ not filecmp.cmp(preprocessed_file, tmp_preprocessed_file): if os.path.isfile(preprocessed_file): os.remove(preprocessed_file) os.rename(tmp_preprocessed_file, preprocessed_file) if os.path.isfile(object_file): os.remove(object_file) sys.stdout.write("compiling " + relative_file + " " * 64 + "\n") result = max(result, subprocess.call(gcc + ["-c", preprocessed_file, "-shared", "-o", object_file])) if result != CODE_OK: if os.path.isfile(object_file): os.remove(object_file) overall_result = result else: recompiled_count += 1 else: if os.path.isfile(object_file): os.remove(object_file) overall_result = result print(" " * 128) if overall_result != CODE_OK: print("failed to compile", overall_result) return overall_result else: print(f"recompiled {recompiled_count}/{len(object_files)} files with result {overall_result}") ensure_file_dir(targets[abi]) command = [] command += gcc command += object_files command.append("-shared") command.append("-Wl,-soname=" + soname) command.append("-o") command.append(targets[abi]) command += includes command += dependencies print("linking object files...") result = subprocess.call(command) if result == CODE_OK: print("build successful") else: print("linker failed with result code", result) overall_result = result return overall_result return overall_result
import numpy as np import pickle from scipy.io import wavfile from feature_extraction import extract_features, OUTPUT_DIM from utils import file_listing, dir_listing, last_component, relative_path, file_exists DATASET_TRAIN_PATH = relative_path('../data/train/') TRAIN_PERSIST_PATH = relative_path('../data/train.npy') def read_wav(filepath): freq, audio = wavfile.read(filepath) return extract_features(audio, freq) def read_dataset_dir(base_dir): dirs = dir_listing(base_dir) X = {last_component(dir): np.empty(OUTPUT_DIM) for dir in dirs} for dir_path in dirs: for file in file_listing(dir_path, 'wav'): speaker = last_component(dir_path) audio_np = read_wav(file) X[speaker] = np.vstack((X[speaker], audio_np)) return X def get_speakers(base_dir=DATASET_TRAIN_PATH): return set( map(lambda dir_path: last_component(dir_path), dir_listing(base_dir)))
# Save model if batch_idx % LOG_PERIOD == 0: path_suffix = f'{args.img_w}x{args.img_h}_epoch-{epoch + 1}_{int(batch_idx / 1000)}K.h5' gan.save_weights(f'{args.model_save_path}/gan_{path_suffix}') generator.save_weights( f'{args.model_save_path}/generator_{path_suffix}') if __name__ == '__main__': # Command line arguments parsing parser = argparse.ArgumentParser( description='Train a colorization deep learning model') parser.add_argument( '--train-dataset', type=str, default=relative_path('../data/train/'), help= 'Train dataset base path. Folder should contain subfolder for each class.' ) parser.add_argument( '--validation-path', type=str, default=relative_path('../data/val/'), help= 'Path to directory with validation images that will be uploaded to comet after each epoch' ) parser.add_argument('--batch-size', type=int, default=128, help='Batch size used during training') parser.add_argument('--img-w', type=int, default=256, help='Image width')
def __get_explicits_path(cls): """ Return path of encrypted explicits file """ path_to_enc_file = relative_path('../data/explicits-list') return path_to_enc_file
# Hyperparams ARCFACE_M = 0.5 ARCFACE_S = 10. CENTERLOSS_ALPHA = 0.008 CENTERLOSS_LAMBDA = 0.5 EMBEDDING_SIZE = 256 MIN_FACES_PER_PERSON = 5 # Min num of samples per class - or class is removed MAX_FACES_PER_PERSON = 200 # Max num of samples per class - additional samples are removed MIN_FACES_UNSAMPLE = 5 # All classes with lower num of samples are upscaled to this num of samples DEV_FACES_PER_PERSON = 2 # Number of images per person in dev data BATCH_SIZE = 256 EPOCHS = 50 TARGET_IMG_WIDTH = 96 TARGET_IMG_HEIGHT = 112 MIN_IMG_WIDTH = TARGET_IMG_WIDTH # no image upscale allowed MIN_IMG_HEIGHT = TARGET_IMG_HEIGHT # no image upscale allowed INPUT_SHAPE = (TARGET_IMG_HEIGHT, TARGET_IMG_WIDTH, 3) # Paths MODEL_SAVE_PATH = os.environ.get('MODEL_SAVE_PATH', relative_path('../model/')) VGG_TRAIN_PATH = os.environ.get('VGG_DATASET', relative_path('../data/VGGFace2/')) + '/train/' VGG_TEST_PATH = os.environ.get('VGG_DATASET', relative_path('../data/VGGFace2/')) + '/test/' VGG_BB_TRAIN_MAP = os.environ.get('BB_TRAIN', relative_path('../data/vggface_bb_landmark/loose_bb_train.csv')) VGG_BB_TEST_MAP = os.environ.get('BB_TEST', relative_path('../data/vggface_bb_landmark/loose_bb_test.csv')) CASIA_PATH = os.environ.get('CASIA_DATASET', relative_path('../data/CASIA-WebFace/')) CASIA_BB_MAP = os.environ.get('CASIA_BB', relative_path('../data/casia_landmark.csv')) LFW_PATH = os.environ.get('LFW_DATASET', relative_path('../data/lfw/')) LFW_BB_MAP = os.environ.get('LFW_BB', relative_path('../data/lfw_landmark.csv')) LFW_PAIRS_PATH = os.environ.get('LFW_PAIRS', relative_path('../data/lfw_pairs.txt'))
import os import pickle from numpy import inf from sklearn.mixture import GaussianMixture as GMM from dataset import load_local_dataset, read_wav from utils import file_listing, dir_listing, last_component, relative_path # Hyperparameters MIXTURE_COMPONENTS = 24 MAX_ITERS = 300 NUM_INITS = 10 MODEL_PERSIST_PATH = relative_path('../model/gmm/') def get_gmm_path(speaker): format_tuple = (speaker, MIXTURE_COMPONENTS, MAX_ITERS, NUM_INITS) return MODEL_PERSIST_PATH + 'speaker%s_%scomps_%smaxiter_%sninit.gmm' % format_tuple def load_gmm(speaker): with open(get_gmm_path(speaker), 'rb') as f: gmm = pickle.load(f) return gmm def load_models(speakers): if len(file_listing(MODEL_PERSIST_PATH, 'gmm')) > 0: print('Loading saved GMM models from file') return {speaker: load_gmm(speaker) for speaker in speakers} else: print('GMMs need to be trained first')
"coordinates": [float(d['lon']), float(d['lat'])] }, "properties": { "city": d['city'], "other_cities": d['other_cities'], "state": d['state'], "county": d['county'], } } if __name__ == "__main__": if len(sys.argv) > 1: fn = sys.argv[1] else: fn = relative_path("../raw/zip_code_database.csv") try: zipcode_db = load_csv_columns(fn, ZIPCODE_COLUMNS) except IOError: print "unable to load", fn sys.exit(-1) print "loaded %s zipcodes" % len(zipcode_db) for (abbr, data) in split_dict_by(zipcode_db, 'state').items(): state_name = STATE_ABBR.get(abbr, '').replace(' ', '_') geojson_collection = {"type": "FeatureCollection", "features": list()} for place in data: geojson_collection['features'].append(geojson_feature(place))
import os from gmm_train import load_models, get_gmm_path, predict from dataset import get_speakers from utils import file_listing, dir_listing, relative_path, get_file_name EVAL_DIR_PATH = relative_path('../data/eval/') if __name__ == '__main__': speakers = get_speakers() gmm_models = load_models(speakers) for file in file_listing(EVAL_DIR_PATH, 'wav'): speaker = predict(gmm_models, file) print('%s -> %s' % (get_file_name(file), speaker))
validation_steps=64, callbacks=[tb_callback, cp_callback]) print('TRAINING COMPLETE') model.save(MODEL_STRUCT_PATH) for dp in glob(os.path.join(TEST_DATA_PATH, '*')): for fp in glob(os.path.join(TEST_DATA_PATH, dp, '*')): (fn, _) = os.path.splitext(fp) arr = numpy.array( load_img(fp, target_size=(SIGN_IMG_HEIGHT, SIGN_IMG_WIDTH), grayscale=False, color_mode='rgb', interpolation='nearest')) a = get_activations(model, [[arr]], auto_compile=True) rp = os.path.join(KERACT_PATH, relative_path(fn, TEST_DATA_PATH)) display_activations(a, directory=rp, save=True) print(f'VISUALIZATION SAVED: {rp}') break print('DONE') yp = model.predict_generator(test_generator) yp = numpy.argmax(yp, axis=1) print('CONFUSION MATRIX:') print(confusion_matrix(test_generator.classes, yp)) print('Classification Report') print(classification_report(test_generator.classes, yp, target_names=TYPES))
import configparser import multiprocessing from pathlib import Path import sys from utils import relative_path CONFIG_FILE = relative_path("config.ini") class Config: # In theory this could be a dict but setting/accessing things globally in python is rather messy # better to follow OOP and encapsulate as much as I can def __init__(self): self.values = {} if Path(CONFIG_FILE).is_file(): self.read_config() return config = configparser.ConfigParser(allow_no_value=True) # TODO - now that we use asynchronous downloads default these to the number of cpus available on the system # and only have a parsing thread count, but include a semaphore for the async stuff to avoid overloading # the server config["processes"] = { "# 0 or less defaults to number of cpus on the system": None, "parsing": -1 } config[ "curseforge"] = { # TODO - support for games other than minecraft? "# can be found in curseforge url for mods listing": None,