def set_strategies(self, coordinator_strategy, taxi_strategy, passenger_strategy): self.coordinator_strategy = load_class(coordinator_strategy) self.taxi_strategy = load_class(taxi_strategy) self.passenger_strategy = load_class(passenger_strategy) logger.debug("Loaded strategy classes: {}, {} and {}".format( self.coordinator_strategy, self.taxi_strategy, self.passenger_strategy))
def __load_module(self, alias): """ Load module class by alias :param alias: str :return: class """ if alias in self.modules: return self.modules[alias] mod_conf = self.config.get('modules') if alias not in mod_conf: msg = "Module '%s' not found in list of available aliases %s" % (alias, sorted(mod_conf.keys())) raise TaurusConfigError(msg) settings = ensure_is_dict(mod_conf, alias, "class") acopy = copy.deepcopy(settings) BetterDict.traverse(acopy, Configuration.masq_sensitive) self.log.debug("Module config: %s %s", alias, acopy) err = TaurusConfigError("Class name for alias '%s' is not found in module settings: %s" % (alias, settings)) clsname = settings.get('class', err) self.modules[alias] = load_class(clsname) if not issubclass(self.modules[alias], EngineModule): # raise TaurusInternalException("Module class does not inherit from EngineModule: %s" % clsname) pass return self.modules[alias]
def load_tactic(dpath, basedir): """Load a tactic from the current layer using a dotted path. The last element in the path should be a Tactic subclass """ obj = utils.load_class(dpath, basedir) if not issubclass(obj, Tactic): raise ValueError("Expected to load a tactic for %s" % dpath) return obj
def get_model(model, dataset, ckpt, image_shape): model_module = importlib.import_module('models.facial.%s' % model) model_func = getattr(model_module, model) classes = load_class(dataset, 'facial') model = model_func(image_shape, len(classes)) model.load_weights( os.path.join('results', 'facial', model, '%s.ckpt' % ckpt)) return model, classes
def get_model(model, dataset, ckpt, signal_len, n_mfcc): model_module = importlib.import_module('models.voice.%s' % model) model_func = getattr(model_module, model) classes = load_class(dataset, 'voice') model = model_func(signal_len, n_mfcc, len(classes)) model.load_weights( os.path.join('results', 'voice', model, '%s.ckpt' % ckpt)) return model, classes
def get_model(model, dataset, ckpt, max_seq_len): model_module = importlib.import_module('models.text.%s' % model) model_func = getattr(model_module, model) vocab = load_vocab(dataset) classes = load_class(dataset, 'text') model = model_func(max_seq_len, len(vocab), len(classes)) model.load_weights(os.path.join('results', 'text', model, '%s.ckpt' % ckpt)) return model, vocab, classes
def init_storage(self, name, config, baseclass): if 'class_name' in config: class_name = config.pop('class_name') classobject = utils.load_class(class_name, baseclass) else: classobject = utils.discover_class('.'.join(('storage', name)), baseclass) io_loop = IOLoop.current() try: return classobject(io_loop, **config) except: self.log.exception('Error initializing storage {} config {}' \ .format(classobject, config))
def prepare_opt_env( self, experimental_settings: NamedTuple) -> ObjectiveFuncType: """The function to create ConfigSpace and load the objective function's class""" with open("params.json") as f: json_params = json.load(f)[self.obj_name] config_info = json_params["config"] self.y_names = json_params["y_names"] self.y_upper_bounds = json_params["y_upper_bounds"] if "y_upper_bounds" in json_params.keys() \ else [1.0e+8 for _ in range(len(self.y_names))] self.in_fmt = json_params["in_fmt"] self.waiting_time = json_params["waiting_time"] if len(self.y_names) != len(self.y_upper_bounds): raise ValueError( "The shape of y_names and y_upper_bounds in params.json must be same." ) if experimental_settings.dim is not None: try: v = config_info['x'] except ValueError: raise ValueError( "'x' is allowed to be the name of hyperparameters of benchmark functions " "ONLY in params.json.") lb, ub = v["lower"], v["upper"] for i in range(experimental_settings.dim): var_name = 'x{:0>5}'.format(i) hp = create_hyperparameter(float, var_name, lower=lb, upper=ub) self.config_space.add_hyperparameter(hp) else: if 'x' in config_info.keys(): raise ValueError( "-dim is required in the command line e.g.) 'python main.py -dim 3 -ini 3'" " when optimizing benchmark functions.") for var_name, v in config_info.items(): if "ignore" in v.keys(): if v["ignore"] != "True" and v["ignore"] != "False": raise ValueError( "ignore must be 'True' or 'False', but {} was given." .format(v["ignore"])) elif v["ignore"] == "True": continue hp = self.get_hp_info_from_json(v, var_name) self.config_space.add_hyperparameter(hp) return utils.load_class("obj_functions.{}".format( self.obj_name))(experimental_settings)
def load_package(self, pkg_name, explicit=False): # Load the package class. pkg_class = load_class(pkg_name) # Do we already have this package loaded? if pkg_class not in self._pkg_map: # Instantiate the package and insert into mapping. pkg = pkg_class(self, explicit) self._pkg_map[pkg_class] = pkg self.packages.append(pkg) # Use the existing package. else: pkg = self._pkg_map[pkg_class] pkg.explicit = True if explicit else pkg.explicit return pkg
import json try: from django.contrib.auth import get_user_model User = get_user_model() username_field = User.USERNAME_FIELD except ImportError: from django.contrib.auth.models import User username_field = 'username' IMAGESTORE_IMAGES_ON_PAGE = getattr(settings, 'IMAGESTORE_IMAGES_ON_PAGE', 20) IMAGESTORE_ON_PAGE = getattr(settings, 'IMAGESTORE_ON_PAGE', 20) ImageForm = load_class(getattr(settings, 'IMAGESTORE_IMAGE_FORM', 'imagestore.forms.ImageForm')) AlbumForm = load_class(getattr(settings, 'IMAGESTORE_ALBUM_FORM', 'imagestore.forms.AlbumForm')) class AjaxableResponseMixin(object): """ Mixin to add AJAX support to a form. Must be used with an object-based FormView (e.g. CreateView) """ def render_to_json_response(self, context, **response_kwargs): data = json.dumps(context) response_kwargs['content_type'] = 'application/json' return HttpResponse(data, **response_kwargs) def form_invalid(self, form): response = super(AjaxableResponseMixin, self).form_invalid(form)
class SimpleView(views.APIView): """ My attempt at a base for a set of generic classes that provide super simplified methods compared to DRF's. When subclassing this, set multi = False for single-object views. """ queryset = None filter_class = load_class( settings.REST_FRAMEWORK['DEFAULT_FILTER_BACKENDS'][0]) serializer_class = None ordering = [] filter_fields = [] api_version = 1 resource_name = 'resources' multi = True def get_queryset(self): return self.queryset def get_serializer(self, **kwargs): serializer = getattr(self, 'serializer', None) force_refresh = kwargs.pop('force_refresh', False) if force_refresh == True or serializer is None: self.serializer = self.serializer_class(**kwargs) return self.serializer def get_object_id(self, obj): return obj.id def update_object(self, request, new_data): obj = self.get_object() serializer = self.get_serializer(force_refresh=True, instance=obj, context={ 'request': request, 'view': self }, data=new_data) if serializer.is_valid(): serializer.save() else: msg = ('Attempting to save the object produced the following ' 'errors. {}'.format(' '.join(serializer.errors))) raise exceptions.BadUpdate(detail=msg) url = request.build_absolute_uri() content = { 'status': 200, 'details': 'The resource at {} was updated with the ' 'fields or sub-resource content provided in the request.' ''.format(url), 'links': { 'self': { 'href': url, 'id': self.get_object_id(obj) } } } return Response(content, status=status.HTTP_200_OK)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--model', type=str, required=True) parser.add_argument('--mode', type=str, required=True, choices=['infer', 'train', 'test']) parser.add_argument('--dataset', type=str, required=True) parser.add_argument('--ckpt', type=str) parser.add_argument('--signal_len', type=int, default=262143) parser.add_argument('--n_mfcc', type=int, default=64) parser.add_argument('--batch_size', type=int, default=128) args = parser.parse_args() model_module = importlib.import_module('models.voice.%s' % args.model) model_func = getattr(model_module, args.model) if args.mode == 'infer': assert args.ckpt, 'Missing argument "--ckpt".' classes = load_class(args.dataset, 'voice') model = model_func(args.signal_len, args.n_mfcc, len(classes)) model.load_weights( os.path.join('results', 'voice', args.model, '%s.ckpt' % args.ckpt)) # import numpy as np # while True: # s = input('>>> ') # encoded = [vocab.get(w, vocab['<UNK>']) for w in jieba.cut(s)] # for _ in range(len(encoded), args.max_seq_len): # encoded.append(vocab['<PAD>']) # y = model.predict(np.array([encoded])) # print(list(zip(classes, y[0]))) elif args.mode == 'train': train_x, train_y, classes = prepare_voice_data(args.dataset, args.signal_len, args.n_mfcc, 'train') model = model_func(args.signal_len, args.n_mfcc, len(classes)) if args.ckpt: model.load_weights( os.path.join('results', 'voice', args.model, '%s.ckpt' % args.ckpt)) from keras.callbacks import EarlyStopping try: model.fit(train_x, train_y, batch_size=args.batch_size, epochs=10000, validation_split=0.1, callbacks=[ EarlyStopping(patience=20, restore_best_weights=True) ]) except KeyboardInterrupt: pass output_path = os.path.join('results', 'voice', args.model) os.makedirs(output_path, exist_ok=True) num = 0 for filename in os.listdir(output_path): name, suffix = filename.split('.') if suffix == 'ckpt' and name.isdigit(): idx = int(name) if idx > num: num = idx num += 1 model.save_weights(os.path.join(output_path, '%d.ckpt' % num)) print('Save to %d.ckpt' % num) elif args.mode == 'test': assert args.ckpt, 'Missing argument "--ckpt".' test_x, test_y, classes = prepare_voice_data(args.dataset, args.signal_len, args.n_mfcc, 'test') model = model_func(args.signal_len, args.n_mfcc, len(classes)) model.load_weights( os.path.join('results', 'voice', args.model, '%s.ckpt' % args.ckpt)) loss, acc = model.evaluate(test_x, test_y, batch_size=args.batch_size) print('loss: %f, acc: %f' % (loss, acc))
if args.dry_run: print('Dry run, exiting.') exit(0) del args, rest random.seed(config.opts.seed) np.random.seed(config.opts.seed) torch.random.manual_seed(config.opts.seed) folder_base = (Path(config.opts.folder)).expanduser().resolve() folder_data = folder_base / 'data' folder_run = folder_base / 'runs' / config.opts.session saver = Saver(folder_run) logger = SummaryWriter(folder_run.as_posix()) ModelClass = load_class(config.model._class_) net: nn.Module = ModelClass( **{k: v for k, v in config.model.items() if k != '_class_'}) net.to(config.opts.device) OptimizerClass = load_class(config.optimizer._class_) optimizer: optim.Optimizer = OptimizerClass( params=net.parameters(), **{k: v for k, v in config.optimizer.items() if k != '_class_'}) if config.training.restore: train_state = saver.load(model=net, optimizer=optimizer, device=config.training.device)
if len(sys.argv) < 2: print 'Usage: {} <config>'.format(sys.argv[0]) sys.exit(-1) config = ConfigParser.SafeConfigParser() config.read(sys.argv[1]) #initalize data_feed data_feed = KafkaClient(config.get('data_feed', 'host')) consumer = SimpleConsumer(data_feed, group=config.get('data_feed', 'group'), topic=config.get('data_feed', 'topic')) fetch_num_messages = config.getint('data_feed', 'fetch_num_messages') #initialize data_sink sink_config = dict(config.items('data_sink')) class_name = sink_config.pop('class') classobject = utils.load_class(class_name, storage.VerdictDB) db = classobject(None, **sink_config) removed = added = 0 for msg in consumer.get_messages(count=fetch_num_messages, block=False): try: msg = json.loads(msg.message.value) blacklist_type = msg['type'] map(partial(db.insert, blacklist_type), msg['add']) added += len(msg['add']) map(partial(db.delete, blacklist_type), msg['remove']) removed += len(msg['remove']) except Exception as e: print '[{}] Error ingesting msg {}'.format(e, msg) print 'added %d urls removed %d urls' % (added, removed)
from utils import load_class from django.db.models import Q try: from django.contrib.auth import get_user_model User = get_user_model() username_field = User.USERNAME_FIELD except ImportError: from django.contrib.auth.models import User username_field = 'username' IMAGESTORE_IMAGES_ON_PAGE = getattr(settings, 'IMAGESTORE_IMAGES_ON_PAGE', 20) IMAGESTORE_ON_PAGE = getattr(settings, 'IMAGESTORE_ON_PAGE', 20) ImageForm = load_class( getattr(settings, 'IMAGESTORE_IMAGE_FORM', 'imagestore.forms.ImageForm')) AlbumForm = load_class( getattr(settings, 'IMAGESTORE_ALBUM_FORM', 'imagestore.forms.AlbumForm')) class AlbumListView(ListView): context_object_name = 'album_list' template_name = 'imagestore/album_list.html' paginate_by = getattr(settings, 'IMAGESTORE_ALBUMS_ON_PAGE', 20) allow_empty = True def get_queryset(self): albums = Album.objects.filter(is_public=True).select_related('head') self.e_context = dict() if 'username' in self.kwargs: user = get_object_or_404(**{
def _new_exporter(et_code, ef_code, st_code, options={}): instance = new_export_instance(et_code, ef_code, st_code) exp_class = utils.load_class(instance.export_type.path) return exp_class(instance.pk, ef_code, et_code, options)
def package(self, name): pkg_class = load_class(name) return self._pkg_map[pkg_class]
dir_config = agent.AgentConfig() dir_config.setName("Director") dir_config.setBindAddress(sys.argv[4]) dir_config.setPort(int(sys.argv[5])) pid = os.fork() if pid == 0: try: os.close(1) os.close(2) os.close(3) log = setup_logger("", "log/%s.log" % config.name) log.debug("Instantiating agent") if dir_config is None: my_agent = load_class(agent_name)(config) else: my_agent = load_class(agent_name)(config, dir_config) log.debug("Starting Agent") my_agent.run() except Exception, e: log.exception("Exception thrown, caught in start_agent") log.debug("Agent exiting") sys._exit(0) else: log.debug("Agent Daemon %s started. PID: %d" % (agent_name, pid)) print "Done (%d)" % (pid) sys.exit(0)