def set_strategies(self, coordinator_strategy, taxi_strategy,
passenger_strategy):
self.coordinator_strategy = load_class(coordinator_strategy)
self.taxi_strategy = load_class(taxi_strategy)
self.passenger_strategy = load_class(passenger_strategy)
logger.debug("Loaded strategy classes: {}, {} and {}".format(
self.coordinator_strategy, self.taxi_strategy,
self.passenger_strategy))
def __load_module(self, alias):
"""
Load module class by alias
:param alias: str
:return: class
"""
if alias in self.modules:
return self.modules[alias]
mod_conf = self.config.get('modules')
if alias not in mod_conf:
msg = "Module '%s' not found in list of available aliases %s" % (alias, sorted(mod_conf.keys()))
raise TaurusConfigError(msg)
settings = ensure_is_dict(mod_conf, alias, "class")
acopy = copy.deepcopy(settings)
BetterDict.traverse(acopy, Configuration.masq_sensitive)
self.log.debug("Module config: %s %s", alias, acopy)
err = TaurusConfigError("Class name for alias '%s' is not found in module settings: %s" % (alias, settings))
clsname = settings.get('class', err)
self.modules[alias] = load_class(clsname)
if not issubclass(self.modules[alias], EngineModule):
# raise TaurusInternalException("Module class does not inherit from EngineModule: %s" % clsname)
pass
return self.modules[alias]
def load_tactic(dpath, basedir):
"""Load a tactic from the current layer using a dotted path. The last
element in the path should be a Tactic subclass
"""
obj = utils.load_class(dpath, basedir)
if not issubclass(obj, Tactic):
raise ValueError("Expected to load a tactic for %s" % dpath)
return obj
def get_model(model, dataset, ckpt, image_shape):
model_module = importlib.import_module('models.facial.%s' % model)
model_func = getattr(model_module, model)
classes = load_class(dataset, 'facial')
model = model_func(image_shape, len(classes))
model.load_weights(
os.path.join('results', 'facial', model, '%s.ckpt' % ckpt))
return model, classes
def load_tactic(dpath, basedir):
"""Load a tactic from the current layer using a dotted path. The last
element in the path should be a Tactic subclass
"""
obj = utils.load_class(dpath, basedir)
if not issubclass(obj, Tactic):
raise ValueError("Expected to load a tactic for %s" % dpath)
return obj
def get_model(model, dataset, ckpt, signal_len, n_mfcc):
model_module = importlib.import_module('models.voice.%s' % model)
model_func = getattr(model_module, model)
classes = load_class(dataset, 'voice')
model = model_func(signal_len, n_mfcc, len(classes))
model.load_weights(
os.path.join('results', 'voice', model, '%s.ckpt' % ckpt))
return model, classes
def get_model(model, dataset, ckpt, max_seq_len):
model_module = importlib.import_module('models.text.%s' % model)
model_func = getattr(model_module, model)
vocab = load_vocab(dataset)
classes = load_class(dataset, 'text')
model = model_func(max_seq_len, len(vocab), len(classes))
model.load_weights(os.path.join('results', 'text', model,
'%s.ckpt' % ckpt))
return model, vocab, classes
def init_storage(self, name, config, baseclass):
if 'class_name' in config:
class_name = config.pop('class_name')
classobject = utils.load_class(class_name, baseclass)
else:
classobject = utils.discover_class('.'.join(('storage', name)), baseclass)
io_loop = IOLoop.current()
try:
return classobject(io_loop, **config)
except:
self.log.exception('Error initializing storage {} config {}' \
.format(classobject, config))
def prepare_opt_env(
self, experimental_settings: NamedTuple) -> ObjectiveFuncType:
"""The function to create ConfigSpace and load the objective function's class"""
with open("params.json") as f:
json_params = json.load(f)[self.obj_name]
config_info = json_params["config"]
self.y_names = json_params["y_names"]
self.y_upper_bounds = json_params["y_upper_bounds"] if "y_upper_bounds" in json_params.keys() \
else [1.0e+8 for _ in range(len(self.y_names))]
self.in_fmt = json_params["in_fmt"]
self.waiting_time = json_params["waiting_time"]
if len(self.y_names) != len(self.y_upper_bounds):
raise ValueError(
"The shape of y_names and y_upper_bounds in params.json must be same."
)
if experimental_settings.dim is not None:
try:
v = config_info['x']
except ValueError:
raise ValueError(
"'x' is allowed to be the name of hyperparameters of benchmark functions "
"ONLY in params.json.")
lb, ub = v["lower"], v["upper"]
for i in range(experimental_settings.dim):
var_name = 'x{:0>5}'.format(i)
hp = create_hyperparameter(float, var_name, lower=lb, upper=ub)
self.config_space.add_hyperparameter(hp)
else:
if 'x' in config_info.keys():
raise ValueError(
"-dim is required in the command line e.g.) 'python main.py -dim 3 -ini 3'"
" when optimizing benchmark functions.")
for var_name, v in config_info.items():
if "ignore" in v.keys():
if v["ignore"] != "True" and v["ignore"] != "False":
raise ValueError(
"ignore must be 'True' or 'False', but {} was given."
.format(v["ignore"]))
elif v["ignore"] == "True":
continue
hp = self.get_hp_info_from_json(v, var_name)
self.config_space.add_hyperparameter(hp)
return utils.load_class("obj_functions.{}".format(
self.obj_name))(experimental_settings)
def load_package(self, pkg_name, explicit=False):
# Load the package class.
pkg_class = load_class(pkg_name)
# Do we already have this package loaded?
if pkg_class not in self._pkg_map:
# Instantiate the package and insert into mapping.
pkg = pkg_class(self, explicit)
self._pkg_map[pkg_class] = pkg
self.packages.append(pkg)
# Use the existing package.
else:
pkg = self._pkg_map[pkg_class]
pkg.explicit = True if explicit else pkg.explicit
return pkg
import json try: from django.contrib.auth import get_user_model User = get_user_model() username_field = User.USERNAME_FIELD except ImportError: from django.contrib.auth.models import User username_field = 'username' IMAGESTORE_IMAGES_ON_PAGE = getattr(settings, 'IMAGESTORE_IMAGES_ON_PAGE', 20) IMAGESTORE_ON_PAGE = getattr(settings, 'IMAGESTORE_ON_PAGE', 20) ImageForm = load_class(getattr(settings, 'IMAGESTORE_IMAGE_FORM', 'imagestore.forms.ImageForm')) AlbumForm = load_class(getattr(settings, 'IMAGESTORE_ALBUM_FORM', 'imagestore.forms.AlbumForm')) class AjaxableResponseMixin(object): """ Mixin to add AJAX support to a form. Must be used with an object-based FormView (e.g. CreateView) """ def render_to_json_response(self, context, **response_kwargs): data = json.dumps(context) response_kwargs['content_type'] = 'application/json' return HttpResponse(data, **response_kwargs) def form_invalid(self, form): response = super(AjaxableResponseMixin, self).form_invalid(form)
class SimpleView(views.APIView):
"""
My attempt at a base for a set of generic classes that provide
super simplified methods compared to DRF's. When subclassing this,
set multi = False for single-object views.
"""
queryset = None
filter_class = load_class(
settings.REST_FRAMEWORK['DEFAULT_FILTER_BACKENDS'][0])
serializer_class = None
ordering = []
filter_fields = []
api_version = 1
resource_name = 'resources'
multi = True
def get_queryset(self):
return self.queryset
def get_serializer(self, **kwargs):
serializer = getattr(self, 'serializer', None)
force_refresh = kwargs.pop('force_refresh', False)
if force_refresh == True or serializer is None:
self.serializer = self.serializer_class(**kwargs)
return self.serializer
def get_object_id(self, obj):
return obj.id
def update_object(self, request, new_data):
obj = self.get_object()
serializer = self.get_serializer(force_refresh=True,
instance=obj,
context={
'request': request,
'view': self
},
data=new_data)
if serializer.is_valid():
serializer.save()
else:
msg = ('Attempting to save the object produced the following '
'errors. {}'.format(' '.join(serializer.errors)))
raise exceptions.BadUpdate(detail=msg)
url = request.build_absolute_uri()
content = {
'status':
200,
'details':
'The resource at {} was updated with the '
'fields or sub-resource content provided in the request.'
''.format(url),
'links': {
'self': {
'href': url,
'id': self.get_object_id(obj)
}
}
}
return Response(content, status=status.HTTP_200_OK)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--model', type=str, required=True)
parser.add_argument('--mode',
type=str,
required=True,
choices=['infer', 'train', 'test'])
parser.add_argument('--dataset', type=str, required=True)
parser.add_argument('--ckpt', type=str)
parser.add_argument('--signal_len', type=int, default=262143)
parser.add_argument('--n_mfcc', type=int, default=64)
parser.add_argument('--batch_size', type=int, default=128)
args = parser.parse_args()
model_module = importlib.import_module('models.voice.%s' % args.model)
model_func = getattr(model_module, args.model)
if args.mode == 'infer':
assert args.ckpt, 'Missing argument "--ckpt".'
classes = load_class(args.dataset, 'voice')
model = model_func(args.signal_len, args.n_mfcc, len(classes))
model.load_weights(
os.path.join('results', 'voice', args.model,
'%s.ckpt' % args.ckpt))
# import numpy as np
# while True:
# s = input('>>> ')
# encoded = [vocab.get(w, vocab['<UNK>']) for w in jieba.cut(s)]
# for _ in range(len(encoded), args.max_seq_len):
# encoded.append(vocab['<PAD>'])
# y = model.predict(np.array([encoded]))
# print(list(zip(classes, y[0])))
elif args.mode == 'train':
train_x, train_y, classes = prepare_voice_data(args.dataset,
args.signal_len,
args.n_mfcc, 'train')
model = model_func(args.signal_len, args.n_mfcc, len(classes))
if args.ckpt:
model.load_weights(
os.path.join('results', 'voice', args.model,
'%s.ckpt' % args.ckpt))
from keras.callbacks import EarlyStopping
try:
model.fit(train_x,
train_y,
batch_size=args.batch_size,
epochs=10000,
validation_split=0.1,
callbacks=[
EarlyStopping(patience=20, restore_best_weights=True)
])
except KeyboardInterrupt:
pass
output_path = os.path.join('results', 'voice', args.model)
os.makedirs(output_path, exist_ok=True)
num = 0
for filename in os.listdir(output_path):
name, suffix = filename.split('.')
if suffix == 'ckpt' and name.isdigit():
idx = int(name)
if idx > num:
num = idx
num += 1
model.save_weights(os.path.join(output_path, '%d.ckpt' % num))
print('Save to %d.ckpt' % num)
elif args.mode == 'test':
assert args.ckpt, 'Missing argument "--ckpt".'
test_x, test_y, classes = prepare_voice_data(args.dataset,
args.signal_len,
args.n_mfcc, 'test')
model = model_func(args.signal_len, args.n_mfcc, len(classes))
model.load_weights(
os.path.join('results', 'voice', args.model,
'%s.ckpt' % args.ckpt))
loss, acc = model.evaluate(test_x, test_y, batch_size=args.batch_size)
print('loss: %f, acc: %f' % (loss, acc))
if args.dry_run:
print('Dry run, exiting.')
exit(0)
del args, rest
random.seed(config.opts.seed)
np.random.seed(config.opts.seed)
torch.random.manual_seed(config.opts.seed)
folder_base = (Path(config.opts.folder)).expanduser().resolve()
folder_data = folder_base / 'data'
folder_run = folder_base / 'runs' / config.opts.session
saver = Saver(folder_run)
logger = SummaryWriter(folder_run.as_posix())
ModelClass = load_class(config.model._class_)
net: nn.Module = ModelClass(
**{k: v
for k, v in config.model.items() if k != '_class_'})
net.to(config.opts.device)
OptimizerClass = load_class(config.optimizer._class_)
optimizer: optim.Optimizer = OptimizerClass(
params=net.parameters(),
**{k: v
for k, v in config.optimizer.items() if k != '_class_'})
if config.training.restore:
train_state = saver.load(model=net,
optimizer=optimizer,
device=config.training.device)
if len(sys.argv) < 2:
print 'Usage: {} <config>'.format(sys.argv[0])
sys.exit(-1)
config = ConfigParser.SafeConfigParser()
config.read(sys.argv[1])
#initalize data_feed
data_feed = KafkaClient(config.get('data_feed', 'host'))
consumer = SimpleConsumer(data_feed,
group=config.get('data_feed', 'group'),
topic=config.get('data_feed', 'topic'))
fetch_num_messages = config.getint('data_feed', 'fetch_num_messages')
#initialize data_sink
sink_config = dict(config.items('data_sink'))
class_name = sink_config.pop('class')
classobject = utils.load_class(class_name, storage.VerdictDB)
db = classobject(None, **sink_config)
removed = added = 0
for msg in consumer.get_messages(count=fetch_num_messages, block=False):
try:
msg = json.loads(msg.message.value)
blacklist_type = msg['type']
map(partial(db.insert, blacklist_type), msg['add'])
added += len(msg['add'])
map(partial(db.delete, blacklist_type), msg['remove'])
removed += len(msg['remove'])
except Exception as e:
print '[{}] Error ingesting msg {}'.format(e, msg)
print 'added %d urls removed %d urls' % (added, removed)
from utils import load_class
from django.db.models import Q
try:
from django.contrib.auth import get_user_model
User = get_user_model()
username_field = User.USERNAME_FIELD
except ImportError:
from django.contrib.auth.models import User
username_field = 'username'
IMAGESTORE_IMAGES_ON_PAGE = getattr(settings, 'IMAGESTORE_IMAGES_ON_PAGE', 20)
IMAGESTORE_ON_PAGE = getattr(settings, 'IMAGESTORE_ON_PAGE', 20)
ImageForm = load_class(
getattr(settings, 'IMAGESTORE_IMAGE_FORM', 'imagestore.forms.ImageForm'))
AlbumForm = load_class(
getattr(settings, 'IMAGESTORE_ALBUM_FORM', 'imagestore.forms.AlbumForm'))
class AlbumListView(ListView):
context_object_name = 'album_list'
template_name = 'imagestore/album_list.html'
paginate_by = getattr(settings, 'IMAGESTORE_ALBUMS_ON_PAGE', 20)
allow_empty = True
def get_queryset(self):
albums = Album.objects.filter(is_public=True).select_related('head')
self.e_context = dict()
if 'username' in self.kwargs:
user = get_object_or_404(**{
def _new_exporter(et_code, ef_code, st_code, options={}):
instance = new_export_instance(et_code, ef_code, st_code)
exp_class = utils.load_class(instance.export_type.path)
return exp_class(instance.pk, ef_code, et_code, options)
def package(self, name):
pkg_class = load_class(name)
return self._pkg_map[pkg_class]
dir_config = agent.AgentConfig()
dir_config.setName("Director")
dir_config.setBindAddress(sys.argv[4])
dir_config.setPort(int(sys.argv[5]))
pid = os.fork()
if pid == 0:
try:
os.close(1)
os.close(2)
os.close(3)
log = setup_logger("", "log/%s.log" % config.name)
log.debug("Instantiating agent")
if dir_config is None:
my_agent = load_class(agent_name)(config)
else:
my_agent = load_class(agent_name)(config, dir_config)
log.debug("Starting Agent")
my_agent.run()
except Exception, e:
log.exception("Exception thrown, caught in start_agent")
log.debug("Agent exiting")
sys._exit(0)
else:
log.debug("Agent Daemon %s started. PID: %d" % (agent_name, pid))
print "Done (%d)" % (pid)
sys.exit(0)
