def test_clone(self):
saved_locale = LocalizedTextFactory()
cloned_locale = saved_locale.clone()
self.assertEqual(
omit(saved_locale.__dict__, ['_state', 'id', 'created_at', 'internal_reference_id']),
omit(cloned_locale.__dict__, ['_state', 'id', 'created_at', 'internal_reference_id'])
)
self.assertNotEqual(saved_locale.id, cloned_locale.id)
self.assertIsNone(cloned_locale.internal_reference_id)
async def handle_request(request: web.Request, service: object, endpoint_cacher: object):
req_ctx = {
'method': request.method,
'url': service['targets'][service['cur_target_index']],
'params': dict(request.rel_url.query),
'data': await request.text(),
'cookies': dict(request.cookies),
'headers': pydash.omit(dict(request.headers), 'Host'),
}
req = None
req_cache = None
req_ctx_hash = None
if not pydash.is_empty(endpoint_cacher):
req_ctx_hash = Hasher.hash_sha_256(json.dumps(req_ctx))
req_cache = await EndpointCacher.get_cache(req_ctx_hash, DB.get_redis(request))
if pydash.is_empty(req_cache):
req = await Api.call(**req_ctx)
if pydash.is_empty(req_ctx_hash):
req_ctx_hash = Hasher.hash_sha_256(json.dumps(req_ctx))
not pydash.is_empty(endpoint_cacher) and queue_async_func.s({
'func': 'EndpointCacher.set_cache',
'args': [req_ctx_hash, req, int(endpoint_cacher['timeout']), 'redis'],
'kwargs': {}
}).apply_async()
else:
req = json.loads(req_cache)
cache_hit = True if not pydash.is_empty(req_cache) else False
return req, cache_hit
def main():
p = get_cli_args(args)
x_train, y_train, qid_train = load_svmlight_file(
p.train.xgboost_train_path, query_id=True) # pylint: disable=unbalanced-tuple-unpacking
x_test, y_test, qid_test = load_svmlight_file(p.train.xgboost_test_path,
query_id=True) # pylint: disable=unbalanced-tuple-unpacking
x_train = x_train.todense()
x_train = np.concatenate([
x_train, x_train[:, -2] / x_train[:, 2], x_train[:, -1] / x_train[:, 4]
], 1)
x_test = x_test.todense()
x_test = np.concatenate(
[x_test, x_test[:, -2] / x_test[:, 2], x_test[:, -1] / x_test[:, 4]],
1)
train_dmatrix = DMatrix(x_train, y_train)
test_dmatrix = DMatrix(x_test, y_test)
train_dmatrix.set_group([len(list(g)) for __, g in groupby(qid_train)])
test_dmatrix.set_group([len(list(g)) for __, g in groupby(qid_test)])
params = {
'objective': 'rank:pairwise',
'eval_metric': ['error', 'map@1'],
'tree_method': 'exact',
'eta': 0.1,
'gamma': 1.0,
'min_child_weight': 0.1,
'max_depth': 6
}
xgb_model = xgb.train(params,
train_dmatrix,
num_boost_round=100,
evals=[(test_dmatrix, 'validation')])
xgb_train_str = items_to_str(_.omit(params, 'objective',
'eval_metric').items(),
sort_by=itemgetter(0))
xgb_model.save_model(xgb_train_str + '_model.xgb')
def populate(self, **kwargs):
"""Extended ndb.Model populate method, so it can ignore properties, which are not
defined in model class without throwing error
"""
kwargs = _.omit(kwargs, Base.PUBLIC_PROPERTIES + ['key', 'id']) # We don't want to populate those properties
kwargs = _.pick(kwargs, _.keys(self._properties)) # We want to populate only real model properties
super(Base, self).populate(**kwargs)
def test_project_update(self):
# create instance
new_project = ProjectFactory.create(
visibility=VisibilityCharChoices.PUBLIC)
# authenticate
self.authenticate()
new_project = pydash.omit(
new_project,
[
"_state", "modified_at", "user", "event", "dtype",
"regional_project"
],
)
# update project name
new_project_name = "Mock Project for Update API Test"
new_country = country.CountryFactory()
new_district = district.DistrictFactory(country=new_country)
new_project["name"] = new_project_name
new_project["reporting_ns"] = new_country.id
new_project["project_country"] = new_country.id
new_project["event"] = new_project["event_id"]
new_project["project_districts"] = [new_district.id]
# submit update request
response = self.client.put(f"/api/v2/project/{new_project['id']}/",
new_project,
format='json')
# check response
self.assert_200(response)
self.assertMatchSnapshot(json.loads(response.content))
self.assertTrue(Project.objects.get(name=new_project_name))
def get_optim(cls, optim_param):
'''Helper to parse optim param and construct optim for net'''
optim_param = optim_param or {}
OptimClass = getattr(torch.optim, _.get(optim_param, 'name', 'Adam'))
optim_param = _.omit(optim_param, 'name')
optim = OptimClass(cls.parameters(), **optim_param)
return optim
def cli(ctx, config_path, broadcast: bool):
config = parse_config(ctx, config_path, Config)
config.contract_bin = StrictStr(
file_validator(config.contract_bin, config_path))
constructor_data = ""
if config.constructor_types and config.constructor_values:
constructor_types = yaml.full_load(config.constructor_types)
constructor_values = yaml.full_load(config.constructor_values)
constructor_data = eth_abi.encode_data(constructor_types,
constructor_values)[2:]
if ctx.obj.get("config"):
# noinspection PyUnresolvedReferences
print_json(config.dict() | {"constructor_data": constructor_data})
exit(0)
tx_params: Any = pydash.omit(config.dict(), "node", "contract_bin",
"constructor_types", "constructor_values")
tx_params["data"] = config.contract_bin + constructor_data
if tx_params["nonce"] is None:
tx_params["nonce"] = get_nonce_or_exit(
config.node, eth_account.private_to_address(config.private_key))
signed_tx = eth_tx.sign_tx(**tx_params)
if broadcast:
res = eth_rpc.eth_send_raw_transaction(config.node, signed_tx.raw_tx)
print_json({"result": res.ok_or_error})
else:
decoded = eth_utils.to_human_readable_tx(
eth_tx.decode_raw_tx(signed_tx.raw_tx))
print_json(signed_tx.dict() | {"decoded": decoded})
async def proxy(request: web.Request, handler: web.RequestHandler):
try:
req_start_time = time()
if pydash.starts_with(request.path_qs, '/raven'):
return await handler(request)
service = Regex.best_match(await Regex.get_matched_paths(request.path, DB.get(request, service_controller.table)))
await handle_service(service, request.remote)
rate_limiter_rules = await RateLimiter.get_rule_by_service_id(str(service['_id']), DB.get_redis(request))
rate_limiter_rule = rate_limiter_rules[0] if rate_limiter_rules else None
await handle_rate_limiter(request, str(service['_id']), rate_limiter_rule)
breakers = await CircuitBreaker.get_by_service_id(str(service['_id']), DB.get(request, circuit_breaker_controller.table))
breaker = breakers[0] if breakers else None
request_validators = await RequestValidator.get_by_service_id(str(service['_id']), DB.get(request, request_validator_controller.table))
request_validator = request_validators[0] if request_validators else None
endpoint_cachers = not pydash.is_empty(service) and await EndpointCacher.get_by_service_id(str(service['_id']), DB.get_redis(request)) or None
endpoint_cacher = endpoint_cachers[0] if endpoint_cachers else None
await handle_request_validator(request_validator, json.loads(await request.text()), request.method)
req, req_cache_hit = await handle_request(request, service, endpoint_cacher)
checks = []
if not pydash.is_empty(
breaker) and breaker['status'] == CircuitBreakerStatus.ON.name:
if req['status'] in breaker['status_codes']:
checks.append(handle_circuit_breaker(
breaker, service, request, req))
else:
await CircuitBreaker.incr_count(str(breaker['_id']), DB.get_redis(request))
queue_async_func.s({
'func': 'Service.advance_target',
'args': [str(service['_id']), f'mongo:{service_controller.table}'],
'kwargs': {}
}).apply_async()
req_finish_time = time()
req_elapsed_time = req_finish_time - req_start_time
checks.append(handle_insights(request, req, str(
service['_id']), req_elapsed_time, req_cache_hit))
await Async.all(checks)
return web.Response(
body=Bytes.decode_bytes(
req['body_bytes']),
status=req['status'],
content_type=req['content_type'],
headers=CIMultiDict(
pydash.omit(
req['headers'],
'Content-Type',
'Transfer-Encoding',
'Content-Encoding')))
except Exception as err:
return Error.handle(err)
def get_loss_fn(cls, loss_param):
'''Helper to parse loss param and construct loss_fn for net'''
loss_param = loss_param or {}
loss_fn = getattr(F, _.get(loss_param, 'name', 'mse_loss'))
loss_param = _.omit(loss_param, 'name')
if not _.is_empty(loss_param):
loss_fn = partial(loss_fn, **loss_param)
return loss_fn
def get_object_from_id(oid, pid):
hit_oid = py_.find(objects['thing-descriptions'], {'oid': oid})
if hit_oid:
hit_values = py_.find(hit_oid['values'], {'pid': pid})
return jsonify(py_.omit(hit_values,'pid'))
else:
content = {'Device or property not found'}
abort(404)
def get_lr_scheduler(cls, lr_scheduler_spec):
'''Helper to parse lr_scheduler param and construct Pytorch optim.lr_scheduler'''
if ps.is_empty(lr_scheduler_spec):
lr_scheduler = NoOpLRScheduler()
else:
LRSchedulerClass = getattr(torch.optim.lr_scheduler, lr_scheduler_spec['name'])
lr_scheduler_spec = ps.omit(lr_scheduler_spec, 'name')
lr_scheduler = LRSchedulerClass(cls.optim, **lr_scheduler_spec)
return lr_scheduler
def test_update(bibliography_repository):
bibliography_entry = BibliographyEntryFactory.build()
updated_entry = pydash.omit({**bibliography_entry, "title": "New Title"}, "PMID")
bibliography_repository.create(bibliography_entry)
bibliography_repository.update(updated_entry)
assert (
bibliography_repository.query_by_id(bibliography_entry["id"]) == updated_entry
)
def format_document(document: object) -> object:
"""
formats mongo document
@param document: (object) document to format
"""
formatted = pydash.omit(document, '_id')
formatted['_id'] = document['_id']['$oid']
return formatted
def get_optim(net, optim_spec):
'''Helper to parse optim param and construct optim for net'''
OptimClass = getattr(torch.optim, optim_spec['name'])
optim_spec = ps.omit(optim_spec, 'name')
if torch.is_tensor(net): # for non-net tensor variable
optim = OptimClass([net], **optim_spec)
else:
optim = OptimClass(net.parameters(), **optim_spec)
return optim
def __init__(self, n_estimators=24, max_depth=10, **kwargs):
super().__init__()
self.kwargs = {
**self.xgb_defaults,
'n_estimators': n_estimators,
'max_depth': max_depth,
**kwargs,
}
if self.kwargs.get('booster') == 'gblinear':
self.kwargs = pydash.omit(self.kwargs, *['max_depth'])
def ActiveDiscovery(self):
self._logger.info("Active discovery - (" + str(len(objects['thing-descriptions'])) + ' nodes)')
#Send request to agent
r = requests.post(globals.config['agent_endpoint'] + '/agent/objects', \
data=json.dumps(py_.omit(objects, 'values')), \
headers={"Content-Type": "application/json"})
# ToDo - Handle response (for potential errors)
pass
async def generate_token(payload: dict, db):
sanitized_payload = pydash.merge(
pydash.omit(payload, 'password', 'token'),
{'timestamp': repr(time.time())}
)
token = Token.generate(sanitized_payload)
update_ctx = {
'token': token
}
await Admin.update(payload['_id'], update_ctx, db)
async def set_cache(_hash: str, ctx: object, timeout: int, db: AioRedis):
"""
sets cache
@param _hash: (str) hash of request
@param ctx: (object) body of response
@param db: redis instance
"""
omit_keys = list(filter(lambda key: ctx[key] is None, ctx.keys()))
await db.set(_hash, json.dumps(pydash.omit(ctx, *omit_keys)))
await db.expire(_hash, timeout)
def update_object_property(oid, pid):
body = request.json
hit_oid = py_.find(objects['thing-descriptions'], {'oid': oid})
if hit_oid:
hit_values = py_.find(hit_oid['values'], {'pid': pid})
hit_values['value'] = body['value']
hit_values['timestamp'] = datetime.datetime.now().isoformat()
return jsonify(py_.omit(hit_values,'pid'))
else:
content = {'please move along': 'nothing to see here'}
abort(404)
def __init__(self, *args, **kwargs):
"""
this class will be used to pass allong values to the argparse add_argument function
"""
#group is seperated out because it is not an original argument to the argparse add_argument function call
#seperating it out allows us to select a group for our argument to be placed in if it is indicated
group = get(kwargs, 'group')
if group:
self.group = group
self.args = args
self.kwargs = omit(kwargs, 'group')
def response_for_exception(request, exception):
response = base_response_for_exception(request, exception)
if isinstance(exception, ValidationError):
dont_flash = [
'password',
'password_confirmation',
]
old_input = pydash.merge(parser.parse(request.GET.urlencode()),
parser.parse(request.POST.urlencode()))
errors = exception.message_dict if hasattr(
exception, 'error_dict') else {
'__all__': exception.messages
}
response = HttpRedirector(request) \
.back() \
.with_input(pydash.omit(old_input, dont_flash)) \
.with_errors(errors)
if request.match('api/*') and not isinstance(response, JsonResponse):
message = str(exception)
data = {}
if isinstance(exception, ValidationError):
status_code = 422
message = _('Invalid data!')
data['errors'] = exception.message_dict if hasattr(
exception, 'error_dict') else {
'__all__': exception.messages
}
elif isinstance(exception, Http404):
status_code = 404
elif isinstance(exception, BadRequest):
status_code = 400
elif isinstance(exception, AuthenticationException):
status_code = 401
elif isinstance(exception, PermissionDenied):
status_code = 403
else:
status_code = 500
if not settings.DEBUG:
message = _('Something went wrong')
if len(message.strip()) == 0:
message = _(status_codes._codes[status_code][0])
return JsonResponse(pydash.merge({'message': message}, data),
status=status_code)
return response
async def patch_handler(request: web.Request):
try:
ctx = json.loads(await request.text())
circuit_breaker_id = request.rel_url.query['id']
Validate.validate_schema(ctx, circuit_breaker_validator)
Validate.validate_object_id(circuit_breaker_id)
await CircuitBreaker.update(circuit_breaker_id, pydash.omit(ctx, 'id'), DB.get(request, table))
return web.json_response({
'message': 'Circuit breaker updated',
})
except Exception as err:
return Error.handle(err)
def main():
p = get_cli_args(args)
try:
open('train.bin').close()
open('eval.bin').close()
lgb_train = lgb.Dataset('train.bin')
lgb_eval = lgb.Dataset('eval.bin', reference=lgb_train)
except:
x_train, y_train, qid_train = load_svmlight_file(
p.train.xgboost_train_path, query_id=True) # pylint: disable=unbalanced-tuple-unpacking
x_test, y_test, qid_test = load_svmlight_file(
p.train.xgboost_test_path, query_id=True) # pylint: disable=unbalanced-tuple-unpacking
x_train = x_train.todense()
x_test = x_test.todense()
lgb_train = lgb.Dataset(
np.array(x_train),
np.array(y_train.squeeze()),
group=[len(list(g)) for __, g in groupby(qid_train)])
lgb_eval = lgb.Dataset(
np.array(x_test),
np.array(y_test.squeeze()),
reference=lgb_train,
group=[len(list(g)) for __, g in groupby(qid_test)])
lgb_train.save_binary("train.bin")
lgb_eval.save_binary("eval.bin")
params = {
'boosting_type': 'gbdt',
'objective': 'lambdarank',
'metric': {'ndcg'},
'ndcg_eval_at': [1],
'metric_freq': 1,
'max_bin': 255,
'num_trees': 100,
'num_leaves': 100,
'learning_rate': 0.1,
'num_iterations': 100,
'num_threads': 8,
'feature_fraction': 1.0,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'verbose': 0,
}
gbm = lgb.train(params,
lgb_train,
num_boost_round=100,
valid_sets=lgb_eval)
xgb_train_str = items_to_str(_.omit(params, 'objective',
'eval_metric').items(),
sort_by=itemgetter(0))
preds = gbm.predict(lgb_eval)
print((y_test != preds).nonzero())
gbm.save_model('model' + xgb_train_str + '.light')
async def patch_handler(request: web.Request):
try:
ctx = json.loads(await request.text())
_id = request.rel_url.query.get('id')
Validate.validate_schema(ctx, endpoint_cache_validator)
Validate.validate_object_id(_id)
await EndpointCacher.update(_id, pydash.omit(ctx, 'service_id', 'response_codes'), DB.get_redis(request))
return web.json_response({
'message': 'Endpoint cache updated',
'status_code': 200
})
except Exception as err:
return Error.handle(err)
def get_lr_scheduler(optim, lr_scheduler_spec):
'''Helper to parse lr_scheduler param and construct Pytorch optim.lr_scheduler'''
if ps.is_empty(lr_scheduler_spec):
lr_scheduler = NoOpLRScheduler(optim)
elif lr_scheduler_spec['name'] == 'LinearToZero':
LRSchedulerClass = getattr(torch.optim.lr_scheduler, 'LambdaLR')
frame = float(lr_scheduler_spec['frame'])
lr_scheduler = LRSchedulerClass(optim, lr_lambda=lambda x: 1 - x / frame)
else:
LRSchedulerClass = getattr(torch.optim.lr_scheduler, lr_scheduler_spec['name'])
lr_scheduler_spec = ps.omit(lr_scheduler_spec, 'name')
lr_scheduler = LRSchedulerClass(optim, **lr_scheduler_spec)
return lr_scheduler
async def patch_handler(request: web.Request):
try:
ctx = json.loads(await request.text())
service_id = request.rel_url.query['id']
Validate.validate_object_id(service_id)
Validate.validate_schema(ctx, insights_validator)
await Insights.update(service_id, pydash.omit(ctx, 'id'),
DB.get(request, table))
return web.json_response({
'message': 'insight updated',
})
except Exception as err:
return Error.handle(err)
def connect(self):
"""Connect to database"""
if self._conx:
raise ex.ConnectionError('Database already connected')
if 'file_options' in self.connect_args:
file = 'file:{}?{}'.format(self.file,
self.connect_args['file_options'])
else:
file = self.file
# self._conx = sqlite3.connect(file, isolation_level=None, **pydash.omit(kwargs, 'file_options'))
self._conx = sqlite3.connect(
file, **pydash.omit(self.connect_args, 'file_options'))
self._c = self._conx.cursor()
return self
def put(self):
data = request.get_json(force=True)
objToPut = []
for item in data['body']:
filters = self.makeFilter(item)
item = omit(item, ['_id', 'updated_at'])
item = map_values_deep(item, updaterIds)
objToPut.append({'filter': filters, 'data': item})
return self.entity().batch_process(objToPut)
async def login_handler(request: web.Request):
try:
ctx = json.loads(await request.text())
verified = await Admin.verify_password(ctx['username'],
ctx['password'],
DB.get(request, table))
if not verified:
raise Exception({'message': 'Unathorized', 'status_code': 401})
admin = await Admin.get_by_username(ctx['username'],
DB.get(request, table))
sanitized_admin = pydash.omit(admin, 'password')
return web.json_response(
{'data': DB.format_document(Bson.to_json(sanitized_admin))})
except Exception as err:
return Error.handle(err)
def get_data(col_name, output='df', **kwargs):
"""
从 mongodb 下载 股票 日数据
mongolib需要知道column
:param col_name:
:return:
list: OCLH
"""
query = {}
date_query = {}
if 'fromdate' in kwargs.keys() and kwargs['fromdate']:
date_query['$gte'] = kwargs['fromdate']
if 'todate' in kwargs.keys() and kwargs['todate']:
date_query['$lt'] = kwargs['todate']
if date_query:
query['date'] = date_query
# print('query', query)
col = db[col_name]
cursor = col.find(query)
if output == 'df':
df = pd.DataFrame(list(cursor))
del df['_id']
del df['code']
df['date'] = df['date'].astype('datetime64')
df = df.set_index('date')
cols = ['open', 'high', 'close', 'low', 'volume']
df = df.ix[:, cols]
return df
elif output == 'obj':
obj_list = list(cursor)
return [pydash.omit(item, '_id', 'code') for item in obj_list]
elif output == 'list':
obj_list = list(cursor)
return [[
item['date'], item['open'], item['close'], item['low'],
item['high'], item['volume']
] for item in obj_list]
def test_project_create(self):
# authenticate
new_user = UserFactory.create()
self.authenticate(new_user)
# create project
new_project_name = "Mock Project for Create API Test"
new_project = ProjectFactory.stub(
name=new_project_name,
visibility=VisibilityCharChoices.PUBLIC,
user=new_user,
)
new_country = country.CountryFactory()
new_district = district.DistrictFactory(country=new_country)
new_project = pydash.omit(
new_project,
[
"user",
"reporting_ns",
"project_country",
"event",
"dtype",
"regional_project",
],
)
new_project["reporting_ns"] = new_country.id
new_project["project_country"] = new_country.id
new_project["project_districts"] = [new_district.id]
# submit create request
response = self.client.post("/api/v2/project/",
new_project,
format='json')
# check response
self.assert_201(response)
self.assertMatchSnapshot(json.loads(response.content))
self.assertTrue(Project.objects.get(name=new_project_name))
def set(self, obj):
source.set(self.collection, obj['id'], omit(obj, 'id'))
def get_loss_fn(cls, loss_spec):
'''Helper to parse loss param and construct loss_fn for net'''
LossClass = getattr(nn, loss_spec['name'])
loss_spec = ps.omit(loss_spec, 'name')
loss_fn = LossClass(**loss_spec)
return loss_fn
def get_optim(cls, optim_spec):
'''Helper to parse optim param and construct optim for net'''
OptimClass = getattr(torch.optim, optim_spec['name'])
optim_spec = ps.omit(optim_spec, 'name')
optim = OptimClass(cls.parameters(), **optim_spec)
return optim
def test_omit(case, expected):
assert _.omit(*case) == expected
