def test_binary(self):
array = np.array([
bson.Binary(b"binary_0"),
bson.Binary(b"bin1"),
bson.Binary(b"utf8-2")
],
dtype=np.dtype('<V15'))
bsonnumpy.ndarray_to_sequence(array)
def test_list(self):
"list type"
self.query('DEL/hello')
self.query('RPUSH/hello/abc')
self.query('RPUSH/hello/def')
f = self.query('LRANGE/hello/0/-1.bson')
obj = bson.decode_all(f.read())
self.assertTrue(obj == [{u'LRANGE': [bson.Binary('abc', 0), bson.Binary('def', 0)]}])
def test_GET_returns_all_details(self):
node = factory.make_node()
lshw_result = self.make_lshw_result(node)
lldp_result = self.make_lldp_result(node)
self.assertDictEqual(
{"lshw": bson.Binary(lshw_result.data),
"lldp": bson.Binary(lldp_result.data)},
self.get_details(node))
def serialize(self):
output = super().serialize()
output['trans_error_diff'] = bson.Binary(
pickle.dumps(self.translational_error_difference,
protocol=pickle.HIGHEST_PROTOCOL))
output['rot_error_diff'] = bson.Binary(
pickle.dumps(self.rotational_error_difference,
protocol=pickle.HIGHEST_PROTOCOL))
output['settings'] = self.settings
return output
def serialize(self):
serialized = super().serialize()
serialized['ground_truth_trajectory'] = bson.Binary(
pickle.dumps(self.ground_truth_trajectory,
protocol=pickle.HIGHEST_PROTOCOL))
serialized['trajectory'] = bson.Binary(
pickle.dumps(self.trajectory, protocol=pickle.HIGHEST_PROTOCOL))
serialized['tracking_stats'] = bson.Binary(
pickle.dumps(self.tracking_stats,
protocol=pickle.HIGHEST_PROTOCOL))
return serialized
def run_task(self, fw_spec):
self.xml = self.get("xml", None)
self.kmesh_factor = self.get("kmesh_factor", 1)
with open("KPOINTS", "r") as f:
kpoints = f.readlines()
with open("INCAR", "r") as f:
incar = f.readlines()
if self.kmesh_factor > 1:
shutil.copyfile("INCAR", "INCAR.nscf")
shutil.copyfile("KPOINTS", "KPOINTS.nscf")
shutil.copyfile("INCAR.Static", "INCAR")
shutil.copyfile("KPOINTS.Static", "KPOINTS")
if self.xml is not None:
with open(self.xml, 'rb') as f:
xmldata = f.read()
binxmldata = gzip.compress(bytes(xmldata))
with open("DOSCAR", 'rb') as f:
doscar = f.read()
bindoscar = gzip.compress(bytes(doscar))
#with gzip.open("zen.txt.gz", "wb") as f:
#f.write(bindata)
self.db_file = env_chk(self.get("db_file"), fw_spec)
self.vasp_db = VaspCalcDb.from_db_file(self.db_file, admin=True)
structure = self.get('structure', Structure.from_file('POSCAR'))
xml_data = {
'metadata': {
'tag': self.get('tag')
},
'hostname':
socket.gethostname(),
self.xml.replace(".", "_") + "_gz":
bson.Binary(pickle.dumps(binxmldata)),
'DOSCAR_gz':
bson.Binary(pickle.dumps(bindoscar)),
'volume':
structure.volume,
'INCAR':
incar,
'KPOINTS':
kpoints,
'VASP':
get_code_version(),
'last_updated':
datetime.datetime.utcnow(),
'US_eastern_time':
datetime.datetime.now(pytz.timezone('US/Eastern')),
'structure':
structure.as_dict(),
'formula_pretty':
structure.composition.reduced_formula
}
self.vasp_db.db['xmlgz'].insert_one(xml_data)
def schedule(cls, subtask, first, interval, *args, **kwargs):
'''First is a timestamp, interval is in seconds'''
t = cls.n()
t._state.data.update(next=first,
interval=interval,
subtask_id=subtask.id,
subtask_repr=repr(subtask),
args=bson.Binary(dumps(args)),
kwargs=bson.Binary(dumps(kwargs)))
t._state.status = 'active'
t._state.m.save()
t._schedule_subtask(subtask)
return t
def _wait_for_reroute(self, migration_opts):
self.logger.info(
"Waiting for donor replica set '%s' for migration '%s' to reroute at least one "
+ "conflicting command", migration_opts.get_donor_name(),
migration_opts.migration_id)
while True:
donor_primary = migration_opts.get_donor_primary()
donor_primary_client = donor_primary.mongo_client()
try:
doc = donor_primary_client["testTenantMigration"][
"rerouted"].find_one({
"_id":
bson.Binary(migration_opts.migration_id.bytes, 4)
})
if doc is not None:
return
except pymongo.errors.PyMongoError:
self.logger.exception(
"Error running find command on donor primary on port %d of replica set '%s'.",
donor_primary.port, migration_opts.get_donor_name())
raise
time.sleep(
_TenantMigrationThread.MIGRATION_STATE_POLL_INTERVAL_SECS)
def schedule_next(self, status):
if self.check_timings:
self.logger.info(
"Timings: %s", ", ".join("%s = %.2fms" % (n, t * 1000)
for n, t in self.check_timings))
super(MODiscoveryJob, self).schedule_next(status)
# Update alarm statuses
self.update_alarms()
# Write job log
key = "discovery-%s-%s" % (self.attrs[self.ATTR_CLASS],
self.attrs[self.ATTR_KEY])
problems = {}
for p in self.problems:
if p["check"] in problems and p["path"]:
problems[p["check"]][p["path"]] = p["message"]
elif p["check"] in problems and not p["path"]:
# p["path"] == ""
problems[p["check"]][p["path"]] += "; %s" % p["message"]
else:
problems[p["check"]] = {p["path"]: p["message"]}
get_db()["noc.joblog"].update({"_id": key}, {
"$set": {
"log": bson.Binary(zlib.compress(self.out_buffer.getvalue())),
"problems": problems
}
},
upsert=True)
async def __GenSwitchMonthlyUsage(para, result_no):
"""
:param para:
:param result_no: url or bytes
:return: {"url" or "bytes"}
"""
from io import BytesIO
result = {}
arg = await ListSwitchMonthlyUsage(para)
arg['year'], arg['month'] = para['year'], para['month']
if result_no == 'url':
file_name = "{0}.xlsx".format(bson.ObjectId())
file_path = os_path.join(g_vars.g_cfg['web']['export_path'], file_name)
await threads.deferToThread(monthly_switch_usage_report.Make,
file_path, arg)
result['url'] = os_path.join(
'/', rgw_consts.Node_URLs.EXPORT_FMT.format(file_name))
else:
io_obj = BytesIO()
await threads.deferToThread(monthly_switch_usage_report.Make, io_obj,
arg)
io_obj.seek(0)
result['bytes'] = bson.Binary(io_obj.read())
io_obj.close()
return result
def test_set(self):
"success type (+OK)"
self.query('DEL/hello')
f = self.query('SET/hello/world.bson')
self.assertTrue(f.headers.getheader('Content-Type') == 'application/bson')
obj = bson.decode_all(f.read())
self.assertTrue(obj == [{u'SET': [True, bson.Binary('OK', 0)]}])
def serialize(self):
serialized = super().serialize()
serialized['trajectory'] = bson.Binary(
pickle.dumps(self._trajectory, protocol=pickle.HIGHEST_PROTOCOL))
serialized['trajectory_source'] = self._trajectory_source
serialized['sequence_type'] = self._sequence_type.value
return serialized
def _cast_value_for_type(self, type, value):
if value is None:
return None
def _check(*types):
return type in types or isinstance(type, types)
if _check(S.DateTime, datetime.datetime):
if isinstance(value, string_type):
return datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S")
else:
return value
elif _check(S.Binary):
return bson.Binary(value)
elif _check(S.Int, int):
return int(value)
elif _check(S.Bool, bool):
if value in ('true', 'false'):
return value == 'true' and True or False
else:
return bool(value)
elif _check(S.Object, dict):
if isinstance(type, S.Object):
type = type.fields
return dict((k, self._cast_value_for_type(type[k], v))
for k, v in value.items())
elif _check(S.Array, list):
if not isinstance(value, (list, tuple)):
value = [value]
if isinstance(type, S.Array):
type = [type.field_type]
return [self._cast_value_for_type(type[0], v) for v in value]
else:
return value
def test_GET_returns_only_those_details_that_exist(self):
node = factory.make_node()
lshw_result = self.make_lshw_result(node)
self.assertDictEqual(
{"lshw": bson.Binary(lshw_result.data),
"lldp": None},
self.get_details(node))
def replace_media_links(body, media_library, db, destination_url,
source_base_url):
for link in media_library:
if link in body:
# This is making some big assumptions about the structure
# of the media URL, that it's like
# http://emptysquare.net/blog/wp-content/uploads/2011/10/img.png
url = link.split('/uploads/')[-1]
media_doc = db.media.find_one({'_id': link})
if not media_doc:
# TODO: remove
cache_path = os.path.join('cache', slugify(link))
if os.path.exists(cache_path):
content, content_type = pickle.load(open(cache_path))
else:
r = requests.get(link)
content = r.content
content_type = r.headers['content-type']
if not os.path.exists('cache'):
os.mkdir('cache')
with open(cache_path, 'w+') as f:
pickle.dump((content, content_type), f)
db.media.insert({
'content': bson.Binary(content),
'type': content_type,
'_id': url,
'mod': datetime.datetime.utcnow(),
})
body = body.replace(link,
os.path.join(destination_url, 'media', url))
return body
def insert_one(self):
print('upsert one')
try:
tic = time.time()
# 11kb ascii block random worst case scenario for compression
noise = ''.join(
random.choices(string.ascii_uppercase + string.digits +
string.ascii_lowercase,
k=1024 * 11))
# 16^7 = 268,435,456 possible _id values
id = ''.join(
random.choices(os.environ['KEYSPACE'],
k=int(os.environ['KEYCOUNT'])))
obj = {
"value": bson.Binary(pickle.dumps(noise)),
"created": datetime.datetime.now(),
"location": os.environ['ISOLOC']
}
output = self.db.queue.update_one({"_id": id}, {"$set": obj},
upsert=True)
self.environment.events.request_success.fire(
request_type="pymongo",
name="uc_write",
response_time=(time.time() - tic),
response_length=0)
except KeyboardInterrupt:
print
sys.exit(0)
except Exception as e:
print(f'{datetime.datetime.now()} - DB-CONNECTION-PROBLEM: '
f'{str(e)}')
connect_problem = True
def test_Keep_accepts_bsonBinary(self):
from niprov.pictures import PictureCache
myImg = Mock()
myImg.provenance = {'id': '007'}
pictures = PictureCache(sentinel.dependencies)
pictures.keep(bson.Binary('/x10/x05/x5f'), for_=myImg)
self.assertEqual('/x10/x05/x5f', pictures.getBytes(for_=myImg))
async def __GenSensorMinsAvgLog(para, result_no):
"""
:param para: {"sensorids":[], "start_ts": timestamp, "stop_ts": timestamp,
"mins_interval": integer, 'tz_offset': integer}
or {"sensorids":[], "hours": hours range, "mins_interval": integer,
'tz_offset': integer}
:param result_no: url or bytes
:return: {"url" or "bytes"}
"""
from io import BytesIO
result = {}
log_tbl = await FindSensorMinsAvgLog2(para)
tz_obj = await api_core.SysCfg.GetTimezone()
log_tbl['tz_offset'] = tz_obj.utcoffset(
rg_lib.DateTime.utc()).total_seconds()
if result_no == 'url':
file_name = "{0}.xlsx".format(bson.ObjectId())
file_path = os_path.join(g_vars.g_cfg['web']['export_path'], file_name)
await threads.deferToThread(sensor_log_report.Make, file_path, log_tbl)
result['url'] = os_path.join(
'/', rgw_consts.Node_URLs.EXPORT_FMT.format(file_name))
else:
io_obj = BytesIO()
await threads.deferToThread(sensor_log_report.Make, io_obj, log_tbl)
io_obj.seek(0)
result['bytes'] = bson.Binary(io_obj.read())
io_obj.close()
return result
def set(self, key, value, waitIndex=False, session=None):
if 'passwd' in value:
passwd = j.core.osis.encrypt(value['passwd'] or '')
value['passwd'] = bson.Binary(passwd)
guid, new, changed = super(parentclass, self).set(key,
value,
session=session)
g = j.core.osis.cmds._getOsisInstanceForCat("system", "group")
if changed:
for group in value['groups']:
grkey = "%s_%s" % (value['gid'], group)
if g.exists(grkey, session=session) == False:
#group does not exist yet, create
grnew = g.getObject()
grnew.id = group
grnew.gid = value['gid']
grnew.domain = value['domain']
grnew.users = [value['id']]
grguid, a, b = g.set(grnew.guid,
grnew.__dict__,
session=session)
else:
gr = g.get(grkey, session=session)
if value['id'] not in gr['users']:
gr['users'].append(value['id'])
g.set(gr['guid'], gr, session=session)
return guid, new, changed
def _wait_for_reroute_or_test_completion(self, migration_opts):
self.logger.info(
"Waiting for donor replica set '%s' for migration '%s' to reroute at least one "
+ "conflicting command. Stop waiting when the test finishes.",
migration_opts.get_donor_name(), migration_opts.migration_id)
start_time = time.time()
while not self.__lifecycle.is_test_finished():
donor_primary = migration_opts.get_donor_primary()
donor_primary_client = donor_primary.mongo_client()
try:
doc = donor_primary_client["testTenantMigration"][
"rerouted"].find_one({
"_id":
bson.Binary(migration_opts.migration_id.bytes, 4)
})
if doc is not None:
return
except pymongo.errors.PyMongoError:
end_time = time.time()
self.logger.exception(
"Error running find command on donor primary on port %d of replica set '%s' "
+ "after waiting for reroute for %0d ms",
donor_primary.port, migration_opts.get_donor_name(),
(end_time - start_time) * 1000)
raise
time.sleep(self.MIGRATION_STATE_POLL_INTERVAL_SECS)
def to_dict(self, convert_numpy_arrays_to=None, fields_to_ignore=None, nan_to_none=False,
use_decimal=False):
result = {}
if fields_to_ignore is None:
fields_to_ignore = tuple()
for k, v in self.get_fields_data():
if k in fields_to_ignore:
continue
if isinstance(v, Model):
result[k] = v.to_dict(convert_numpy_arrays_to=convert_numpy_arrays_to,
fields_to_ignore=fields_to_ignore,
nan_to_none=nan_to_none,
use_decimal=use_decimal)
elif isinstance(v, list):
result[k] = [el.to_dict(convert_numpy_arrays_to=convert_numpy_arrays_to,
fields_to_ignore=fields_to_ignore,
nan_to_none=nan_to_none,
use_decimal=use_decimal) for el in v]
elif isinstance(v, np.ndarray) and convert_numpy_arrays_to is not None:
if convert_numpy_arrays_to == 'list':
if use_decimal:
result[k] = [decimal.Decimal("%f" % a) if isinstance(a, float) else a for a in v.tolist()]
else:
result[k] = v.tolist()
elif convert_numpy_arrays_to == 'bytes':
result[k] = bson.Binary(v.tostring())
else:
raise ValueError('convert_numpy_arrays_to must be "list" or "bytes"')
elif nan_to_none and isinstance(v, float) and not np.isfinite(v):
result[k] = None
elif use_decimal and isinstance(v, float):
result[k] = decimal.Decimal("%f" % v)
else:
result[k] = v
return result
def _forget_migration(self, migration_opts):
"""Run donorForgetMigration to garbage collection the tenant migration denoted by migration_opts'."""
self.logger.info("Forgetting tenant migration: %s.",
str(migration_opts))
donor_primary = migration_opts.get_donor_primary()
donor_primary_client = donor_primary.mongo_client()
try:
donor_primary_client.admin.command(
{
"donorForgetMigration":
1,
"migrationId":
bson.Binary(migration_opts.migration_id.bytes, 4)
},
bson.codec_options.CodecOptions(
uuid_representation=bson.binary.UUID_SUBTYPE))
except pymongo.errors.OperationFailure as err:
if err.code != self.NO_SUCH_MIGRATION_ERR_CODE:
raise
# The fixture was restarted.
self.logger.info(
"Could not find tenant migration '%s' on donor primary on" +
" port %d of replica set '%s'.", migration_opts.migration_id,
donor_primary.port, migration_opts.get_donor_name())
except pymongo.errors.PyMongoError:
self.logger.exception(
"Error forgetting tenant migration '%s' on donor primary on" +
" port %d of replica set '%s'.", migration_opts.migration_id,
donor_primary.port, migration_opts.get_donor_name())
raise
def main():
"""For audio downloading and persistence"""
con = Connection()
db = con.audiodb
if use_proxy == True:
install_proxy()
words = open(filename).read().split()
for wordname in words:
if wordname.isalpha() == False:
continue
wordfilename = wordname.lower() + '.mp3'
wordrecord = wordname.lower() + '_mp3'
num = db.audiodb.find({"file_name": wordrecord}).count()
if num > 0:
print("Already got " + "\"" + wordname + "\"")
continue
try:
mp3file = get_data_from_req(
"https://ssl.gstatic.com/dictionary/static/sounds/de/0/" +
wordfilename)
if not is_file_valid(mp3file):
continue
bin = bson.Binary(mp3file.read())
db.audiodb.save({"file_name": wordrecord, "mp3": bin})
print("Done!")
except Exception as e:
print(e)
def build_image(image_data, image_content_type):
built_image = {
'image_data': bson.Binary(image_data),
'image_content_type': image_content_type
}
return built_image
def _start_and_wait_for_migration(self, migration_opts): # noqa: D205,D400
"""Run donorStartMigration to start a tenant migration based on 'migration_opts', wait for
the migration decision and return the last response for donorStartMigration.
"""
cmd_obj = {
"donorStartMigration":
1,
"migrationId":
bson.Binary(migration_opts.migration_id.bytes, 4),
"recipientConnectionString":
migration_opts.recipient_rs.get_driver_connection_url(),
"tenantId":
migration_opts.tenant_id,
"readPreference":
migration_opts.read_preference,
"donorCertificateForRecipient":
get_certificate_and_private_key("jstests/libs/tenant_migration_donor.pem"),
"recipientCertificateForDonor":
get_certificate_and_private_key("jstests/libs/tenant_migration_recipient.pem"),
}
donor_primary = migration_opts.get_donor_primary()
self.logger.info(
"Starting tenant migration '%s' on donor primary on port %d of replica set '%s'.",
migration_opts.migration_id, donor_primary.port, migration_opts.get_donor_name())
while True:
try:
# Keep polling the migration state until the migration completes.
donor_primary_client = self._create_client(donor_primary)
res = donor_primary_client.admin.command(
cmd_obj,
bson.codec_options.CodecOptions(uuid_representation=bson.binary.UUID_SUBTYPE))
except (pymongo.errors.AutoReconnect, pymongo.errors.NotMasterError):
donor_primary = migration_opts.get_donor_primary()
self.logger.info(
"Retrying tenant migration '%s' against donor primary on port %d of replica " +
"set '%s'.", migration_opts.migration_id, donor_primary.port,
migration_opts.get_donor_name())
continue
if res["state"] == "committed":
self.logger.info(
"Tenant migration '%s' with donor primary on port %d of replica set '%s' has " +
"committed.", migration_opts.migration_id, donor_primary.port,
migration_opts.get_donor_name())
return res
if res["state"] == "aborted":
self.logger.info(
"Tenant migration '%s' with donor primary on port %d of replica set '%s' has " +
"aborted: %s.", migration_opts.migration_id, donor_primary.port,
migration_opts.get_donor_name(), str(res))
return res
if not res["ok"]:
self.logger.info(
"Tenant migration '%s' with donor primary on port %d of replica set '%s' has " +
"failed: %s.", migration_opts.migration_id, donor_primary.port,
migration_opts.get_donor_name(), str(res))
return res
time.sleep(self.POLL_INTERVAL_SECS)
def recalc_score_doc(self, doc):
scores = []
for sig, count in doc["sigs"].items():
sig = self.sigset.get_sig(bson.Binary(binascii.unhexlify(sig)))
scores.append(sig["score"] * 4 / (1 + 3 / count))
score = pow(sum(score**2 for score in scores), 0.5)
doc["score"] = score
conn[self.db][self.collection].save(doc)
def save_definitions_to_meta_data():
from DB import MetaData
import bson
data = {'elements': _elements, 'queries': _queries}
MetaData.set_data('definitions', bson.Binary(pickle.dumps(data)))
MetaData.set_data('definitionsDate', datetime.datetime.utcnow())
def _start_and_wait_for_migration(self, migration_opts): # noqa: D205,D400
"""Run donorStartMigration to start a tenant migration based on 'migration_opts', wait for
the migration decision and return the last response for donorStartMigration.
"""
cmd_obj = {
"donorStartMigration":
1,
"migrationId":
bson.Binary(migration_opts.migration_id.bytes, 4),
"recipientConnectionString":
migration_opts.recipient_rs.get_driver_connection_url(),
"tenantId":
migration_opts.tenant_id,
"readPreference":
migration_opts.read_preference,
"donorCertificateForRecipient":
get_certificate_and_private_key(
"jstests/libs/tenant_migration_donor.pem"),
"recipientCertificateForDonor":
get_certificate_and_private_key(
"jstests/libs/tenant_migration_recipient.pem"),
}
donor_primary = migration_opts.get_donor_primary()
donor_primary_client = donor_primary.mongo_client()
self.logger.info(
"Starting tenant migration on donor primary on port %d of replica set '%s'.",
donor_primary.port, migration_opts.get_donor_name())
while True:
# Keep polling the migration state until the migration completes.
res = donor_primary_client.admin.command(
cmd_obj,
bson.codec_options.CodecOptions(
uuid_representation=bson.binary.UUID_SUBTYPE))
if res["state"] == "committed":
self.logger.info(
"Tenant migration with donor primary on port " +
str(donor_primary.port) + " of replica set '" +
migration_opts.get_donor_name() + "' has committed.")
return res
if res["state"] == "aborted":
self.logger.info(
"Tenant migration with donor primary on port " +
str(donor_primary.port) + " of replica set '" +
migration_opts.get_donor_name() + "' has aborted: " +
str(res))
return res
if not res["ok"]:
self.logger.info(
"Tenant migration with donor primary on port " +
str(donor_primary.port) + " of replica set '" +
migration_opts.get_donor_name() + "' has failed: " +
str(res))
return res
time.sleep(self.MIGRATION_STATE_POLL_INTERVAL_SECS)
def send(self, data, address=None, broadcast=False, delay=0):
import bson
timestamp = self.database.eval(self.get_time)
ready_date = timestamp + datetime.timedelta(seconds=delay)
self.collection.insert({
'timestamp': timestamp,
'arrival_date': ready_date,
'message': bson.Binary(data=data)
})
def Int128(value):
val = value
if val < 0:
val += (1 << 128)
return {
"binary":
bson.Binary((0).to_bytes(1, "big") + val.to_bytes(16, "big")),
"string": "%d" % value
}
