def test_binary(self): array = np.array([ bson.Binary(b"binary_0"), bson.Binary(b"bin1"), bson.Binary(b"utf8-2") ], dtype=np.dtype('<V15')) bsonnumpy.ndarray_to_sequence(array)
def test_list(self): "list type" self.query('DEL/hello') self.query('RPUSH/hello/abc') self.query('RPUSH/hello/def') f = self.query('LRANGE/hello/0/-1.bson') obj = bson.decode_all(f.read()) self.assertTrue(obj == [{u'LRANGE': [bson.Binary('abc', 0), bson.Binary('def', 0)]}])
def test_GET_returns_all_details(self): node = factory.make_node() lshw_result = self.make_lshw_result(node) lldp_result = self.make_lldp_result(node) self.assertDictEqual( {"lshw": bson.Binary(lshw_result.data), "lldp": bson.Binary(lldp_result.data)}, self.get_details(node))
def serialize(self): output = super().serialize() output['trans_error_diff'] = bson.Binary( pickle.dumps(self.translational_error_difference, protocol=pickle.HIGHEST_PROTOCOL)) output['rot_error_diff'] = bson.Binary( pickle.dumps(self.rotational_error_difference, protocol=pickle.HIGHEST_PROTOCOL)) output['settings'] = self.settings return output
def serialize(self): serialized = super().serialize() serialized['ground_truth_trajectory'] = bson.Binary( pickle.dumps(self.ground_truth_trajectory, protocol=pickle.HIGHEST_PROTOCOL)) serialized['trajectory'] = bson.Binary( pickle.dumps(self.trajectory, protocol=pickle.HIGHEST_PROTOCOL)) serialized['tracking_stats'] = bson.Binary( pickle.dumps(self.tracking_stats, protocol=pickle.HIGHEST_PROTOCOL)) return serialized
def run_task(self, fw_spec): self.xml = self.get("xml", None) self.kmesh_factor = self.get("kmesh_factor", 1) with open("KPOINTS", "r") as f: kpoints = f.readlines() with open("INCAR", "r") as f: incar = f.readlines() if self.kmesh_factor > 1: shutil.copyfile("INCAR", "INCAR.nscf") shutil.copyfile("KPOINTS", "KPOINTS.nscf") shutil.copyfile("INCAR.Static", "INCAR") shutil.copyfile("KPOINTS.Static", "KPOINTS") if self.xml is not None: with open(self.xml, 'rb') as f: xmldata = f.read() binxmldata = gzip.compress(bytes(xmldata)) with open("DOSCAR", 'rb') as f: doscar = f.read() bindoscar = gzip.compress(bytes(doscar)) #with gzip.open("zen.txt.gz", "wb") as f: #f.write(bindata) self.db_file = env_chk(self.get("db_file"), fw_spec) self.vasp_db = VaspCalcDb.from_db_file(self.db_file, admin=True) structure = self.get('structure', Structure.from_file('POSCAR')) xml_data = { 'metadata': { 'tag': self.get('tag') }, 'hostname': socket.gethostname(), self.xml.replace(".", "_") + "_gz": bson.Binary(pickle.dumps(binxmldata)), 'DOSCAR_gz': bson.Binary(pickle.dumps(bindoscar)), 'volume': structure.volume, 'INCAR': incar, 'KPOINTS': kpoints, 'VASP': get_code_version(), 'last_updated': datetime.datetime.utcnow(), 'US_eastern_time': datetime.datetime.now(pytz.timezone('US/Eastern')), 'structure': structure.as_dict(), 'formula_pretty': structure.composition.reduced_formula } self.vasp_db.db['xmlgz'].insert_one(xml_data)
def schedule(cls, subtask, first, interval, *args, **kwargs): '''First is a timestamp, interval is in seconds''' t = cls.n() t._state.data.update(next=first, interval=interval, subtask_id=subtask.id, subtask_repr=repr(subtask), args=bson.Binary(dumps(args)), kwargs=bson.Binary(dumps(kwargs))) t._state.status = 'active' t._state.m.save() t._schedule_subtask(subtask) return t
def _wait_for_reroute(self, migration_opts): self.logger.info( "Waiting for donor replica set '%s' for migration '%s' to reroute at least one " + "conflicting command", migration_opts.get_donor_name(), migration_opts.migration_id) while True: donor_primary = migration_opts.get_donor_primary() donor_primary_client = donor_primary.mongo_client() try: doc = donor_primary_client["testTenantMigration"][ "rerouted"].find_one({ "_id": bson.Binary(migration_opts.migration_id.bytes, 4) }) if doc is not None: return except pymongo.errors.PyMongoError: self.logger.exception( "Error running find command on donor primary on port %d of replica set '%s'.", donor_primary.port, migration_opts.get_donor_name()) raise time.sleep( _TenantMigrationThread.MIGRATION_STATE_POLL_INTERVAL_SECS)
def schedule_next(self, status): if self.check_timings: self.logger.info( "Timings: %s", ", ".join("%s = %.2fms" % (n, t * 1000) for n, t in self.check_timings)) super(MODiscoveryJob, self).schedule_next(status) # Update alarm statuses self.update_alarms() # Write job log key = "discovery-%s-%s" % (self.attrs[self.ATTR_CLASS], self.attrs[self.ATTR_KEY]) problems = {} for p in self.problems: if p["check"] in problems and p["path"]: problems[p["check"]][p["path"]] = p["message"] elif p["check"] in problems and not p["path"]: # p["path"] == "" problems[p["check"]][p["path"]] += "; %s" % p["message"] else: problems[p["check"]] = {p["path"]: p["message"]} get_db()["noc.joblog"].update({"_id": key}, { "$set": { "log": bson.Binary(zlib.compress(self.out_buffer.getvalue())), "problems": problems } }, upsert=True)
async def __GenSwitchMonthlyUsage(para, result_no): """ :param para: :param result_no: url or bytes :return: {"url" or "bytes"} """ from io import BytesIO result = {} arg = await ListSwitchMonthlyUsage(para) arg['year'], arg['month'] = para['year'], para['month'] if result_no == 'url': file_name = "{0}.xlsx".format(bson.ObjectId()) file_path = os_path.join(g_vars.g_cfg['web']['export_path'], file_name) await threads.deferToThread(monthly_switch_usage_report.Make, file_path, arg) result['url'] = os_path.join( '/', rgw_consts.Node_URLs.EXPORT_FMT.format(file_name)) else: io_obj = BytesIO() await threads.deferToThread(monthly_switch_usage_report.Make, io_obj, arg) io_obj.seek(0) result['bytes'] = bson.Binary(io_obj.read()) io_obj.close() return result
def test_set(self): "success type (+OK)" self.query('DEL/hello') f = self.query('SET/hello/world.bson') self.assertTrue(f.headers.getheader('Content-Type') == 'application/bson') obj = bson.decode_all(f.read()) self.assertTrue(obj == [{u'SET': [True, bson.Binary('OK', 0)]}])
def serialize(self): serialized = super().serialize() serialized['trajectory'] = bson.Binary( pickle.dumps(self._trajectory, protocol=pickle.HIGHEST_PROTOCOL)) serialized['trajectory_source'] = self._trajectory_source serialized['sequence_type'] = self._sequence_type.value return serialized
def _cast_value_for_type(self, type, value): if value is None: return None def _check(*types): return type in types or isinstance(type, types) if _check(S.DateTime, datetime.datetime): if isinstance(value, string_type): return datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S") else: return value elif _check(S.Binary): return bson.Binary(value) elif _check(S.Int, int): return int(value) elif _check(S.Bool, bool): if value in ('true', 'false'): return value == 'true' and True or False else: return bool(value) elif _check(S.Object, dict): if isinstance(type, S.Object): type = type.fields return dict((k, self._cast_value_for_type(type[k], v)) for k, v in value.items()) elif _check(S.Array, list): if not isinstance(value, (list, tuple)): value = [value] if isinstance(type, S.Array): type = [type.field_type] return [self._cast_value_for_type(type[0], v) for v in value] else: return value
def test_GET_returns_only_those_details_that_exist(self): node = factory.make_node() lshw_result = self.make_lshw_result(node) self.assertDictEqual( {"lshw": bson.Binary(lshw_result.data), "lldp": None}, self.get_details(node))
def replace_media_links(body, media_library, db, destination_url, source_base_url): for link in media_library: if link in body: # This is making some big assumptions about the structure # of the media URL, that it's like # http://emptysquare.net/blog/wp-content/uploads/2011/10/img.png url = link.split('/uploads/')[-1] media_doc = db.media.find_one({'_id': link}) if not media_doc: # TODO: remove cache_path = os.path.join('cache', slugify(link)) if os.path.exists(cache_path): content, content_type = pickle.load(open(cache_path)) else: r = requests.get(link) content = r.content content_type = r.headers['content-type'] if not os.path.exists('cache'): os.mkdir('cache') with open(cache_path, 'w+') as f: pickle.dump((content, content_type), f) db.media.insert({ 'content': bson.Binary(content), 'type': content_type, '_id': url, 'mod': datetime.datetime.utcnow(), }) body = body.replace(link, os.path.join(destination_url, 'media', url)) return body
def insert_one(self): print('upsert one') try: tic = time.time() # 11kb ascii block random worst case scenario for compression noise = ''.join( random.choices(string.ascii_uppercase + string.digits + string.ascii_lowercase, k=1024 * 11)) # 16^7 = 268,435,456 possible _id values id = ''.join( random.choices(os.environ['KEYSPACE'], k=int(os.environ['KEYCOUNT']))) obj = { "value": bson.Binary(pickle.dumps(noise)), "created": datetime.datetime.now(), "location": os.environ['ISOLOC'] } output = self.db.queue.update_one({"_id": id}, {"$set": obj}, upsert=True) self.environment.events.request_success.fire( request_type="pymongo", name="uc_write", response_time=(time.time() - tic), response_length=0) except KeyboardInterrupt: print sys.exit(0) except Exception as e: print(f'{datetime.datetime.now()} - DB-CONNECTION-PROBLEM: ' f'{str(e)}') connect_problem = True
def test_Keep_accepts_bsonBinary(self): from niprov.pictures import PictureCache myImg = Mock() myImg.provenance = {'id': '007'} pictures = PictureCache(sentinel.dependencies) pictures.keep(bson.Binary('/x10/x05/x5f'), for_=myImg) self.assertEqual('/x10/x05/x5f', pictures.getBytes(for_=myImg))
async def __GenSensorMinsAvgLog(para, result_no): """ :param para: {"sensorids":[], "start_ts": timestamp, "stop_ts": timestamp, "mins_interval": integer, 'tz_offset': integer} or {"sensorids":[], "hours": hours range, "mins_interval": integer, 'tz_offset': integer} :param result_no: url or bytes :return: {"url" or "bytes"} """ from io import BytesIO result = {} log_tbl = await FindSensorMinsAvgLog2(para) tz_obj = await api_core.SysCfg.GetTimezone() log_tbl['tz_offset'] = tz_obj.utcoffset( rg_lib.DateTime.utc()).total_seconds() if result_no == 'url': file_name = "{0}.xlsx".format(bson.ObjectId()) file_path = os_path.join(g_vars.g_cfg['web']['export_path'], file_name) await threads.deferToThread(sensor_log_report.Make, file_path, log_tbl) result['url'] = os_path.join( '/', rgw_consts.Node_URLs.EXPORT_FMT.format(file_name)) else: io_obj = BytesIO() await threads.deferToThread(sensor_log_report.Make, io_obj, log_tbl) io_obj.seek(0) result['bytes'] = bson.Binary(io_obj.read()) io_obj.close() return result
def set(self, key, value, waitIndex=False, session=None): if 'passwd' in value: passwd = j.core.osis.encrypt(value['passwd'] or '') value['passwd'] = bson.Binary(passwd) guid, new, changed = super(parentclass, self).set(key, value, session=session) g = j.core.osis.cmds._getOsisInstanceForCat("system", "group") if changed: for group in value['groups']: grkey = "%s_%s" % (value['gid'], group) if g.exists(grkey, session=session) == False: #group does not exist yet, create grnew = g.getObject() grnew.id = group grnew.gid = value['gid'] grnew.domain = value['domain'] grnew.users = [value['id']] grguid, a, b = g.set(grnew.guid, grnew.__dict__, session=session) else: gr = g.get(grkey, session=session) if value['id'] not in gr['users']: gr['users'].append(value['id']) g.set(gr['guid'], gr, session=session) return guid, new, changed
def _wait_for_reroute_or_test_completion(self, migration_opts): self.logger.info( "Waiting for donor replica set '%s' for migration '%s' to reroute at least one " + "conflicting command. Stop waiting when the test finishes.", migration_opts.get_donor_name(), migration_opts.migration_id) start_time = time.time() while not self.__lifecycle.is_test_finished(): donor_primary = migration_opts.get_donor_primary() donor_primary_client = donor_primary.mongo_client() try: doc = donor_primary_client["testTenantMigration"][ "rerouted"].find_one({ "_id": bson.Binary(migration_opts.migration_id.bytes, 4) }) if doc is not None: return except pymongo.errors.PyMongoError: end_time = time.time() self.logger.exception( "Error running find command on donor primary on port %d of replica set '%s' " + "after waiting for reroute for %0d ms", donor_primary.port, migration_opts.get_donor_name(), (end_time - start_time) * 1000) raise time.sleep(self.MIGRATION_STATE_POLL_INTERVAL_SECS)
def to_dict(self, convert_numpy_arrays_to=None, fields_to_ignore=None, nan_to_none=False, use_decimal=False): result = {} if fields_to_ignore is None: fields_to_ignore = tuple() for k, v in self.get_fields_data(): if k in fields_to_ignore: continue if isinstance(v, Model): result[k] = v.to_dict(convert_numpy_arrays_to=convert_numpy_arrays_to, fields_to_ignore=fields_to_ignore, nan_to_none=nan_to_none, use_decimal=use_decimal) elif isinstance(v, list): result[k] = [el.to_dict(convert_numpy_arrays_to=convert_numpy_arrays_to, fields_to_ignore=fields_to_ignore, nan_to_none=nan_to_none, use_decimal=use_decimal) for el in v] elif isinstance(v, np.ndarray) and convert_numpy_arrays_to is not None: if convert_numpy_arrays_to == 'list': if use_decimal: result[k] = [decimal.Decimal("%f" % a) if isinstance(a, float) else a for a in v.tolist()] else: result[k] = v.tolist() elif convert_numpy_arrays_to == 'bytes': result[k] = bson.Binary(v.tostring()) else: raise ValueError('convert_numpy_arrays_to must be "list" or "bytes"') elif nan_to_none and isinstance(v, float) and not np.isfinite(v): result[k] = None elif use_decimal and isinstance(v, float): result[k] = decimal.Decimal("%f" % v) else: result[k] = v return result
def _forget_migration(self, migration_opts): """Run donorForgetMigration to garbage collection the tenant migration denoted by migration_opts'.""" self.logger.info("Forgetting tenant migration: %s.", str(migration_opts)) donor_primary = migration_opts.get_donor_primary() donor_primary_client = donor_primary.mongo_client() try: donor_primary_client.admin.command( { "donorForgetMigration": 1, "migrationId": bson.Binary(migration_opts.migration_id.bytes, 4) }, bson.codec_options.CodecOptions( uuid_representation=bson.binary.UUID_SUBTYPE)) except pymongo.errors.OperationFailure as err: if err.code != self.NO_SUCH_MIGRATION_ERR_CODE: raise # The fixture was restarted. self.logger.info( "Could not find tenant migration '%s' on donor primary on" + " port %d of replica set '%s'.", migration_opts.migration_id, donor_primary.port, migration_opts.get_donor_name()) except pymongo.errors.PyMongoError: self.logger.exception( "Error forgetting tenant migration '%s' on donor primary on" + " port %d of replica set '%s'.", migration_opts.migration_id, donor_primary.port, migration_opts.get_donor_name()) raise
def main(): """For audio downloading and persistence""" con = Connection() db = con.audiodb if use_proxy == True: install_proxy() words = open(filename).read().split() for wordname in words: if wordname.isalpha() == False: continue wordfilename = wordname.lower() + '.mp3' wordrecord = wordname.lower() + '_mp3' num = db.audiodb.find({"file_name": wordrecord}).count() if num > 0: print("Already got " + "\"" + wordname + "\"") continue try: mp3file = get_data_from_req( "https://ssl.gstatic.com/dictionary/static/sounds/de/0/" + wordfilename) if not is_file_valid(mp3file): continue bin = bson.Binary(mp3file.read()) db.audiodb.save({"file_name": wordrecord, "mp3": bin}) print("Done!") except Exception as e: print(e)
def build_image(image_data, image_content_type): built_image = { 'image_data': bson.Binary(image_data), 'image_content_type': image_content_type } return built_image
def _start_and_wait_for_migration(self, migration_opts): # noqa: D205,D400 """Run donorStartMigration to start a tenant migration based on 'migration_opts', wait for the migration decision and return the last response for donorStartMigration. """ cmd_obj = { "donorStartMigration": 1, "migrationId": bson.Binary(migration_opts.migration_id.bytes, 4), "recipientConnectionString": migration_opts.recipient_rs.get_driver_connection_url(), "tenantId": migration_opts.tenant_id, "readPreference": migration_opts.read_preference, "donorCertificateForRecipient": get_certificate_and_private_key("jstests/libs/tenant_migration_donor.pem"), "recipientCertificateForDonor": get_certificate_and_private_key("jstests/libs/tenant_migration_recipient.pem"), } donor_primary = migration_opts.get_donor_primary() self.logger.info( "Starting tenant migration '%s' on donor primary on port %d of replica set '%s'.", migration_opts.migration_id, donor_primary.port, migration_opts.get_donor_name()) while True: try: # Keep polling the migration state until the migration completes. donor_primary_client = self._create_client(donor_primary) res = donor_primary_client.admin.command( cmd_obj, bson.codec_options.CodecOptions(uuid_representation=bson.binary.UUID_SUBTYPE)) except (pymongo.errors.AutoReconnect, pymongo.errors.NotMasterError): donor_primary = migration_opts.get_donor_primary() self.logger.info( "Retrying tenant migration '%s' against donor primary on port %d of replica " + "set '%s'.", migration_opts.migration_id, donor_primary.port, migration_opts.get_donor_name()) continue if res["state"] == "committed": self.logger.info( "Tenant migration '%s' with donor primary on port %d of replica set '%s' has " + "committed.", migration_opts.migration_id, donor_primary.port, migration_opts.get_donor_name()) return res if res["state"] == "aborted": self.logger.info( "Tenant migration '%s' with donor primary on port %d of replica set '%s' has " + "aborted: %s.", migration_opts.migration_id, donor_primary.port, migration_opts.get_donor_name(), str(res)) return res if not res["ok"]: self.logger.info( "Tenant migration '%s' with donor primary on port %d of replica set '%s' has " + "failed: %s.", migration_opts.migration_id, donor_primary.port, migration_opts.get_donor_name(), str(res)) return res time.sleep(self.POLL_INTERVAL_SECS)
def recalc_score_doc(self, doc): scores = [] for sig, count in doc["sigs"].items(): sig = self.sigset.get_sig(bson.Binary(binascii.unhexlify(sig))) scores.append(sig["score"] * 4 / (1 + 3 / count)) score = pow(sum(score**2 for score in scores), 0.5) doc["score"] = score conn[self.db][self.collection].save(doc)
def save_definitions_to_meta_data(): from DB import MetaData import bson data = {'elements': _elements, 'queries': _queries} MetaData.set_data('definitions', bson.Binary(pickle.dumps(data))) MetaData.set_data('definitionsDate', datetime.datetime.utcnow())
def _start_and_wait_for_migration(self, migration_opts): # noqa: D205,D400 """Run donorStartMigration to start a tenant migration based on 'migration_opts', wait for the migration decision and return the last response for donorStartMigration. """ cmd_obj = { "donorStartMigration": 1, "migrationId": bson.Binary(migration_opts.migration_id.bytes, 4), "recipientConnectionString": migration_opts.recipient_rs.get_driver_connection_url(), "tenantId": migration_opts.tenant_id, "readPreference": migration_opts.read_preference, "donorCertificateForRecipient": get_certificate_and_private_key( "jstests/libs/tenant_migration_donor.pem"), "recipientCertificateForDonor": get_certificate_and_private_key( "jstests/libs/tenant_migration_recipient.pem"), } donor_primary = migration_opts.get_donor_primary() donor_primary_client = donor_primary.mongo_client() self.logger.info( "Starting tenant migration on donor primary on port %d of replica set '%s'.", donor_primary.port, migration_opts.get_donor_name()) while True: # Keep polling the migration state until the migration completes. res = donor_primary_client.admin.command( cmd_obj, bson.codec_options.CodecOptions( uuid_representation=bson.binary.UUID_SUBTYPE)) if res["state"] == "committed": self.logger.info( "Tenant migration with donor primary on port " + str(donor_primary.port) + " of replica set '" + migration_opts.get_donor_name() + "' has committed.") return res if res["state"] == "aborted": self.logger.info( "Tenant migration with donor primary on port " + str(donor_primary.port) + " of replica set '" + migration_opts.get_donor_name() + "' has aborted: " + str(res)) return res if not res["ok"]: self.logger.info( "Tenant migration with donor primary on port " + str(donor_primary.port) + " of replica set '" + migration_opts.get_donor_name() + "' has failed: " + str(res)) return res time.sleep(self.MIGRATION_STATE_POLL_INTERVAL_SECS)
def send(self, data, address=None, broadcast=False, delay=0): import bson timestamp = self.database.eval(self.get_time) ready_date = timestamp + datetime.timedelta(seconds=delay) self.collection.insert({ 'timestamp': timestamp, 'arrival_date': ready_date, 'message': bson.Binary(data=data) })
def Int128(value): val = value if val < 0: val += (1 << 128) return { "binary": bson.Binary((0).to_bytes(1, "big") + val.to_bytes(16, "big")), "string": "%d" % value }