def show(id): ingredient = Ingredient.find_one({ "_id": bson.ObjectId(oid=str(id)) }) #get id ObjectId and the informartions sent return dumps(ingredient) #dumps all to json
def handle_Str(self, node): return bson.ObjectId(node.s)
def containers(data_builder, as_admin, file_form, api_db): """Populate DB with test dataset including deleted and non-deleted entries.""" p_1 = data_builder.create_project() s_1_1 = data_builder.create_session(project=p_1) c_1_1_1 = data_builder.create_collection() an_1_1_1 = as_admin.post('/sessions/' + s_1_1 + '/analyses', files=file_form('analysis.csv', meta={ 'label': 'no-job', 'inputs': [{ 'name': 'analysis.csv' }] })).json()['_id'] ac_1_1_1 = data_builder.create_acquisition(session=s_1_1) ac_1_1_2 = data_builder.create_acquisition(session=s_1_1) s_1_2 = data_builder.create_session(project=p_1) ac_1_2_1 = data_builder.create_acquisition(session=s_1_2) p_2 = data_builder.create_project() s_2_1 = data_builder.create_session(project=p_2) ac_2_1_1 = data_builder.create_acquisition(session=s_2_1) assert as_admin.post('/acquisitions/' + ac_1_1_1 + '/files', files=file_form('f_1_1_1_1')).ok assert as_admin.post('/acquisitions/' + ac_1_1_1 + '/files', files=file_form('f_1_1_1_2')).ok assert as_admin.post('/acquisitions/' + ac_2_1_1 + '/files', files=file_form('f_2_1_1_1')).ok assert as_admin.delete('/sessions/' + s_1_1 + '/analyses/' + an_1_1_1).ok assert as_admin.delete('/collections/' + c_1_1_1).ok assert as_admin.delete('/acquisitions/' + ac_1_1_1 + '/files/f_1_1_1_1').ok assert as_admin.delete('/acquisitions/' + ac_1_1_1).ok assert as_admin.delete('/sessions/' + s_1_1).ok assert as_admin.delete('/projects/' + p_1).ok containers = attrdict.AttrDict( p_1=p_1, s_1_1=s_1_1, c_1_1_1=c_1_1_1, an_1_1_1=an_1_1_1, ac_1_1_1=ac_1_1_1, ac_1_1_2=ac_1_1_2, s_1_2=s_1_2, ac_1_2_1=ac_1_2_1, p_2=p_2, s_2_1=s_2_1, ac_2_1_1=ac_2_1_1, ) def is_deleted(cont_key, filename=None): cont_name = { 'p': 'projects', 's': 'sessions', 'ac': 'acquisitions', 'an': 'analyses', 'c': 'collections', }[cont_key.split('_')[0]] url = '/{}/{}'.format(cont_name, containers[cont_key]) if filename is None: return as_admin.get(url).status_code == 404 else: return as_admin.get(url + '/files/' + filename).status_code == 404 containers['is_deleted'] = is_deleted yield containers api_db.analyses.delete_one({'_id': bson.ObjectId(an_1_1_1)})
from my_mongodb import MyMongo import zlib import bson from cStringIO import StringIO # f = open('../test1/tianyancha/detail.html') # s = f.read() # content = StringIO(f.read()) db = MyMongo().get_db() # db['test'].save({'source': bson.binary.Binary(zlib.compress(s))}) # db['test'].save({'source': content.getvalue()}) print '' print db['raw_result_gzcc'].find({'_id': {'$gt': bson.ObjectId('5bd90a48c02bfe18842558e9')}}).count()
def decode(obj: Any) -> Any: if "__datetime__" in obj: return datetime.datetime.strptime(obj["as_str"], "%Y%m%dT%H:%M:%S.%f") if "__object_id__" in obj: return bson.ObjectId(obj["as_str"]) return obj
async def convert(self, ctx, argument): try: return bson.ObjectId(argument) except bson.errors.InvalidId: raise commands.BadArgument
def __project_refs(self, project_id, operation): project_id = bson.ObjectId(project_id) return self.db.collection.find_one_and_update( {"_id": self.user_id}, {operation: { "project_refs": project_id }})
def make_metadata(self, **kwargs): kwargs = du.defaults( kwargs, { 'hash_': b'\xa5\xc9\x08\xaf$\x0b\x116', 'source_type': imeta.ImageSourceType.SYNTHETIC, 'environment_type': imeta.EnvironmentType.INDOOR_CLOSE, 'light_level': imeta.LightingLevel.WELL_LIT, 'time_of_day': imeta.TimeOfDay.DAY, 'camera_pose': tf.Transform((1, 3, 4), (0.2, 0.8, 0.2, -0.7)), 'right_camera_pose': tf.Transform((-10, -20, -30), (0.9, -0.7, 0.5, -0.3)), 'intrinsics': cam_intr.CameraIntrinsics(700, 700, 654.2, 753.3, 400, 300), 'right_intrinsics': cam_intr.CameraIntrinsics(700, 710, 732.1, 612.3, 400, 300), 'lens_focal_distance': 5, 'aperture': 22, 'simulator': bson.ObjectId('5a14cf0e36ed1e17a55f1e35'), 'simulation_world': 'TestSimulationWorld', 'lighting_model': imeta.LightingModel.LIT, 'texture_mipmap_bias': 1, 'normal_maps_enabled': True, 'roughness_enabled': True, 'geometry_decimation': 0.8, 'procedural_generation_seed': 16234, 'labelled_objects': [ imeta.LabelledObject(class_names=('cup', ), bounding_box=(142, 280, 54, 78), label_color=(2, 227, 34), relative_pose=tf.Transform( location=(-246, 468, 4), rotation=(0.2, 0.3, 0.4)), object_id='LabelledObject-68478'), imeta.LabelledObject(class_names=('car', ), bounding_box=(542, 83, 63, 123), label_color=(26, 12, 212), relative_pose=tf.Transform( location=(61, -717, 161), rotation=(0.7, 0.6, 0.3)), object_id='LabelledObject-8246'), imeta.LabelledObject(class_names=('cow', ), bounding_box=(349, 672, 124, 208), label_color=(162, 134, 163), relative_pose=tf.Transform( location=(286, -465, -165), rotation=(0.9, 0.1, 0.5)), object_id='LabelledObject-56485') ], 'average_scene_depth': 90.12, 'base_image': self.parent_image, 'transformation_matrix': np.array([[0.19882871, 0.58747441, 0.90084303], [0.6955363, 0.48193339, 0.09503605], [0.20549805, 0.6110534, 0.61145574]]) }) return imeta.ImageMetadata(**kwargs)
def test_clone(self): alt_metadata = { 'hash_': [b'\x1f`\xa8\x8aR\xed\x9f\x0b'], 'source_type': [imeta.ImageSourceType.REAL_WORLD], 'environment_type': [ imeta.EnvironmentType.INDOOR, imeta.EnvironmentType.OUTDOOR_URBAN, imeta.EnvironmentType.OUTDOOR_LANDSCAPE ], 'light_level': [ imeta.LightingLevel.PITCH_BLACK, imeta.LightingLevel.DIM, imeta.LightingLevel.EVENLY_LIT, imeta.LightingLevel.BRIGHT ], 'time_of_day': [ imeta.TimeOfDay.DAWN, imeta.TimeOfDay.MORNING, imeta.TimeOfDay.AFTERNOON, imeta.TimeOfDay.TWILIGHT, imeta.TimeOfDay.NIGHT ], 'camera_pose': [tf.Transform((12, 13, 14), (-0.5, 0.3, 0.8, -0.9))], 'right_camera_pose': [tf.Transform((11, 15, 19), (-0.2, 0.4, 0.6, -0.8))], 'intrinsics': [cam_intr.CameraIntrinsics(900, 910, 894.7, 861.2, 640, 360)], 'right_intrinsics': [cam_intr.CameraIntrinsics(900, 890, 760.45, 405.1, 640, 360)], 'lens_focal_distance': [22], 'aperture': [1.2], 'simulator': [bson.ObjectId()], 'simulation_world': ['TestSimulationWorld2'], 'lighting_model': [imeta.LightingModel.UNLIT], 'texture_mipmap_bias': [2], 'normal_maps_enabled': [False], 'roughness_enabled': [False], 'geometry_decimation': [0.3], 'procedural_generation_seed': [7329], 'average_scene_depth': [102.33], 'base_image': [mock.create_autospec(core.image.Image)], 'transformation_matrix': [np.random.uniform(0, 1, (3, 3))], 'labelled_objects': [ tuple(), (imeta.LabelledObject(class_names=('cup', ), bounding_box=(142, 280, 54, 78), label_color=(2, 227, 34), relative_pose=tf.Transform( location=(-246, 468, 4), rotation=(0.2, 0.3, 0.4)), object_id='LabelledObject-68478'), imeta.LabelledObject(class_names=('cat', ), bounding_box=(542, 83, 63, 123), label_color=(26, 12, 212), relative_pose=tf.Transform( location=(61, -717, 161), rotation=(0.7, 0.6, 0.3)), object_id='LabelledObject-8246'), imeta.LabelledObject(class_names=('cow', ), bounding_box=(349, 672, 124, 208), label_color=(162, 134, 163), relative_pose=tf.Transform( location=(286, -465, -165), rotation=(0.9, 0.1, 0.5)), object_id='LabelledObject-56485')), (imeta.LabelledObject(class_names=('cup', ), bounding_box=(142, 12, 54, 78), label_color=(2, 227, 34), relative_pose=tf.Transform( location=(-246, 468, 4), rotation=(0.2, 0.3, 0.4)), object_id='LabelledObject-68478'), imeta.LabelledObject(class_names=('car', ), bounding_box=(542, 83, 63, 123), label_color=(26, 12, 212), relative_pose=tf.Transform( location=(61, -717, 161), rotation=(0.7, 0.6, 0.3)), object_id='LabelledObject-8246'), imeta.LabelledObject(class_names=('cow', ), bounding_box=(349, 672, 124, 208), label_color=(162, 134, 163), relative_pose=tf.Transform( location=(286, -465, -165), rotation=(0.9, 0.1, 0.5)), object_id='LabelledObject-56485')), (imeta.LabelledObject(class_names=('cup', ), bounding_box=(142, 280, 54, 78), label_color=(2, 227, 34), relative_pose=tf.Transform( location=(-246, 468, 4), rotation=(0.2, 0.3, 0.4)), object_id='LabelledObject-68478'), imeta.LabelledObject(class_names=('car', ), bounding_box=(542, 83, 63, 123), label_color=(26, 12, 212), relative_pose=tf.Transform( location=(61, -717, 161), rotation=(0.7, 0.6, 0.3)), object_id='LabelledObject-8246'), imeta.LabelledObject(class_names=('cow', ), bounding_box=(349, 672, 124, 208), label_color=(162, 134, 255), relative_pose=tf.Transform( location=(286, -465, -165), rotation=(0.9, 0.1, 0.5)), object_id='LabelledObject-56485')) ] } a = self.make_metadata() b = a.clone() self.assert_metadata_equal(a, b) # Change single keys, and make sure it is no longer equal for key, values in alt_metadata.items(): for val in values: b = a.clone(**{key: val}) if key == 'hash_': self.assertEqual(val, b.hash) self.assertNotEqual(a.hash, b.hash) else: self.assertEqual(a.hash, b.hash) if key == 'source_type': self.assertEqual(val, b.source_type) self.assertNotEqual(a.source_type, b.source_type) else: self.assertEqual(a.source_type, b.source_type) if key == 'environment_type': self.assertEqual(val, b.environment_type) self.assertNotEqual(a.environment_type, b.environment_type) else: self.assertEqual(a.environment_type, b.environment_type) if key == 'light_level': self.assertEqual(val, b.light_level) self.assertNotEqual(a.light_level, b.light_level) else: self.assertEqual(a.light_level, b.light_level) if key == 'time_of_day': self.assertEqual(val, b.time_of_day) self.assertNotEqual(a.time_of_day, b.time_of_day) else: self.assertEqual(a.time_of_day, b.time_of_day) if key == 'camera_pose': self.assertEqual(val, b.camera_pose) self.assertNotEqual(a.camera_pose, b.camera_pose) else: self.assertEqual(a.camera_pose, b.camera_pose) if key == 'right_camera_pose': self.assertEqual(val, b.right_camera_pose) self.assertNotEqual(a.right_camera_pose, b.right_camera_pose) else: self.assertEqual(a.right_camera_pose, b.right_camera_pose) if key == 'intrinsics': self.assertEqual(val, b.camera_intrinsics) self.assertNotEqual(a.camera_intrinsics, b.camera_intrinsics) else: self.assertEqual(a.camera_intrinsics, b.camera_intrinsics) self.assertEqual(a.width, b.width) self.assertEqual(a.height, b.height) if key == 'right_intrinsics': self.assertEqual(val, b.right_camera_intrinsics) self.assertNotEqual(a.right_camera_intrinsics, b.right_camera_intrinsics) else: self.assertEqual(a.right_camera_intrinsics, b.right_camera_intrinsics) if key == 'lens_focal_distance': self.assertEqual(val, b.lens_focal_distance) self.assertNotEqual(a.lens_focal_distance, b.lens_focal_distance) else: self.assertEqual(a.lens_focal_distance, b.lens_focal_distance) if key == 'aperture': self.assertEqual(val, b.aperture) self.assertNotEqual(a.aperture, b.aperture) else: self.assertEqual(a.aperture, b.aperture) if key == 'simulation_world': self.assertEqual(val, b.simulation_world) self.assertNotEqual(a.simulation_world, b.simulation_world) else: self.assertEqual(a.simulation_world, b.simulation_world) if key == 'lighting_model': self.assertEqual(val, b.lighting_model) self.assertNotEqual(a.lighting_model, b.lighting_model) else: self.assertEqual(a.lighting_model, b.lighting_model) if key == 'texture_mipmap_bias': self.assertEqual(val, b.texture_mipmap_bias) self.assertNotEqual(a.texture_mipmap_bias, b.texture_mipmap_bias) else: self.assertEqual(a.texture_mipmap_bias, b.texture_mipmap_bias) if key == 'normal_maps_enabled': self.assertEqual(val, b.normal_maps_enabled) self.assertNotEqual(a.normal_maps_enabled, b.normal_maps_enabled) else: self.assertEqual(a.normal_maps_enabled, b.normal_maps_enabled) if key == 'roughness_enabled': self.assertEqual(val, b.roughness_enabled) self.assertNotEqual(a.roughness_enabled, b.roughness_enabled) else: self.assertEqual(a.roughness_enabled, b.roughness_enabled) if key == 'geometry_decimation': self.assertEqual(val, b.geometry_decimation) self.assertNotEqual(a.geometry_decimation, b.geometry_decimation) else: self.assertEqual(a.geometry_decimation, b.geometry_decimation) if key == 'procedural_generation_seed': self.assertEqual(val, b.procedural_generation_seed) self.assertNotEqual(a.procedural_generation_seed, b.procedural_generation_seed) else: self.assertEqual(a.procedural_generation_seed, b.procedural_generation_seed) if key == 'labelled_objects': self.assertEqual(val, b.labelled_objects) self.assertNotEqual(a.labelled_objects, b.labelled_objects) else: self.assertEqual(a.labelled_objects, b.labelled_objects) if key == 'average_scene_depth': self.assertEqual(val, b.average_scene_depth) self.assertNotEqual(a.average_scene_depth, b.average_scene_depth) else: self.assertEqual(a.average_scene_depth, b.average_scene_depth)
def post(self, cont_name): self.config = self.container_handler_configurations[cont_name] self.storage = self.config['storage'] mongo_validator, payload_validator = self._get_validators() payload = self.request.json_body #validate the input payload payload_validator(payload, 'POST') if cont_name == 'subjects': if 'project' not in payload: # The new POST /subjects reuses json schema used for "embedded" subject creation, # but requires project in the payload, too raise APIValidationException('project required') subject_code = payload.get('code') or payload.get('label') if not subject_code: raise APIValidationException('label or code required') if self.storage.get_all_el( { 'project': bson.ObjectId(payload['project']), 'code': subject_code, }, None, {'_id': 1}): raise APIValidationException( 'subject code "{}" already exists in project {}'.format( subject_code, payload['project'])) # Load the parent container in which the new container will be created # to check permissions. parent_container, parent_id_property = self._get_parent_container( payload) # Always add the id of the parent to the container payload[parent_id_property] = parent_container['_id'] # If the new container is a session add the group of the parent project in the payload if cont_name == 'sessions': payload['group'] = parent_container['group'] # Optionally inherit permissions of a project from the parent group. The default behaviour # for projects is to give admin permissions to the requestor. # The default for other containers is to inherit. if self.is_true('inherit') and cont_name == 'projects': payload['permissions'] = parent_container.get('permissions') elif cont_name == 'projects': payload['permissions'] = [{ '_id': self.uid, 'access': 'admin' }] if self.uid else [] # Unsorted projects are reserved for reaper uploads if payload['label'] in PROJECT_BLACKLIST: self.abort( 400, 'The project "{}" can\'t be created as it is integral within the API' .format(payload['label'])) else: payload['permissions'] = parent_container.get('permissions', []) # Created and modified timestamps are added here to the payload payload['created'] = payload['modified'] = datetime.datetime.utcnow() if payload.get('timestamp'): payload['timestamp'] = dateutil.parser.parse(payload['timestamp']) permchecker = self._get_permchecker(parent_container=parent_container) if cont_name == 'projects': # Validate any changes to storage providers providers.validate_provider_updates({}, payload.get('providers'), self.user_is_admin) # Handle embedded subjects for backwards-compatibility if cont_name == 'sessions': self._handle_embedded_subject(payload, parent_container) # This line exec the actual request validating the payload that will create the new container # and checking permissions using respectively the two decorators, mongo_validator and permchecker result = mongo_validator(permchecker(self.storage.exec_op))( 'POST', payload=payload) if result.acknowledged: return {'_id': result.inserted_id} else: self.abort( 404, 'Element not added in container {}'.format( self.storage.cont_name))
def put(self, cont_name, **kwargs): _id = kwargs.pop('cid') self.config = self.container_handler_configurations[cont_name] self.storage = self.config['storage'] container = self._get_container(_id) mongo_validator, payload_validator = self._get_validators() payload = self.request.json_body payload_validator(payload, 'PUT') # Check if any payload keys are any propogated property, add to r_payload rec = False r_payload = {} prop_keys = set(payload.keys()).intersection( set(self.config.get('propagated_properties', []))) if prop_keys: rec = True for key in prop_keys: r_payload[key] = payload[key] if cont_name == 'projects': # Validate any changes to storage providers providers.validate_provider_updates(container, payload.get('providers'), self.user_is_admin) if cont_name == 'subjects': # Check for code collision if changing code/label or moving to a new project # TODO: Minor duplication of code below, resolve when ability to edit subject # via session is resolved current_project, _ = self._get_parent_container(container) target_project, _ = self._get_parent_container(payload) project_id = ( target_project or current_project )['_id'] # It's current project or the new project it is moving to subject_code = payload.get('code') or payload.get( 'label') or container.get('code') or container.get( 'label' ) # It's current label or the new label it is moving to # Check for subject code collision 1st when changing project and/or subject code if subject_code and self.storage.get_all_el( { 'project': project_id, 'code': subject_code, '_id': { '$ne': container['_id'] } # Make sure that if neither code nor project changed, we allow it }, None, {'_id': 1}): raise APIValidationException( 'subject code "{}" already exists in project {}'.format( subject_code, project_id)) payload['code'] = subject_code payload['label'] = subject_code # Handle embedded subjects for backwards-compatibility if cont_name == 'sessions': current_project, _ = self._get_parent_container(container) target_project, _ = self._get_parent_container(payload) project_id = (target_project or current_project)['_id'] current_subject = container['subject'] payload_subject = payload.get('subject', {}) target_subject_id = payload_subject.get('_id') target_subject_code = payload_subject.get( 'code') or payload_subject.get('label') subject_code = target_subject_code or container['subject'].get( 'code') subject_storage = containerstorage.SubjectStorage() # Check for subject code collision 1st when changing project and/or subject code if ((target_project and project_id != current_project['_id']) or (target_subject_code and subject_code != current_subject.get('code'))): if subject_storage.get_all_el( { 'project': project_id, 'code': subject_code }, None, {'_id': 1}): raise APIValidationException( 'subject code "{}" already exists in project {}'. format(subject_code, project_id)) # Handle changing subject id (moving session to another subject) if target_subject_id: target_subject = subject_storage.get_container( target_subject_id) # If payload also contains project, verify that the target_subject is in it if target_project and project_id != target_subject['project']: raise APIValidationException( 'subject {} is not in project {}'.format( target_subject_id, project_id)) # Make sure session.project is also updated if not target_project: payload['project'] = target_subject['project'] target_project, _ = self._get_parent_container(payload) # Handle changing project (moving session and subject to another project) # * Copy subject into target project if there are other sessions on it # * Move if this is the only session on it (else branch) elif (target_project and project_id != current_project['_id'] ) and config.db.sessions.count( {'subject': container['subject']['_id']}) > 1: subject = copy.deepcopy(container['subject']) subject.pop('parents') subject.update(payload_subject ) # Still apply any embedded subject changes subject['_id'] = bson.ObjectId( ) # Causes new subject creation via extract_subject payload['subject'] = subject # Enable embedded subject updates via session updates: match on subject._id else: payload.setdefault('subject', {})['_id'] = container['subject']['_id'] self._handle_embedded_subject(payload, target_project or current_project) # Check if we are updating the parent container of the element (ie we are moving the container) # In this case, we will check permissions on it. target_parent_container, parent_id_property = self._get_parent_container( payload) if target_parent_container: if cont_name in ['sessions', 'acquisitions']: payload[parent_id_property] = bson.ObjectId( payload[parent_id_property]) parent_perms = target_parent_container.get('permissions', []) payload['permissions'] = parent_perms if cont_name == 'sessions': payload['group'] = target_parent_container['group'] # Propagate permissions down to acquisitions rec = True r_payload['permissions'] = parent_perms payload['modified'] = datetime.datetime.utcnow() if payload.get('timestamp'): payload['timestamp'] = dateutil.parser.parse(payload['timestamp']) permchecker = self._get_permchecker(container, target_parent_container) # Specifies wether the metadata fields should be replaced or patched with payload value replace_metadata = self.get_param('replace_metadata', default=False) # This line exec the actual request validating the payload that will update the container # and checking permissions using respectively the two decorators, mongo_validator and permchecker result = mongo_validator(permchecker(self.storage.exec_op))( 'PUT', _id=_id, payload=payload, recursive=rec, r_payload=r_payload, replace_metadata=replace_metadata) if result.modified_count == 1: return {'modified': result.modified_count} else: self.abort( 404, 'Element not updated in container {} {}'.format( self.storage.cont_name, _id))
def get_jobs(self, cid): # Only enabled for sessions container type per url rule in api.py self.config = self.container_handler_configurations["sessions"] self.storage = self.config['storage'] cont = self._get_container(cid, projection={ 'files': 0, 'metadata': 0 }, get_children=True) permchecker = self._get_permchecker(cont) permchecker(noop)('GET', cid) analyses = AnalysisStorage().get_analyses(None, 'session', cont['_id']) acquisitions = cont.get('acquisitions', []) # Get query params states = self.request.GET.getall('states') tags = self.request.GET.getall('tags') join_cont = 'containers' in self.request.params.getall('join') join_gears = 'gears' in self.request.params.getall('join') limit = int(self.request.params.get('limit', 10000)) skip = int(self.request.params.get('skip', 0)) cont_refs = [ containerutil.ContainerReference(cont_type, str(cont_id)) for cont_type, cont_id in [('session', cont['_id'])] + [('analysis', an['_id']) for an in analyses] + [('acquisition', aq['_id']) for aq in acquisitions] ] jobs = Queue.search_containers(cont_refs, states=states, tags=tags, limit=limit, skip=skip) unique_jobs = {} gear_ids = set() for job in jobs: if job['_id'] not in unique_jobs: clean_job = remove_potential_phi_from_job(job) unique_jobs[job['_id']] = Job.load(clean_job) if clean_job.get( 'gear_id') and clean_job['gear_id'] not in gear_ids: gear_ids.add(clean_job['gear_id']) #response = {'jobs': sorted(unique_jobs.values(), key=lambda job: job.created)} response = {'jobs': unique_jobs.values()} if join_gears: gears = config.db.gears.find({ '_id': { '$in': [bson.ObjectId(gear_id) for gear_id in gear_ids] } }) response['gears'] = {str(gear['_id']): gear for gear in gears} if join_cont: # create a map of analyses and acquisitions by _id containers = { str(cont['_id']): cont for cont in analyses + acquisitions } for container in containers.itervalues(): # No need to return perm arrays container.pop('permissions', None) response['containers'] = containers return response
def validate(cls, v: Any) -> bson.ObjectId: if isinstance(v, (bson.ObjectId, cls)): return v if isinstance(v, str) and bson.ObjectId.is_valid(v): return bson.ObjectId(v) raise TypeError("invalid ObjectId specified")
def delete(id): delete = Ingredient.delete_one({ "_id": bson.ObjectId(oid=str(id)) }) #Take the ObjectId from recepie and delete that documnet delete_count = delete.deleted_count return 'Delete ' + dumps(delete_count) + ' document'
def find_by_session_id(self, session_id, *args, **kwargs): """Returns matched TopicStore documents collected in the same session""" if isinstance(session_id, str): session_id = bson.ObjectId(session_id) return self.find({"_ts_meta.session": session_id}, *args, **kwargs)
def ObjectIdSilent(oid=None): if oid is not None: oid = str(oid) if oid is None or len(oid) != 32: return bson.ObjectId(oid) return oid
class MailMessage(object): FOLDER_INBOX = bson.ObjectId('4f168258e6cdaf1620000000') FOLDER_SENT = bson.ObjectId('4f16826ce6cdaf1620000001') FOLDER_SPAM = bson.ObjectId('4f16826ce6cdaf1620000002') FOLDER_DRAFT = bson.ObjectId('4f16826ce6cdaf1620000003') #Warning! This-deleted folder is not folder! Only for client side FOLDER_DELETED = bson.ObjectId('4f16826ce6cdaf1620000004') def __init__(self, owner, text, company_rek='', subject='Без темы', folder=FOLDER_DRAFT, headers=None, attaches=None, input_number=0, output_number=0, send_date=mongodb_datetime_now(), is_important=False, is_official=False, is_auto_answer=False, is_read=False, is_deleted=False): attaches = attaches or list( ) #Attache name, attache_id, attache size ... headers = headers or [{ 'FROM': owner, 'TO': '', 'DATE': send_date }] #FROM: TO: self._id = None self.owner = owner # FROM USER COMPANY if user has many companies - REK self.company_rek = company_rek self.text = text self.subject = subject self.folder = folder self.headers = headers self.attaches = attaches self.input_number = input_number self.output_number = output_number self.send_date = send_date self.is_important = is_important self.is_official = is_official self.is_auto_answer = is_auto_answer self.is_read = is_read self.is_deleted = is_deleted def __eq__(self, other): if not isinstance(other, MailMessage): return False return self.owner == other.owner and self.company_rek == other.company_rek and self.text == other.text and self.folder == other.folder \ and self._id == other._id and self.headers == other.headers and self.attaches == other.attaches \ and self.input_number == other.output_number and self.send_date == other.send_date and self.is_important == other.is_important \ and self.is_official == other.is_official and self.is_auto_answer == other.is_auto_answer and self.is_read == other.is_read and self.is_deleted == other.is_deleted def save(self): if self._id: if messages_manager.update_message(self._id, self): result = self._id else: raise rek.mail_messages.messages_exceptions.CantSaveMessage( 'Can not save message') else: result = messages_manager.save_message(self) return result def clone(self): return messages_manager.save_message(self) def drop(self): messages_manager.del_message(self._id, self.owner) def _get_last_recipient(self): if len(self.headers): return self.headers[-1][u'TO'] else: return '' to = property(_get_last_recipient) def _get_last_sender(self): #sender if len(self.headers): header_item = self.headers[-1] return header_item['FROM'] else: return self.owner sender = property(_get_last_sender) def set_sender_and_recipient(self, sender, to, send_date=None): if (self.to == to or self.to == '') and self.sender == sender: #edit last header self.update_or_add_last_header(to) else: #add_new_header self.add_new_header(sender, to, send_date) def add_new_header(self, sender=None, to='', send_date=mongodb_datetime_now()): # when new message saved as draft massage have not from or to header data sender = sender or self.owner return self.headers.append({ u'FROM': sender, u'TO': to, u'DATE': send_date }) def update_or_add_last_header(self, to, sender=None): sender = sender or self.owner if len(self.headers): last_header = self.headers[-1] last_header[u'TO'] = to last_header[u'FROM'] = sender return True else: return self.add_new_header(self, to)
def ParseObjectId(oid): if oid: return bson.ObjectId(str(oid))
def reset_active_project(self): project_id = bson.ObjectId(project_id) return self.db.collection.find_one_and_update( {"_id": self.user_id}, {"$set": { "active_project": None }})
def verify_otp_code(self, code, remote_ip=None): if remote_ip: doc = self.otp_cache_collection.find_one({ '_id': bson.ObjectId(self.id), }) if doc: _, hash_salt, cur_otp_hash = doc['otp_hash'].split('$') hash_salt = base64.b64decode(hash_salt) else: hash_salt = os.urandom(8) cur_otp_hash = None otp_hash = hashlib.sha512() otp_hash.update(code + remote_ip) otp_hash.update(hash_salt) otp_hash = base64.b64encode(otp_hash.digest()) if otp_hash == cur_otp_hash: self.otp_cache_collection.update( { '_id': bson.ObjectId(self.id), }, {'$currentDate': { 'timestamp': True, }}) return True otp_hash = '$'.join(( '1', base64.b64encode(hash_salt), otp_hash, )) otp_secret = self.otp_secret padding = 8 - len(otp_secret) % 8 if padding != 8: otp_secret = otp_secret.ljust(len(otp_secret) + padding, '=') otp_secret = base64.b32decode(otp_secret.upper()) valid_codes = [] epoch = int(time.time() / 30) for epoch_offset in range(-1, 2): value = struct.pack('>q', epoch + epoch_offset) hmac_hash = hmac.new(otp_secret, value, hashlib.sha1).digest() offset = ord(hmac_hash[-1]) & 0x0F truncated_hash = hmac_hash[offset:offset + 4] truncated_hash = struct.unpack('>L', truncated_hash)[0] truncated_hash &= 0x7FFFFFFF truncated_hash %= 1000000 valid_codes.append('%06d' % truncated_hash) if code not in valid_codes: return False response = self.otp_collection.update( { '_id': { 'user_id': self.id, 'code': code, }, }, {'$currentDate': { 'timestamp': True, }}, upsert=True) if response['updatedExisting']: return False if remote_ip: self.otp_cache_collection.update({ '_id': bson.ObjectId(self.id), }, { '$set': { 'otp_hash': otp_hash, }, '$currentDate': { 'timestamp': True, }, }, upsert=True) return True
def __init__(self, user_id): self.user_id = bson.ObjectId(user_id) self.db = DB("users")
def get_pk(self, pk): try: return bson.ObjectId(pk) except bson.errors.InvalidId: raise NotFoundError()
def create_el(self, analysis, parent_type, parent_id, origin, uid=None): """ Create an analysis. * Fill defaults if not provided * Flatten input filerefs using `FileReference.get_file()` If `analysis` has a `job` key, create a "job-based" analysis: * Analysis inputs will be copied from the job inputs * Create analysis and job, both referencing each other * Do not create (remove) analysis if can't enqueue job """ parent_type = containerutil.singularize(parent_type) parent = self.get_parent( None, cont={'parent': { 'type': parent_type, 'id': parent_id }}) defaults = { '_id': bson.ObjectId(), 'parent': { 'type': parent_type, 'id': bson.ObjectId(parent_id) }, 'created': datetime.datetime.utcnow(), 'modified': datetime.datetime.utcnow(), 'user': origin.get('id') } for key in defaults: analysis.setdefault(key, defaults[key]) if 'public' in parent: analysis.setdefault('public', parent['public']) job = analysis.pop('job', None) if job is not None: if parent_type not in [ 'project', 'session', 'subject', 'acquisition' ]: raise APIValidationException( reason='Cannot create analysis via job at the {} level'. format(parent_type)) analysis.setdefault('inputs', []) for key, fileref_dict in job['inputs'].iteritems(): analysis['inputs'].append(fileref_dict) # Verify and flatten input filerefs for i, fileref_dict in enumerate(analysis.get('inputs', [])): try: fileref = containerutil.create_filereference_from_dictionary( fileref_dict) except KeyError: # Legacy analyses already have fileinfos as inputs instead of filerefs pass else: analysis['inputs'][i] = fileref.get_file() if analysis.get('info') is not None: analysis['info'] = util.mongo_sanitize_fields(analysis['info']) result = super(AnalysisStorage, self).create_el(analysis, origin) if not result.acknowledged: raise APIStorageException( 'Analysis not created for container {} {}'.format( parent_type, parent_id)) if job is not None: # Create job job['destination'] = { 'type': 'analysis', 'id': str(analysis['_id']) } tags = job.get('tags', []) if 'analysis' not in tags: tags.append('analysis') job['tags'] = tags try: job = Queue.enqueue_job(job, origin, perm_check_uid=uid) job.insert() # Copy gear info and add id gear_info = job.gear_info.copy() gear_info['id'] = job.gear_id self.update_el(analysis['_id'], { 'job': job.id_, 'gear_info': gear_info }, None) except: # NOTE #775 remove unusable analysis - until jobs have a 'hold' state self.delete_el(analysis['_id']) raise return result
def read_data(temp_id,backupi): db2 = client.odm collection=db2.html_jobs post_id=bson.ObjectId(temp_id) job=collection.find_one({"_id":post_id}) url1=str(job['url']) url=str(job['cat_url'].encode('utf-8')) id1=str(job['step']) afterid=str(job['afterurl']) dataset_keyword2=str(job['identifier']) if ',' in dataset_keyword2: dataset_keyword3=dataset_keyword2.split(',') dataset_keyword=dataset_keyword3[0] dataset_keyword1=dataset_keyword3[1] else: dataset_keyword=dataset_keyword2 dataset_keyword1="nothingatall" title=job['title'] title1=title.split('@/@') if title1[0]!="": commands.append(title1[0]) ckantitle=title1[0] notes=job['notes'] notes1=notes.split('@/@') if notes1[1]=='value' and notes1[0]!='': commands.append(notes1[0]) if notes1[1]=='label' and notes1[0]!='': label.append(notes1[0]) if notes1[1]=='link' and notes1[0]!='': links.append(notes1[0]) ckannotes=notes1[0] author=job['author'] author1=author.split('@/@') if author1[1]=='value' and author1[0]!='': commands.append(author1[0]) if author1[1]=='label' and author1[0]!='': label.append(author1[0]) if author1[1]=='link' and author1[0]!='': links.append(author1[0]) ckanauthor=author1[0] country=job['country'] country1=country.split('@/@') if country1[1]=='value' and country1[0]!='': commands.append(country1[0]) if country1[1]=='label' and country1[0]!='': label.append(country1[0]) if country1[1]=='link' and country1[0]!='': links.append(country1[0]) ckancountry=country1[0] temporal_coverage=job['temporal_coverage'] temporal_coverage1=temporal_coverage.split('@/@') if temporal_coverage1[1]=='value' and temporal_coverage1[0]!='': commands.append(temporal_coverage1[0]) if temporal_coverage1[1]=='label' and temporal_coverage1[0]!='': label.append(temporal_coverage1[0]) if temporal_coverage1[1]=='link' and temporal_coverage1[0]!='': links.append(temporal_coverage1[0]) ckantemporalcoverage=temporal_coverage1[0] date_released=job['date_released'] date_released1=date_released.split('@/@') if date_released1[1]=='value' and date_released1[0]!='': commands.append(date_released1[0]) if date_released1[1]=='label' and date_released1[0]!='': label.append(date_released1[0]) if date_released1[1]=='link' and date_released1[0]!='': links.append(date_released1[0]) ckandate_released=date_released1[0] author_email=job['author_email'] author_email1=author_email.split('@/@') if author_email1[1]=='value' and author_email1[0]!='': commands.append(author_email1[0]) if author_email1[1]=='label' and author_email1[0]!='': label.append(author_email1[0]) if author_email1[1]=='link' and author_email1[0]!='': links.append(author_email1[0]) ckanauthor_email=author_email1[0] tags=job['tags'] tags1=tags.split('@/@') if tags1[1]=='value' and tags1[0]!='': commands.append(tags1[0]) if tags1[1]=='label' and tags1[0]!='': label.append(tags1[0]) if tags1[1]=='link' and tags1[0]!='': links.append(tags1[0]) ckantags=tags1[0] resource=job['resource'] resource1=resource.split('@/@') if resource1[1]=='value' and resource1[0]!='': commands.append(resource1[0]) if resource1[1]=='label' and resource1[0]!='': label.append(resource1[0]) if resource1[1]=='link' and resource1[0]!='': links.append(resource1[0]) ckanresource=resource1[0] licence=job['license'] licence1=licence.split('@/@') if licence1[1]=='label' and licence1[0]!='': label.append(licence1[0]) if licence1[1]=='value' and licence1[0]!='': commands.append(licence1[0]) if licence1[1]=='link' and licence1[0]!='': links.append(licence1[0]) ckanlicense=licence1[0] date_updated=job['date_updated'] date_updated1=date_updated.split('@/@') if date_updated1[1]=='value' and date_updated1[0]!='': commands.append(date_updated1[0]) if date_updated1[1]=='label' and date_updated1[0]!='': label.append(date_updated1[0]) if date_updated1[1]=='link' and date_updated1[0]!='': links.append(date_updated1[0]) ckandate_updated=date_updated1[0] organization=job['organization'] organization1=organization.split('@/@') if organization1[1]=='value' and organization1[0]!='': commands.append(organization1[0]) if organization1[1]=='label' and organization1[0]!='': label.append(organization1[0]) if organization1[1]=='link' and organization1[0]!='': links.append(organization1[0]) ckanorganization=organization1[0] maintainer_email=job['maintainer_email'] maintainer_email1=maintainer_email.split('@/@') if maintainer_email1[1]=='value' and maintainer_email1[0]!='': commands.append(maintainer_email1[0]) if maintainer_email1[1]=='label' and maintainer_email1[0]!='': label.append(maintainer_email1[0]) if maintainer_email1[1]=='link' and maintainer_email1[0]!='': links.append(maintainer_email1[0]) ckanmaintainer_email=maintainer_email1[0] state=job['state'] state1=state.split('@/@') if state1[1]=='value' and state1[0]!='': commands.append(state1[0]) if state1[1]=='label' and state1[0]!='': label.append(state1[0]) if state1[1]=='link' and state1[0]!='': links.append(state1[0]) ckanstate=state1[0] city=job['city'] city1=city.split('@/@') if city1[1]=='value' and city1[0]!='': commands.append(city1[0]) if city1[1]=='label' and city1[0]!='': label.append(city1[0]) if city1[1]=='link' and city1[0]!='': links.append(city1[0]) ckancity=city1[0] category=job['category'] category1=category.split('@/@') if category1[1]=='value' and category1[0]!='': commands.append(category1[0]) if category1[1]=='label' and category1[0]!='': label.append(category1[0]) if category1[1]=='link' and category1[0]!='': links.append(category1[0]) ckanExtrasCategory=category1[0] frequency=job['frequency'] frequency1=frequency.split('@/@') if frequency1[1]=='value' and frequency1[0]!='': commands.append(frequency1[0]) if frequency1[1]=='label' and frequency1[0]!='': label.append(frequency1[0]) if frequency1[1]=='link' and frequency1[0]!='': links.append(frequency1[0]) ckanExtrasFrequency=frequency1[0] language=job['language'] ckanExtrasLanguage=language maintainer=job['maintainer'] maintainer1=maintainer.split('@/@') if maintainer1[1]=='value' and maintainer1[0]!='': commands.append(maintainer1[0]) if maintainer1[1]=='label' and maintainer1[0]!='': label.append(maintainer1[0]) if maintainer1[1]=='link' and maintainer1[0]!='': links.append(maintainer1[0]) ckanMaintainer=maintainer1[0] try: step=int(id1) except ValueError: step="" ## call function StartHarvestProcedure dataset_urls=StartHarvestProcedure(commands,label,links,url,step,afterid,endpoint,dataset_keyword,dataset_keyword1,ckannotes,ckanlicense,ckanresource,ckantags,ckanauthor_email,ckanauthor,ckantitle,ckandate_updated,ckanExtrasCategory ,ckanExtrasFrequency,ckanExtrasLanguage,ckanMaintainer,ckandate_released,ckancountry,ckantemporalcoverage,backupi,ckanorganization,ckanmaintainer_email,ckanstate,ckancity) return dataset_urls
def setUp(self): super(RuleEnforcementPermissionsResolverTestCase, self).setUp() register_internal_trigger_types() # Create some mock users user_1_db = UserDB(name='1_role_rule_pack_grant') user_1_db = User.add_or_update(user_1_db) self.users['custom_role_rule_pack_grant'] = user_1_db user_2_db = UserDB(name='1_role_rule_grant') user_2_db = User.add_or_update(user_2_db) self.users['custom_role_rule_grant'] = user_2_db user_3_db = UserDB(name='custom_role_pack_rule_all_grant') user_3_db = User.add_or_update(user_3_db) self.users['custom_role_pack_rule_all_grant'] = user_3_db user_4_db = UserDB(name='custom_role_rule_all_grant') user_4_db = User.add_or_update(user_4_db) self.users['custom_role_rule_all_grant'] = user_4_db user_5_db = UserDB(name='custom_role_rule_modify_grant') user_5_db = User.add_or_update(user_5_db) self.users['custom_role_rule_modify_grant'] = user_5_db user_6_db = UserDB(name='rule_pack_rule_create_grant') user_6_db = User.add_or_update(user_6_db) self.users['rule_pack_rule_create_grant'] = user_6_db user_7_db = UserDB(name='rule_pack_rule_all_grant') user_7_db = User.add_or_update(user_7_db) self.users['rule_pack_rule_all_grant'] = user_7_db user_8_db = UserDB(name='rule_rule_create_grant') user_8_db = User.add_or_update(user_8_db) self.users['rule_rule_create_grant'] = user_8_db user_9_db = UserDB(name='rule_rule_all_grant') user_9_db = User.add_or_update(user_9_db) self.users['rule_rule_all_grant'] = user_9_db user_10_db = UserDB(name='custom_role_rule_list_grant') user_10_db = User.add_or_update(user_10_db) self.users['custom_role_rule_list_grant'] = user_10_db # Create some mock resources on which permissions can be granted rule_1_db = RuleDB(pack='test_pack_1', name='rule1', action={'ref': 'core.local'}, trigger='core.st2.key_value_pair.create') rule_1_db = Rule.add_or_update(rule_1_db) self.resources['rule_1'] = rule_1_db rule_enforcement_1_db = RuleEnforcementDB( trigger_instance_id=str(bson.ObjectId()), execution_id=str(bson.ObjectId()), rule={ 'ref': rule_1_db.ref, 'uid': rule_1_db.uid, 'id': str(rule_1_db.id) }) rule_enforcement_1_db = RuleEnforcement.add_or_update( rule_enforcement_1_db) self.resources['rule_enforcement_1'] = rule_enforcement_1_db rule_2_db = RuleDB(pack='test_pack_1', name='rule2') rule_2_db = Rule.add_or_update(rule_2_db) self.resources['rule_2'] = rule_2_db rule_enforcement_2_db = RuleEnforcementDB( trigger_instance_id=str(bson.ObjectId()), execution_id=str(bson.ObjectId()), rule={ 'ref': rule_2_db.ref, 'uid': rule_2_db.uid, 'id': str(rule_2_db.id) }) rule_enforcement_2_db = RuleEnforcement.add_or_update( rule_enforcement_2_db) self.resources['rule_enforcement_2'] = rule_enforcement_2_db rule_3_db = RuleDB(pack='test_pack_2', name='rule3') rule_3_db = Rule.add_or_update(rule_3_db) self.resources['rule_3'] = rule_3_db rule_enforcement_3_db = RuleEnforcementDB( trigger_instance_id=str(bson.ObjectId()), execution_id=str(bson.ObjectId()), rule={ 'ref': rule_3_db.ref, 'uid': rule_3_db.uid, 'id': str(rule_3_db.id) }) rule_enforcement_3_db = RuleEnforcement.add_or_update( rule_enforcement_3_db) self.resources['rule_enforcement_3'] = rule_enforcement_3_db # Create some mock roles with associated permission grants # Custom role 2 - one grant on parent pack # "rule_view" on pack_1 grant_db = PermissionGrantDB( resource_uid=self.resources['pack_1'].get_uid(), resource_type=ResourceType.PACK, permission_types=[PermissionType.RULE_VIEW]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_3_db = RoleDB(name='custom_role_rule_pack_grant', permission_grants=permission_grants) role_3_db = Role.add_or_update(role_3_db) self.roles['custom_role_rule_pack_grant'] = role_3_db # Custom role 4 - one grant on rule # "rule_view on rule_3 grant_db = PermissionGrantDB( resource_uid=self.resources['rule_3'].get_uid(), resource_type=ResourceType.RULE, permission_types=[PermissionType.RULE_VIEW]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_4_db = RoleDB(name='custom_role_rule_grant', permission_grants=permission_grants) role_4_db = Role.add_or_update(role_4_db) self.roles['custom_role_rule_grant'] = role_4_db # Custom role - "rule_all" grant on a parent rule pack grant_db = PermissionGrantDB( resource_uid=self.resources['pack_1'].get_uid(), resource_type=ResourceType.PACK, permission_types=[PermissionType.RULE_ALL]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_4_db = RoleDB(name='custom_role_pack_rule_all_grant', permission_grants=permission_grants) role_4_db = Role.add_or_update(role_4_db) self.roles['custom_role_pack_rule_all_grant'] = role_4_db # Custom role - "rule_all" grant on a rule grant_db = PermissionGrantDB( resource_uid=self.resources['rule_1'].get_uid(), resource_type=ResourceType.RULE, permission_types=[PermissionType.RULE_ALL]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_4_db = RoleDB(name='custom_role_rule_all_grant', permission_grants=permission_grants) role_4_db = Role.add_or_update(role_4_db) self.roles['custom_role_rule_all_grant'] = role_4_db # Custom role - "rule_modify" on role_1 grant_db = PermissionGrantDB( resource_uid=self.resources['rule_1'].get_uid(), resource_type=ResourceType.RULE, permission_types=[PermissionType.RULE_MODIFY]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_5_db = RoleDB(name='custom_role_rule_modify_grant', permission_grants=permission_grants) role_5_db = Role.add_or_update(role_5_db) self.roles['custom_role_rule_modify_grant'] = role_5_db # Custom role - "rule_create" grant on pack_1 grant_db = PermissionGrantDB( resource_uid=self.resources['pack_1'].get_uid(), resource_type=ResourceType.PACK, permission_types=[PermissionType.RULE_CREATE]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_6_db = RoleDB(name='rule_pack_rule_create_grant', permission_grants=permission_grants) role_6_db = Role.add_or_update(role_6_db) self.roles['rule_pack_rule_create_grant'] = role_6_db # Custom role - "rule_all" grant on pack_1 grant_db = PermissionGrantDB( resource_uid=self.resources['pack_1'].get_uid(), resource_type=ResourceType.PACK, permission_types=[PermissionType.RULE_ALL]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_7_db = RoleDB(name='rule_pack_rule_all_grant', permission_grants=permission_grants) role_7_db = Role.add_or_update(role_7_db) self.roles['rule_pack_rule_all_grant'] = role_7_db # Custom role - "rule_create" grant on rule_1 grant_db = PermissionGrantDB( resource_uid=self.resources['rule_1'].get_uid(), resource_type=ResourceType.RULE, permission_types=[PermissionType.RULE_CREATE]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_8_db = RoleDB(name='rule_rule_create_grant', permission_grants=permission_grants) role_8_db = Role.add_or_update(role_8_db) self.roles['rule_rule_create_grant'] = role_8_db # Custom role - "rule_all" grant on rule_1 grant_db = PermissionGrantDB( resource_uid=self.resources['rule_1'].get_uid(), resource_type=ResourceType.RULE, permission_types=[PermissionType.RULE_ALL]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_9_db = RoleDB(name='rule_rule_all_grant', permission_grants=permission_grants) role_9_db = Role.add_or_update(role_9_db) self.roles['rule_rule_all_grant'] = role_9_db # Custom role - "rule_list" grant grant_db = PermissionGrantDB( resource_uid=None, resource_type=None, permission_types=[PermissionType.RULE_LIST]) grant_db = PermissionGrant.add_or_update(grant_db) permission_grants = [str(grant_db.id)] role_10_db = RoleDB(name='custom_role_rule_list_grant', permission_grants=permission_grants) role_10_db = Role.add_or_update(role_10_db) self.roles['custom_role_rule_list_grant'] = role_10_db # Create some mock role assignments user_db = self.users['custom_role_rule_pack_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_rule_pack_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_rule_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_rule_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_pack_rule_all_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_pack_rule_all_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_rule_all_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_rule_all_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_rule_modify_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_rule_modify_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['rule_pack_rule_create_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['rule_pack_rule_create_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['rule_pack_rule_all_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['rule_pack_rule_all_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['rule_rule_create_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['rule_rule_create_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['rule_rule_all_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['rule_rule_all_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db) user_db = self.users['custom_role_rule_list_grant'] role_assignment_db = UserRoleAssignmentDB( user=user_db.name, role=self.roles['custom_role_rule_list_grant'].name, source='assignments/%s.yaml' % user_db.name) UserRoleAssignment.add_or_update(role_assignment_db)
def test_save_id(self): doc = {'_id': bson.ObjectId(), 'x': 1} self.bind.db.coll.save(doc)
def setUp(self): self.guid = "urn:newsml:localhost:2016-09-12T12:11:40.160498:7237e59f-c42d-4865-aee5-e364aeb2966a" self.archived_only_data = [ { "_id": bson.ObjectId("588c1b901d41c805dce70df0"), "guid": self.guid, "item_id": self.guid, "_current_version": 3, "type": "text", "abstract": "test", "state": "fetched", "slugline": "slugline", "headline": "headline", "flags": {"marked_archived_only": True}, "subject": [{"qcode": "17004000", "name": "Statistics"}], "body_html": "Test Document body", "source": "AAP", "linked_in_packages": [ { "package": "urn:newsml:localhost:2017-01-28T15:28:33.535974:30c01757-cfb5-4985-a8f4-c3ecb253a244", "package_type": "takes", } ], }, { "_id": bson.ObjectId("57d224de069b7f038e9d2a53"), "guid": "urn:newsml:localhost:2016-08-30T06:15:35.379754:3d68cc4c-1f16-4a7f-bfca-92ca50bcdd8f", "item_id": "urn:newsml:localhost:2016-08-30T06:15:35.379754:3d68cc4c-1f16-4a7f-bfca-92ca50bcdd8f", "_current_version": 4, "type": "text", "abstract": "test", "state": "fetched", "slugline": "slugline", "headline": "headline", "flags": {"marked_archived_only": True}, "subject": [{"qcode": "17004000", "name": "Statistics"}], "body_html": "Test Document body", "source": "AAP", }, { "_id": "213456", "guid": self.guid, "item_id": "urn:newsml:localhost:2017-01-28T15:28:33.535974:30c01757-cfb5-4985-a8f4-c3ecb253a244", "_current_version": 1, "type": "composite", "abstract": "test", "state": "fetched", "slugline": "slugline", "headline": "headline", "groups": [ {"id": "root", "refs": [{"idRef": "main"}], "role": "grpRole:NEP"}, { "id": "main", "refs": [ { "residRef": "urn:newsml:localhost:2016-09-12T12:11:40.160498:7237e59f-c42d-4865-aee5-e364aeb2966a", "guid": "urn:newsml:localhost:2016-09-12T12:11:40.160498:7237e59f-c42d-4865-aee5-e364aeb2966a", "itemClass": "icls:text", "headline": "Testing takes", "type": "text", "renditions": {}, "_current_version": 3, "is_published": True, "slugline": "Takes test", "sequence": 1, "location": "archived", }, { "residRef": "urn:newsml:localhost:2016-08-30T06:15:35.379754:3d68cc4c-1f16-4a7f-bfca-92ca50bcdd8f", "guid": "urn:newsml:localhost:2016-08-30T06:15:35.379754:3d68cc4c-1f16-4a7f-bfca-92ca50bcdd8f", "itemClass": "icls:text", "headline": "Testing takes", "type": "text", "renditions": {}, "_current_version": 4, "is_published": True, "slugline": "Takes test", "sequence": 2, "location": "archived", }, ], "role": "grpRole:main", }, ], }, ] self.archivedService = get_resource_service("archived")
def test_online_analysis(data_builder, as_admin, as_drone, file_form, api_db): gear = data_builder.create_gear(gear={'inputs': {'csv': {'base': 'file'}}}) group = data_builder.create_group() project = data_builder.create_project() session = data_builder.create_session() acquisition = data_builder.create_acquisition() assert as_admin.post('/acquisitions/' + acquisition + '/files', files=file_form('input.csv')).ok # Try to create job-based analysis with invalid fileref r = as_admin.post('/sessions/' + session + '/analyses', json={ 'label': 'online', 'job': {'gear_id': gear, 'inputs': {'csv': {'type': 'acquisition', 'id': acquisition, 'name': 'nosuch.csv'}}} }) assert r.status_code == 404 # Try to create job-based analysis with invalid gear id r = as_admin.post('/sessions/' + session + '/analyses', json={ 'label': 'online', 'job': {'gear_id': '000000000000000000000000', 'inputs': {'csv': {'type': 'acquisition', 'id': acquisition, 'name': 'input.csv'}}} }) assert r.status_code == 404 # Try to create job-based analysis at group level r = as_admin.post('/groups/' + group + '/analyses', json={ 'label': 'online', 'job': {'gear_id': gear, 'inputs': {'csv': {'type': 'acquisition', 'id': acquisition, 'name': 'input.csv'}}} }) # No endpoint to make group analyses but the handler/dao layer specifically allows it assert r.status_code == 404 # Create analysis job at project level r = as_admin.post('/projects/' + project + '/analyses', json={ 'label': 'online', 'job': {'gear_id': gear, 'inputs': {'csv': {'type': 'acquisition', 'id': acquisition, 'name': 'input.csv'}}} }) assert r.ok analysis = r.json()['_id'] # Test that permission updates don't make it to analyses r = as_admin.post('/projects/' + project + '/permission', json={ '_id': '*****@*****.**', 'access': 'ro'}) # Verify job was created with it r = as_admin.get('/analyses/' + analysis) assert r.ok job = r.json().get('job') assert job assert not r.json().get('permissions') # Engine upload r = as_drone.post('/engine', params={'level': 'analysis', 'id': analysis, 'job': job}, files=file_form('output.csv', meta={'type': 'tabular data'})) assert r.ok check_files(as_admin, analysis, 'files', 'output.csv') api_db.analyses.delete_one({'_id': bson.ObjectId(analysis)}) # Create job-based analysis at acquisition level r = as_admin.post('/acquisitions/' + acquisition + '/analyses', json={ 'label': 'online', 'job': {'gear_id': gear, 'inputs': {'csv': {'type': 'acquisition', 'id': acquisition, 'name': 'input.csv'}}} }) assert r.ok analysis = r.json()['_id'] # Verify job was created with it r = as_admin.get('/analyses/' + analysis) assert r.ok job = r.json().get('job') assert job # Engine upload r = as_drone.post('/engine', params={'level': 'analysis', 'id': analysis, 'job': job}, files=file_form('output.csv', meta={'type': 'tabular data'})) assert r.ok check_files(as_admin, analysis, 'files', 'output.csv') api_db.analyses.delete_one({'_id': bson.ObjectId(analysis)}) # Create job-based analysis r = as_admin.post('/sessions/' + session + '/analyses', json={ 'label': 'online', 'job': {'gear_id': gear, 'inputs': {'csv': {'type': 'acquisition', 'id': acquisition, 'name': 'input.csv'}}} }) assert r.ok analysis = r.json()['_id'] # Verify job was created with it r = as_admin.get('/analyses/' + analysis) assert r.ok job = r.json().get('job') assert job # Verify that gear info was stored gear_info = r.json().get('gear_info') assert gear_info.get('id') == gear assert gear_info.get('name') assert gear_info.get('version') == '0.0.1' check_files(as_admin, analysis, 'inputs', 'input.csv') # Try manual upload - not allowed for job-based analysis r = as_admin.post('/analyses/' + analysis + '/files', files=file_form('output.csv')) assert r.status_code == 400 # Engine upload r = as_drone.post('/engine', params={'level': 'analysis', 'id': analysis, 'job': job}, files=file_form('output.csv', meta={'type': 'tabular data'})) assert r.ok check_files(as_admin, analysis, 'files', 'output.csv') # Force retry the job r = as_drone.get('/jobs/next') assert r.ok r = as_drone.get('/jobs/next') assert r.ok r = as_drone.get('/jobs/next') assert r.ok # set job to failed r = as_drone.put('/jobs/' + job, json={'state': 'failed'}) assert r.ok r = as_admin.post('/jobs/' + job + '/retry', params={'ignoreState': True, 'root': True}) assert r.ok retried_job_id = r.json()['_id'] r = as_drone.get('/jobs/next') assert r.ok # Verify job was created with it r = as_admin.get('/analyses/' + analysis) assert r.ok print job assert r.json().get('job') == retried_job_id # Engine upload r = as_drone.post('/engine', params={'level': 'analysis', 'id': analysis, 'job': retried_job_id}, files=file_form('output.csv', meta={'type': 'tabular data'})) assert r.ok check_files(as_admin, analysis, 'files', 'output.csv') # Verify job was created with it r = as_admin.get('/analyses/' + analysis) assert r.ok print job assert r.json().get('job') == retried_job_id r = as_admin.get('/jobs/' + retried_job_id) assert r.ok print r.json()['saved_files'] assert r.json()['saved_files'] != [] check_files(as_admin, analysis, 'files', 'output.csv') api_db.analyses.delete_one({'_id': bson.ObjectId(analysis)})
def _deserialize(self, value, attr, data, **kwargs): try: return bson.ObjectId(value) except BSONError: raise ValidationError('invalid ObjectId `%s`' % value)
def get_todo_by_id(id): todos = mongo.db.todos todo = todos.find_one({'_id': bson.ObjectId(oid=str(id))}) sanitized = json.loads(json_util.dumps(todo)) return jsonify(sanitized)