def entity_updated(self, entity, attr): """an entity using this storage for attr has been updated""" # get the name of the previous file containing the value oldpath = self.current_fs_path(entity, attr) if entity._cw.transaction_data.get('fs_importing'): # If we are importing from the filesystem, the file already exists. # We do not need to create it but we need to fetch the content of # the file as the actual content of the attribute fpath = entity.cw_edited[attr].getvalue() entity._cw_dont_cache_attribute(attr, repo_side=True) assert fpath is not None binary = Binary.from_file(fpath) else: # We must store the content of the attributes # into a file to stay consistent with the behaviour of entity_add. # Moreover, the BytesFileSystemStorage expects to be able to # retrieve the current value of the attribute at anytime by reading # the file on disk. To be able to rollback things, use a new file # and keep the old one that will be removed on commit if everything # went ok. # # fetch the current attribute value in memory binary = entity.cw_edited.pop(attr) if binary is None: fpath = None else: # Get filename for it fd, fpath = self.new_fs_path(entity, attr) # write attribute value on disk self._writecontent(fd, binary) # Mark the new file as added during the transaction. # The file will be removed on rollback AddFileOp.get_instance(entity._cw).add_data(fpath) # reinstall poped value if fpath is None: entity.cw_edited.edited_attribute(attr, None) else: # register the new location for the file. binary_obj = Binary(fpath.encode('utf-8')) entity.cw_edited.edited_attribute(attr, binary_obj) if oldpath is not None and oldpath != fpath: # Mark the old file as useless so the file will be removed at # commit. DeleteFileOp.get_instance(entity._cw).add_data(oldpath) return binary
def prepare(self, schema): """Prepare an external entity for later insertion: * ensure attributes and inlined relations have a single value * turn set([value]) into value and remove key associated to empty set * remove non inlined relations and return them as a [(e1key, relation, e2key)] list Return a list of non inlined relations that may be inserted later, each relations defined by a 3-tuple (subject extid, relation type, object extid). The instance's schema is given as argument. Take care the importer may call this method several times. """ assert self._schema is None, 'prepare() has already been called for %s' % self self._schema = schema eschema = schema.eschema(self.etype) deferred = [] entity_dict = self.values for key, rtype, role in self.iter_rdefs(): rschema = schema.rschema(rtype) if rschema.final or (rschema.inlined and role == 'subject'): assert len(entity_dict[key]) <= 1, \ "more than one value for %s: %s (%s)" % (rtype, entity_dict[key], self.extid) if entity_dict[key]: entity_dict[rtype] = entity_dict[key].pop() if key != rtype: del entity_dict[key] if (rschema.final and eschema.has_metadata(rtype, 'format') and not rtype + '_format' in entity_dict): entity_dict[rtype + '_format'] = u'text/plain' if (rschema.final and eschema.rdef(rtype).object.type == 'Bytes' and not isinstance(entity_dict[rtype], Binary)): entity_dict[rtype] = Binary(entity_dict[rtype]) else: del entity_dict[key] else: for target_extid in entity_dict.pop(key): if role == 'subject': deferred.append((self.extid, rtype, target_extid)) else: deferred.append((target_extid, rtype, self.extid)) return deferred
def grok_data(self): """ self.data is something such as an excel file or CSV data or a pickled numpy array or an already processed binary. Ensure it's a pickle numpy array before storing object in db. If data seems to be already processed, return True, else return False. """ entity = self.entity try: filename = entity.data.filename.lower() except AttributeError: data = entity.data if isinstance(data, Binary): return True # if not isinstance(data, numpy.ndarray): # raise TypeError('data is neither a Binary nor a numpy array (%s)' % type(data)) numpy_array = data else: adapter = self._cw.vreg['adapters'].select_or_none( 'source_to_numpy_array', self._cw, entity=entity, filename=filename) if adapter is None: msg = self._cw._( 'Unsupported file type %s') % entity.data.filename raise ValidationError(entity.eid, {'data': msg}) numpy_array = adapter.to_numpy_array(entity.data, filename) if numpy_array.ndim != 1: raise ValidationError( entity.eid, {'data': _('data must be a 1-dimensional array')}) if numpy_array.size == 0: raise ValidationError( entity.eid, {'data': _('data must have at least one value')}) data = Binary() compressed_data = zlib.compress(pickle.dumps(numpy_array, protocol=2)) data.write(compressed_data) entity.cw_edited['data'] = data entity.array = numpy_array return False
def _rdef_values(rdef): amap = {'order': 'ordernum', 'default': 'defaultval'} values = {} extra = {} for prop in rdef.rproperty_defs(rdef.object): if prop in _IGNORED_PROPS: continue value = getattr(rdef, prop) if prop not in KNOWN_RPROPERTIES: extra[prop] = value continue # XXX type cast really necessary? if prop in ('indexed', 'fulltextindexed', 'internationalizable'): value = bool(value) elif prop == 'ordernum': value = int(value) if value is not None and prop == 'default': value = Binary.zpickle(value) values[amap.get(prop, prop)] = value if extra: values['extra_props'] = Binary(json.dumps(extra).encode('ascii')) relations = ['X %s %%(%s)s' % (attr, attr) for attr in sorted(values)] return relations, values
def _process_ldap_item(self, dn, iterator): """Turn an ldap received item into a proper dict.""" itemdict = {'dn': dn} for key, value in iterator: if self.user_attrs.get( key) == 'upassword': # XXx better password detection value = value[0].encode('utf-8') # we only support ldap_salted_sha1 for ldap sources, see: server/utils.py if not value.startswith(b'{SSHA}'): value = utils.crypt_password(value) itemdict[key] = Binary(value) elif self.user_attrs.get(key) == 'modification_date': itemdict[key] = datetime.strptime(value[0], '%Y%m%d%H%M%SZ') else: if len(value) == 1: itemdict[key] = value = value[0] else: itemdict[key] = value # we expect memberUid to be a list of user ids, make sure of it member = self.group_rev_attrs['member'] if isinstance(itemdict.get(member), str): itemdict[member] = [itemdict[member]] return itemdict
def __call__(self): """ Before adding the CWUpload entity, create a CWProcessing entity. """ # Here we want to execute rql request with user permissions: user who # is creating this entity with self._cw.security_enabled(read=True, write=True): # Create the associated processing task processing_eid = self._cw.create_entity( "CWProcessing", status=u"scheduled", uploaded_by=self._cw.user.eid).eid # Link the processing with the submitted task self._cw.add_relation( self.entity.eid, "related_processing", processing_eid) # Create an entity to store the task result result_eid = self._cw.create_entity( "UploadForm", data=Binary(json.dumps({})), data_format=u"text/json", data_name=u"result.json", uploaded_by=self._cw.user.eid).eid # Link the processing with the final results self._cw.add_relation(processing_eid, "result_form", result_eid)
def test_forge_concerned_by_links(self): with self.admin_access.client_cnx() as cnx: ea = cnx.create_entity('EmailAddress', address=u'*****@*****.**', alias=u'First Last') cnx.commit() p = cnx.find('Person', firstname=u'First').one() self.assertFalse(p.concerned_by) e = cnx.create_entity('Email', subject=u'none', messageid=u'toto') f = cnx.create_entity('File', data=Binary('Some content'), data_name=u'some part', data_format=u'text/plain', data_encoding=u'ascii') e.cw_set(attachment=f, sender=ea) cnx.commit() # now check the file has been linked to the person by hooks with self.admin_access.client_cnx() as cnx: p = cnx.find('Person', firstname=u'First').one() self.assertTrue(p.concerned_by) self.assertEqual('some part', p.concerned_by[0].data_name)
import sys import getpass from cubicweb import Binary from cubicweb.server.utils import crypt_password if __args__: login = __args__.pop() else: login = raw_input("login? ") rset = rql('Any U WHERE U is CWUser, U login %(login)s', {'login': login}) if len(rset) != 1: sys.exit("user '%s' does not exist!" % login) pass1 = getpass.getpass(prompt='Enter new password? ') pass2 = getpass.getpass(prompt='Confirm? ') if pass1 != pass2: sys.exit("passwords don't match!") crypted = crypt_password(pass1) cwuser = rset.get_entity(0,0) cwuser.cw_set(upassword=Binary(crypted)) commit() print("password updated.")
def test_pickleable(self): b = Binary(b'toto') bb = pickle.loads(pickle.dumps(b)) self.assertEqual(b, bb)
def _process_form_value(self, form): value = form._cw.form.get(self.input_name(form)) if isinstance(value, str): # file modified using a text widget return Binary(value.encode(self.encoding(form))) return super(EditableFileField, self)._process_form_value(form)
def create_file(self, cnx, content=b'the-data'): return cnx.create_entity('File', data=Binary(content), data_format=u'text/plain', data_name=u'foo.pdf')
def test_bfss_update_to_None(self): with self.admin_access.repo_cnx() as cnx: f = cnx.create_entity('Affaire', opt_attr=Binary(b'toto')) cnx.commit() f.cw_set(opt_attr=None) cnx.commit()
def import_data(self): """ Method that import the users in cw. .. note:: This procedure create a user as a 'CWUser' entity which is related to different groups through the 'in_group' relation. | .. image:: ../schemas/users.png :width: 500px :align: center :alt: schema .. warning:: This procedure expect that the 'CWGroup' of interest are already created. """ # Get the activated State entity rql = "Any X Where X is State, X name 'activated'" rset = self.session.execute(rql) if rset.rowcount != 1: logging.error( "Can't insert users, no activated State entity detected.") raise Exception("Can't insert users, no activated State entity " "detected.") state_eid = rset[0][0] # Go through the goup names nb_of_users = float(len(self.users)) maxsize = max([len(name) for name in self.users]) cnt_user = 1. for user_name, user_item in self.users.iteritems(): # Print a progress bar self._progress_bar( cnt_user / nb_of_users, title="{0}(users)".format(user_name), bar_length=40, maxsize=maxsize + 7) cnt_user += 1. # Create the user crypted = crypt_password(user_item["password"]) user_entity, is_created = self._get_or_create_unique_entity( rql=("Any X Where X is CWUser, X login " "'{0}'".format(user_item["login"])), check_unicity=True, entity_name="CWUser", login=unicode(user_item["login"]), upassword=Binary(crypted)) # If the user is created, add relation with the State entity if is_created: self._set_unique_relation(user_entity.eid, "in_state", state_eid, check_unicity=False) # Go through group names for group_name in user_item["group_names"]: # Check if the group exists rql = "Any X Where X is CWGroup, X name '{0}'".format(group_name) # Execute the rql request rset = self.session.execute(rql) # The request returns some data -> do nothing if rset.rowcount != 1: logging.error( "Can't insert user '{0}' in group '{1}', got {2} " "matches.".format(user_name, group_name, rset.rowcount)) continue # Get the group entity group_eid = rset[0][0] # > add relation with the user self._set_unique_relation(user_entity.eid, "in_group", group_eid, check_unicity=True) print # new line after last progress bar update
def generate_bytes(self, entity, attrname, index, format=None): fakefile = Binary(("%s%s" % (attrname, index)).encode('ascii')) fakefile.filename = u"file_%s" % attrname return fakefile
def __call__(self): """ Before adding the CWSearch entity, create a 'rset' and a 'result.json' File entities that contain all the filepath attached to the current rql request. Filepath are found by patching the rql request with the declared 'rqldownload-adaptors' actions. The '__rset_type__' adaptor attribute is used to export the rset. When an 'ecsvexport' is used, no file are then attached in the 'result.json' file. .. warning:: For the moment we assume the database intergrity (ie. all file paths inserted in the db exist on the file system) and thus do not check to speed up the hook. """ # Get the rql/export type from the CWSearch form rql = self.entity.cw_edited.get("path") # Execute the rql # ToDo: try to get the current request cw_rset rset = self._cw.execute(rql) # Get all the entities entities = {} if rset.rowcount > 0: for rowindex in range(len(rset[0])): try: entity = rset.get_entity(0, rowindex) entity_type = entity.__class__.__name__ entities[rowindex] = entity_type except NotAnEntity: pass except: raise if len(entities) == 0: raise ValidationError( "CWSearch", { "entities": _('cannot find any entity for the request ' '{0}'.format(rql))}) # Find the constant nodes constant_nodes = {} self._find_constant_nodes(rset.syntax_tree().children, constant_nodes) # Check we can associated rset entities with their rql labels actions = [] rql_etypes = constant_nodes.get("etype", {}) for etype in entities.values(): if etype not in rql_etypes or len(rql_etypes[etype]) != 1: raise ValidationError( "CWSearch", { "rql": _('cannot find entity description in the ' 'request {0}. Expect something like "Any X ' 'Where X is ' '{1}, ..."'.format(rql, etype))}) # Get all the rqldownload declared adapters possible_actions = self._cw.vreg["actions"]["rqldownload-adapters"] # Keep only actions that respect the current context actions = {} export_vids = set() for index, etype in entities.items(): entity_label = rql_etypes[etype][0] for action in possible_actions: for selector in action.__select__.selectors: if (isinstance(selector, is_instance) and etype in selector.expected_etypes): actions.setdefault(etype, []).append( (action, entity_label)) export_vids.add(unicode(action.__rset_type__)) # Check that at least one action has been found for this request if actions == []: raise ValidationError( "CWSearch", { "actions": _('cannot find an action for this request ' '{0}'.format(rql))}) # Check that the export types are homogeneous if len(export_vids) != 1: raise ValidationError( "CWSearch", { "actions": _('cannot deal with different or no action ' 'export types: {0}'.format(export_vids))}) export_vid = export_vids.pop() # Create an empty result structure result = {"rql": rql, "files": [], "nonexistent-files": [], "upper_file_index": 0} # Here we want to execute rql request with user permissions: user who # is creating this entity with self._cw.security_enabled(read=True, write=True): # Set the adaptor rset type self.entity.cw_edited["rset_type"] = export_vid # Create the global rql from the declared actions global_rql = rql cnt = 1 upper_file_index = 0 for etype, action_item in actions.items(): for action, entity_label in action_item: global_rql, nb_files = action(self._cw).rql( global_rql, entity_label, cnt) upper_file_index += nb_files cnt += 1 result["upper_file_index"] = upper_file_index # Execute the global rql rset = self._cw.execute(global_rql) result["rql"] = global_rql # Because self._cw is not a cubicwebRequest add an empty form # parameter self._cw.__dict__["form"] = {} try: view = self._cw.vreg["views"].select( export_vid, self._cw, rset=rset) rset_view = Binary() view.w = rset_view.write view.call() except: raise ValidationError( "CWSearch", { "rset_type": _('cannot apply this view "{0}" on this ' 'rset, choose another view ' 'id'.format(export_vid))}) # Save the rset in a File entity f_eid = self._cw.create_entity( "File", data=rset_view, data_format=u"text", data_name=u"rset").eid # Entity modification related event: specify that the rset has been # modified self.entity.cw_edited["rset"] = f_eid # Get all the files attached to the current request # Note: we assume the database integrity (ie. all file paths # inserted in the db exist on the file system) and thus do not # check to speed up this process. files_set = set() non_existent_files_set = set() if export_vid != "ecsvexport": for rset_row in rset.rows: for rset_index in range(upper_file_index): files_set.add(rset_row[rset_index]) # Update the result structure result["files"] = list(files_set) result["nonexistent-files"] = list(non_existent_files_set) # Save the result in a File entity f_eid = self._cw.create_entity( "File", data=Binary(json.dumps(result)), data_format=u"text/json", data_name=u"result.json").eid # Entity modification related event: specify that the result has # been modified self.entity.cw_edited["result"] = f_eid
def test_encoding_binary(self): for content in (b'he he', b'h\xe9 hxe9'): with self.subTest(content=content): encoded = self.encode(Binary(content)) self.assertEqual(base64.b64decode(encoded), content)
def get_file_data(self, file_eid, rset_file, session_index, search_name=None): return Binary("nothing in %s" % file_eid)
def _create_questionnaire(self, questionnaire_name, q_items, identifier_prefix, subject_id, subject_eid, study_eid, assessment_eid, questionnaire_eids, question_eids): """ Create a scans and its associated relations. """ # Create a questionnaire run qr_id = self._md5_sum(identifier_prefix + "_" + questionnaire_name) qr_entity, is_created = self._get_or_create_unique_entity( rql=("Any X Where X is QuestionnaireRun, X identifier " "'{0}'".format(qr_id)), check_unicity=True, entity_name="QuestionnaireRun", identifier=unicode(qr_id), label=unicode(questionnaire_name) ) # If we just create the questionnaire run, specify and relate the entity if is_created: # > add relation with the questionnaire self._set_unique_relation( qr_entity.eid, "questionnaire", questionnaire_eids[questionnaire_name], check_unicity=False) self._set_unique_relation( questionnaire_eids[questionnaire_name], "questionnaire_questionnaire_runs", qr_entity.eid, check_unicity=False) # > add relation with the assessment self._set_unique_relation( assessment_eid, "questionnaire_runs", qr_entity.eid, check_unicity=False) self._set_unique_relation( qr_entity.eid, "in_assessment", assessment_eid, check_unicity=False, subjtype="QuestionnaireRun") # > add relation with the study self._set_unique_relation( qr_entity.eid, "study", study_eid, check_unicity=False) self._set_unique_relation( study_eid, "study_questionnaire_runs", qr_entity.eid, check_unicity=False) # > add relation with the subject self._set_unique_relation( qr_entity.eid, "subject", subject_eid, check_unicity=False) self._set_unique_relation( subject_eid, "subject_questionnaire_runs", qr_entity.eid, check_unicity=False) if self.use_openanswer: # Go through all answers for question_name, answer in q_items.iteritems(): # Get the question entity question_eid = question_eids[questionnaire_name][ question_name] # Create an open answer answer_entity, _ = self._get_or_create_unique_entity( rql="", check_unicity=False, entity_name="OpenAnswer", identifier=unicode(self._md5_sum( qr_id + "_" + question_name)), value=unicode(answer)) # > add relation with the question self._set_unique_relation( answer_entity.eid, "question", question_eid, check_unicity=False, subjtype="OpenAnswer") self._set_unique_relation( question_eid, "question_open_answers", answer_entity.eid, check_unicity=False) # > add relation with the questionnaire run self._set_unique_relation( answer_entity.eid, "questionnaire_run", qr_entity.eid, check_unicity=False, subjtype="OpenAnswer") self._set_unique_relation( qr_entity.eid, "open_answers", answer_entity.eid, check_unicity=False) # > add relation with the assessment self._set_unique_relation( answer_entity.eid, "in_assessment", assessment_eid, check_unicity=False, subjtype="OpenAnswer") else: f_entity, _ = self._get_or_create_unique_entity( rql="", entity_name="RestrictedFile", title=u"{0} ({1})".format(questionnaire_name, subject_id), data=Binary(json.dumps(q_items)), data_format=u"text/json", data_name=u"result.json", check_unicity=False) self._set_unique_relation( qr_entity.eid, "file", f_entity.eid, check_unicity=False, subjtype="QuestionnaireRun") self._set_unique_relation( f_entity.eid, "in_assessment", assessment_eid, check_unicity=False, subjtype="RestrictedFile") return qr_entity.eid