def entity_updated(self, entity, attr):
"""an entity using this storage for attr has been updated"""
# get the name of the previous file containing the value
oldpath = self.current_fs_path(entity, attr)
if entity._cw.transaction_data.get('fs_importing'):
# If we are importing from the filesystem, the file already exists.
# We do not need to create it but we need to fetch the content of
# the file as the actual content of the attribute
fpath = entity.cw_edited[attr].getvalue()
entity._cw_dont_cache_attribute(attr, repo_side=True)
assert fpath is not None
binary = Binary.from_file(fpath)
else:
# We must store the content of the attributes
# into a file to stay consistent with the behaviour of entity_add.
# Moreover, the BytesFileSystemStorage expects to be able to
# retrieve the current value of the attribute at anytime by reading
# the file on disk. To be able to rollback things, use a new file
# and keep the old one that will be removed on commit if everything
# went ok.
#
# fetch the current attribute value in memory
binary = entity.cw_edited.pop(attr)
if binary is None:
fpath = None
else:
# Get filename for it
fd, fpath = self.new_fs_path(entity, attr)
# write attribute value on disk
self._writecontent(fd, binary)
# Mark the new file as added during the transaction.
# The file will be removed on rollback
AddFileOp.get_instance(entity._cw).add_data(fpath)
# reinstall poped value
if fpath is None:
entity.cw_edited.edited_attribute(attr, None)
else:
# register the new location for the file.
binary_obj = Binary(fpath.encode('utf-8'))
entity.cw_edited.edited_attribute(attr, binary_obj)
if oldpath is not None and oldpath != fpath:
# Mark the old file as useless so the file will be removed at
# commit.
DeleteFileOp.get_instance(entity._cw).add_data(oldpath)
return binary
def prepare(self, schema):
"""Prepare an external entity for later insertion:
* ensure attributes and inlined relations have a single value
* turn set([value]) into value and remove key associated to empty set
* remove non inlined relations and return them as a [(e1key, relation, e2key)] list
Return a list of non inlined relations that may be inserted later, each relations defined by
a 3-tuple (subject extid, relation type, object extid).
The instance's schema is given as argument.
Take care the importer may call this method several times.
"""
assert self._schema is None, 'prepare() has already been called for %s' % self
self._schema = schema
eschema = schema.eschema(self.etype)
deferred = []
entity_dict = self.values
for key, rtype, role in self.iter_rdefs():
rschema = schema.rschema(rtype)
if rschema.final or (rschema.inlined and role == 'subject'):
assert len(entity_dict[key]) <= 1, \
"more than one value for %s: %s (%s)" % (rtype, entity_dict[key], self.extid)
if entity_dict[key]:
entity_dict[rtype] = entity_dict[key].pop()
if key != rtype:
del entity_dict[key]
if (rschema.final
and eschema.has_metadata(rtype, 'format')
and not rtype + '_format' in entity_dict):
entity_dict[rtype + '_format'] = u'text/plain'
if (rschema.final
and eschema.rdef(rtype).object.type == 'Bytes'
and not isinstance(entity_dict[rtype], Binary)):
entity_dict[rtype] = Binary(entity_dict[rtype])
else:
del entity_dict[key]
else:
for target_extid in entity_dict.pop(key):
if role == 'subject':
deferred.append((self.extid, rtype, target_extid))
else:
deferred.append((target_extid, rtype, self.extid))
return deferred
def grok_data(self):
""" self.data is something such as an excel file or CSV data or a pickled
numpy array or an already processed binary.
Ensure it's a pickle numpy array before storing object in db.
If data seems to be already processed, return True, else return False.
"""
entity = self.entity
try:
filename = entity.data.filename.lower()
except AttributeError:
data = entity.data
if isinstance(data, Binary):
return True
# if not isinstance(data, numpy.ndarray):
# raise TypeError('data is neither a Binary nor a numpy array (%s)' % type(data))
numpy_array = data
else:
adapter = self._cw.vreg['adapters'].select_or_none(
'source_to_numpy_array',
self._cw,
entity=entity,
filename=filename)
if adapter is None:
msg = self._cw._(
'Unsupported file type %s') % entity.data.filename
raise ValidationError(entity.eid, {'data': msg})
numpy_array = adapter.to_numpy_array(entity.data, filename)
if numpy_array.ndim != 1:
raise ValidationError(
entity.eid, {'data': _('data must be a 1-dimensional array')})
if numpy_array.size == 0:
raise ValidationError(
entity.eid, {'data': _('data must have at least one value')})
data = Binary()
compressed_data = zlib.compress(pickle.dumps(numpy_array, protocol=2))
data.write(compressed_data)
entity.cw_edited['data'] = data
entity.array = numpy_array
return False
def _rdef_values(rdef):
amap = {'order': 'ordernum', 'default': 'defaultval'}
values = {}
extra = {}
for prop in rdef.rproperty_defs(rdef.object):
if prop in _IGNORED_PROPS:
continue
value = getattr(rdef, prop)
if prop not in KNOWN_RPROPERTIES:
extra[prop] = value
continue
# XXX type cast really necessary?
if prop in ('indexed', 'fulltextindexed', 'internationalizable'):
value = bool(value)
elif prop == 'ordernum':
value = int(value)
if value is not None and prop == 'default':
value = Binary.zpickle(value)
values[amap.get(prop, prop)] = value
if extra:
values['extra_props'] = Binary(json.dumps(extra).encode('ascii'))
relations = ['X %s %%(%s)s' % (attr, attr) for attr in sorted(values)]
return relations, values
def _process_ldap_item(self, dn, iterator):
"""Turn an ldap received item into a proper dict."""
itemdict = {'dn': dn}
for key, value in iterator:
if self.user_attrs.get(
key) == 'upassword': # XXx better password detection
value = value[0].encode('utf-8')
# we only support ldap_salted_sha1 for ldap sources, see: server/utils.py
if not value.startswith(b'{SSHA}'):
value = utils.crypt_password(value)
itemdict[key] = Binary(value)
elif self.user_attrs.get(key) == 'modification_date':
itemdict[key] = datetime.strptime(value[0], '%Y%m%d%H%M%SZ')
else:
if len(value) == 1:
itemdict[key] = value = value[0]
else:
itemdict[key] = value
# we expect memberUid to be a list of user ids, make sure of it
member = self.group_rev_attrs['member']
if isinstance(itemdict.get(member), str):
itemdict[member] = [itemdict[member]]
return itemdict
def __call__(self):
""" Before adding the CWUpload entity, create a CWProcessing entity.
"""
# Here we want to execute rql request with user permissions: user who
# is creating this entity
with self._cw.security_enabled(read=True, write=True):
# Create the associated processing task
processing_eid = self._cw.create_entity(
"CWProcessing", status=u"scheduled",
uploaded_by=self._cw.user.eid).eid
# Link the processing with the submitted task
self._cw.add_relation(
self.entity.eid, "related_processing", processing_eid)
# Create an entity to store the task result
result_eid = self._cw.create_entity(
"UploadForm", data=Binary(json.dumps({})),
data_format=u"text/json", data_name=u"result.json",
uploaded_by=self._cw.user.eid).eid
# Link the processing with the final results
self._cw.add_relation(processing_eid, "result_form", result_eid)
def test_forge_concerned_by_links(self):
with self.admin_access.client_cnx() as cnx:
ea = cnx.create_entity('EmailAddress',
address=u'*****@*****.**',
alias=u'First Last')
cnx.commit()
p = cnx.find('Person', firstname=u'First').one()
self.assertFalse(p.concerned_by)
e = cnx.create_entity('Email', subject=u'none', messageid=u'toto')
f = cnx.create_entity('File',
data=Binary('Some content'),
data_name=u'some part',
data_format=u'text/plain',
data_encoding=u'ascii')
e.cw_set(attachment=f, sender=ea)
cnx.commit()
# now check the file has been linked to the person by hooks
with self.admin_access.client_cnx() as cnx:
p = cnx.find('Person', firstname=u'First').one()
self.assertTrue(p.concerned_by)
self.assertEqual('some part', p.concerned_by[0].data_name)
import sys
import getpass
from cubicweb import Binary
from cubicweb.server.utils import crypt_password
if __args__:
login = __args__.pop()
else:
login = raw_input("login? ")
rset = rql('Any U WHERE U is CWUser, U login %(login)s', {'login': login})
if len(rset) != 1:
sys.exit("user '%s' does not exist!" % login)
pass1 = getpass.getpass(prompt='Enter new password? ')
pass2 = getpass.getpass(prompt='Confirm? ')
if pass1 != pass2:
sys.exit("passwords don't match!")
crypted = crypt_password(pass1)
cwuser = rset.get_entity(0,0)
cwuser.cw_set(upassword=Binary(crypted))
commit()
print("password updated.")
def test_pickleable(self):
b = Binary(b'toto')
bb = pickle.loads(pickle.dumps(b))
self.assertEqual(b, bb)
def _process_form_value(self, form):
value = form._cw.form.get(self.input_name(form))
if isinstance(value, str):
# file modified using a text widget
return Binary(value.encode(self.encoding(form)))
return super(EditableFileField, self)._process_form_value(form)
def create_file(self, cnx, content=b'the-data'):
return cnx.create_entity('File',
data=Binary(content),
data_format=u'text/plain',
data_name=u'foo.pdf')
def test_bfss_update_to_None(self):
with self.admin_access.repo_cnx() as cnx:
f = cnx.create_entity('Affaire', opt_attr=Binary(b'toto'))
cnx.commit()
f.cw_set(opt_attr=None)
cnx.commit()
def import_data(self):
""" Method that import the users in cw.
.. note::
This procedure create a user as a 'CWUser' entity which is related
to different groups through the 'in_group' relation.
|
.. image:: ../schemas/users.png
:width: 500px
:align: center
:alt: schema
.. warning::
This procedure expect that the 'CWGroup' of interest are already
created.
"""
# Get the activated State entity
rql = "Any X Where X is State, X name 'activated'"
rset = self.session.execute(rql)
if rset.rowcount != 1:
logging.error(
"Can't insert users, no activated State entity detected.")
raise Exception("Can't insert users, no activated State entity "
"detected.")
state_eid = rset[0][0]
# Go through the goup names
nb_of_users = float(len(self.users))
maxsize = max([len(name) for name in self.users])
cnt_user = 1.
for user_name, user_item in self.users.iteritems():
# Print a progress bar
self._progress_bar(
cnt_user / nb_of_users,
title="{0}(users)".format(user_name),
bar_length=40, maxsize=maxsize + 7)
cnt_user += 1.
# Create the user
crypted = crypt_password(user_item["password"])
user_entity, is_created = self._get_or_create_unique_entity(
rql=("Any X Where X is CWUser, X login "
"'{0}'".format(user_item["login"])),
check_unicity=True,
entity_name="CWUser",
login=unicode(user_item["login"]),
upassword=Binary(crypted))
# If the user is created, add relation with the State entity
if is_created:
self._set_unique_relation(user_entity.eid,
"in_state", state_eid, check_unicity=False)
# Go through group names
for group_name in user_item["group_names"]:
# Check if the group exists
rql = "Any X Where X is CWGroup, X name '{0}'".format(group_name)
# Execute the rql request
rset = self.session.execute(rql)
# The request returns some data -> do nothing
if rset.rowcount != 1:
logging.error(
"Can't insert user '{0}' in group '{1}', got {2} "
"matches.".format(user_name, group_name, rset.rowcount))
continue
# Get the group entity
group_eid = rset[0][0]
# > add relation with the user
self._set_unique_relation(user_entity.eid,
"in_group", group_eid, check_unicity=True)
print # new line after last progress bar update
def generate_bytes(self, entity, attrname, index, format=None):
fakefile = Binary(("%s%s" % (attrname, index)).encode('ascii'))
fakefile.filename = u"file_%s" % attrname
return fakefile
def __call__(self):
""" Before adding the CWSearch entity, create a 'rset' and a
'result.json' File entities that contain all the filepath attached
to the current rql request.
Filepath are found by patching the rql request with the declared
'rqldownload-adaptors' actions.
The '__rset_type__' adaptor attribute is used to export the rset.
When an 'ecsvexport' is used, no file are then attached in
the 'result.json' file.
.. warning::
For the moment we assume the database intergrity (ie. all file
paths inserted in the db exist on the file system) and thus do not
check to speed up the hook.
"""
# Get the rql/export type from the CWSearch form
rql = self.entity.cw_edited.get("path")
# Execute the rql
# ToDo: try to get the current request cw_rset
rset = self._cw.execute(rql)
# Get all the entities
entities = {}
if rset.rowcount > 0:
for rowindex in range(len(rset[0])):
try:
entity = rset.get_entity(0, rowindex)
entity_type = entity.__class__.__name__
entities[rowindex] = entity_type
except NotAnEntity:
pass
except:
raise
if len(entities) == 0:
raise ValidationError(
"CWSearch", {
"entities": _('cannot find any entity for the request '
'{0}'.format(rql))})
# Find the constant nodes
constant_nodes = {}
self._find_constant_nodes(rset.syntax_tree().children, constant_nodes)
# Check we can associated rset entities with their rql labels
actions = []
rql_etypes = constant_nodes.get("etype", {})
for etype in entities.values():
if etype not in rql_etypes or len(rql_etypes[etype]) != 1:
raise ValidationError(
"CWSearch", {
"rql": _('cannot find entity description in the '
'request {0}. Expect something like "Any X '
'Where X is '
'{1}, ..."'.format(rql, etype))})
# Get all the rqldownload declared adapters
possible_actions = self._cw.vreg["actions"]["rqldownload-adapters"]
# Keep only actions that respect the current context
actions = {}
export_vids = set()
for index, etype in entities.items():
entity_label = rql_etypes[etype][0]
for action in possible_actions:
for selector in action.__select__.selectors:
if (isinstance(selector, is_instance) and
etype in selector.expected_etypes):
actions.setdefault(etype, []).append(
(action, entity_label))
export_vids.add(unicode(action.__rset_type__))
# Check that at least one action has been found for this request
if actions == []:
raise ValidationError(
"CWSearch", {
"actions": _('cannot find an action for this request '
'{0}'.format(rql))})
# Check that the export types are homogeneous
if len(export_vids) != 1:
raise ValidationError(
"CWSearch", {
"actions": _('cannot deal with different or no action '
'export types: {0}'.format(export_vids))})
export_vid = export_vids.pop()
# Create an empty result structure
result = {"rql": rql, "files": [], "nonexistent-files": [],
"upper_file_index": 0}
# Here we want to execute rql request with user permissions: user who
# is creating this entity
with self._cw.security_enabled(read=True, write=True):
# Set the adaptor rset type
self.entity.cw_edited["rset_type"] = export_vid
# Create the global rql from the declared actions
global_rql = rql
cnt = 1
upper_file_index = 0
for etype, action_item in actions.items():
for action, entity_label in action_item:
global_rql, nb_files = action(self._cw).rql(
global_rql, entity_label, cnt)
upper_file_index += nb_files
cnt += 1
result["upper_file_index"] = upper_file_index
# Execute the global rql
rset = self._cw.execute(global_rql)
result["rql"] = global_rql
# Because self._cw is not a cubicwebRequest add an empty form
# parameter
self._cw.__dict__["form"] = {}
try:
view = self._cw.vreg["views"].select(
export_vid, self._cw, rset=rset)
rset_view = Binary()
view.w = rset_view.write
view.call()
except:
raise ValidationError(
"CWSearch", {
"rset_type": _('cannot apply this view "{0}" on this '
'rset, choose another view '
'id'.format(export_vid))})
# Save the rset in a File entity
f_eid = self._cw.create_entity(
"File", data=rset_view,
data_format=u"text",
data_name=u"rset").eid
# Entity modification related event: specify that the rset has been
# modified
self.entity.cw_edited["rset"] = f_eid
# Get all the files attached to the current request
# Note: we assume the database integrity (ie. all file paths
# inserted in the db exist on the file system) and thus do not
# check to speed up this process.
files_set = set()
non_existent_files_set = set()
if export_vid != "ecsvexport":
for rset_row in rset.rows:
for rset_index in range(upper_file_index):
files_set.add(rset_row[rset_index])
# Update the result structure
result["files"] = list(files_set)
result["nonexistent-files"] = list(non_existent_files_set)
# Save the result in a File entity
f_eid = self._cw.create_entity(
"File", data=Binary(json.dumps(result)),
data_format=u"text/json", data_name=u"result.json").eid
# Entity modification related event: specify that the result has
# been modified
self.entity.cw_edited["result"] = f_eid
def test_encoding_binary(self):
for content in (b'he he', b'h\xe9 hxe9'):
with self.subTest(content=content):
encoded = self.encode(Binary(content))
self.assertEqual(base64.b64decode(encoded), content)
def get_file_data(self,
file_eid,
rset_file,
session_index,
search_name=None):
return Binary("nothing in %s" % file_eid)
def _create_questionnaire(self, questionnaire_name, q_items,
identifier_prefix, subject_id, subject_eid,
study_eid, assessment_eid, questionnaire_eids,
question_eids):
""" Create a scans and its associated relations.
"""
# Create a questionnaire run
qr_id = self._md5_sum(identifier_prefix + "_" + questionnaire_name)
qr_entity, is_created = self._get_or_create_unique_entity(
rql=("Any X Where X is QuestionnaireRun, X identifier "
"'{0}'".format(qr_id)),
check_unicity=True,
entity_name="QuestionnaireRun",
identifier=unicode(qr_id),
label=unicode(questionnaire_name)
)
# If we just create the questionnaire run, specify and relate the entity
if is_created:
# > add relation with the questionnaire
self._set_unique_relation(
qr_entity.eid, "questionnaire",
questionnaire_eids[questionnaire_name], check_unicity=False)
self._set_unique_relation(
questionnaire_eids[questionnaire_name],
"questionnaire_questionnaire_runs", qr_entity.eid,
check_unicity=False)
# > add relation with the assessment
self._set_unique_relation(
assessment_eid, "questionnaire_runs", qr_entity.eid,
check_unicity=False)
self._set_unique_relation(
qr_entity.eid, "in_assessment", assessment_eid,
check_unicity=False, subjtype="QuestionnaireRun")
# > add relation with the study
self._set_unique_relation(
qr_entity.eid, "study", study_eid, check_unicity=False)
self._set_unique_relation(
study_eid, "study_questionnaire_runs", qr_entity.eid,
check_unicity=False)
# > add relation with the subject
self._set_unique_relation(
qr_entity.eid, "subject", subject_eid, check_unicity=False)
self._set_unique_relation(
subject_eid, "subject_questionnaire_runs", qr_entity.eid,
check_unicity=False)
if self.use_openanswer:
# Go through all answers
for question_name, answer in q_items.iteritems():
# Get the question entity
question_eid = question_eids[questionnaire_name][
question_name]
# Create an open answer
answer_entity, _ = self._get_or_create_unique_entity(
rql="",
check_unicity=False,
entity_name="OpenAnswer",
identifier=unicode(self._md5_sum(
qr_id + "_" + question_name)),
value=unicode(answer))
# > add relation with the question
self._set_unique_relation(
answer_entity.eid, "question", question_eid,
check_unicity=False, subjtype="OpenAnswer")
self._set_unique_relation(
question_eid, "question_open_answers",
answer_entity.eid, check_unicity=False)
# > add relation with the questionnaire run
self._set_unique_relation(
answer_entity.eid, "questionnaire_run", qr_entity.eid,
check_unicity=False, subjtype="OpenAnswer")
self._set_unique_relation(
qr_entity.eid, "open_answers", answer_entity.eid,
check_unicity=False)
# > add relation with the assessment
self._set_unique_relation(
answer_entity.eid, "in_assessment", assessment_eid,
check_unicity=False, subjtype="OpenAnswer")
else:
f_entity, _ = self._get_or_create_unique_entity(
rql="",
entity_name="RestrictedFile",
title=u"{0} ({1})".format(questionnaire_name, subject_id),
data=Binary(json.dumps(q_items)),
data_format=u"text/json",
data_name=u"result.json",
check_unicity=False)
self._set_unique_relation(
qr_entity.eid, "file", f_entity.eid,
check_unicity=False, subjtype="QuestionnaireRun")
self._set_unique_relation(
f_entity.eid, "in_assessment", assessment_eid,
check_unicity=False, subjtype="RestrictedFile")
return qr_entity.eid