def right_dict(self):
key = self._cache_prefix + str(self.pk)
cache = type(self)._dict_cache
if key not in cache:
from wq.db.rest.models import get_object_id, get_ct
obj = self.right
oid = get_object_id(obj)
ct = get_ct(obj)
cache[key] = {
'item_id': get_object_id(obj),
'item_label': str(obj),
'item_url': '%s/%s' % (ct.urlbase, oid),
'item_page': ct.identifier
}
return cache[key]
def right_dict(self):
key = self._cache_prefix + str(self.pk)
cache = type(self)._dict_cache
if key not in cache:
from wq.db.rest.models import get_object_id, get_ct
obj = self.right
oid = get_object_id(obj)
ct = get_ct(obj)
cache[key] = {
'item_id': get_object_id(obj),
'item_label': str(obj),
'item_url': '%s/%s' % (ct.urlbase, oid),
'item_page': ct.identifier
}
return cache[key]
def to_representation(self, obj):
if hasattr(obj, 'content_type_id') and hasattr(obj, 'content_object'):
ctype = ContentType.objects.get(pk=obj.content_type_id)
if isinstance(obj, Identifier):
match_str = "%s (%s)" % (obj.name, obj.slug)
elif isinstance(obj, Annotation):
match_str = obj.value
else:
match_str = str(obj)
obj = obj.content_object
else:
ctype = get_ct(obj)
match_str = str(obj)
if not ctype.is_registered():
raise ImproperlyConfigured(
"Register %s to include it in search results"
% ctype.model_class()
)
obj_id = get_object_id(obj)
url = ctype.urlbase
if url:
url += '/'
url += str(obj_id)
return {
'id': obj_id,
'url': url,
'type': str(ctype),
'label': str(obj),
'match': match_str,
}
def get_value(self, dictionary):
# Handle attachments that are submitted together with their parent
# Ideal case: an array of dicts; this can be handled by the default
# implementation. HTML JSON forms will use this approach.
if self.field_name in dictionary:
return super(TypedAttachmentListSerializer, self).get_value(dictionary)
# Deprecated/"classic" form style, where each attachment is submitted
# as form fields with names in the format [model]_[typeid] e.g.
# annotation_23=30.5 will become {'type': 23, 'value': '30.5'}
# Retrieve form values into more usable array/dict format
attachments = []
ct = get_ct(self.child.Meta.model)
i = 0
for atype in self.child.expected_types:
fields = {
afield: "%s-%s-%s" % (ct.identifier, get_object_id(atype) if atype else "", afield)
for afield in self.child.attachment_fields
}
found = False
for key in fields.values():
if key in dictionary:
found = True
if found:
attachment = self.child.create_dict(atype, dictionary, fields, i)
if attachment:
attachments.append(attachment)
i += 1
# Return extracted values to default implementation
return attachments
def to_representation(self, obj):
if hasattr(obj, 'content_type_id') and hasattr(obj, 'content_object'):
ctype = ContentType.objects.get(pk=obj.content_type_id)
if isinstance(obj, Identifier):
match_str = "%s (%s)" % (obj.name, obj.slug)
elif isinstance(obj, Annotation):
match_str = obj.value
else:
match_str = str(obj)
obj = obj.content_object
else:
ctype = get_ct(obj)
match_str = str(obj)
if not ctype.is_registered():
raise ImproperlyConfigured(
"Register %s to include it in search results" %
ctype.model_class())
obj_id = get_object_id(obj)
url = ctype.urlbase
if url:
url += '/'
url += str(obj_id)
return {
'id': obj_id,
'url': url,
'type': str(ctype),
'label': str(obj),
'match': match_str,
}
def to_representation(self, obj):
data = super(TypedAttachmentSerializer, self).to_representation(obj)
has_parent = (self.parent and self.parent.parent
and hasattr(self.parent.parent.Meta, 'model'))
if has_parent:
# This is being serialized with its parent object, don't need to
# reference the parent again.
pass
else:
# Include pointer to parent object. Collapse content_type and
# object_id into a single attribute: the identifier for the content
# type plus '_id' e.g. if contenttype's name is 'visit' then the
# attribute is 'visit_id'. This lets us pretend the generic
# foreign key is a regular one in client. For the value, Use
# get_object_id insted of obj.object_id, in case the the content
# object is an IdentifiedModel
parent_obj = getattr(obj, self.object_field)
idname = get_ct(parent_obj).identifier
data[idname + '_id'] = get_object_id(parent_obj)
# In detail views, include full parent object (without _id suffix)
if self.is_detail:
from wq.db import rest
data[idname] = rest.router.serialize(parent_obj)
return data
def to_representation(self, obj):
data = super(TypedAttachmentSerializer, self).to_representation(obj)
has_parent = (
self.parent and self.parent.parent and
hasattr(self.parent.parent.Meta, 'model')
)
if has_parent:
# This is being serialized with its parent object, don't need to
# reference the parent again.
pass
else:
# Include pointer to parent object. Collapse content_type and
# object_id into a single attribute: the identifier for the content
# type plus '_id' e.g. if contenttype's name is 'visit' then the
# attribute is 'visit_id'. This lets us pretend the generic
# foreign key is a regular one in client. For the value, Use
# get_object_id insted of obj.object_id, in case the the content
# object is an IdentifiedModel
parent_obj = getattr(obj, self.Meta.object_field)
if parent_obj is not None:
idname = get_ct(parent_obj).identifier
data[idname + '_id'] = get_object_id(parent_obj)
# In detail views, include full parent object (without _id
# suffix)
if self.is_detail:
from wq.db import rest
data[idname] = rest.router.serialize(parent_obj)
return data
def user_info(self, request):
user_dict = rest.router.serialize(request.user)
user_dict['id'] = get_object_id(request.user)
return Response({
'user': user_dict,
'config': rest.router.get_config(request.user),
'csrftoken': csrf.get_token(request),
})
def to_native(self, loc):
has_parent = self.parent and hasattr(self.parent.opts, 'model')
if self.as_geometry and has_parent:
return json.loads(loc.geometry.geojson)
data = super(LocationSerializer, self).to_native(loc)
if has_parent:
pass
else:
# Include pointer to parent object (see annotate/serializers.py)
idname = get_ct(loc.content_object).identifier + '_id'
data[idname] = get_object_id(loc.content_object)
return data
def get_object_id(self, obj):
# Try to use the same identifier that the webservice uses for
# this parameter/site/etc.
if get_ct(obj).is_identified:
idents = obj.identifiers.filter(
authority=self.webservice.authority)
else:
idents = []
if len(idents) > 0:
return idents[0].slug
else:
# No authority-specific IDs found, use default ID for object
return get_object_id(obj)
def to_representation(self, loc):
has_parent = (self.parent and self.parent.parent
and hasattr(self.parent.parent.Meta, 'model'))
if self.as_geometry and has_parent:
return json.loads(loc.geometry.geojson)
data = super(LocationSerializer, self).to_representation(loc)
if has_parent:
pass
else:
# Include pointer to parent object (see annotate/serializers.py)
idname = get_ct(loc.content_object).identifier + '_id'
data[idname] = get_object_id(loc.content_object)
return data
def get_object_id(self, obj):
# Try to use the same identifier that the webservice uses for
# this parameter/site/etc.
if get_ct(obj).is_identified:
idents = obj.identifiers.filter(
authority=self.webservice.authority
)
else:
idents = []
if len(idents) > 0:
return idents[0].slug
else:
# No authority-specific IDs found, use default ID for object
return get_object_id(obj)
def make_list(cls, name):
rows = cls.objects.all()
ct = CONTENT_TYPES[cls]
result = [{
'url': '%s/%s' % (ct.urlbase, get_object_id(row)),
'label': str(row)
} for row in rows]
result.insert(0, {
'url': '%s/new' % ct.urlbase,
'label': "New %s" % name,
})
return {
'name': name,
'choices': result
}
def to_native(self, obj):
if hasattr(obj, 'content_type_id') and hasattr(obj, 'content_object'):
ctype = ContentType.objects.get(pk=obj.content_type_id)
obj = obj.content_object
else:
ctype = get_ct(obj)
url = ctype.urlbase
if url:
url += '/'
url += unicode(get_object_id(obj))
return {
'url': url,
'type': unicode(ctype),
'label': unicode(obj)
}
def to_representation(self, obj):
if hasattr(obj, 'content_type_id') and hasattr(obj, 'content_object'):
ctype = ContentType.objects.get(pk=obj.content_type_id)
obj = obj.content_object
else:
ctype = get_ct(obj)
obj_id = get_object_id(obj)
url = ctype.urlbase
if url:
url += '/'
url += str(obj_id)
return {
'id': obj_id,
'url': url,
'type': str(ctype),
'label': str(obj)
}
def to_native(self, obj):
data = super(TypedAttachmentSerializer, self).to_native(obj)
has_parent = self.parent and hasattr(self.parent.opts, 'model')
if has_parent:
# This is being serialized with its parent object, don't need to
# reference the parent again.
pass
else:
# Include pointer to parent object. Collapse content_type and
# object_id into a single attribute: the identifier for the content
# type plus '_id' e.g. if contenttype's name is 'visit' then the
# attribute is 'visit_id'. This lets us pretend the generic
# foreign key is a regular one in client. For the value, Use
# get_object_id insted of obj.object_id, in case the the content
# object is an IdentifiedModel
parent_obj = getattr(obj, self.object_field)
idname = get_ct(parent_obj).identifier + '_id'
data[idname] = get_object_id(parent_obj)
return data
def field_from_native(self, data, files, field_name, into):
# Handle attachments that are submitted together with their parent
# Ideal case: an array of dicts, this can be handled by the default
# implementation
if field_name in data:
return super(TypedAttachmentSerializer, self).field_from_native(
data, files, field_name, into
)
# Common case: form submission. In this case each attachment should
# be submitted as form fields with names in the format [model]_[typeid]
# e.g. annotation_23=30.5 will become {'type': 23, 'value': '30.5'}
# Retrieve form values into more usable array/dict format
attachments = []
ct = get_ct(self.opts.model)
i = 0
for atype in self.expected_types:
fields = {
afield: '%s-%s-%s' % (
ct.identifier,
get_object_id(atype) if atype else '',
afield
) for afield in self.attachment_fields
}
found = False
for key in fields.values():
if key in data:
found = True
if found:
attachment = self.create_dict(atype, data, fields, i)
if attachment:
attachments.append(attachment)
i += 1
# Send modified object to default implementation
return super(TypedAttachmentSerializer, self).field_from_native(
{field_name: attachments}, files, field_name, into
)
def load_columns(instance):
rels = instance.relationships.filter(type__name='Contains Column')
table = instance.load_io()
cols = list(table.field_map.keys())
matched = []
for rel in rels:
item = rel.right
info = {
'match': str(item),
'rel_id': rel.pk,
}
if isinstance(item, UnknownItem):
info['type'] = "unknown"
info['value'] = item.name
elif isinstance(item, Parameter):
info['type'] = "parameter_value"
info['parameter_id'] = get_object_id(item)
elif isinstance(item, MetaColumn):
info['type'] = item.type
info['field_name'] = item.name
if rel.range_set.filter(type='list').exists():
col = rel.range_set.get(type='list').start_column
info['name'] = cols[col].replace('\n', ' - ')
info['column'] = colname(col)
info['colnum'] = col
elif rel.range_set.filter(type='value').exists():
info['name'] = get_range_value(table, rel.range_set.get(
type='head'
))
info['value'] = get_range_value(table, rel.range_set.get(
type='value'
))
matched.append(info)
matched.sort(key=lambda info: info.get('colnum', -1))
return matched
def get_value(self, dictionary):
# Handle attachments that are submitted together with their parent
# Ideal case: an array of dicts; this can be handled by the default
# implementation. HTML JSON forms will use this approach.
if self.field_name in dictionary:
return super(TypedAttachmentListSerializer,
self).get_value(dictionary)
# Deprecated/"classic" form style, where each attachment is submitted
# as form fields with names in the format [model]_[typeid] e.g.
# annotation_23=30.5 will become {'type': 23, 'value': '30.5'}
# Retrieve form values into more usable array/dict format
attachments = []
ct = get_ct(self.child.Meta.model)
i = 0
for atype in self.child.expected_types:
fields = {
afield: '%s-%s-%s' %
(ct.identifier, get_object_id(atype) if atype else '', afield)
for afield in self.child.attachment_fields
}
found = False
for key in fields.values():
if key in dictionary:
found = True
if found:
attachment = self.child.create_dict(atype, dictionary, fields,
i)
if attachment:
attachments.append(attachment)
i += 1
# Return extracted values to default implementation
return attachments
def import_data(file, user):
matched = read_columns(file)
table = load_file(file)
if jc_backend:
jc_backend.unpatch()
for col in matched:
col['item'] = file.relationships.get(pk=col['rel_id']).right
col['item_id'] = get_object_id(col['item'])
file_globals = {
'event_key': {},
'report_meta': {
'user': user,
'status': DEFAULT_STATUS,
},
'param_vals': {}
}
for field_name in EVENT_KEY:
info = Event._meta.get_field_by_name(field_name)
field = info[0]
if field.null:
file_globals['event_key'][field_name] = None
def save_value(col, val, obj):
item = col['item']
if isinstance(item, Parameter):
if col['item_id'] in obj['param_vals']:
obj['param_vals'][col['item_id']] = "%s %s" % (
obj['param_vals'][col['item_id']],
val
)
else:
obj['param_vals'][col['item_id']] = val
elif isinstance(item, MetaColumn):
if val is None or val == '':
return
if item.type == 'event':
if '.' in item.name:
name, part = item.name.split('.')
else:
name, part = item.name, None
fld = Event._meta.get_field_by_name(
name
)[0].get_internal_type()
if (fld in DATE_FIELDS and isinstance(val, basestring)):
from dateutil.parser import parse
val = parse(val)
if fld == 'DateField':
val = val.date()
# Handle date & time being separate columns
if obj['event_key'].get(name) is not None:
other_val = obj['event_key'][name]
if not part:
raise Exception(
'Expected multi-column date and time for %s' % name
)
if part not in ('date', 'time'):
raise Exception(
'Unexpected field name: %s.%s!' % (name, part)
)
if part == 'date':
date, time = val, other_val
else:
date, time = other_val, val
if not isinstance(date, datetime.date):
raise Exception("Expected date but got %s!" % date)
if not isinstance(time, datetime.time):
if (isinstance(time, float)
and time >= 100 and time <= 2400):
time = str(time)
if len(time) == 3:
time = datetime.time(
int(time[0]),
int(time[1:2])
)
else:
time = datetime.time(
int(time[0:1]),
int(time[2:3])
)
else:
raise Exception("Expected time but got %s!" % time)
val = datetime.datetime.combine(date, time)
obj['event_key'][name] = val
elif item.type == 'report':
obj['report_meta'][item.name] = val
for col in matched:
if 'value' in col:
save_value(col, col['value'], file_globals)
def add_record(row):
record = {
key: file_globals[key].copy()
for key in file_globals
}
for col in matched:
if 'colnum' in col:
save_value(col, row[col['colnum']], record)
if len(record['event_key'].keys()) < len(EVENT_KEY):
raise Exception('Incomplete Record')
return Report.objects.create_report(
EventKey(**record['event_key']),
record['param_vals'],
**record['report_meta']
)
rows = len(table)
errors = []
skipped = []
def rownum(i):
return i + table.start_row + 1
for i, row in enumerate(table):
current_task.update_state(state='PROGRESS', meta={
'current': i + 1,
'total': rows,
'skipped': skipped
})
skipreason = None
try:
report = add_record(row)
except Exception as e:
skipreason = repr(e)
skipped.append({'row': rownum(i), 'reason': skipreason})
report, is_new = SkippedRecord.objects.get_or_create(
reason=skipreason
)
rel = file.create_relationship(
report,
'Contains Row',
'Row In'
)
Range.objects.create(
relationship=rel,
type='row',
start_row=i + table.start_row,
start_column=0,
end_row=i + table.start_row,
end_column=len(row) - 1
)
status = {
'current': i + 1,
'total': rows,
'skipped': skipped
}
if jc_backend:
jc_backend.patch()
if rows and rows > len(skipped):
from johnny.cache import invalidate
invalidate(*[
cls._meta.db_table for cls in
(File, Site, Event, Report, Parameter, Result, Relationship)
])
import_complete.send(sender=import_data, file=file, status=status)
return status
def read_row_identifiers(instance, user):
coltypes = get_meta_columns(instance)
ids = {}
for mtype in coltypes:
ids[mtype] = Counter()
for row in instance.load_io():
for mtype, cols in coltypes.items():
counter = ids[mtype]
meta = OrderedDict()
for col in cols:
meta[col['field_name']] = row[col['colnum']]
key = tuple(meta.items())
counter[key] += 1
idgroups = []
unknown_ids = 0
for mtype in ids:
cls = META_CLASSES[mtype]
idinfo = {
'type_id': mtype,
'type_label': mtype.capitalize(),
'ids': []
}
for key, count in ids[mtype].items():
meta = OrderedDict(key)
try:
obj = cls.objects.get_by_natural_key(meta['id'])
except cls.DoesNotExist:
obj = None
info = {
'value': meta['id'],
'count': count,
'meta': [{
'name': k,
'value': v
} for k, v in meta.items() if k != 'id']
}
if obj is not None:
info['match'] = str(obj)
# FIXME: Confirm that metadata hasn't changed
else:
info['ident_id'] = unknown_ids
unknown_ids += 1
info['unknown'] = True
choices = instance.get_id_choices(cls, meta)
info['choices'] = [{
'id': get_object_id(obj),
'label': str(obj),
} for obj in choices]
info['choices'].insert(0, {
'id': 'new',
'label': "New %s" % idinfo['type_label'],
})
idinfo['ids'].append(info)
idinfo['ids'].sort(key=lambda info: info['value'])
idgroups.append(idinfo)
if not unknown_ids:
# Create relationships after all IDs are resolved.
# FIXME: parse_columns() always creates relationships, using
# UnknownItem to stand in for unknown column identifiers.
# Use UnknownItem here as well?
from_type = get_ct(instance)
for idinfo in idgroups:
cls = META_CLASSES[idinfo['type_id']]
to_type = get_ct(cls)
reltype, is_new = RelationshipType.objects.get_or_create(
from_type=from_type,
to_type=to_type,
name='Contains Identifier',
inverse_name='Identifier In'
)
for info in idinfo['ids']:
obj = cls.objects.get_by_natural_key(info['value'])
rel, isnew = instance.relationships.get_or_create(
type=reltype,
to_content_type=to_type,
to_object_id=obj.pk,
from_content_type=from_type,
from_object_id=instance.pk,
)
# FIXME: create equivalent of Range objects here?
return {
'unknown_count': unknown_ids,
'types': idgroups,
}
def get_id(self, instance):
return get_object_id(instance)