def admin_add_data_json(entity_id):
ent = Entity.query.filter(Entity.id == entity_id).first()
if not ent:
return json.dumps({'errors': [{'detail': 'Entity not found'}]})
form = DataForm()
if not form.validate():
return form_errors_json(form)
docs = Data.query.filter(Document.entity_id == ent.id,
Data.format == form.format.data).all()
if len(docs) != 0:
return json.dumps({
'errors': [{
'title': 'Wrong format',
'detail': 'Duplicate format'
}]
})
doc = Data(ent.id,
form.format.data,
url=form.url.data,
enabled=form.enabled.data,
notes=form.notes.data)
log(doc.entity_id, "Added data document `%s'" % doc)
db.session.add(doc)
db.session.commit()
return json.dumps({'success': True})
def title(self, value):
if self._title != value:
log(self.id,
"Changed title from '%s' to '%s'" % (self._title, value))
self._title = value
self.slug = slugify(value)[:64] if value else SLUG_DEFAULT
def admin_new_entity():
form = EntityForm()
if form.validate():
try:
ent = Entity(type=form.type.data,
title=form.title.data,
id=form.id.data)
except EntityPIDExistsException:
flash("There already exists and entity with this PID.", "warning")
return admin_list_entities(form=form)
except EntityCollisionException:
flash("PID collision detected!", "warning")
return admin_list_entities(form=form)
db.session.add(ent)
db.session.flush()
db.session.add(Data(ent.id, 'html'))
db.session.add(Representation(ent.id, 1, reference=True))
db.session.commit()
log(ent.id, "Created entity `%s'" % ent)
return redirect("/resolver/entity/%s" % ent.id)
else:
return admin_list_entities(form=form, show_form=True)
def admin_delete_document(id):
doc = Document.query.filter(Document.id == id).first()
entity_id = doc.entity_id
if not doc:
flash("Document not found", "warning")
return redirect("/resolver/entity")
if type(doc) == Representation:
if doc.reference:
count = Representation.query.\
filter(Document.entity_id == entity_id).count()
if count > 1:
flash("Please select another representation to be reference first",
'warning')
return redirect("/resolver/entity/%s" % entity_id)
db.session.delete(doc)
db.session.flush()
i = 1
reps = Representation.query.\
filter(Document.entity_id == entity_id).\
order_by(Representation.order.asc()).all()
for rep in reps:
rep.order = i
i += 1
else:
db.session.delete(doc)
log(entity_id, "Removed the document '%s' %s" %
(doc, doc.entity))
db.session.commit()
flash("Document deleted succesfully", "success")
return redirect("/resolver/entity/%s" % entity_id)
def admin_new_entity():
form = EntityForm()
if form.validate():
try:
ent = Entity(type=form.type.data,
title=form.title.data,
id=form.id.data)
except EntityPIDExistsException:
flash("There already exists and entity with this PID.", "warning")
return admin_list_entities(form=form)
except EntityCollisionException:
flash("PID collision detected!", "warning")
return admin_list_entities(form=form)
db.session.add(ent)
db.session.flush()
db.session.add(Data(ent.id, 'html'))
db.session.add(Representation(ent.id, 1, reference=True))
db.session.commit()
log(ent.id, "Created entity `%s'" % ent)
return redirect("/resolver/entity/%s" % ent.id)
else:
return admin_list_entities(form=form, show_form=True)
def delete_document(id):
doc = Document.query.filter(Document.id == id).first()
if not doc:
return ErrorRestApi().response(status=404, errors=['Document not found.'])
if isinstance(doc, Representation):
if doc.reference:
count = Representation.query. \
filter(Document.entity_id == doc.entity_id).count()
if count > 1:
return ErrorRestApi().response(status=409, errors=['Can not delete reference representation.'])
db.session.delete(doc)
db.session.flush()
i = 1
reps = Representation.query. \
filter(Document.entity_id == doc.entity_id). \
order_by(Representation.order.asc()).all()
for rep in reps:
rep.order = i
i += 1
else:
db.session.delete(doc)
log(doc.entity_id, "Removed the document '%s' %s" %
(doc, doc.entity))
db.session.commit()
return "", 204
def admin_add_representation_json(entity_id):
ent = Entity.query.filter(Entity.id == entity_id).first()
if not ent:
return json.dumps({'errors':[{'detail':'Entity not found'}]})
form = RepresentationForm()
if not form.validate():
return form_errors_json(form)
ref = Representation.query.filter(Document.entity_id == ent.id,
Representation.reference == True).first()
if form.reference.data:
if ref:
ref.reference = False
elif not ref:
return json.dumps({'errors':[{'title':'Reference error',
'detail':'At least one representation has to be the reference image.'}]})
highest = Representation.query.\
filter(Document.entity_id == ent.id).\
order_by(Representation.order.desc()).first()
if highest:
order = highest.order + 1
else:
order = 1
rep = Representation(ent.id, order, url=form.url.data, label=form.label.data,
enabled=form.enabled.data, notes=form.notes.data,
reference=form.reference.data)
log(rep.entity_id, "Added representation document `%s'" % rep)
db.session.add(rep)
db.session.commit()
return json.dumps({'success':True})
def create_entity():
form = EntityForm(csrf_enabled=False)
if form.validate_on_submit():
try:
ent = Entity(type=form.type.data,
title=form.title.data,
id=form.id.data)
except EntityPIDExistsException:
return ErrorRestApi().response(status=409,
errors=['Duplicate ID for entity.'])
except EntityCollisionException as e:
return ErrorRestApi().response(
status=409,
errors=[{
'title':
'ID Collision',
'detail':
'The provided ID \'{0}\' collides with the existing ID \'{1}\''
.format(form.id.data, e.original_id)
}])
db.session.add(ent)
db.session.flush()
db.session.add(Data(ent.id, 'html'))
db.session.add(Representation(ent.id, 1, reference=True))
db.session.commit()
log(ent.id, "Created entity `%s'" % ent)
return RestApi().response(
status=201, data={'data': EntityViewApi().output(entity=ent)})
errors = [{
'title': 'Malformed parameter',
'detail': 'Field %s: %s' % (field, ' '.join(error))
} for field, error in form.errors.iteritems()]
return ErrorRestApi().response(status=400, errors=errors)
def create_entity():
form = EntityForm(csrf_enabled=False)
if form.validate_on_submit():
try:
ent = Entity(type=form.type.data,
title=form.title.data,
id=form.id.data)
except EntityPIDExistsException:
return ErrorRestApi().response(status=409, errors=['Duplicate ID for entity.'])
except EntityCollisionException as e:
return ErrorRestApi().response(status=409, errors=[{
'title': 'ID Collision',
'detail': 'The provided ID \'{0}\' collides with the existing ID \'{1}\''.format(form.id.data,
e.original_id)
}])
db.session.add(ent)
db.session.flush()
db.session.add(Data(ent.id, 'html'))
db.session.add(Representation(ent.id, 1, reference=True))
db.session.commit()
log(ent.id, "Created entity `%s'" % ent)
return RestApi().response(status=201, data={'data': EntityViewApi().output(entity=ent)})
errors = [{'title': 'Malformed parameter',
'detail': 'Field %s: %s' % (field, ' '.join(error))}
for field, error in form.errors.iteritems()]
return ErrorRestApi().response(status=400, errors=errors)
def delete_document(id):
doc = Document.query.filter(Document.id == id).first()
if not doc:
return ErrorRestApi().response(status=404,
errors=['Document not found.'])
if isinstance(doc, Representation):
if doc.reference:
count = Representation.query. \
filter(Document.entity_id == doc.entity_id).count()
if count > 1:
return ErrorRestApi().response(
status=409,
errors=['Can not delete reference representation.'])
db.session.delete(doc)
db.session.flush()
i = 1
reps = Representation.query. \
filter(Document.entity_id == doc.entity_id). \
order_by(Representation.order.asc()).all()
for rep in reps:
rep.order = i
i += 1
else:
db.session.delete(doc)
log(doc.entity_id, "Removed the document '%s' %s" % (doc, doc.entity))
db.session.commit()
return "", 204
def admin_delete_document(id):
doc = Document.query.filter(Document.id == id).first()
entity_id = doc.entity_id
if not doc:
flash("Document not found", "warning")
return redirect("/resolver/entity")
if type(doc) == Representation:
if doc.reference:
count = Representation.query.\
filter(Document.entity_id == entity_id).count()
if count > 1:
flash(
"Please select another representation to be reference first",
'warning')
return redirect("/resolver/entity/%s" % entity_id)
db.session.delete(doc)
db.session.flush()
i = 1
reps = Representation.query.\
filter(Document.entity_id == entity_id).\
order_by(Representation.order.asc()).all()
for rep in reps:
rep.order = i
i += 1
else:
db.session.delete(doc)
log(entity_id, "Removed the document '%s' %s" % (doc, doc.entity))
db.session.commit()
flash("Document deleted succesfully", "success")
return redirect("/resolver/entity/%s" % entity_id)
def create_document():
try:
data = json.loads(request.data)
validate(data, document_schema)
ent = Entity.query.filter(Entity.id == data['entity']).first()
if not ent:
return ErrorRestApi().response(status=400, errors=['Entity not found.'])
if data['type'] == 'data':
docs = Data.query.filter(Document.entity_id == ent.id,
Data.format == data['format']).all()
if len(docs) != 0:
return ErrorRestApi().response(status=400, errors=['Duplicate data format \'{0}\' for entity.'
.format(data['format'])])
doc = Data(ent.id, data['format'], data.get('url', ''),
data['enabled'], data.get('notes', ''))
else:
ref = Representation.query.filter(Document.entity_id == ent.id,
Representation.reference == True).first()
if data['reference']:
if ref:
ref.reference = False
elif not ref:
errors = [{
'title': 'Reference error',
'detail': 'At least one reference representation is required.'
}]
return ErrorRestApi().response(status=400, errors=errors)
highest = Representation.query. \
filter(Document.entity_id == ent.id). \
order_by(Representation.order.desc()).first()
if highest:
order = highest.order + 1
else:
order = 1
doc = Representation(ent.id, order, url=data.get('url', ''),
label=data.get('label', ''), enabled=data['enabled'],
notes=data.get('form', ''), reference=data['reference'])
db.session.add(doc)
db.session.commit()
log(doc.entity_id, "Added %s document `%s'" % (data['type'], doc))
return RestApi().response(status=201, data={'data': doc.to_dict()})
except ValueError as e:
errors = [{
'title': 'Malformed request',
'detail': 'Expected correctly formatted JSON data: {0}'.format(e)
}]
return ErrorRestApi().response(status=400, errors=errors)
except ValidationError as e:
errors = [{
'title': 'Malformed request',
'detail': 'JSON data does not conform to schema: {0}'.format(e)
}]
return ErrorRestApi().response(status=400, errors=errors)
def enabled(self, value):
value = bool(value)
if self._enabled and (not value):
log(self.entity_id, "Disabled document %s" % self)
elif (not self._enabled) and value:
log(self.entity_id, "Enabled document %s" % self)
self._enabled = value
def format(self, value):
if value not in data_formats:
raise Exception("Incorrect data format")
if self._format != value:
log(self.entity_id, "Changed format from '%s' to '%s' for %s" %
(self._format, value, self))
self._format = value
def reference(self, value):
value = bool(value)
if self._reference and (not value):
log(self.entity_id, "Removed as reference: %s" % self)
elif (not self._reference) and value:
log(self.entity_id, "Set as reference: %s" % self)
self._reference = value
def reference(self, value):
value = bool(value)
if self._reference and (not value):
log(self.entity_id, "Removed as reference: %s" % self)
elif (not self._reference) and value:
log(self.entity_id, "Set as reference: %s" % self)
self._reference = value
def format(self, value):
if value not in data_formats:
raise Exception("Incorrect data format")
if self._format != value:
log(
self.entity_id, "Changed format from '%s' to '%s' for %s" %
(self._format, value, self))
self._format = value
def delete_entity(id):
ent = Entity.query.filter(Entity.id == id).first()
if ent:
db.session.delete(ent)
db.session.commit()
log(id, "Removed the entity `%s'" % ent)
return "", 204
else:
return ErrorRestApi().response(status=404, errors=['Entity not found.'])
def type(self, value):
if value not in entity_types:
raise Exception("Wrong entity type")
if self._type != value:
log(self.id,
"Changed type from '%s' to '%s'" % (self._type, value))
self._type = value
def admin_delete_entity(id):
ent = Entity.query.filter(Entity.id == id).first()
if not ent:
flash("Entity not found!", "danger")
else:
db.session.delete(ent)
db.session.commit()
log(id, "Removed the entity `%s'" % ent)
flash("Entity deleted succesfully!", "success")
return redirect("/resolver/entity")
def admin_delete_entity(id):
ent = Entity.query.filter(Entity.id == id).first()
if not ent:
flash("Entity not found!", "danger")
else:
db.session.delete(ent)
db.session.commit()
log(id, "Removed the entity `%s'" % ent)
flash("Entity deleted succesfully!", "success")
return redirect("/resolver/entity")
def delete_entity(id):
ent = Entity.query.filter(Entity.id == id).first()
if ent:
db.session.delete(ent)
db.session.commit()
log(id, "Removed the entity `%s'" % ent)
return "", 204
else:
return ErrorRestApi().response(status=404,
errors=['Entity not found.'])
def update_entity(id):
# With form-data, you also get the boundary after the first ';'
content_type = request.headers['Content-Type'].split(';')
if content_type[0] == 'application/x-www-form-urlencoded' \
or content_type[0] == 'multipart/form-data':
form = EntityForm(csrf_enabled=False)
data = {
'id': form.id.data,
'title': form.title.data,
'type': form.type.data
}
else:
try:
data = json.loads(request.data)
validate(data, entity_schema)
except ValueError as e:
errors = [{
'title': 'Malformed request',
'detail': 'Expected correctly formatted JSON data: {0}'.format(e)
}]
return ErrorRestApi().response(status=400, errors=errors)
except ValidationError as e:
errors = [{
'title': 'Malformed request',
'detail': 'JSON data does not conform to schema: {0}'.format(e)
}]
return ErrorRestApi().response(status=400, errors=errors)
ent = Entity.query.filter(Entity.id == id).first()
if not ent:
return ErrorRestApi().response(status=404, errors=['Entity not found.'])
ent_str = str(ent)
try:
ent.id = data["id"]
ent.title = data.get("title", "")
ent.type = data["type"]
except EntityPIDExistsException:
db.session.rollback()
return ErrorRestApi().response(status=409, errors=['Duplicate ID \'{0}\' for entity.'.format(data['id'])])
except EntityCollisionException as e:
db.session.rollback()
errors = [{
'title': 'ID collision',
'detail': 'The provided ID \'{0}\' collides with the existing ID \'{0}\'.'.format(data['id'], e.original_id)
}]
return ErrorRestApi().response(status=409, errors=errors)
db.session.commit()
log(ent.id, "Changed entity from `%s' to `%s'" % (ent_str, ent))
return RestApi().response(data={'data': EntityViewApi().output(entity=ent)})
def url(self, url):
# TODO: URL validation here
old = self._url
if url:
u = urlparse(url)
if u.scheme:
self._url = url
else:
self._url = urlunparse(('http',)+u[1:])
else:
self._url = None
if old != self._url:
log(self.entity_id, "Changed URL from '%s' to '%s' for %s" %
(old, self._url, self))
def admin_add_representation_json(entity_id):
ent = Entity.query.filter(Entity.id == entity_id).first()
if not ent:
return json.dumps({'errors': [{'detail': 'Entity not found'}]})
form = RepresentationForm()
if not form.validate():
return form_errors_json(form)
ref = Representation.query.filter(
Document.entity_id == ent.id,
Representation.reference == True).first()
if form.reference.data:
if ref:
ref.reference = False
elif not ref:
return json.dumps({
'errors': [{
'title':
'Reference error',
'detail':
'At least one representation has to be the reference image.'
}]
})
highest = Representation.query.\
filter(Document.entity_id == ent.id).\
order_by(Representation.order.desc()).first()
if highest:
order = highest.order + 1
else:
order = 1
rep = Representation(ent.id,
order,
url=form.url.data,
label=form.label.data,
enabled=form.enabled.data,
notes=form.notes.data,
reference=form.reference.data)
log(rep.entity_id, "Added representation document `%s'" % rep)
db.session.add(rep)
db.session.commit()
return json.dumps({'success': True})
def admin_representation_down(id):
doc = Document.query.filter(Document.id == id).first()
if (not doc) or (doc.type != 'representation'):
flash('Document not found or wrong type', 'danger')
return redirect("/resolver/entity/%s" % doc.entity_id)\
if doc else redirect("/resolver/entity")
next = Representation.query.filter(Document.entity_id == doc.entity_id,
Representation.order == (doc.order + 1)).\
first()
if next:
doc.order = doc.order + 1
next.order = next.order - 1
db.session.commit()
log(doc.entity_id, "Moved document `%s' down one position" % doc)
return redirect("/resolver/entity/%s" % doc.entity_id)
def id(self, value):
new_id = cleanID(value)
ent = Entity.query.filter(Entity.id == new_id).first()
# check against self (in case we're changing an existing entity's PID)
if ent and not ent == self:
# PID already in DB. If the unclean PID is not equal to the existing
# entity's original PID, this is probably a collision.
if value == ent.original_id:
raise EntityPIDExistsException()
else:
raise EntityCollisionException(ent.original_id)
if self._id != new_id:
log(self.id, "Changed id from '%s' to '%s'" % (self._id, new_id))
self.original_id = value
self._id = new_id
def admin_representation_down(id):
doc = Document.query.filter(Document.id == id).first()
if (not doc) or (doc.type != 'representation'):
flash('Document not found or wrong type', 'danger')
return redirect("/resolver/entity/%s" % doc.entity_id)\
if doc else redirect("/resolver/entity")
next = Representation.query.filter(Document.entity_id == doc.entity_id,
Representation.order == (doc.order + 1)).\
first()
if next:
doc.order = doc.order + 1
next.order = next.order - 1
db.session.commit()
log(doc.entity_id, "Moved document `%s' down one position" % doc)
return redirect("/resolver/entity/%s" % doc.entity_id)
def admin_add_data_json(entity_id):
ent = Entity.query.filter(Entity.id == entity_id).first()
if not ent:
return json.dumps({'errors':[{'detail':'Entity not found'}]})
form = DataForm()
if not form.validate():
return form_errors_json(form)
docs = Data.query.filter(Document.entity_id == ent.id,
Data.format == form.format.data).all()
if len(docs) != 0:
return json.dumps({'errors':[{'title':'Wrong format',
'detail':'Duplicate format'}]})
doc = Data(ent.id, form.format.data, url=form.url.data,
enabled=form.enabled.data, notes=form.notes.data)
log(doc.entity_id, "Added data document `%s'" % doc)
db.session.add(doc)
db.session.commit()
return json.dumps({'success':True})
def create_document():
try:
data = json.loads(request.data)
validate(data, document_schema)
ent = Entity.query.filter(Entity.id == data['entity']).first()
if not ent:
return ErrorRestApi().response(status=400,
errors=['Entity not found.'])
if data['type'] == 'data':
docs = Data.query.filter(Document.entity_id == ent.id,
Data.format == data['format']).all()
if len(docs) != 0:
return ErrorRestApi().response(
status=400,
errors=[
'Duplicate data format \'{0}\' for entity.'.format(
data['format'])
])
doc = Data(ent.id, data['format'], data.get('url', ''),
data['enabled'], data.get('notes', ''))
else:
ref = Representation.query.filter(
Document.entity_id == ent.id,
Representation.reference == True).first()
if data['reference']:
if ref:
ref.reference = False
elif not ref:
errors = [{
'title':
'Reference error',
'detail':
'At least one reference representation is required.'
}]
return ErrorRestApi().response(status=400, errors=errors)
highest = Representation.query. \
filter(Document.entity_id == ent.id). \
order_by(Representation.order.desc()).first()
if highest:
order = highest.order + 1
else:
order = 1
doc = Representation(ent.id,
order,
url=data.get('url', ''),
label=data.get('label', ''),
enabled=data['enabled'],
notes=data.get('form', ''),
reference=data['reference'])
db.session.add(doc)
db.session.commit()
log(doc.entity_id, "Added %s document `%s'" % (data['type'], doc))
return RestApi().response(status=201, data={'data': doc.to_dict()})
except ValueError as e:
errors = [{
'title':
'Malformed request',
'detail':
'Expected correctly formatted JSON data: {0}'.format(e)
}]
return ErrorRestApi().response(status=400, errors=errors)
except ValidationError as e:
errors = [{
'title':
'Malformed request',
'detail':
'JSON data does not conform to schema: {0}'.format(e)
}]
return ErrorRestApi().response(status=400, errors=errors)
def update_entity(id):
# With form-data, you also get the boundary after the first ';'
content_type = request.headers['Content-Type'].split(';')
if content_type[0] == 'application/x-www-form-urlencoded' \
or content_type[0] == 'multipart/form-data':
form = EntityForm(csrf_enabled=False)
data = {
'id': form.id.data,
'title': form.title.data,
'type': form.type.data
}
else:
try:
data = json.loads(request.data)
validate(data, entity_schema)
except ValueError as e:
errors = [{
'title':
'Malformed request',
'detail':
'Expected correctly formatted JSON data: {0}'.format(e)
}]
return ErrorRestApi().response(status=400, errors=errors)
except ValidationError as e:
errors = [{
'title':
'Malformed request',
'detail':
'JSON data does not conform to schema: {0}'.format(e)
}]
return ErrorRestApi().response(status=400, errors=errors)
ent = Entity.query.filter(Entity.id == id).first()
if not ent:
return ErrorRestApi().response(status=404,
errors=['Entity not found.'])
ent_str = str(ent)
try:
ent.id = data["id"]
ent.title = data.get("title", "")
ent.type = data["type"]
except EntityPIDExistsException:
db.session.rollback()
return ErrorRestApi().response(
status=409,
errors=['Duplicate ID \'{0}\' for entity.'.format(data['id'])])
except EntityCollisionException as e:
db.session.rollback()
errors = [{
'title':
'ID collision',
'detail':
'The provided ID \'{0}\' collides with the existing ID \'{0}\'.'.
format(data['id'], e.original_id)
}]
return ErrorRestApi().response(status=409, errors=errors)
db.session.commit()
log(ent.id, "Changed entity from `%s' to `%s'" % (ent_str, ent))
return RestApi().response(
data={'data': EntityViewApi().output(entity=ent)})
def import_file(file):
"""
Import a CSV file
CSV layout: PID, entity_type, title, document_type, URL, enabled, notes, format, reference, order
0 1 2 3 4 5 6 7 8 9
:param file:
:return:
"""
reader = UnicodeReader(file)
# NOTE: we always assume the first row is a header
# As this feature is mainly used for imports/edits from Excel, it is
# possible that Excel uses `;' as a separator instead of `,' ...
if len(reader.next()) != 10:
file.seek(0)
reader = UnicodeReader(file, delimiter=';')
reader.next() # Skip header again
# Create id for the import logging function (id = unique identifier of this import action)
import_id = str(time.time())
rows = 0
count_pids = 0
records = {}
failures = []
bad_records = []
for record in reader:
record_id = record[0]
# Skip wrong types now
# TODO: do we actually fail on importing a wrong type?
if not record[1] in entity_types:
failures.append((record_id, "Wrong entity type `%s'" % record[1]))
bad_records.append(record)
continue
if not record[3] in document_types:
failures.append(
(record_id, "Wrong document type `%s'" % record[3]))
bad_records.append(record)
continue
rows += 1
# Check whether the record_id is already in records: if it is, we append this record. Else, we add it (as a list)
if records.get(record_id, False):
records[record_id].append(record)
import_log(import_id, "Appended document to PID %s" % record_id)
else:
records[record_id] = [record]
count_pids += 1
import_log(import_id, "Added new PID %s" % record_id)
for record_id, record_list in records.iteritems():
clean_id = cleanID(record_id)
ent = Entity.query. \
filter(Entity.id == clean_id).first()
if ent:
if not ent.original_id == record_id:
failures.append(
(clean_id, "PID collision with `%s'" % ent.original_id))
bad_records += record_list
continue
else:
ent = Entity(clean_id)
db.session.add(ent)
db.session.flush()
log(ent.id, "Added entity `%s'" % ent)
# All records in the list have the same title and type
ent.title = record_list[0][2]
ent.type = record_list[0][1]
for record in record_list:
if record[4] == 'None':
url = ''
else:
url = record[4]
if record[5] == '1':
enabled = '1'
else:
enabled = '0'
# record[3] = document_type
if record[3] == 'data':
if not (record[7] and record[7] in data_formats):
failures.append(
(clean_id,
"Format missing or invalid for PID `%s'" % ent.id))
bad_records.append(record)
continue
# record[7] = format
doc = Data.query.filter(Data.format == record[7],
Document.entity_id == ent.id).first()
if doc:
doc.url = url
doc.enabled = enabled
doc.notes = record[6]
else:
doc = Data(ent.id,
record[7],
url=url,
enabled=enabled,
notes=record[6])
db.session.add(doc)
log(id, "Added data document `%s'" % doc)
elif record[3] == 'representation':
# This function expects order in the database to be "" (empty) when not set,
# but in reality it is set to count(documents) + 1 when not provided.
# When we search for it with an unset order, we will never find it and thus
# create a new representation which is not needed.
# See https://github.com/PACKED-vzw/resolver/issues/50
if record[4] == "" or not record[4]:
# r_url is None (NULL) in the DB, but "" in the CSV
r_url = None
else:
r_url = record[4]
doc = Representation.query.filter(
Document.entity_id == ent.id, Document.url == r_url,
Document.type == record[3]).first()
if doc:
doc.url = url
doc.enabled = enabled
doc.notes = record[6]
else:
if record[9] and record[9] != "":
order = int(record[9])
else:
# We set order to the total amount representation documents for this entity + 1
order = Representation.query \
.filter(Document.entity_id == ent.id).count() + 1
doc = Representation(ent.id,
order,
url=url,
enabled=enabled,
notes=record[6])
db.session.add(doc)
log(clean_id, "Added representation document `%s'" % doc)
reference = record[8] == '1'
if reference:
ref = Representation.query. \
filter(Document.entity_id == ent.id,
Representation.reference == True).first()
if ref:
ref.reference = False
doc.reference = True
db.session.flush()
for record_id in records:
reps = Representation.query. \
filter(Document.entity_id == record_id). \
order_by(Representation.order.asc()).all()
i = 1
has_reference = False
for rep in reps:
rep.order = i
i += 1
has_reference = rep.reference
if (not has_reference) and i > 1:
reps[0].reference = True
db.session.commit()
return (import_id, rows, count_pids, failures, bad_records)
def import_file(file):
"""
Import a CSV file
CSV layout: PID, entity_type, title, document_type, URL, enabled, notes, format, reference, order
0 1 2 3 4 5 6 7 8 9
:param file:
:return:
"""
reader = UnicodeReader(file)
# NOTE: we always assume the first row is a header
# As this feature is mainly used for imports/edits from Excel, it is
# possible that Excel uses `;' as a separator instead of `,' ...
if len(reader.next()) != 10:
file.seek(0)
reader = UnicodeReader(file, delimiter=';')
reader.next() # Skip header again
# Create id for the import logging function (id = unique identifier of this import action)
import_id = str(time.time())
rows = 0
count_pids = 0
records = {}
failures = []
bad_records = []
for record in reader:
record_id = record[0]
# Skip wrong types now
# TODO: do we actually fail on importing a wrong type?
if not record[1] in entity_types:
failures.append((record_id, "Wrong entity type `%s'" % record[1]))
bad_records.append(record)
continue
if not record[3] in document_types:
failures.append((record_id, "Wrong document type `%s'" % record[3]))
bad_records.append(record)
continue
rows += 1
# Check whether the record_id is already in records: if it is, we append this record. Else, we add it (as a list)
if records.get(record_id, False):
records[record_id].append(record)
import_log(import_id, "Appended document to PID %s" % record_id)
else:
records[record_id] = [record]
count_pids += 1
import_log(import_id, "Added new PID %s" % record_id)
for record_id, record_list in records.iteritems():
clean_id = cleanID(record_id)
ent = Entity.query. \
filter(Entity.id == clean_id).first()
if ent:
if not ent.original_id == record_id:
failures.append((clean_id, "PID collision with `%s'" % ent.original_id))
bad_records += record_list
continue
else:
ent = Entity(clean_id)
db.session.add(ent)
db.session.flush()
log(ent.id, "Added entity `%s'" % ent)
# All records in the list have the same title and type
ent.title = record_list[0][2]
ent.type = record_list[0][1]
for record in record_list:
if record[4] == 'None':
url = ''
else:
url = record[4]
if record[5] == '1':
enabled = '1'
else:
enabled = '0'
# record[3] = document_type
if record[3] == 'data':
if not (record[7] and record[7] in data_formats):
failures.append((clean_id, "Format missing or invalid for PID `%s'" % ent.id))
bad_records.append(record)
continue
# record[7] = format
doc = Data.query.filter(Data.format == record[7],
Document.entity_id == ent.id).first()
if doc:
doc.url = url
doc.enabled = enabled
doc.notes = record[6]
else:
doc = Data(ent.id, record[7], url=url, enabled=enabled,
notes=record[6])
db.session.add(doc)
log(id, "Added data document `%s'" % doc)
elif record[3] == 'representation':
# This function expects order in the database to be "" (empty) when not set,
# but in reality it is set to count(documents) + 1 when not provided.
# When we search for it with an unset order, we will never find it and thus
# create a new representation which is not needed.
# See https://github.com/PACKED-vzw/resolver/issues/50
if record[4] == "" or not record[4]:
# r_url is None (NULL) in the DB, but "" in the CSV
r_url = None
else:
r_url = record[4]
doc = Representation.query.filter(Document.entity_id == ent.id,
Document.url == r_url,
Document.type == record[3]).first()
if doc:
doc.url = url
doc.enabled = enabled
doc.notes = record[6]
else:
if record[9] and record[9] != "":
order = int(record[9])
else:
# We set order to the total amount representation documents for this entity + 1
order = Representation.query \
.filter(Document.entity_id == ent.id).count() + 1
doc = Representation(ent.id, order, url=url,
enabled=enabled, notes=record[6])
db.session.add(doc)
log(clean_id, "Added representation document `%s'" % doc)
reference = record[8] == '1'
if reference:
ref = Representation.query. \
filter(Document.entity_id == ent.id,
Representation.reference == True).first()
if ref:
ref.reference = False
doc.reference = True
db.session.flush()
for record_id in records:
reps = Representation.query. \
filter(Document.entity_id == record_id). \
order_by(Representation.order.asc()).all()
i = 1
has_reference = False
for rep in reps:
rep.order = i
i += 1
has_reference = rep.reference
if (not has_reference) and i > 1:
reps[0].reference = True
db.session.commit()
return (import_id, rows, count_pids, failures, bad_records)
