def admin_add_data_json(entity_id): ent = Entity.query.filter(Entity.id == entity_id).first() if not ent: return json.dumps({'errors': [{'detail': 'Entity not found'}]}) form = DataForm() if not form.validate(): return form_errors_json(form) docs = Data.query.filter(Document.entity_id == ent.id, Data.format == form.format.data).all() if len(docs) != 0: return json.dumps({ 'errors': [{ 'title': 'Wrong format', 'detail': 'Duplicate format' }] }) doc = Data(ent.id, form.format.data, url=form.url.data, enabled=form.enabled.data, notes=form.notes.data) log(doc.entity_id, "Added data document `%s'" % doc) db.session.add(doc) db.session.commit() return json.dumps({'success': True})
def title(self, value): if self._title != value: log(self.id, "Changed title from '%s' to '%s'" % (self._title, value)) self._title = value self.slug = slugify(value)[:64] if value else SLUG_DEFAULT
def admin_new_entity(): form = EntityForm() if form.validate(): try: ent = Entity(type=form.type.data, title=form.title.data, id=form.id.data) except EntityPIDExistsException: flash("There already exists and entity with this PID.", "warning") return admin_list_entities(form=form) except EntityCollisionException: flash("PID collision detected!", "warning") return admin_list_entities(form=form) db.session.add(ent) db.session.flush() db.session.add(Data(ent.id, 'html')) db.session.add(Representation(ent.id, 1, reference=True)) db.session.commit() log(ent.id, "Created entity `%s'" % ent) return redirect("/resolver/entity/%s" % ent.id) else: return admin_list_entities(form=form, show_form=True)
def admin_delete_document(id): doc = Document.query.filter(Document.id == id).first() entity_id = doc.entity_id if not doc: flash("Document not found", "warning") return redirect("/resolver/entity") if type(doc) == Representation: if doc.reference: count = Representation.query.\ filter(Document.entity_id == entity_id).count() if count > 1: flash("Please select another representation to be reference first", 'warning') return redirect("/resolver/entity/%s" % entity_id) db.session.delete(doc) db.session.flush() i = 1 reps = Representation.query.\ filter(Document.entity_id == entity_id).\ order_by(Representation.order.asc()).all() for rep in reps: rep.order = i i += 1 else: db.session.delete(doc) log(entity_id, "Removed the document '%s' %s" % (doc, doc.entity)) db.session.commit() flash("Document deleted succesfully", "success") return redirect("/resolver/entity/%s" % entity_id)
def delete_document(id): doc = Document.query.filter(Document.id == id).first() if not doc: return ErrorRestApi().response(status=404, errors=['Document not found.']) if isinstance(doc, Representation): if doc.reference: count = Representation.query. \ filter(Document.entity_id == doc.entity_id).count() if count > 1: return ErrorRestApi().response(status=409, errors=['Can not delete reference representation.']) db.session.delete(doc) db.session.flush() i = 1 reps = Representation.query. \ filter(Document.entity_id == doc.entity_id). \ order_by(Representation.order.asc()).all() for rep in reps: rep.order = i i += 1 else: db.session.delete(doc) log(doc.entity_id, "Removed the document '%s' %s" % (doc, doc.entity)) db.session.commit() return "", 204
def admin_add_representation_json(entity_id): ent = Entity.query.filter(Entity.id == entity_id).first() if not ent: return json.dumps({'errors':[{'detail':'Entity not found'}]}) form = RepresentationForm() if not form.validate(): return form_errors_json(form) ref = Representation.query.filter(Document.entity_id == ent.id, Representation.reference == True).first() if form.reference.data: if ref: ref.reference = False elif not ref: return json.dumps({'errors':[{'title':'Reference error', 'detail':'At least one representation has to be the reference image.'}]}) highest = Representation.query.\ filter(Document.entity_id == ent.id).\ order_by(Representation.order.desc()).first() if highest: order = highest.order + 1 else: order = 1 rep = Representation(ent.id, order, url=form.url.data, label=form.label.data, enabled=form.enabled.data, notes=form.notes.data, reference=form.reference.data) log(rep.entity_id, "Added representation document `%s'" % rep) db.session.add(rep) db.session.commit() return json.dumps({'success':True})
def create_entity(): form = EntityForm(csrf_enabled=False) if form.validate_on_submit(): try: ent = Entity(type=form.type.data, title=form.title.data, id=form.id.data) except EntityPIDExistsException: return ErrorRestApi().response(status=409, errors=['Duplicate ID for entity.']) except EntityCollisionException as e: return ErrorRestApi().response( status=409, errors=[{ 'title': 'ID Collision', 'detail': 'The provided ID \'{0}\' collides with the existing ID \'{1}\'' .format(form.id.data, e.original_id) }]) db.session.add(ent) db.session.flush() db.session.add(Data(ent.id, 'html')) db.session.add(Representation(ent.id, 1, reference=True)) db.session.commit() log(ent.id, "Created entity `%s'" % ent) return RestApi().response( status=201, data={'data': EntityViewApi().output(entity=ent)}) errors = [{ 'title': 'Malformed parameter', 'detail': 'Field %s: %s' % (field, ' '.join(error)) } for field, error in form.errors.iteritems()] return ErrorRestApi().response(status=400, errors=errors)
def create_entity(): form = EntityForm(csrf_enabled=False) if form.validate_on_submit(): try: ent = Entity(type=form.type.data, title=form.title.data, id=form.id.data) except EntityPIDExistsException: return ErrorRestApi().response(status=409, errors=['Duplicate ID for entity.']) except EntityCollisionException as e: return ErrorRestApi().response(status=409, errors=[{ 'title': 'ID Collision', 'detail': 'The provided ID \'{0}\' collides with the existing ID \'{1}\''.format(form.id.data, e.original_id) }]) db.session.add(ent) db.session.flush() db.session.add(Data(ent.id, 'html')) db.session.add(Representation(ent.id, 1, reference=True)) db.session.commit() log(ent.id, "Created entity `%s'" % ent) return RestApi().response(status=201, data={'data': EntityViewApi().output(entity=ent)}) errors = [{'title': 'Malformed parameter', 'detail': 'Field %s: %s' % (field, ' '.join(error))} for field, error in form.errors.iteritems()] return ErrorRestApi().response(status=400, errors=errors)
def delete_document(id): doc = Document.query.filter(Document.id == id).first() if not doc: return ErrorRestApi().response(status=404, errors=['Document not found.']) if isinstance(doc, Representation): if doc.reference: count = Representation.query. \ filter(Document.entity_id == doc.entity_id).count() if count > 1: return ErrorRestApi().response( status=409, errors=['Can not delete reference representation.']) db.session.delete(doc) db.session.flush() i = 1 reps = Representation.query. \ filter(Document.entity_id == doc.entity_id). \ order_by(Representation.order.asc()).all() for rep in reps: rep.order = i i += 1 else: db.session.delete(doc) log(doc.entity_id, "Removed the document '%s' %s" % (doc, doc.entity)) db.session.commit() return "", 204
def admin_delete_document(id): doc = Document.query.filter(Document.id == id).first() entity_id = doc.entity_id if not doc: flash("Document not found", "warning") return redirect("/resolver/entity") if type(doc) == Representation: if doc.reference: count = Representation.query.\ filter(Document.entity_id == entity_id).count() if count > 1: flash( "Please select another representation to be reference first", 'warning') return redirect("/resolver/entity/%s" % entity_id) db.session.delete(doc) db.session.flush() i = 1 reps = Representation.query.\ filter(Document.entity_id == entity_id).\ order_by(Representation.order.asc()).all() for rep in reps: rep.order = i i += 1 else: db.session.delete(doc) log(entity_id, "Removed the document '%s' %s" % (doc, doc.entity)) db.session.commit() flash("Document deleted succesfully", "success") return redirect("/resolver/entity/%s" % entity_id)
def create_document(): try: data = json.loads(request.data) validate(data, document_schema) ent = Entity.query.filter(Entity.id == data['entity']).first() if not ent: return ErrorRestApi().response(status=400, errors=['Entity not found.']) if data['type'] == 'data': docs = Data.query.filter(Document.entity_id == ent.id, Data.format == data['format']).all() if len(docs) != 0: return ErrorRestApi().response(status=400, errors=['Duplicate data format \'{0}\' for entity.' .format(data['format'])]) doc = Data(ent.id, data['format'], data.get('url', ''), data['enabled'], data.get('notes', '')) else: ref = Representation.query.filter(Document.entity_id == ent.id, Representation.reference == True).first() if data['reference']: if ref: ref.reference = False elif not ref: errors = [{ 'title': 'Reference error', 'detail': 'At least one reference representation is required.' }] return ErrorRestApi().response(status=400, errors=errors) highest = Representation.query. \ filter(Document.entity_id == ent.id). \ order_by(Representation.order.desc()).first() if highest: order = highest.order + 1 else: order = 1 doc = Representation(ent.id, order, url=data.get('url', ''), label=data.get('label', ''), enabled=data['enabled'], notes=data.get('form', ''), reference=data['reference']) db.session.add(doc) db.session.commit() log(doc.entity_id, "Added %s document `%s'" % (data['type'], doc)) return RestApi().response(status=201, data={'data': doc.to_dict()}) except ValueError as e: errors = [{ 'title': 'Malformed request', 'detail': 'Expected correctly formatted JSON data: {0}'.format(e) }] return ErrorRestApi().response(status=400, errors=errors) except ValidationError as e: errors = [{ 'title': 'Malformed request', 'detail': 'JSON data does not conform to schema: {0}'.format(e) }] return ErrorRestApi().response(status=400, errors=errors)
def enabled(self, value): value = bool(value) if self._enabled and (not value): log(self.entity_id, "Disabled document %s" % self) elif (not self._enabled) and value: log(self.entity_id, "Enabled document %s" % self) self._enabled = value
def format(self, value): if value not in data_formats: raise Exception("Incorrect data format") if self._format != value: log(self.entity_id, "Changed format from '%s' to '%s' for %s" % (self._format, value, self)) self._format = value
def reference(self, value): value = bool(value) if self._reference and (not value): log(self.entity_id, "Removed as reference: %s" % self) elif (not self._reference) and value: log(self.entity_id, "Set as reference: %s" % self) self._reference = value
def format(self, value): if value not in data_formats: raise Exception("Incorrect data format") if self._format != value: log( self.entity_id, "Changed format from '%s' to '%s' for %s" % (self._format, value, self)) self._format = value
def delete_entity(id): ent = Entity.query.filter(Entity.id == id).first() if ent: db.session.delete(ent) db.session.commit() log(id, "Removed the entity `%s'" % ent) return "", 204 else: return ErrorRestApi().response(status=404, errors=['Entity not found.'])
def type(self, value): if value not in entity_types: raise Exception("Wrong entity type") if self._type != value: log(self.id, "Changed type from '%s' to '%s'" % (self._type, value)) self._type = value
def admin_delete_entity(id): ent = Entity.query.filter(Entity.id == id).first() if not ent: flash("Entity not found!", "danger") else: db.session.delete(ent) db.session.commit() log(id, "Removed the entity `%s'" % ent) flash("Entity deleted succesfully!", "success") return redirect("/resolver/entity")
def update_entity(id): # With form-data, you also get the boundary after the first ';' content_type = request.headers['Content-Type'].split(';') if content_type[0] == 'application/x-www-form-urlencoded' \ or content_type[0] == 'multipart/form-data': form = EntityForm(csrf_enabled=False) data = { 'id': form.id.data, 'title': form.title.data, 'type': form.type.data } else: try: data = json.loads(request.data) validate(data, entity_schema) except ValueError as e: errors = [{ 'title': 'Malformed request', 'detail': 'Expected correctly formatted JSON data: {0}'.format(e) }] return ErrorRestApi().response(status=400, errors=errors) except ValidationError as e: errors = [{ 'title': 'Malformed request', 'detail': 'JSON data does not conform to schema: {0}'.format(e) }] return ErrorRestApi().response(status=400, errors=errors) ent = Entity.query.filter(Entity.id == id).first() if not ent: return ErrorRestApi().response(status=404, errors=['Entity not found.']) ent_str = str(ent) try: ent.id = data["id"] ent.title = data.get("title", "") ent.type = data["type"] except EntityPIDExistsException: db.session.rollback() return ErrorRestApi().response(status=409, errors=['Duplicate ID \'{0}\' for entity.'.format(data['id'])]) except EntityCollisionException as e: db.session.rollback() errors = [{ 'title': 'ID collision', 'detail': 'The provided ID \'{0}\' collides with the existing ID \'{0}\'.'.format(data['id'], e.original_id) }] return ErrorRestApi().response(status=409, errors=errors) db.session.commit() log(ent.id, "Changed entity from `%s' to `%s'" % (ent_str, ent)) return RestApi().response(data={'data': EntityViewApi().output(entity=ent)})
def url(self, url): # TODO: URL validation here old = self._url if url: u = urlparse(url) if u.scheme: self._url = url else: self._url = urlunparse(('http',)+u[1:]) else: self._url = None if old != self._url: log(self.entity_id, "Changed URL from '%s' to '%s' for %s" % (old, self._url, self))
def admin_add_representation_json(entity_id): ent = Entity.query.filter(Entity.id == entity_id).first() if not ent: return json.dumps({'errors': [{'detail': 'Entity not found'}]}) form = RepresentationForm() if not form.validate(): return form_errors_json(form) ref = Representation.query.filter( Document.entity_id == ent.id, Representation.reference == True).first() if form.reference.data: if ref: ref.reference = False elif not ref: return json.dumps({ 'errors': [{ 'title': 'Reference error', 'detail': 'At least one representation has to be the reference image.' }] }) highest = Representation.query.\ filter(Document.entity_id == ent.id).\ order_by(Representation.order.desc()).first() if highest: order = highest.order + 1 else: order = 1 rep = Representation(ent.id, order, url=form.url.data, label=form.label.data, enabled=form.enabled.data, notes=form.notes.data, reference=form.reference.data) log(rep.entity_id, "Added representation document `%s'" % rep) db.session.add(rep) db.session.commit() return json.dumps({'success': True})
def admin_representation_down(id): doc = Document.query.filter(Document.id == id).first() if (not doc) or (doc.type != 'representation'): flash('Document not found or wrong type', 'danger') return redirect("/resolver/entity/%s" % doc.entity_id)\ if doc else redirect("/resolver/entity") next = Representation.query.filter(Document.entity_id == doc.entity_id, Representation.order == (doc.order + 1)).\ first() if next: doc.order = doc.order + 1 next.order = next.order - 1 db.session.commit() log(doc.entity_id, "Moved document `%s' down one position" % doc) return redirect("/resolver/entity/%s" % doc.entity_id)
def id(self, value): new_id = cleanID(value) ent = Entity.query.filter(Entity.id == new_id).first() # check against self (in case we're changing an existing entity's PID) if ent and not ent == self: # PID already in DB. If the unclean PID is not equal to the existing # entity's original PID, this is probably a collision. if value == ent.original_id: raise EntityPIDExistsException() else: raise EntityCollisionException(ent.original_id) if self._id != new_id: log(self.id, "Changed id from '%s' to '%s'" % (self._id, new_id)) self.original_id = value self._id = new_id
def admin_add_data_json(entity_id): ent = Entity.query.filter(Entity.id == entity_id).first() if not ent: return json.dumps({'errors':[{'detail':'Entity not found'}]}) form = DataForm() if not form.validate(): return form_errors_json(form) docs = Data.query.filter(Document.entity_id == ent.id, Data.format == form.format.data).all() if len(docs) != 0: return json.dumps({'errors':[{'title':'Wrong format', 'detail':'Duplicate format'}]}) doc = Data(ent.id, form.format.data, url=form.url.data, enabled=form.enabled.data, notes=form.notes.data) log(doc.entity_id, "Added data document `%s'" % doc) db.session.add(doc) db.session.commit() return json.dumps({'success':True})
def create_document(): try: data = json.loads(request.data) validate(data, document_schema) ent = Entity.query.filter(Entity.id == data['entity']).first() if not ent: return ErrorRestApi().response(status=400, errors=['Entity not found.']) if data['type'] == 'data': docs = Data.query.filter(Document.entity_id == ent.id, Data.format == data['format']).all() if len(docs) != 0: return ErrorRestApi().response( status=400, errors=[ 'Duplicate data format \'{0}\' for entity.'.format( data['format']) ]) doc = Data(ent.id, data['format'], data.get('url', ''), data['enabled'], data.get('notes', '')) else: ref = Representation.query.filter( Document.entity_id == ent.id, Representation.reference == True).first() if data['reference']: if ref: ref.reference = False elif not ref: errors = [{ 'title': 'Reference error', 'detail': 'At least one reference representation is required.' }] return ErrorRestApi().response(status=400, errors=errors) highest = Representation.query. \ filter(Document.entity_id == ent.id). \ order_by(Representation.order.desc()).first() if highest: order = highest.order + 1 else: order = 1 doc = Representation(ent.id, order, url=data.get('url', ''), label=data.get('label', ''), enabled=data['enabled'], notes=data.get('form', ''), reference=data['reference']) db.session.add(doc) db.session.commit() log(doc.entity_id, "Added %s document `%s'" % (data['type'], doc)) return RestApi().response(status=201, data={'data': doc.to_dict()}) except ValueError as e: errors = [{ 'title': 'Malformed request', 'detail': 'Expected correctly formatted JSON data: {0}'.format(e) }] return ErrorRestApi().response(status=400, errors=errors) except ValidationError as e: errors = [{ 'title': 'Malformed request', 'detail': 'JSON data does not conform to schema: {0}'.format(e) }] return ErrorRestApi().response(status=400, errors=errors)
def update_entity(id): # With form-data, you also get the boundary after the first ';' content_type = request.headers['Content-Type'].split(';') if content_type[0] == 'application/x-www-form-urlencoded' \ or content_type[0] == 'multipart/form-data': form = EntityForm(csrf_enabled=False) data = { 'id': form.id.data, 'title': form.title.data, 'type': form.type.data } else: try: data = json.loads(request.data) validate(data, entity_schema) except ValueError as e: errors = [{ 'title': 'Malformed request', 'detail': 'Expected correctly formatted JSON data: {0}'.format(e) }] return ErrorRestApi().response(status=400, errors=errors) except ValidationError as e: errors = [{ 'title': 'Malformed request', 'detail': 'JSON data does not conform to schema: {0}'.format(e) }] return ErrorRestApi().response(status=400, errors=errors) ent = Entity.query.filter(Entity.id == id).first() if not ent: return ErrorRestApi().response(status=404, errors=['Entity not found.']) ent_str = str(ent) try: ent.id = data["id"] ent.title = data.get("title", "") ent.type = data["type"] except EntityPIDExistsException: db.session.rollback() return ErrorRestApi().response( status=409, errors=['Duplicate ID \'{0}\' for entity.'.format(data['id'])]) except EntityCollisionException as e: db.session.rollback() errors = [{ 'title': 'ID collision', 'detail': 'The provided ID \'{0}\' collides with the existing ID \'{0}\'.'. format(data['id'], e.original_id) }] return ErrorRestApi().response(status=409, errors=errors) db.session.commit() log(ent.id, "Changed entity from `%s' to `%s'" % (ent_str, ent)) return RestApi().response( data={'data': EntityViewApi().output(entity=ent)})
def import_file(file): """ Import a CSV file CSV layout: PID, entity_type, title, document_type, URL, enabled, notes, format, reference, order 0 1 2 3 4 5 6 7 8 9 :param file: :return: """ reader = UnicodeReader(file) # NOTE: we always assume the first row is a header # As this feature is mainly used for imports/edits from Excel, it is # possible that Excel uses `;' as a separator instead of `,' ... if len(reader.next()) != 10: file.seek(0) reader = UnicodeReader(file, delimiter=';') reader.next() # Skip header again # Create id for the import logging function (id = unique identifier of this import action) import_id = str(time.time()) rows = 0 count_pids = 0 records = {} failures = [] bad_records = [] for record in reader: record_id = record[0] # Skip wrong types now # TODO: do we actually fail on importing a wrong type? if not record[1] in entity_types: failures.append((record_id, "Wrong entity type `%s'" % record[1])) bad_records.append(record) continue if not record[3] in document_types: failures.append( (record_id, "Wrong document type `%s'" % record[3])) bad_records.append(record) continue rows += 1 # Check whether the record_id is already in records: if it is, we append this record. Else, we add it (as a list) if records.get(record_id, False): records[record_id].append(record) import_log(import_id, "Appended document to PID %s" % record_id) else: records[record_id] = [record] count_pids += 1 import_log(import_id, "Added new PID %s" % record_id) for record_id, record_list in records.iteritems(): clean_id = cleanID(record_id) ent = Entity.query. \ filter(Entity.id == clean_id).first() if ent: if not ent.original_id == record_id: failures.append( (clean_id, "PID collision with `%s'" % ent.original_id)) bad_records += record_list continue else: ent = Entity(clean_id) db.session.add(ent) db.session.flush() log(ent.id, "Added entity `%s'" % ent) # All records in the list have the same title and type ent.title = record_list[0][2] ent.type = record_list[0][1] for record in record_list: if record[4] == 'None': url = '' else: url = record[4] if record[5] == '1': enabled = '1' else: enabled = '0' # record[3] = document_type if record[3] == 'data': if not (record[7] and record[7] in data_formats): failures.append( (clean_id, "Format missing or invalid for PID `%s'" % ent.id)) bad_records.append(record) continue # record[7] = format doc = Data.query.filter(Data.format == record[7], Document.entity_id == ent.id).first() if doc: doc.url = url doc.enabled = enabled doc.notes = record[6] else: doc = Data(ent.id, record[7], url=url, enabled=enabled, notes=record[6]) db.session.add(doc) log(id, "Added data document `%s'" % doc) elif record[3] == 'representation': # This function expects order in the database to be "" (empty) when not set, # but in reality it is set to count(documents) + 1 when not provided. # When we search for it with an unset order, we will never find it and thus # create a new representation which is not needed. # See https://github.com/PACKED-vzw/resolver/issues/50 if record[4] == "" or not record[4]: # r_url is None (NULL) in the DB, but "" in the CSV r_url = None else: r_url = record[4] doc = Representation.query.filter( Document.entity_id == ent.id, Document.url == r_url, Document.type == record[3]).first() if doc: doc.url = url doc.enabled = enabled doc.notes = record[6] else: if record[9] and record[9] != "": order = int(record[9]) else: # We set order to the total amount representation documents for this entity + 1 order = Representation.query \ .filter(Document.entity_id == ent.id).count() + 1 doc = Representation(ent.id, order, url=url, enabled=enabled, notes=record[6]) db.session.add(doc) log(clean_id, "Added representation document `%s'" % doc) reference = record[8] == '1' if reference: ref = Representation.query. \ filter(Document.entity_id == ent.id, Representation.reference == True).first() if ref: ref.reference = False doc.reference = True db.session.flush() for record_id in records: reps = Representation.query. \ filter(Document.entity_id == record_id). \ order_by(Representation.order.asc()).all() i = 1 has_reference = False for rep in reps: rep.order = i i += 1 has_reference = rep.reference if (not has_reference) and i > 1: reps[0].reference = True db.session.commit() return (import_id, rows, count_pids, failures, bad_records)
def import_file(file): """ Import a CSV file CSV layout: PID, entity_type, title, document_type, URL, enabled, notes, format, reference, order 0 1 2 3 4 5 6 7 8 9 :param file: :return: """ reader = UnicodeReader(file) # NOTE: we always assume the first row is a header # As this feature is mainly used for imports/edits from Excel, it is # possible that Excel uses `;' as a separator instead of `,' ... if len(reader.next()) != 10: file.seek(0) reader = UnicodeReader(file, delimiter=';') reader.next() # Skip header again # Create id for the import logging function (id = unique identifier of this import action) import_id = str(time.time()) rows = 0 count_pids = 0 records = {} failures = [] bad_records = [] for record in reader: record_id = record[0] # Skip wrong types now # TODO: do we actually fail on importing a wrong type? if not record[1] in entity_types: failures.append((record_id, "Wrong entity type `%s'" % record[1])) bad_records.append(record) continue if not record[3] in document_types: failures.append((record_id, "Wrong document type `%s'" % record[3])) bad_records.append(record) continue rows += 1 # Check whether the record_id is already in records: if it is, we append this record. Else, we add it (as a list) if records.get(record_id, False): records[record_id].append(record) import_log(import_id, "Appended document to PID %s" % record_id) else: records[record_id] = [record] count_pids += 1 import_log(import_id, "Added new PID %s" % record_id) for record_id, record_list in records.iteritems(): clean_id = cleanID(record_id) ent = Entity.query. \ filter(Entity.id == clean_id).first() if ent: if not ent.original_id == record_id: failures.append((clean_id, "PID collision with `%s'" % ent.original_id)) bad_records += record_list continue else: ent = Entity(clean_id) db.session.add(ent) db.session.flush() log(ent.id, "Added entity `%s'" % ent) # All records in the list have the same title and type ent.title = record_list[0][2] ent.type = record_list[0][1] for record in record_list: if record[4] == 'None': url = '' else: url = record[4] if record[5] == '1': enabled = '1' else: enabled = '0' # record[3] = document_type if record[3] == 'data': if not (record[7] and record[7] in data_formats): failures.append((clean_id, "Format missing or invalid for PID `%s'" % ent.id)) bad_records.append(record) continue # record[7] = format doc = Data.query.filter(Data.format == record[7], Document.entity_id == ent.id).first() if doc: doc.url = url doc.enabled = enabled doc.notes = record[6] else: doc = Data(ent.id, record[7], url=url, enabled=enabled, notes=record[6]) db.session.add(doc) log(id, "Added data document `%s'" % doc) elif record[3] == 'representation': # This function expects order in the database to be "" (empty) when not set, # but in reality it is set to count(documents) + 1 when not provided. # When we search for it with an unset order, we will never find it and thus # create a new representation which is not needed. # See https://github.com/PACKED-vzw/resolver/issues/50 if record[4] == "" or not record[4]: # r_url is None (NULL) in the DB, but "" in the CSV r_url = None else: r_url = record[4] doc = Representation.query.filter(Document.entity_id == ent.id, Document.url == r_url, Document.type == record[3]).first() if doc: doc.url = url doc.enabled = enabled doc.notes = record[6] else: if record[9] and record[9] != "": order = int(record[9]) else: # We set order to the total amount representation documents for this entity + 1 order = Representation.query \ .filter(Document.entity_id == ent.id).count() + 1 doc = Representation(ent.id, order, url=url, enabled=enabled, notes=record[6]) db.session.add(doc) log(clean_id, "Added representation document `%s'" % doc) reference = record[8] == '1' if reference: ref = Representation.query. \ filter(Document.entity_id == ent.id, Representation.reference == True).first() if ref: ref.reference = False doc.reference = True db.session.flush() for record_id in records: reps = Representation.query. \ filter(Document.entity_id == record_id). \ order_by(Representation.order.asc()).all() i = 1 has_reference = False for rep in reps: rep.order = i i += 1 has_reference = rep.reference if (not has_reference) and i > 1: reps[0].reference = True db.session.commit() return (import_id, rows, count_pids, failures, bad_records)