def _to_python(self, name, state): entity = Entity.by_name(state.dataset, name) if entity is None: return name if state.entity and entity.id == state.entity.id: return name raise Invalid('Entity already exists.', name, None)
def import_upload(dataset_name, id, account_id, entity_col, alias_col): dataset = Dataset.find(dataset_name) account = Account.by_id(account_id) metadata, row_set = parse_upload(dataset, id) headers = detect_headers(row_set) for row in row_set: data = dict([(c.column, c.value) for c in row]) entity = data.pop(entity_col) if entity_col else None alias = data.pop(alias_col) if alias_col else None if alias_col and alias is not None and len(alias) and alias != entity: d = {'name': alias, 'data': data} alias_obj = Alias.lookup(dataset, d, account, match_entity=False) data = {} if entity_col and entity is not None and len(entity): d = {'name': entity, 'data': data} entity_obj = Entity.by_name(dataset, entity) if entity_obj is None: entity_obj = Entity.create(dataset, d, account) entity_obj.data = data if alias_col and entity_col: alias_obj.match(dataset, {'choice': entity_obj.id}, account) db.session.commit() flush_cache(dataset)
def _to_python(self, value, state): if isinstance(value, dict): value = value.get('id') entity = Entity.by_id(value) if entity is None: entity = Entity.by_name(state.dataset, value) if entity is None: raise Invalid('Entity does not exist: %s' % value, value, None) if entity == state.entity: return None if entity.dataset != state.dataset: raise Invalid('Entity belongs to a different dataset.', value, None) return entity
def import_upload(upload_id, account_id, mapping): upload = Upload.all().filter_by(id=upload_id).first() account = Account.by_id(account_id) mapped = mapping['columns'].values() rows = [apply_mapping(r, mapping) for r in upload.tab.dict] # put aliases second. rows = sorted(rows, key=lambda r: 2 if r.get('canonical') else 1) for i, row in enumerate(rows): try: entity = None if row.get('id'): entity = Entity.by_id(row.get('id')) if entity is None: entity = Entity.by_name(upload.dataset, row.get('name')) if entity is None: entity = Entity.create(upload.dataset, row, account) # restore some defaults: if entity.canonical_id and 'canonical' not in mapped: row['canonical'] = entity.canonical_id if entity.invalid and 'invalid' not in mapped: row['invalid'] = entity.invalid if entity.attributes: attributes = entity.attributes.copy() else: attributes = {} attributes.update(row['attributes']) row['attributes'] = attributes entity.update(row, account) print(entity) if i % 100 == 0: db.session.commit() logging.debug('Commit') except Invalid as inv: logging.warning('Exception during import: {}'.format(str(inv))) db.session.commit() logging.info('Import Completed')
def import_upload(upload_id, account_id, mapping): upload = Upload.all().filter_by(id=upload_id).first() account = Account.by_id(account_id) mapped = mapping['columns'].values() rows = [apply_mapping(r, mapping) for r in upload.tab.dict] # put aliases second. rows = sorted(rows, key=lambda r: 2 if r.get('canonical') else 1) for i, row in enumerate(rows): try: entity = None if row.get('id'): entity = Entity.by_id(row.get('id')) if entity is None: entity = Entity.by_name(upload.dataset, row.get('name')) if entity is None: entity = Entity.create(upload.dataset, row, account) # restore some defaults: if entity.canonical_id and 'canonical' not in mapped: row['canonical'] = entity.canonical_id if entity.invalid and 'invalid' not in mapped: row['invalid'] = entity.invalid if entity.attributes: attributes = entity.attributes.copy() else: attributes = {} attributes.update(row['attributes']) row['attributes'] = attributes entity.update(row, account) print entity if i % 100 == 0: print 'COMMIT' db.session.commit() except Invalid, inv: # TODO: logging. print inv
def import_upload(dataset_name, sig, account_id, entity_col, alias_col): dataset = Dataset.find(dataset_name) account = Account.by_id(account_id) metadata, row_set = parse_upload(dataset, sig) headers = detect_headers(row_set) for row in row_set: data = dict([(c.column, c.value) for c in row]) entity = data.pop(entity_col) if entity_col else None alias = data.pop(alias_col) if alias_col else None if alias_col and alias is not None and len(alias) and alias != entity: d = {'name': alias, 'data': data} alias_obj = Alias.lookup(dataset, d, account, match_entity=False) data = {} if entity_col and entity is not None and len(entity): d = {'name': entity, 'data': data} entity_obj = Entity.by_name(dataset, entity) if entity_obj is None: entity_obj = Entity.create(dataset, d, account) entity_obj.data = data if alias_col and entity_col: alias_obj.match(dataset, {'choice': entity_obj.id}, account) db.session.commit() flush_cache()
def view_by_name(dataset): dataset = Dataset.find(dataset) entity = Entity.by_name(dataset, request.args.get('name')) if entity is None: raise NotFound("No such entity: %s" % request.args.get('name')) return view(dataset.name, entity.id)
def by_name(dataset): dataset = Dataset.find(dataset) name = request.args.get('name') entity = object_or_404(Entity.by_name(dataset, name)) return jsonify(entity)
def view_by_name(dataset): dataset = Dataset.find(dataset) entity = Entity.by_name(dataset, request.args.get('name')) if entity is None: raise NotFound("No such entity: %s" % request.args.get('name')) return view(dataset.name, entity.id)
def by_name(dataset): dataset = Dataset.find(dataset) name = request.args.get('name') entity = object_or_404(Entity.by_name(dataset, name)) return jsonify(entity)