class LeadTypeAttachment(LeadType): validate_schema = {} validate_save = validator.parser(copy.copy(validate_schema), flip_hash='+', additional_properties=True) validate_update = validator.parser(copy.copy(validate_schema), flip_hash='?', additional_properties=True)
class LeadTypeManual(LeadType): validate_schema = {"#description": validator.String()} validate_save = validator.parser(copy.copy(validate_schema), flip_hash='+', additional_properties=True) validate_update = validator.parser(copy.copy(validate_schema), flip_hash='?', additional_properties=True)
class LeadTypeURL(LeadType): validate_schema = { "#url": validator.URL(), "#website": validator.String(), } validate_save = validator.parser(copy.copy(validate_schema), flip_hash='+', additional_properties=True) validate_update = validator.parser(copy.copy(validate_schema), flip_hash='?', additional_properties=True)
class Domain(ObjectTable): __tablename__ = 'domain' __export__ = { const.ACL_READ: ['id', 'name', 'name_display', 'description', 'restrict_countries', 'event_types'] } name = db.Column(db.String(255)) name_display = db.Column(db.String(255)) description = db.Column(db.Text) status = db.Column(db.SmallInteger, default=const.STATUS_ACTIVE) restrict_countries = db.Column(sa_utils.JSONType()) event_types = db.Column(sa_utils.JSONType()) validate_schema = { "#name": "string", "#name_display": "string", "?description": "string", "?restrict_countries": [validator.Enum([country.alpha2 for country in pycountry.countries])], "?event_types": [validator.Tag(tag_class='event_type')] } validate_save = validator.parser(validate_schema, flip_hash='+') validate_update = validator.parser(validate_schema, flip_hash='?') @classmethod def af_save(cls_, current_user, data, obj_id=None): if obj_id is not None: data = cls_.validate_update.validate(data) domain = cls_.get(obj_id, required=True) domain.update(**data) else: data = cls_.validate_save.validate(data) domain = cls_(**data) domain.save() return domain.jsonify(acl=const.ACL_READ) @classmethod def af_find(cls_, current_user, data): q = DomainQuery(current_user) q.assign_request(data) return q.execute()
class Binbag(ObjectTable): __tablename__ = 'binbag' __export__ = {const.ACL_READ: ['id', 'name', 'mime', 'reference']} name = db.Column(db.String(255)) reference = db.Column(db.String(64)) mime = db.Column(db.String(255)) validate_schema = {"#name": "string", "#mime": "string"} validate_save = validator.parser(validate_schema, flip_hash='+') validate_update = validator.parser(validate_schema, flip_hash='?') @classmethod def af_save(cls_, current_user, data, file=None, obj_id=None): if obj_id is not None: data = cls_.validate_update.validate(data) binbag = cls_.get(obj_id, required=True) binbag.update(**data) else: data = cls_.validate_save.validate(data) data['reference'] = str(uuid.uuid4()) binbag = cls_(**data) binbag.save() if file is not None: # we do it through a file because mysql is shit file.save("%s/%s" % (current_app.config['BINBAG_DIR'], binbag.reference)) return binbag.jsonify(acl=const.ACL_READ) def get_content(self): with open("%s/%s" % (current_app.config['BINBAG_DIR'], self.reference), 'rb') as file_: return file_.read()
def validate(self, data, obj_id=None): vobj = { "?tag_class": validator.Enum(tag_class_dict.keys()), "?parent_id": validator.Integer() } for name, props in self.__structure__.items(): if props['edit'] == 'text': vobj[('+' if obj_id is None else '?') + name] = "string" data = validator.parser(vobj).validate(data) # restructure rs_data = dict() for k, v in data.items(): if k in ['tag_class', 'name', 'restricted_domains', 'id', 'parent_id']: rs_data[k] = v else: if 'data' not in rs_data: rs_data['data'] = dict() rs_data['data'][k] = v return rs_data
def validate(self, data, obj_id=None): vobj = { "?tag_class": validator.Enum(tag_class_dict.keys()), "?parent_id": validator.Integer() } for name, props in self.__structure__.items(): if props['edit'] == 'text': vobj[('+' if obj_id is None else '?') + name] = "string" data = validator.parser(vobj).validate(data) # restructure rs_data = dict() for k, v in data.items(): if k in [ 'tag_class', 'name', 'restricted_domains', 'id', 'parent_id' ]: rs_data[k] = v else: if 'data' not in rs_data: rs_data['data'] = dict() rs_data['data'][k] = v return rs_data
def post(self): validate_row = validator.parser({ "+ADM_CODE": "string", "+ADM_PARENT_CODE": "string", "+COUNTRY_CODE": "string", "+ADM_NAME": "string" }) try: ref_map = {} id_map = {} records = [] file = request.files['file'] reader = csv.DictReader(file.read().decode("utf-8").split('\n'), delimiter=',') for row in reader: row = validate_row.validate(row) id_map[row['ADM_CODE']] = True if len(row['ADM_PARENT_CODE']) > 0: ref_map[row['ADM_PARENT_CODE']] = True records.append(row) for parent_code in ref_map.keys(): if parent_code not in id_map: raise ApiError('Missing parent code for: %s' % parent_code) for record in records: models.Location.bsave({ 'name': record['ADM_NAME'], 'code': record['ADM_CODE'], 'parent_code': record['ADM_PARENT_CODE'], 'country_code': record['COUNTRY_CODE'] }) return self.respond({'success': True}) except csv.Error as e: raise ApiError(str(e))
class Entry(ObjectTable): __tablename__ = 'entry' __export__ = { const.ACL_READ: ['id', 'user_id', 'domain_id', 'lead_id', 'status', 'name', 'country_code', 'created_at', 'excerpt', 'severity', 'reliability', 'status_ord', 'timeline', 'information_at'] } status = db.Column(db.SmallInteger, default=const.STATUS_ACTIVE) user_id = db.Column(db.BigInteger, db.ForeignKey('user.id'), index=True) lead_id = db.Column(db.BigInteger, db.ForeignKey('lead.id'), index=True) domain_id = db.Column(db.BigInteger, db.ForeignKey('domain.id'), index=True) name = db.Column(db.Text) country_code = db.Column(db.String(3)) excerpt = db.Column(db.Text) tags = relationship("EntryTag", backref="entry") locations = relationship("EntryLocation", backref="entry") severity = db.Column(db.Integer) reliability = db.Column(db.Integer) status_ord = db.Column(db.Integer) timeline = db.Column(db.Integer) information_at = db.Column(db.DateTime) validate_schema = { "#lead_id": "integer", "#severity": "integer", "#reliability": "integer", "?status": validator.Enum([const.STATUS_ACTIVE, const.STATUS_INACTIVE, const.STATUS_DELETED]), "?status_ord": validator.Tag(tag_class='status'), "?timeline": validator.Tag(tag_class='timeline'), "?country_code": validator.Enum([None] + [country.alpha2 for country in pycountry.countries]), "#tags": { '?sector': [validator.Tag(tag_class='sector')], '?vulnerable': [validator.TagBlock(tag_class='vulnerable')], '?affected': [validator.TagBlock(tag_class='affected')], '?underlying': [validator.Tag(tag_class='underlying')], '?crisis_driver': [validator.Tag(tag_class='crisis_driver')] }, "#locations": [{ "+source": validator.Enum([const.LOCATION_SOURCE_GEONAME, const.LOCATION_SOURCE_GOOGLE_MAP_SHAPE, const.LOCATION_SOURCE_SELF]), "+location_id": validator.AnyOf("string", "integer"), "+asciiname": "string", "?data": validator.AnyDict() }], "#excerpt": "string", "?information_at": validator.Timestamp() } validate_save = validator.parser(copy.copy(validate_schema), flip_hash='+') validate_update = validator.parser(copy.copy(validate_schema), flip_hash='?') @classmethod def af_save(cls_, current_user, data, obj_id=None): from sidr import models # cuz js is lame if 'lead_id' in data: data['lead_id'] = int(data['lead_id']) if obj_id is not None: action_type = 'EDIT_ENTRY' data = cls_.validate_update.validate(data) entry_data = {key: value for (key, value) in data.items() if key not in ['tags', 'locations']} entry = cls_.get(obj_id, required=True) entry.user_id = current_user.id entry.update(**entry_data) else: action_type = 'ADD_ENTRY' data = cls_.validate_save.validate(data) lead = models.Lead.get(data['lead_id'], required=True) data['user_id'] = current_user.id data['lead_id'] = lead.id data['domain_id'] = lead.domain_id entry_data = {key: value for (key, value) in data.items() if key not in ['tags', 'locations']} entry = cls_(**entry_data) entry.save() if 'locations' in data: models.EntryLocation.update_locations(entry, data['locations']) if 'tags' in data: models.EntryTag.update_tags(entry, data['tags']) models.Action.mark(current_user, action_type, entry.jsonify(acl=const.ACL_READ), domain_id=entry.domain_id) return entry.jsonify_complete(acl=const.ACL_READ) @classmethod def af_find(cls_, current_user, data, rtype='json'): q = EntryQuery(current_user, rtype=rtype) q.assign_request(data) return q.execute() @classmethod def get_overview(cls_, current_user, domain_id): sql = 'SELECT SUM(IF(TO_DAYS(NOW()) - TO_DAYS( created_at ) <= 1, 1, 0)) AS entries_today, SUM(IF(status=1, 1,0)) AS entries_active, SUM(IF(status=2, 1,0)) AS entries_inactive ' for i in range(1, 7): sql += ' ,SUM(IF(status=1 AND severity={0}, 1,0)) AS severity_{0}_active, SUM(IF(status=2 AND severity={0}, 1,0)) AS severity_{0}_inactive'.format(i) sql += ' FROM entry WHERE domain_id={0}'.format(int(domain_id)) row = db.session.execute(sql).first() if row is None or row['entries_today'] is None: return {} rsp = { 'today': int(row['entries_today']), 'active': int(row['entries_active']), 'inactive': int(row['entries_inactive']), 'severity': {} } for i in range(1, 7): rsp['severity'][i] = { 'active': int(row['severity_%s_active' % i]), 'inactive': int(row['severity_%s_inactive' % i]) } return rsp def jsonify_complete(self, acl): rsp = self.jsonify(acl=acl) if self.tags is not None: utags = {} for tag in self.tags: if tag.tag_class not in utags: utags[tag.tag_class] = [] utag = { 'id': tag.tag_id } if isinstance(tag.data, dict): utag.update(tag.data) utags[tag.tag_class].append(utag) rsp['tags'] = utags if self.locations is not None: rsp['locations'] = [] for location in self.locations: rsp['locations'].append(location.jsonify(acl=const.ACL_READ)) if self.user is not None: rsp['user'] = self.user.jsonify(acl=const.ACL_READ) if self.lead is not None: rsp['lead'] = self.lead.jsonify(acl=const.ACL_READ) return rsp
from flask import current_app from itsdangerous import JSONWebSignatureSerializer from sidr.api import Resource, ApiError, current_user from sidr import models, const, validator authSchema = { "+email": 'string', "+password": '******', } validator = validator.parser(authSchema) class AuthResource(Resource): def post(self): serializer = JSONWebSignatureSerializer(current_app.config['API_KEY']) args = validator.validate(self.get_request()) user = models.User.find_first({'email': args['email']}) if user is None or user.password != args['password']: raise ApiError("Wrong email or password provided for {}".format( args['email'])) if user.status is not const.STATUS_ACTIVE: raise ApiError("User is not active") token = serializer.dumps({ "id": user.id, "signature": user.signature }).decode('utf-8') return self.respond({"token": token, "user": user.jsonify(user)})
class User(ObjectTable, UserMixin): __tablename__ = 'user' __export__ = { const.ACL_OWNER: ['id', 'name', 'email', 'orgnization', 'status', 'role', 'state'], const.ACL_READ: ['id', 'name', 'orgnization', 'role'] } default_cmp_user_id = 'id' email = db.Column(db.String(255), unique=True, index=True) name = db.Column(db.String(255)) orgnization = db.Column(db.String(255)) password = db.Column(sa_utils.PasswordType(schemes=['pbkdf2_sha512'])) signature = db.Column(db.String(64), default=random_string) status = db.Column(db.SmallInteger, default=const.STATUS_ACTIVE) role = db.Column(db.SmallInteger, default=const.ROLE_USER) state = db.Column(sa_utils.JSONType()) leads = db.relationship("Lead", backref="user") entries = db.relationship("Entry", backref="user") actions = db.relationship("Action", backref="user") __table_args__ = (db.UniqueConstraint('email'), ) validate_save = validator.parser({ "+email": validator.Email, "+name": "string", "+password": "******", "?state": { 'focus_domain_id': "integer" } }) validate_update = validator.parser({ "?email": validator.Email, "?name": "string", "?password": "******", "?state": { 'focus_domain_id': "integer" } }) @classmethod def af_save(cls_, current_user, data, obj_id=None): try: if obj_id is not None: user = cls_.get_restricted(current_user, obj_id, 'id') data = cls_.validate_update.validate(data) user.update(**data) return user.jsonify(current_user), user else: data = cls_.validate_save.validate(data) user = cls_(**data) user.save() return user.jsonify(acl=const.ACL_OWNER), user except IntegrityError: raise ApiValidationError(ValidationError('Email already exists')) @classmethod def af_map(cls_, current_user): return cls_.af_find(current_user, {'status': const.STATUS_ACTIVE}) @classmethod def af_find(cls_, current_user, data): q = UserQuery(current_user) q.assign_request(data) return q.execute() @classmethod def af_reset_send(cls_, email, returnurl): user = cls_.find_first({'email': email}) if not user: raise ApiValidationError(ValidationError('Unknown email')) serializer = JSONWebSignatureSerializer(current_app.config['API_KEY']) resettoken = serializer.dumps({ "id": user.id, "signature": user.signature }).decode('utf-8') content = 'Click here to reset your password: %s' % (returnurl + resettoken) mailer.send(user.email, user.name, 'Password reset link', content) return {'success': True} @classmethod def af_reset_recieve(cls_, token, password): serializer = JSONWebSignatureSerializer(current_app.config['API_KEY']) try: payload = serializer.loads(token) user = cls_.get(payload['id']) if user is None or payload[ 'signature'] != user.signature or user.status != const.STATUS_ACTIVE: raise ApiValidationError( ValidationError('User reset not allowed')) user.update(password=password, signature=random_string()) return user.jsonify(acl=const.ACL_OWNER) except (itsdangerous.SignatureExpired, itsdangerous.BadSignature): raise ApiValidationError(ValidationError('Token is broken')) def connect_domain(self, domain_id): from sidr.models import DomainUser return DomainUser.add_connection(self.id, domain_id)
class Lead(ObjectTable): __tablename__ = 'lead' __export__ = { const.ACL_READ: [ 'id', 'user_id', 'assignee_id', 'domain_id', 'lead_type', 'status', 'name', 'data', 'confidentiality', 'created_at', 'published_at', 'binbags', 'source_id', 'content_format_id', 'website', 'url' ] } user_id = db.Column(db.BigInteger, db.ForeignKey('user.id'), index=True) assignee_id = db.Column(db.BigInteger) domain_id = db.Column(db.BigInteger, db.ForeignKey('domain.id'), index=True) source_id = db.Column(db.BigInteger, db.ForeignKey('tag.id'), index=True) content_format_id = db.Column(db.BigInteger, db.ForeignKey('tag.id'), index=True) lead_type = db.Column(db.String(255)) binbags = db.Column(sa_utils.JSONType()) status = db.Column(db.SmallInteger, default=const.STATUS_PENDING) name = db.Column(db.String(255)) description = db.Column(db.Text) website = db.Column(db.String(255)) confidentiality = db.Column(db.SmallInteger, default=const.CONFIDENTIALITY_UNPROTECTED) url = db.Column(db.Text) published_at = db.Column(db.DateTime) entries = db.relationship("Entry", backref="lead") validate_schema = { "?name": "string", "?status": validator.Enum([ const.STATUS_ACTIVE, const.STATUS_INACTIVE, const.STATUS_PENDING, const.STATUS_DELETED ]), "#domain_id": "integer", "?confidentiality": "integer", "#lead_type": validator.Enum(lead_type_dict.keys()), "#source_id": validator.Tag(tag_class='source'), "?content_format_id": validator.Tag(tag_class='content_format'), "?description": "string", "?binbags": [{ "mime": "string", "name": "string", "reference": "string", "id": "integer" }], "?published_at": validator.Timestamp() } validate_save = validator.parser(copy.copy(validate_schema), flip_hash='+', additional_properties=True) validate_update = validator.parser(copy.copy(validate_schema), flip_hash='?', additional_properties=True) @classmethod def af_save(cls_, current_user, data, obj_id=None): from sidr import models if obj_id is not None: action_type = 'EDIT_LEAD' data = cls_.validate_update.validate(data) lead = cls_.get(obj_id, required=True) if lead.lead_type is not None: data = lead_type_dict[lead.lead_type].validate_update.validate( data) lead.user_id = current_user.id lead.update(**data) else: action_type = 'ADD_LEAD' data = cls_.validate_save.validate(data) data = lead_type_dict[data['lead_type']].validate_save.validate( data) data['user_id'] = current_user.id lead = cls_(**data) lead.save() models.Action.mark(current_user, action_type, lead.jsonify(acl=const.ACL_READ), domain_id=lead.domain_id) return lead.jsonify(acl=const.ACL_READ) @classmethod def af_find(cls_, current_user, data, rtype='json'): q = LeadQuery(current_user, rtype=rtype) q.assign_request(data) return q.execute() @classmethod def get_overview(cls_, current_user, domain_id): sql = 'SELECT SUM(IF(TO_DAYS(NOW()) - TO_DAYS( created_at ) <= 1, 1, 0)) AS leads_today, SUM(IF(status=1, 1,0)) AS leads_active, SUM(IF(status=3, 1,0)) AS leads_pending from lead' sql += ' WHERE domain_id={0}'.format(int(domain_id)) row = db.session.execute(sql).first() if row is None or row['leads_today'] is None: return {} return { 'today': int(row['leads_today']), 'active': int(row['leads_active']), 'pending': int(row['leads_pending']), }
class Location(ObjectTable): __tablename__ = 'location' __export__ = { const.ACL_READ: ['id', 'code', 'level', 'parent_id', 'longtitude', 'latitude', 'name', 'country_code'] } name = db.Column(db.String(255)) code = db.Column(db.String(255), index=True) level = db.Column(db.String(255)) parent_code = db.Column(db.String(255), index=True) longtitude = db.Column(db.Integer) latitude = db.Column(db.Integer) country_code = db.Column(db.String(3)) tree = db.Column(sa_utils.JSONType()) status = db.Column(db.SmallInteger, default=const.STATUS_ACTIVE) validate_schema = { "#name": "string", "#code": "string", "?level": "string", "?longtitude": "integer", "?latitude": "integer", "?parent_code": "string", "#country_code": validator.Enum([country.alpha2 for country in pycountry.countries]) } validate_save = validator.parser(copy.copy(validate_schema), flip_hash='+', additional_properties=True) validate_update = validator.parser(copy.copy(validate_schema), flip_hash='?', additional_properties=True) def deduce_tree(self): if len(self.parent_code) < 1: self.update(level=1) return tree = [] level = 2 f_code = self.parent_code while(True): p = Location.find_first({'code': f_code}) tree.insert(0, {'code': p.code, 'name': p.name, 'id': p.id}) if len(p.parent_code) < 1: break else: f_code = p.parent_code level = level + 1 self.update(level=level, tree=tree) @classmethod def af_save(cls_, current_user, data, obj_id=None): if obj_id is not None: data = cls_.validate_update.validate(data) location = cls_.get(obj_id, required=True) location.update(**data) else: data = cls_.validate_save.validate(data) location = cls_(**data) location.save() return location.jsonify(acl=const.ACL_READ) @classmethod def bsave(cls_, data): location = cls_.find_first({'country_code': data['country_code'], 'code': data['code']}) if location is None: location = cls_(**data) location.save() else: location.update(**data) location.deduce_tree() @classmethod def af_find(cls_, current_user, data): q = LocationQuery(current_user) q.assign_request(data) return q.execute() @classmethod def af_autocomplete(cls_, country_code, value): rsp = [] rows = cls_.get_query().filter(db.or_(cls_.name.like("%" + value + "%"), cls_.code.like("%" + value + "%"))).filter(cls_.country_code == country_code).limit(15).all() for row in rows: rsp.append(row.jsonify(acl=const.ACL_READ)) return {'results': rsp}
from flask import current_app from itsdangerous import JSONWebSignatureSerializer from sidr.api import Resource, ApiError, current_user from sidr import models, const, validator authSchema = { "+email": 'string', "+password": '******', } validator = validator.parser(authSchema) class AuthResource(Resource): def post(self): serializer = JSONWebSignatureSerializer( current_app.config['API_KEY']) args = validator.validate(self.get_request()) user = models.User.find_first({'email': args['email']}) if user is None or user.password != args['password']: raise ApiError("Wrong email or password provided for {}".format(args['email'])) if user.status is not const.STATUS_ACTIVE: raise ApiError("User is not active") token = serializer.dumps({"id": user.id, "signature": user.signature}).decode('utf-8') return self.respond({"token": token, "user": user.jsonify(user)}) def init_app(app):
class Tag(ObjectTable): __tablename__ = 'tag' __export__ = { const.ACL_READ: ['id', 'tag_class', 'name', 'data', 'restricted_domains', 'tree'] } tag_class = db.Column(db.String(255)) status = db.Column(db.SmallInteger, default=const.STATUS_ACTIVE) name = db.Column(db.String(255)) data = db.Column(sa_utils.JSONType()) restrict_domains = db.Column(sa_utils.ScalarListType()) parent_id = db.Column(db.BigInteger, db.ForeignKey('tag.id')) tree = db.Column(sa_utils.JSONType()) validate_save = validator.parser( { "+tag_class": validator.Enum(tag_class_dict.keys()), "?parent_id": validator.Integer() }, additional_properties=True) @classmethod def af_tag_classes(cls_): rsp = [] for name, obj in tag_class_dict.items(): rsp.append({'name': name, 'metadata': obj.jsonify_metadata()}) return {'result': rsp, 'total': len(rsp)} @classmethod def af_delete(cls_, obj_id): tag = cls_.get(obj_id, required=True) tag.update(**{'status': const.STATUS_DELETED}) return tag.jsonify(acl=const.ACL_READ) @classmethod def af_save(cls_, current_user, data, obj_id=None): data = cls_.validate_save.validate(data) data = tag_class_dict[data['tag_class']].validate(data, obj_id) if 'parent_id' in data: parents = cls_.get_parenthood(data['parent_id']) tree = [] for parent in parents: tree.append({ 'id': parent.id, 'name': parent.name, 'title': parent.data['title'] }) data['tree'] = tree if obj_id is not None: tag = cls_.get(obj_id, required=True) tag.update(**data) return tag.jsonify(acl=const.ACL_READ) else: tag = cls_(**data) tag.save() return tag.jsonify(acl=const.ACL_READ) @classmethod def af_find(cls_, current_user, data): q = TagQuery(current_user) q.assign_request(data) return q.execute() @classmethod def get_parenthood(cls_, obj_id, parents=None): if parents is None: parents = [] if obj_id == 0 or obj_id is None: return parents obj = cls_.get(obj_id) parents.append(obj) return cls_.get_parenthood(obj.parent_id, parents=parents) @classmethod def get_id_map(cls_): umap = {} rows = cls_.find().all() for row in rows: umap[row.id] = row.name return umap def jsonify(self, acl=None): return tag_class_dict[self.tag_class].jsonify_tag(self)