def test_redacted_changeset(): """Redacted changesets have no metadata so those cases need to be threated to avoid a ZeroDivisionError in the Analyse.count() method. """ ch = Analyse(34495147) ch.full_analysis() assert ch.is_suspect is False
def test_analyse_user_details(): ch = Analyse(31450443) ch.full_analysis() assert ch.user_details assert ch.user_details['contributor_uid'] == 2578646 assert ch.user_details['contributor_name'] == 'Tobsen Laufi' assert ch.user_details['contributor_blocks'] == 0 assert ch.user_details['contributor_since'] == datetime(2015, 01, 15) assert ch.user_details['contributor_traces'] == 0 assert ch.user_details['nodes_c'] == 0 assert ch.user_details['nodes_m'] == 0 assert ch.user_details['nodes_d'] == 975 assert ch.user_details['ways_c'] == 0 assert ch.user_details['ways_m'] == 0 assert ch.user_details['ways_d'] == 43 assert ch.user_details['relations_c'] == 0 assert ch.user_details['relations_m'] == 0 assert ch.user_details['relations_d'] == 1 assert ch.user_details['changesets_no'] == 1 assert ch.user_details['changesets_changes'] == 1019 assert ch.user_details['changesets_f_tstamp'] == datetime(2015, 05, 25, 16, 30, 43) assert ch.user_details['changesets_l_tstamp'] == datetime(2015, 05, 25, 16, 30, 43) assert ch.user_details['changesets_mapping_days'] == '2015=1'
def test_analyse_count(): ch = Analyse(32663070) ch.full_analysis() assert ch.create == 8 assert ch.modify == 3 assert ch.delete == 2 assert ch.is_suspect is False assert len(ch.suspicion_reasons) == 0
def test_new_user_custom_create_value(): """Created: 1900. Modified: 16. Deleted: 320 / JOSM""" ch = Analyse(10013029, create_threshold=2000) ch.full_analysis() assert ch.is_suspect is True assert 'possible import' not in ch.suspicion_reasons assert 'New mapper' in ch.suspicion_reasons assert len(ch.suspicion_reasons) == 1
def cli(id): """Analyse an OpenStreetMap changeset.""" ch = Analyse(id) ch.full_analysis() click.echo( 'Created: %s. Modified: %s. Deleted: %s' % (ch.create, ch.modify, ch.delete) ) if ch.is_suspect: click.echo('The changeset %s is suspect!' % id) else: click.echo('The changeset %s is not suspect!' % id)
def test_custom_top_threshold(): """C/M/D = 1072 124 282 / made with iD""" ch = Analyse(45862717) ch.full_analysis() assert ch.is_suspect assert 'possible import' in ch.suspicion_reasons ch = Analyse(45862717, top_threshold=1100) ch.full_analysis() assert ch.is_suspect is False assert len(ch.suspicion_reasons) == 0
def test_custom_percentage(): """C/M/D = 481 620 80 / JOSM""" ch = Analyse(45082154) ch.full_analysis() assert ch.is_suspect is False assert len(ch.suspicion_reasons) == 0 ch = Analyse(45082154, percentage=0.5) ch.full_analysis() assert ch.is_suspect assert 'mass modification' in ch.suspicion_reasons
def cli(id): """Analyse an OpenStreetMap changeset.""" ch = Analyse(id) ch.full_analysis() click.echo( 'Created: %s. Modified: %s. Deleted: %s' % (ch.create, ch.modify, ch.delete) ) if ch.is_suspect: click.echo('The changeset {} is suspect! Reasons: {}'.format( id, ', '.join(ch.suspicion_reasons) )) else: click.echo('The changeset %s is not suspect!' % id)
def test_changeset_with_warning_tag_almost_junction(): ch_dict = { 'created_by': 'iD', 'created_at': '2019-04-25T18:08:46Z', 'host': 'https://www.openstreetmap.org/edit', 'comment': 'add pois', 'comments_count': '3', 'id': '1', 'user': '******', 'uid': '123123', 'warnings:almost_junction:highway-highway': '1', 'warnings:missing_role': '1', 'warnings:missing_tag:any': '1', 'warnings:private_data': '1', 'warnings:mismatched_geometry': '1', 'warnings:unsquare_way': '1', 'bbox': Polygon([(-71.0646843, 44.2371354), (-71.0048652, 44.2371354), (-71.0048652, 44.2430624), (-71.0646843, 44.2430624), (-71.0646843, 44.2371354)]) } changeset = Analyse(ch_dict) changeset.full_analysis() assert 'Almost junction' in changeset.suspicion_reasons assert 'Missing role' in changeset.suspicion_reasons assert 'Missing tag' in changeset.suspicion_reasons assert 'Private information' in changeset.suspicion_reasons assert 'Mismatched geometry' in changeset.suspicion_reasons assert 'Unsquare corners' in changeset.suspicion_reasons assert changeset.is_suspect
def test_changeset_with_review_requested(): ch_dict = { 'created_by': 'Potlatch 2', 'created_at': '2015-04-25T18:08:46Z', 'comment': 'add pois', 'id': '1', 'user': '******', 'uid': '123123', 'review_requested': 'yes', 'bbox': Polygon([ (-71.0646843, 44.2371354), (-71.0048652, 44.2371354), (-71.0048652, 44.2430624), (-71.0646843, 44.2430624), (-71.0646843, 44.2371354) ]) } changeset = Analyse(ch_dict) changeset.full_analysis() assert 'Review requested' in changeset.suspicion_reasons assert changeset.is_suspect
def test_get_dict(): """Test if get_dict function return only the fields that osmcha-django needs to save in the database. """ # An iD changeset ch = Analyse(46286980) ch.full_analysis() assert 'id' in ch.get_dict().keys() assert 'user' in ch.get_dict().keys() assert 'uid' in ch.get_dict().keys() assert 'editor' in ch.get_dict().keys() assert 'bbox' in ch.get_dict().keys() assert 'date' in ch.get_dict().keys() assert 'comment' in ch.get_dict().keys() assert 'source' in ch.get_dict().keys() assert 'imagery_used' in ch.get_dict().keys() assert 'is_suspect' in ch.get_dict().keys() assert 'powerfull_editor' in ch.get_dict().keys() assert 'suspicion_reasons' in ch.get_dict().keys() assert 'create' in ch.get_dict().keys() assert 'modify' in ch.get_dict().keys() assert 'delete' in ch.get_dict().keys() assert len(ch.get_dict().keys()) == 15 # A JOSM changeset ch = Analyse(46315321) ch.full_analysis() assert 'id' in ch.get_dict().keys() assert 'user' in ch.get_dict().keys() assert 'uid' in ch.get_dict().keys() assert 'editor' in ch.get_dict().keys() assert 'bbox' in ch.get_dict().keys() assert 'date' in ch.get_dict().keys() assert 'comment' in ch.get_dict().keys() assert 'source' in ch.get_dict().keys() assert 'imagery_used' in ch.get_dict().keys() assert 'is_suspect' in ch.get_dict().keys() assert 'powerfull_editor' in ch.get_dict().keys() assert 'suspicion_reasons' in ch.get_dict().keys() assert 'create' in ch.get_dict().keys() assert 'modify' in ch.get_dict().keys() assert 'delete' in ch.get_dict().keys() assert len(ch.get_dict().keys()) == 15
def test_changeset_with_warning_tag_invalid_format(): ch_dict = { 'created_by': 'iD', 'created_at': '2019-04-25T18:08:46Z', 'host': 'https://www.openstreetmap.org/edit', 'comment': 'add pois', 'id': '1', 'user': '******', 'uid': '123123', 'warnings:invalid_format': '0', 'bbox': Polygon([ (-71.0646843, 44.2371354), (-71.0048652, 44.2371354), (-71.0048652, 44.2430624), (-71.0646843, 44.2430624), (-71.0646843, 44.2371354) ]) } changeset = Analyse(ch_dict) changeset.full_analysis() assert changeset.suspicion_reasons == [] assert not changeset.is_suspect
def test_changeset_with_warning_tag_disconnected_way(): ch_dict = { 'created_by': 'iD', 'created_at': '2019-04-25T18:08:46Z', 'host': 'https://www.openstreetmap.org/edit', 'comment': 'add pois', 'comments_count': '2', 'id': '1', 'user': '******', 'uid': '123123', 'warnings:disconnected_way:highway': '4', 'warnings:suspicious_name:generic_name': '4', 'warnings:impossible_oneway:highway': '4', 'warnings:incompatible_source': '4', 'warnings:outdated_tags:incomplete_tags': '9', 'bbox': Polygon([(-71.0646843, 44.2371354), (-71.0048652, 44.2371354), (-71.0048652, 44.2430624), (-71.0646843, 44.2430624), (-71.0646843, 44.2371354)]) } changeset = Analyse(ch_dict) changeset.full_analysis() assert 'Disconnected way' in changeset.suspicion_reasons assert 'Generic name' in changeset.suspicion_reasons assert 'Impossible oneway' in changeset.suspicion_reasons assert 'suspect_word' in changeset.suspicion_reasons assert 'Outdated tags' in changeset.suspicion_reasons assert changeset.is_suspect
def test_save_user_details(self): analyze = Analyse(31450443) # A random changeset analyze.full_analysis() # Removing these values from the object analyze_dict = analyze.__dict__.copy() for key in analyze.__dict__: if analyze.__dict__.get(key) == '': analyze_dict.pop(key) analyze_dict.pop('suspicion_reasons') analyze_dict.pop('user_details') changeset = Changeset.objects.create(**analyze_dict) changeset.save() self.assertIsNone(changeset.user_detail) changeset.save_user_details(analyze) self.assertIsNotNone(changeset.user_detail) self.assertEqual(changeset.user_detail.contributor_name, 'Tobsen Laufi') self.assertEqual(changeset.user_detail.contributor_blocks, 0) self.assertEqual(changeset.user_detail.contributor_since, datetime(2015, 01, 15)) self.assertEqual(changeset.user_detail.contributor_traces, 0) self.assertEqual(changeset.user_detail.nodes_c, 0) self.assertEqual(changeset.user_detail.nodes_m, 0) self.assertEqual(changeset.user_detail.nodes_d, 975) self.assertEqual(changeset.user_detail.ways_c, 0) self.assertEqual(changeset.user_detail.ways_m, 0) self.assertEqual(changeset.user_detail.ways_d, 43) self.assertEqual(changeset.user_detail.relations_c, 0) self.assertEqual(changeset.user_detail.relations_m, 0) self.assertEqual(changeset.user_detail.relations_d, 1) self.assertEqual(changeset.user_detail.changesets_no, 1) self.assertEqual(changeset.user_detail.changesets_changes, 1019) self.assertEqual(changeset.user_detail.changesets_f_tstamp, datetime(2015, 05, 25, 16, 30, 43)) self.assertEqual(changeset.user_detail.changesets_l_tstamp, datetime(2015, 05, 25, 16, 30, 43)) self.assertEqual(changeset.user_detail.changesets_mapping_days, '2015=1')
def create_changeset(changeset_id): """Analyse and create the changeset in the database.""" ch = Analyse(changeset_id) ch.full_analysis() # remove suspicion_reasons and empty values from dict ch_dict = ch.__dict__.copy() for key in ch.__dict__: if ch.__dict__.get(key) == "": ch_dict.pop(key) ch_dict.pop("suspicion_reasons") # save changeset changeset = Changeset(**ch_dict) changeset.save() if ch.suspicion_reasons: for reason in ch.suspicion_reasons: reason, created = SuspicionReasons.objects.get_or_create(name=reason) reason.changesets.add(changeset) print("{c[id]} created".format(c=ch_dict))
def create_changeset(changeset_id): """Analyse and create the changeset in the database.""" ch = Analyse(changeset_id) ch.full_analysis() # remove suspicion_reasons and empty values from dict ch_dict = ch.__dict__.copy() for key in ch.__dict__: if ch.__dict__.get(key) == '': ch_dict.pop(key) ch_dict['score'] = ch_dict['changeset_score'] ch_dict.pop('suspicion_reasons') # ch_dict.pop('user_details') ch_dict.pop('user_score') ch_dict.pop('changeset_score') ch_dict.pop('user_score_details') ch_dict.pop('changeset_score_details') # save changeset changeset, created = Changeset.objects.update_or_create(id=ch_dict['id'], defaults=ch_dict) if ch.suspicion_reasons: for reason in ch.suspicion_reasons: reason, created = SuspicionReasons.objects.get_or_create(name=reason) reason.changesets.add(changeset) SuspicionScore.objects.filter(changeset=changeset).delete() for detail in ch.changeset_score_details: s = SuspicionScore(changeset=changeset) s.score = detail['score'] s.reason = detail['reason'] s.save() # if ch.user_details: # changeset.save_user_details(ch) print('{c[id]} created'.format(c=ch_dict))
def create_changeset(changeset_id): """Analyse and create the changeset in the database.""" ch = Analyse(changeset_id) ch.full_analysis() # remove suspicion_reasons and empty values from dict ch_dict = ch.__dict__.copy() for key in ch.__dict__: if ch.__dict__.get(key) == '': ch_dict.pop(key) ch_dict.pop('suspicion_reasons') # save changeset changeset = Changeset(**ch_dict) changeset.save() if ch.suspicion_reasons: for reason in ch.suspicion_reasons: reason, created = SuspicionReasons.objects.get_or_create( name=reason) reason.changesets.add(changeset) print('{c[id]} created'.format(c=ch_dict))
def test_custom_delete_value(): """C/M/D = 0 0 61 / iD""" ch = Analyse(45901540, delete_threshold=100) ch.full_analysis() assert ch.is_suspect is False assert len(ch.suspicion_reasons) == 0
def test_changeset_without_tags(): ch = Analyse(46755934) ch.full_analysis() assert ch.powerfull_editor assert ch.is_suspect assert 'Software editor was not declared' in ch.suspicion_reasons
def test_custom_modify_value(): """Created: 322. Modified: 1115. Deleted: 140 / Potlatch 2""" ch = Analyse(19863853, modify_threshold=1200) ch.full_analysis() assert ch.is_suspect is False assert len(ch.suspicion_reasons) == 0
def test_no_duplicated_reason(): """Changeset with word import in comment and source fields.""" ch = Analyse(45632780) ch.full_analysis() assert ch.is_suspect assert ch.suspicion_reasons == ['suspect_word']
def test_analyse_mass_modification(): """Created: 322. Modified: 1115. Deleted: 140 / Potlatch 2""" ch = Analyse(19863853) ch.full_analysis() assert ch.is_suspect assert 'mass modification' in ch.suspicion_reasons
def test_analyse_mass_deletion(): """Created: 0. Modified: 0. Deleted: 1019 / Potlatch 2""" ch = Analyse(31450443) ch.full_analysis() assert ch.is_suspect assert 'mass deletion' in ch.suspicion_reasons
def test_changeset_by_new_mapper(): changeset = Analyse(36700893) changeset.full_analysis() print(changeset.suspicion_reasons) assert 'New mapper' in changeset.suspicion_reasons assert changeset.is_suspect
def test_changeset_by_user_with_more_than_one_block(): changeset = Analyse(34879408) changeset.full_analysis() assert 'User has multiple blocks' in changeset.suspicion_reasons assert changeset.is_suspect
def test_prediction_from_gabbar(): changeset = Analyse(46107636) changeset.full_analysis() assert 'Flagged by gabbar' in changeset.suspicion_reasons assert changeset.is_suspect
def test_changeset_by_new_mapper(): changeset = Analyse(46756461) changeset.full_analysis() assert 'New mapper' in changeset.suspicion_reasons assert changeset.is_suspect
def test_changeset_by_old_mapper_with_special_character_username(): changeset = Analyse(46141825) changeset.full_analysis() assert 'New mapper' not in changeset.suspicion_reasons assert not changeset.is_suspect
def test_changeset_by_old_mapper_with_unicode_username(): changeset = Analyse(46790192) changeset.full_analysis() assert 'New mapper' not in changeset.suspicion_reasons assert not changeset.is_suspect
def test_changeset_with_6_mapping_days(): changeset = Analyse(13523366) changeset.full_analysis() assert 'New mapper' not in changeset.suspicion_reasons assert not changeset.is_suspect
def test_analyse_import(): """Created: 1900. Modified: 16. Deleted: 320 / JOSM""" ch = Analyse(10013029) ch.full_analysis() assert ch.is_suspect assert 'possible import' in ch.suspicion_reasons
def test_analyse_mass_modification(): ch = Analyse(19863853) ch.full_analysis() assert ch.is_suspect assert 'mass modification' in ch.suspicion_reasons
def test_analyse_import(): ch = Analyse(10013029) ch.full_analysis() assert ch.is_suspect assert 'possible import' in ch.suspicion_reasons
def test_analyse_mass_deletion(): ch = Analyse(31450443) ch.full_analysis() assert ch.is_suspect assert 'mass deletion' in ch.suspicion_reasons
def test_get_dict(): """Test if get_dict function return only the fields that osmcha-django needs to save in the database. """ # An iD changeset ch = Analyse(46286980) ch.full_analysis() assert 'id' in ch.get_dict().keys() assert 'user' in ch.get_dict().keys() assert 'uid' in ch.get_dict().keys() assert 'editor' in ch.get_dict().keys() assert 'bbox' in ch.get_dict().keys() assert 'date' in ch.get_dict().keys() assert 'comment' in ch.get_dict().keys() assert 'comments_count' in ch.get_dict().keys() assert 'source' in ch.get_dict().keys() assert 'imagery_used' in ch.get_dict().keys() assert 'is_suspect' in ch.get_dict().keys() assert 'powerfull_editor' in ch.get_dict().keys() assert 'suspicion_reasons' in ch.get_dict().keys() assert 'create' in ch.get_dict().keys() assert 'modify' in ch.get_dict().keys() assert 'delete' in ch.get_dict().keys() assert 'metadata' in ch.get_dict().keys() assert ch.get_dict( )['metadata']['host'] == 'https://www.openstreetmap.org/id' assert len(ch.get_dict().keys()) == 17 # An iD changeset with warnings: ch = Analyse(72783703) ch.full_analysis() assert 'id' in ch.get_dict().keys() assert 'user' in ch.get_dict().keys() assert 'uid' in ch.get_dict().keys() assert 'editor' in ch.get_dict().keys() assert 'bbox' in ch.get_dict().keys() assert 'date' in ch.get_dict().keys() assert 'comment' in ch.get_dict().keys() assert 'comments_count' in ch.get_dict().keys() assert 'source' in ch.get_dict().keys() assert 'imagery_used' in ch.get_dict().keys() assert 'is_suspect' in ch.get_dict().keys() assert 'powerfull_editor' in ch.get_dict().keys() assert 'suspicion_reasons' in ch.get_dict().keys() assert 'create' in ch.get_dict().keys() assert 'modify' in ch.get_dict().keys() assert 'delete' in ch.get_dict().keys() assert 'metadata' in ch.get_dict().keys() assert ch.get_dict( )['metadata']['host'] == 'https://www.openstreetmap.org/edit' assert ch.get_dict()['metadata']['locale'] == 'en-US' assert ch.get_dict()['metadata']['warnings:crossing_ways'] == 1 assert ch.get_dict()['metadata']['changesets_count'] == 5970 assert ch.get_dict()['comments_count'] == 2 assert len(ch.get_dict().keys()) == 17 # A JOSM changeset ch = Analyse(46315321) ch.full_analysis() assert 'id' in ch.get_dict().keys() assert 'user' in ch.get_dict().keys() assert 'uid' in ch.get_dict().keys() assert 'editor' in ch.get_dict().keys() assert 'bbox' in ch.get_dict().keys() assert 'date' in ch.get_dict().keys() assert 'comment' in ch.get_dict().keys() assert 'comments_count' in ch.get_dict().keys() assert 'source' in ch.get_dict().keys() assert 'imagery_used' in ch.get_dict().keys() assert 'is_suspect' in ch.get_dict().keys() assert 'powerfull_editor' in ch.get_dict().keys() assert 'suspicion_reasons' in ch.get_dict().keys() assert 'create' in ch.get_dict().keys() assert 'modify' in ch.get_dict().keys() assert 'delete' in ch.get_dict().keys() assert 'metadata' in ch.get_dict().keys() assert ch.get_dict()['metadata'] == {} assert len(ch.get_dict().keys()) == 17