def download_changesets(changeset_ids, print_status): """Downloads changesets and all their contents from API, returns (diffs, changeset_users) tuple.""" ch_users = {} diffs = defaultdict(dict) for changeset_id in changeset_ids: print_status(changeset_id) root = api_download('changeset/{0}/download'.format(changeset_id), sysexit_message='Failed to download changeset {0}'.format(changeset_id)) # Iterate over each object, download previous version (unless it's creation) and make a diff count = total = 0 for action in root: if action.tag != 'create': total += len(action) for action in root: for obj_xml in action: if action.tag != 'create': count += 1 if changeset_id not in ch_users: ch_users[changeset_id] = obj_xml.get('user').encode('utf-8') obj = obj_to_dict(obj_xml) if obj['version'] > 1: print_status(changeset_id, obj['type'], obj['id'], count, total) try: obj_prev = obj_to_dict(api_download('{0}/{1}/{2}'.format( obj['type'], obj['id'], obj['version'] - 1), throw=[403])[0]) except HTTPError: msg = '\nCannot revert redactions, see version {0} at https://openstreetmap.org/{1}/{2}/history' raise RevertError(msg.format(obj['version'] - 1, obj['type'], obj['id'])) else: obj_prev = None diffs[(obj['type'], obj['id'])][obj['version']] = make_diff(obj, obj_prev) print_status('flush') return diffs, ch_users
def revert_changes(diffs, print_status): """Actually reverts changes in diffs dict. Returns a changes list for uploading to API.""" # merge versions of same objects in diffs for k in diffs: diff = None for v in sorted(diffs[k].keys()): diff = merge_diffs(diff, diffs[k][v]) diffs[k] = diff changes = [] count = 0 for kobj, change in diffs.iteritems(): count += 1 if change is None: continue try: # Download the latest version of an object print_status(None, kobj[0], kobj[1], count, len(diffs)) obj = obj_to_dict( api_download('{0}s?{0}s={1}'.format(kobj[0], kobj[1]))[0]) # Apply the change obj_new = None if len(change) == 2 and change[1][0] == 'create': if not obj['deleted']: obj_new = { 'type': obj['type'], 'id': obj['id'], 'deleted': True } elif len(change) == 2 and change[1][0] == 'delete': # Restore only if the object is still absent if obj['deleted']: obj_new = change[1][1] else: # Controversial, but I've decided to replace the object with the old one in this case obj_new = change[1][1] else: obj_new = apply_diff(change, deepcopy(obj)) if obj_new is not None: obj_new['version'] = obj['version'] if obj_new != obj: changes.append(obj_new) except Exception as e: raise RevertError( '\nFailed to download the latest version of {0} {1}: {2}'. format(kobj[0], kobj[1], e)) print_status('flush') return changes
def revert_changes(diffs, print_status): """Actually reverts changes in diffs dict. Returns a changes list for uploading to API.""" # merge versions of same objects in diffs for k in diffs: diff = None for v in sorted(diffs[k].keys()): diff = merge_diffs(diff, diffs[k][v]) diffs[k] = diff changes = [] count = 0 for kobj, change in diffs.iteritems(): count += 1 if change is None: continue try: # Download the latest version of an object print_status(None, kobj[0], kobj[1], count, len(diffs)) obj = obj_to_dict(api_download('{0}s?{0}s={1}'.format(kobj[0], kobj[1]))[0]) # Apply the change obj_new = None if len(change) == 2 and change[1][0] == 'create': if not obj['deleted']: obj_new = {'type': obj['type'], 'id': obj['id'], 'deleted': True} elif len(change) == 2 and change[1][0] == 'delete': # Restore only if the object is still absent if obj['deleted']: obj_new = change[1][1] else: # Controversial, but I've decided to replace the object with the old one in this case obj_new = change[1][1] else: obj_new = apply_diff(change, deepcopy(obj)) if obj_new is not None: obj_new['version'] = obj['version'] if obj_new != obj: changes.append(obj_new) except Exception as e: raise RevertError('\nFailed to download the latest version of {0} {1}: {2}'.format(kobj[0], kobj[1], e)) print_status('flush') return changes
# If we downloaded an incomplete history, add that version vref = None for h in history: if int(h.get('version')) == obj_version: vref = h if vref is None: vref = api_download('{0}/{1}/{2}'.format(obj_type, obj_id, obj_version))[0] history.insert(0, vref) if vref.get('visible') == 'false': safe_print('Will not delete the object, use other means.') sys.exit(1) # Now building a list of changes, traversing all references, finding objects to undelete obj = obj_to_dict(vref) obj['version'] = last_version changes = [obj] queue = deque() processed = {} queue.extend(find_new_refs(obj, obj_to_dict(history[-1]))) singleref = False while len(queue): qobj = queue.popleft() if qobj in processed: continue singleref = True sys.stderr.write( '\rDownloading referenced {0}, {1} left, {2} to undelete{3}'. format(qobj[0], len(queue), len(changes) - 1, ' ' * 10))
# If we downloaded an incomplete history, add that version vref = None for h in history: if int(h.get('version')) == obj_version: vref = h if vref is None: vref = api_download('{0}/{1}/{2}'.format(obj_type, obj_id, obj_version))[0] history.insert(0, vref) if vref.get('visible') == 'false': safe_print('Will not delete the object, use other means.') sys.exit(1) # Now building a list of changes, traversing all references, finding objects to undelete obj = obj_to_dict(vref) obj['version'] = last_version changes = [obj] queue = deque() processed = {} queue.extend(find_new_refs(obj, obj_to_dict(history[-1]))) singleref = False while len(queue): qobj = queue.popleft() if qobj in processed: continue singleref = True sys.stderr.write('\rDownloading referenced {0}, {1} left, {2} to undelete{3}'.format(qobj[0], len(queue), len(changes) - 1, ' ' * 10)) sys.stderr.flush() # Download last version and grab references from it try: