def make_upload_log_internal(resp_root, osm_root, date, cid, user): placesids = {} for child in resp_root: version = None if 'new_version' in child.attrib: version = child.attrib['new_version'] placesids[child.attrib['old_id']] = (child.attrib['new_id'], child.tag, version) gisids = {} for child in osm_root: action = child.tag # create, delete, update for grandchild in child: source_id = None # skip elements with no tags (typically vertices) has_tags = False for tag in grandchild.findall('tag'): has_tags = True if tag.attrib['k'] == 'nps:source_system_key_value': source_id = tag.attrib['v'] break if has_tags: tempid = grandchild.attrib['id'] gisids[tempid] = (action, source_id) data = DataTable() data.fieldnames = ['date_time', 'user_name', 'changeset', 'action', 'element', 'places_id', 'version', 'source_id'] data.fieldtypes = ['DATE', 'TEXT', 'LONG', 'TEXT', 'TEXT', 'TEXT', 'LONG', 'TEXT'] for tempid in gisids: load = gisids[tempid] diff = placesids[tempid] row = [date, user, cid, load[0], diff[1], diff[0], diff[2], load[1]] data.rows.append(row) return data
def make_upload_log_from_changeset_id(cid, server, logger): try: logger.info("Requesting info on changeset {} from server.".format(cid)) except AttributeError: pass # busy wait on server for changeset info elements = None countdown = 10 # times 3 seconds = 30 seconds before giving up. while countdown: elements = server.get_sourceids_for_changeset(cid) if elements: break else: if "404" in server.error: try: logger.info("Changeset not found or not ready. Waiting...") except AttributeError: pass time.sleep(3) countdown -= 1 else: raise UploadError("Server failure requesting source ids for changeset. " + server.error) if not elements: raise UploadError("Changeset not found. It may not be ready yet, try again in a little while.") try: element_root = Et.fromstring(elements) except Et.ParseError as e: raise UploadError("Info returned from server is invalid ({0}).".format(e.message)) if element_root.tag != "osm": raise UploadError("Info returned from server is invalid (no root osm element).") try: logger.info("Building link table.") except AttributeError: pass data = DataTable() data.fieldnames = ['date_time', 'user_name', 'changeset', 'action', 'element', 'places_id', 'version', 'source_id'] data.fieldtypes = ['DATE', 'TEXT', 'LONG', 'TEXT', 'TEXT', 'TEXT', 'LONG', 'TEXT'] try: for element in element_root: cid = int(element.attrib['changeset_id']) version = int(element.attrib['version']) try: date = datetime.datetime.strptime(element.attrib['timestamp'], "%Y-%m-%dT%H:%M:%S.%fZ") except ValueError: # on rare occassions, there are no partial seconds, and the format is truncated date = datetime.datetime.strptime(element.attrib['timestamp'], "%Y-%m-%dT%H:%M:%SZ") row = [date, element.attrib['user'], cid, element.attrib['action'], element.attrib['element'], element.attrib['places_id'], version, element.attrib['gis_id']] data.rows.append(row) except (IndexError, AttributeError, KeyError) as e: raise UploadError("Element info returned from server is invalid ({0}).".format(e.message)) try: logger.info("Created link table.") except AttributeError: pass return data