def get_player_info(player_id): player_lmps_url = PLAYER_LMPS_URL_TEMPLATE.format(player_id) tree = hf.get_web_page_html(player_lmps_url) # Player name on DSDA has no class or id, but always is a th # element and always uses colspan="6" title_elem = tree.xpath('//th[@colspan="6"]') if title_elem is None or len(title_elem) == 0: print('Invalid ID.') return None if title_elem[0][0].tag == 'a': player_name = title_elem[0][0].text_content() else: player_name = title_elem[0].text player_url = PLAYER_URL_TEMPLATE.format(player_id) tree = hf.get_web_page_html(player_url) # Profile text on DSDA is always a div element and has class="textbox" profile_elem = tree.xpath('//div[@class="textbox"]') print(profile_elem) player_profile_text = '' if profile_elem is not None and len(profile_elem) > 0: for elem in profile_elem: player_profile_text += elem.text_content() + '\n' print( OUTPUT_TEMPLATE.format(player_name=player_name, profile_text=player_profile_text)) return None
def guess_iwad_by_wad_url(wad_url): """Guesses IWAD for a wad URL This guess is performed according to whether the majority of the map numbers match one of the following formats: E#M# D1EP# D1ALL Anything with MAP##, D2EP#, or D2ALL is ignored because it could be one of several IWADs. """ doom1_map_counts = 0 other_map_counts = 0 tree = hf.get_web_page_html(wad_url) # Demos for wads on DSDA are placed in a table, and there are at # most two tables on a particular page (demo and page navigation # tables), so the first one is needed to parse the demos tables = tree.xpath('//table') # All demo rows on DSDA have these classes rows = tables[0].xpath('.//tr[@class="row1top" or @class="row2top"]') for row in rows: map_number = row[0].text_content() if (re.match(D1_MAP_NUMBER_REGEX, map_number) or re.match(D1_EPISODE_REGEX, map_number) or map_number == 'D1ALL'): doom1_map_counts += 1 else: other_map_counts += 1 if doom1_map_counts > other_map_counts: return 'doom' else: return 'unknown'
def get_demo_rows_for_wad_url(wad_url): """Gets demo rows for a wad URL""" tree = hf.get_web_page_html(wad_url) # Demos for wads on DSDA are placed in a table, and there are at # most two tables on a particular page (demo and page navigation # tables), so the first one is needed to parse the demos tables = tree.xpath('//table') # All demo rows on DSDA have these classes rows = tables[0].xpath('.//tr[@class="row1" or @class="row2" or' ' @class="row1top" or @class="row2top"]') return rows
def get_demo_pack_info(demo_pack_id): demo_pack_url = DEMO_PACK_URL_TEMPLATE.format(demo_pack_id) demo_pack_tree = hf.get_web_page_html(demo_pack_url) # Demo pack title on DSDA has no class or id, but always # is a th element and always uses colspan="6" demo_pack_title_elem = demo_pack_tree.xpath('//th[@colspan="6"]') if (demo_pack_title_elem is None or len(demo_pack_title_elem) == 0): print('Invalid demo pack ID.') demo_pack_file_name = None else: demo_pack_file_name = demo_pack_title_elem[0][0].text_content() print(OUTPUT_TEMPLATE.format(filename=demo_pack_file_name))
def get_wad_urls_for_paginated_wad(wad_url): """Gets all wad URLs fo a paginated wad""" # Current URL will always be the first page of a paginated wad wad_urls = [wad_url] tree = hf.get_web_page_html(wad_url) # Gets the second table on a page, which will always exist for # paginated wads as the first is the demo table and the second is # the navigation table tables = tree.xpath('//table') rows = tables[1].xpath('.//tr[@class="row1" or @class="row2"]') for row in rows: for col in row: # Sometimes columns in the table will be blank for padding if len(col): page_url = col[0].get('href') wad_urls.append('http://doomedsda.us/{}'.format(page_url)) return wad_urls
def get_main_port_info(): tree = hf.get_web_page_html(MAIN_PAGE) info_lists = tree.xpath('//li') source_ports_elem = None for info_list in info_lists: if len(info_list): if re.match(SOURCE_PORTS_RE, info_list[0].text_content()): source_ports_elem = info_list if source_ports_elem is None or not len(source_ports_elem): return None ports = defaultdict(list) cur_port_family = None for elem in source_ports_elem: if elem.tag == 'span': cur_port_family = elem.text if elem.tag == 'a': if cur_port_family is not None: ports[cur_port_family].append(elem.text) return ports
def get_wad_info(wad_id): wad_url = WAD_URL_TEMPLATE.format(wad_id) tree = hf.get_web_page_html(wad_url) # Wad title on DSDA has no class or id, but always is a th # element and always uses colspan="5" title_elem = tree.xpath('//th[@colspan="5"]') if title_elem is None or len(title_elem) == 0: print('Invalid ID.') return None wad_filename = title_elem[0][0].text_content() if wad_filename in IWADS: print( OUTPUT_TEMPLATE.format(filename=wad_filename, author=IWADS[wad_filename]['author'], compat=IWADS[wad_filename]['compat'], iwad=IWADS[wad_filename]['iwad'])) return None if wad_filename in PAGINATED_WAD_INFO: print( OUTPUT_TEMPLATE.format( filename=wad_filename, author=PAGINATED_WAD_INFO[wad_filename]['author'], compat=PAGINATED_WAD_INFO[wad_filename]['compat'], iwad=PAGINATED_WAD_INFO[wad_filename]['iwad'])) return None wad_author = title_elem[0][2].text_content() guessed_compat = guess_compat_by_wad_url(wad_url) guessed_iwad = guess_iwad_by_wad_url(wad_url) print( OUTPUT_TEMPLATE.format(filename=wad_filename, author=wad_author, compat=guessed_compat, iwad=guessed_iwad)) return None
def get_demo_info(demo_id): wad_url = WAD_URL_TEMPLATE.format(demo_id) tree = hf.get_web_page_html(wad_url) # Wad title on DSDA has no class or id, but always is a th # element and always uses colspan="5" title_elem = tree.xpath('//th[@colspan="5"]') if title_elem is None or len(title_elem) == 0: print('Invalid ID.') return None wad_filename = title_elem[0][0].text_content() if wad_filename in PAGINATED_WADS: wad_urls = get_wad_urls_for_paginated_wad(wad_url) else: wad_urls = [wad_url] demos = [] for wad_url in wad_urls: rows = get_demo_rows_for_wad_url(wad_url) cur_map = None cur_category = None for row in rows[1:]: if len(row) > 2: is_pack = False demo_link = row[-1][0].get('href') if demo_link.startswith('col'): is_pack = True demo_pack_tree = hf.get_web_page_html( 'http://doomedsda.us/{}'.format(demo_link)) # Demo pack title on DSDA has no class or id, but always # is a th element and always uses colspan="6" demo_pack_title_elem = demo_pack_tree.xpath( '//th[@colspan="6"]') if (demo_pack_title_elem is None or len(demo_pack_title_elem) == 0): print('Invalid demo pack ID.') demo_file_name = None else: demo_file_name = demo_pack_title_elem[0][ 0].text_content() else: demo_file_name = demo_link if len(row) > 3: cur_category = row[0].text if len(row) > 4: cur_map = row[0].text cur_category = row[1].text engine = row[-2].text players = row[-3].text time_text = row[-1].text_content() is_tas = False if 'TAS' in time_text: is_tas = True time = time_text.split()[0] cur_demo_info = { 'filename': demo_file_name, 'wad': wad_filename, 'level': cur_map, 'time': time, 'category': cur_category, 'engine': engine, 'players': players, 'is_tas': is_tas, 'is_pack': is_pack } demos.append(cur_demo_info) else: comment = row[0].text_content() last_demo = demos.pop() last_demo['comment'] = comment demos.append(last_demo) return demos