Exemplo n.º 1
0
def get_laowa_lens_list(scraping: IScrapingService) -> DataFrame:
    # レンズのURL一覧を取得する
    lens_list: List[Tuple[str, str]] = []
    page = scraping.get_page('https://www.laowa.jp/cat1/', cache=False)
    for div_element in page.find_all('div.product3'):
        h3_element = div_element.find('h3')
        if h3_element is None:
            continue
        a_element = div_element.find('a')
        if a_element is None:
            continue
        lens_name = h3_element.text
        lens_url = a_element.attrs['href']
        if 'LAOWA' in lens_name and 'mm' in lens_name:
            lens_list.append((lens_name, lens_url))

    # レンズの情報を取得する
    lens_raw_data_list: List[Dict[str, any]] = []
    for lens_name, lens_url in lens_list:
        page = scraping.get_page(lens_url)
        temp: Dict[str, str] = {'レンズ名': lens_name, 'URL': lens_url}
        section_element = page.find('div.productTable')
        if section_element is not None:
            for tr_element in section_element.find_all('tr'):
                td_elements = tr_element.find_all('td')
                if len(td_elements) < 2:
                    continue
                if td_elements[0].full_text is None or td_elements[1].full_text is None:
                    continue
                temp[td_elements[0].full_text] = td_elements[1].full_text
            # 特殊処理
            if temp['レンズ名'] == 'LAOWA 15mm F4 WIDE ANGLE MACRO':
                if 'Nikon' in temp['質量']:
                    # 記述が入れ替わっているので対策
                    temp2 = temp.copy()
                    temp2['マウント'] = temp['質量']
                    temp2['質量'] = temp['マウント']
                    temp = temp2
            lens_raw_data_list.append(temp)
    df = DataFrame.from_records(lens_raw_data_list)

    # 変換用に整形
    df['maker'] = 'LAOWA'
    df = convert_columns(df, {
        'レンズ名': 'name', 'URL': 'url', 'フォーマット': '対応フォーマット', '対応マウント': 'マウント',
        '寸法(鏡筒直径×長さ)': 'サイズ', '最小フォーカシングディスタンス': '最短撮影距離',
        '最大倍率比': '最大撮影倍率', '最大倍率': '最大撮影倍率',
        }, [
        '開放F値', '画角', 'レンズ構成', 'シフト機能', '最大イメージサークル', '絞り羽根枚数', 'フォーカス', 'JAN',
        '発売日', '絞り羽枚数', 'フォーカシング', 'フィルタースレッド', 'ワーキングディスタンス', '最大口径比',
        '絞り羽根枚数(F)', '絞り羽根枚数(T)', 'シフト量', '最小ワーキングディスタンス', '対応フォーマット',
    ])
    if '対応フォーマット' in df:
        del df['対応フォーマット']
    if None in df:
        del df[None]
    if '' in df:
        del df['']

    mount_list: List[str] = []
    for mount_temp in df['マウント']:
        if 'マイクロフォーサーズ' in mount_temp:
            mount_list.append('マイクロフォーサーズ')
        elif 'Leica L' in mount_temp:
            mount_list.append('ライカL')
        else:
            mount_list.append('')
    df['mount'] = mount_list
    df = df[df['mount'] != '']
    del df['マウント']

    w_list, t_list = extract_numbers(df['焦点距離'], [r'(\d+\.?\d*)-(\d+\.?\d*)mm'], [r'(\d+\.?\d*)mm'])
    w_list2: List[int] = []
    t_list2: List[int] = []
    for w, t, mount in zip(w_list, t_list, list(df['mount'])):
        if mount == 'マイクロフォーサーズ':
            w_list2.append(int((Decimal(w) * 2).quantize(Decimal('1'))))
            t_list2.append(int((Decimal(t) * 2).quantize(Decimal('1'))))
        elif mount == 'ライカL':
            w_list2.append(int(w))
            t_list2.append(int(t))
    df['wide_focal_length'] = w_list2
    df['telephoto_focal_length'] = t_list2
    del df['焦点距離']

    w, t = extract_numbers(df['name'], [r'F(\d+\.?\d*)-(\d+\.?\d*)'], [r'F(\d+\.?\d*)'])
    df['wide_f_number'] = [float(x) for x in w]
    df['telephoto_f_number'] = [float(x) for x in t]

    w_fd_list: List[int] = []
    t_fd_list: List[int] = []
    for fd in df['最短撮影距離'].values:
        result = regex(fd, r'(\d+.?\d*)mm~(\d+.?\d*)mm')
        if len(result) > 0:
            w_fd_list.append(int(result[0]))
            t_fd_list.append(int(result[1]))
            continue
        result = regex(fd, r'(\d+.?\d*)cm')
        if len(result) > 0:
            w_fd_list.append(int(Decimal(result[0]) * 10))
            t_fd_list.append(int(Decimal(result[0]) * 10))
            continue
        result = regex(fd, r'(\d+.?\d*)cm')
        if len(result) > 0:
            w_fd_list.append(int(Decimal(result[0]) * 10))
            t_fd_list.append(int(Decimal(result[0]) * 10))
            continue
        result = regex(fd, r'(\d+.?\d*)mm')
        if len(result) > 0:
            w_fd_list.append(int(result[0]))
            t_fd_list.append(int(result[0]))
            continue
        w_fd_list.append(0)
        t_fd_list.append(0)
    df['wide_min_focus_distance'] = w_fd_list
    df['telephoto_min_focus_distance'] = t_fd_list
    del df['最短撮影距離']

    mag_list: List[float] = []
    for val1, val2 in zip(df['最大撮影倍率'].values, df['mount'].values):
        value = Decimal(0)
        while True:
            if val1 != val1:
                break
            result = regex(val1, r'(\d+.?\d*)/(\d+.?\d*)倍')
            if len(result) > 0:
                value = Decimal(result[0]) / Decimal(result[1])
                break
            result = regex(val1, r'(\d+.?\d*):(\d+.?\d*)')
            if len(result) > 0:
                value = Decimal(result[0]) / Decimal(result[1])
                break
            result = regex(val1, r'(\d+.?\d*)倍')
            if len(result) > 0:
                value = Decimal(result[0])
                break
            result = regex(val1, r'(\d+.?\d*)')
            if len(result) > 0:
                value = Decimal(result[0])
                break
            break
        if val2 == 'マイクロフォーサーズ':
            mag_list.append(float(value * 2))
        else:
            mag_list.append(float(value))
    df['max_photographing_magnification'] = mag_list
    del df['最大撮影倍率']

    fd_list: List[float] = []
    for fd, name in zip(df['フィルター径'].values, df['name'].values):
        if fd != fd or name == 'LAOWA 10-18mm F4.5-5.6 FE ZOOM':
            fd_list.append(-1)
            continue
        result = regex(fd, r'(\d+.?\d*)mm')
        if len(result) == 0:
            fd_list.append(-1)
            continue
        fd_list.append(int(result[0]))
    df['filter_diameter'] = fd_list
    del df['フィルター径']

    df['is_drip_proof'] = False
    df['has_image_stabilization'] = False

    i: List[bool] = []
    for record in df.to_records():
        if record['wide_focal_length'] == record['telephoto_focal_length']:
            i.append(True)
            continue
        i.append(False)
    df['is_inner_zoom'] = i

    d, le = extract_numbers(df['サイズ'], [r'(\d+\.?\d*)[^\d]+(\d+\.?\d*)(mm|mm)'], [])
    df['overall_diameter'] = [float(x) for x in d]
    df['overall_length'] = [float(x) for x in le]
    del df['サイズ']

    weight: List[float] = []
    for f in df['質量']:
        result = regex(f, r'([\d,]+)(g|g)')
        if len(result) > 0:
            weight.append(int(result[0].replace(',', '')))
        else:
            weight.append(-1)
    df['weight'] = weight
    del df['質量']

    df['price'] = 0
    return df
Exemplo n.º 2
0
def get_sigma_lens_list(scraping: IScrapingService) -> DataFrame:
    # レンズのURL一覧を取得する
    page = scraping.get_page('https://www.sigma-global.com/jp/lenses/',
                             cache=False)
    lens_list_mft: List[Tuple[str, str]] = []
    lens_list_l: List[Tuple[str, str]] = []
    for li_element in page.find('div.p-lens-search__main').find_all('li'):
        lens_link = li_element.find('a').attrs['href']
        if 'lenses' not in lens_link:
            continue
        h4_element = li_element.find('h4')
        if h4_element is None:
            continue
        lens_name = h4_element.text
        if 'micro-four-thirds' in li_element.attrs['data-lens-mount']:
            lens_list_mft.append((lens_name, lens_link))
        if 'l-mount' in li_element.attrs['data-lens-mount']:
            lens_list_l.append((lens_name, lens_link))
        # 特殊処理
        if '35mm F1.4 DG HSM' in lens_name:
            lens_list_l.append((lens_name, lens_link))

    page: DomObject = scraping.get_page(
        'https://www.sigma-global.com/jp/lenses/discontinued/', cache=False)
    lens_list_old: List[Tuple[str, str]] = []
    for li_element in page.find_all('li.p-support-service__item'):
        a_element = li_element.find('a')
        lens_link = a_element.attrs['href']
        lens_name = a_element.find('h4 > span').text
        lens_list_old.append((lens_name, lens_link))

    # レンズごとに情報を取得する
    lens_raw_data_list: List[Dict[str, any]] = []
    for lens_list, lens_mount in [(lens_list_mft, 'マイクロフォーサーズ'),
                                  (lens_list_l, 'ライカL')]:
        for lens_name, lens_link in lens_list:
            if 'lenses/c' in lens_link and '| Contemporary' not in lens_name:
                lens_name2 = lens_name + ' | Contemporary'
            elif 'lenses/a' in lens_link and '| Art' not in lens_name:
                lens_name2 = lens_name + ' | Art'
            else:
                lens_name2 = lens_name

            page = scraping.get_page(lens_link)
            temp_dict: Dict[str, str] = {
                'mount': lens_mount,
                'name': lens_name2,
                'url': lens_link
            }
            raw_dict = item_page_to_raw_dict(page, lens_mount)
            temp_dict.update(raw_dict)
            lens_raw_data_list.append(temp_dict)
    for lens_name, lens_link in lens_list_old:
        if 'DN' not in lens_name:
            # DNが含まれない=ミラーレス用ではないので除外
            continue
        page = scraping.get_page(lens_link)
        temp_dict: Dict[str, str] = {
            'mount': 'マイクロフォーサーズ',
            'name': lens_name,
            'url': lens_link
        }
        temp_dict2 = item_page_to_raw_dict(page, '')
        if len(temp_dict2) > 0:
            temp_dict.update(temp_dict2)
            lens_raw_data_list.append(temp_dict)
    df = DataFrame.from_records(lens_raw_data_list)

    # 変換用に整形
    df['maker'] = 'SIGMA'
    df['product_number'] = df['エディションナンバー']
    del df['エディションナンバー']
    del df['レンズ構成枚数']
    del df['画角']
    del df['絞り羽根枚数']
    del df['最小絞り']
    del df['付属品']
    del df['対応マウント / バーコード']

    # focal_length
    w, t = extract_numbers(df['name'], [r'(\d+)-(\d+)mm'], [r'(\d+)mm'])
    wide_focal_length: List[int] = []
    telephoto_focal_length: List[int] = []
    for wf, tf, mount, name in zip(w, t, df['mount'], df['name']):
        if mount == 'マイクロフォーサーズ':
            wide_focal_length.append(int(wf) * 2)
            telephoto_focal_length.append(int(tf) * 2)
        else:
            if 'DC' in name:
                wide_focal_length.append(int(1.5 * int(wf)))
                telephoto_focal_length.append(int(1.5 * int(tf)))
            else:
                wide_focal_length.append(int(wf))
                telephoto_focal_length.append(int(tf))
    df['wide_focal_length'] = wide_focal_length
    df['telephoto_focal_length'] = telephoto_focal_length

    # f_number
    w, t = extract_numbers(df['name'], [r'F(\d+\.?\d*)-(\d+\.?\d*)'],
                           [r'F(\d+\.?\d*)'])
    df['wide_f_number'] = [float(x) for x in w]
    df['telephoto_f_number'] = [float(x) for x in t]

    # min_focus_distance
    w, t = extract_numbers(df['最短撮影距離'], [
        r'(\d+\.?\d*)-(\d+\.?\d*)cm', r'(\d+\.?\d*) \(W\)-(\d+\.?\d*) \(T\)cm',
        r'(\d+\.?\d*)\(W\) - (\d+\.?\d*)\(T\)cm',
        f'(\d+\.?\d*)(W)-(\d+\.?\d*)(T)cm'
    ], [r'(\d+\.?\d*)cm'])

    df['wide_min_focus_distance'] = [int(Decimal(x).scaleb(1)) for x in w]
    df['telephoto_min_focus_distance'] = [int(Decimal(x).scaleb(1)) for x in t]
    del df['最短撮影距離']

    # max_photographing_magnification
    m: List[float] = []
    for record in df.to_records():
        temp = regex(record['最大撮影倍率'].replace(':', ':'),
                     r'.*1:(\d+\.?\d*).*1:(\d+\.?\d*).*')
        if len(temp) > 0:
            if float(temp[0]) < float(temp[1]):
                denominator = temp[0]
            else:
                denominator = temp[1]
        else:
            temp = regex(record['最大撮影倍率'].replace(':', ':'),
                         r'.*1:(\d+\.?\d*).*')
            denominator = temp[0]
        if record['mount'] == 'マイクロフォーサーズ':
            m.append(
                float((Decimal('2') / Decimal(denominator)).quantize(
                    Decimal('0.01'))))
        else:
            if 'DC' in record['name']:
                m.append(
                    float((Decimal('1.5') / Decimal(denominator)).quantize(
                        Decimal('0.01'))))
            else:
                m.append(
                    float((Decimal('1') / Decimal(denominator)).quantize(
                        Decimal('0.01'))))
    df['max_photographing_magnification'] = m
    del df['最大撮影倍率']

    # filter_diameter
    filter_diameter: List[float] = []
    for f in df['フィルターサイズ']:
        if f == f:
            result = regex(f, r'(\d+.?\d*)mm')
            if len(result) > 0:
                filter_diameter.append(float(result[0]))
            else:
                filter_diameter.append(-1)
        else:
            filter_diameter.append(-1)
    df['filter_diameter'] = filter_diameter
    del df['フィルターサイズ']

    # is_drip_proof
    df['is_drip_proof'] = df['name'].map(lambda x: 'DC' in x or 'DG' in x)

    # has_image_stabilization
    df['has_image_stabilization'] = df['name'].map(lambda x: 'OS' in x)

    # is_inner_zoom
    is_inner_zoom: List[bool] = []
    for record in df.to_dict(orient='records'):
        is_inner_zoom.append(
            record['wide_focal_length'] == record['telephoto_focal_length'])
    df['is_inner_zoom'] = is_inner_zoom

    # overall_diameter, overall_length
    overall_diameter, overall_length = extract_numbers(
        df['最大径 × 長さ'], [r'(\d+\.?\d*)mm[^\d]*(\d+\.?\d*)mm'], [])
    for i in range(0, len(df)):
        # データが存在しない分については手動で埋める
        if df['name'].values[i] == '19mm F2.8 EX DN':
            overall_diameter[i] = '60.6'
            overall_length[i] = '45.7'
        elif df['name'].values[i] == '30mm F2.8 EX DN':
            overall_diameter[i] = '60.6'
            overall_length[i] = '38.6'
        elif df['name'].values[i] == '19mm F2.8 DN | Art':
            overall_diameter[i] = '60.8'
            overall_length[i] = '45.7'
        elif df['name'].values[i] == '30mm F2.8 DN | Art':
            overall_diameter[i] = '60.8'
            overall_length[i] = '40.5'
        elif df['name'].values[i] == '60mm F2.8 DN | Art':
            overall_diameter[i] = '60.8'
            overall_length[i] = '55.5'
    df['overall_diameter'] = [float(x) for x in overall_diameter]
    df['overall_length'] = [float(x) for x in overall_length]
    del df['最大径 × 長さ']

    # weight
    weight: List[float] = []
    for i in range(0, len(df)):
        f = df['質量'].values[i]
        if f != f:
            if df['name'].values[i] == '19mm F2.8 EX DN':
                weight.append(140)
            elif df['name'].values[i] == '30mm F2.8 EX DN':
                weight.append(130)
            elif df['name'].values[i] == '19mm F2.8 DN | Art':
                weight.append(160)
            elif df['name'].values[i] == '30mm F2.8 DN | Art':
                weight.append(140)
            elif df['name'].values[i] == '60mm F2.8 DN | Art':
                weight.append(190)
            continue
        result = regex(f, r'([\d,]+)g')
        if len(result) > 0:
            weight.append(int(result[0].replace(',', '')))
        else:
            weight.append(int(f))
    df['weight'] = weight
    del df['質量']

    # price
    price: List[float] = []
    for f in df['希望小売価格']:
        result = regex(f, r'([\d,]+) *円')
        if len(result) > 0:
            price.append(int(result[0].replace(',', '')))
        else:
            price.append(26240)  # アドホックな修正
    df['price'] = price
    del df['希望小売価格']

    return df
Exemplo n.º 3
0
def get_samyang_lens_list(scraping: IScrapingService) -> DataFrame:
    # レンズのURL一覧を取得する
    lens_list: List[Tuple[str, str, str]] = []
    page = scraping.get_page(
        'https://www.kenko-tokina.co.jp/camera-lens/samyang/', cache=False)
    for li_element in page.find_all('li.col.list_item'):
        lens_name = li_element.find('h3 > a').text
        lens_url = li_element.find('h3 > a').attrs['href']
        if 'data-spec3' in li_element.attrs:
            mount_info = li_element.attrs['data-spec3']
            if 'マイクロフォーサーズ' in mount_info:
                lens_list.append((lens_name, lens_url, 'マイクロフォーサーズ'))

    # レンズの情報を取得する
    temp_list: List[Dict[str, any]] = []
    for lens_name, lens_url, lens_mount in lens_list:
        page = scraping.get_page(lens_url)
        for table_element in page.find_all('table'):
            temp: Dict[str, any] = {
                'name': lens_name,
                'url': lens_url,
                'mount': lens_mount
            }
            for tr_element in table_element.find_all('tr'):
                th_element = tr_element.find('th')
                td_element = tr_element.find('td')
                if th_element.text == '大きさ' or th_element.text == '全長' or th_element.text == '質量' or th_element.text == '重さ':
                    # 大きさや質量はマウント毎に異なるので特殊処理を実施
                    temp2 = td_element.html.replace('<td>', '').replace(
                        '</td>', '').replace('\n', '').split('<br>')
                    temp3 = ''
                    for temp4 in temp2:
                        if 'マイクロフォーサーズ' in temp4:
                            temp3 = temp4
                            break
                    if temp3 == '':
                        for temp4 in temp2:
                            if 'ソニーE' in temp4:
                                temp3 = temp4
                                break
                        if temp3 == '':
                            temp3 = temp2[0]
                    temp[th_element.text] = temp3.replace('\n', '')
                else:
                    temp[th_element.text] = td_element.text
            if len(temp) > 0:
                temp_list.append(temp)
    df = DataFrame.from_records(temp_list)

    # 変換用に整形
    df['maker'] = 'SAMYANG'
    df['price'] = 0
    del df['レンズ構成']
    del df['レンズフード']
    del df['マウント']
    del df['JANコード']
    del df['フォーマットサイズ']
    del df['絞り羽根']
    del df['絞り羽根枚数']
    del df['付属品']
    del df['JANコード:']

    w, t = extract_numbers(df['焦点距離'], [], [r'(\d+\.?\d*)mm'])
    df['wide_focal_length'] = [int(Decimal(x) * 2) for x in w]
    df['telephoto_focal_length'] = [int(Decimal(x) * 2) for x in w]
    del df['焦点距離']
    del df['画角']

    w, t = extract_numbers(df['name'], [], [r'F(\d+\.?\d*)'])
    df['wide_f_number'] = [float(x) for x in w]
    df['telephoto_f_number'] = [float(x) for x in t]
    del df['明るさ']
    del df['絞り']

    w: List[int] = []
    t: List[int] = []
    for fd in df['最短撮影距離'].values:
        result = regex(fd, r'(\d+.?\d*)cm')
        if len(result) > 0:
            w.append(int(Decimal(result[0]) * 10))
            t.append(int(Decimal(result[0]) * 10))
            continue
        result = regex(fd, r'(\d+.?\d*)( *)m')
        if len(result) > 0:
            w.append(int(Decimal(result[0]) * 1000))
            t.append(int(Decimal(result[0]) * 1000))
            continue
        w.append(0)
        t.append(0)
    df['wide_min_focus_distance'] = w
    df['telephoto_min_focus_distance'] = t
    del df['最短撮影距離']

    mag_list: List[float] = []
    for val1, val2 in zip(df['最大撮影倍率'].values, df['mount'].values):
        value = Decimal(0)
        while True:
            if val1 != val1:
                break
            result = regex(val1, r'(\d+.?\d*)倍')
            if len(result) > 0:
                value = Decimal(result[0])
                break
            break
        if val2 == 'マイクロフォーサーズ':
            mag_list.append(float(value * 2))
        else:
            mag_list.append(float(value))
    df['max_photographing_magnification'] = mag_list
    del df['最大撮影倍率']

    fd_list: List[float] = []
    for record in df.to_dict(orient='records'):
        text = record['フィルターサイズ']
        if text != text:
            text = record['フィルター径']
        result = regex(text, r'(\d+.?\d*)mm')
        if len(result) == 0:
            fd_list.append(-1)
            continue
        fd_list.append(int(result[0]))
    df['filter_diameter'] = fd_list
    del df['フィルターサイズ']
    del df['フィルター径']

    df['is_drip_proof'] = False
    df['has_image_stabilization'] = False

    i: List[bool] = []
    for record in df.to_records():
        if record['wide_focal_length'] == record['telephoto_focal_length']:
            i.append(True)
            continue
        i.append(False)
    df['is_inner_zoom'] = i

    diameter_list: List[float] = []
    length_list: List[float] = []
    for record in df.to_dict(orient='records'):
        text = record['大きさ']
        if text == text:
            result = regex(text, r'(\d+.?\d*)[^\d.]+(\d+.?\d*) *mm')
            if len(result) != 2:
                diameter_list.append(-1)
                length_list.append(-1)
            else:
                length_list.append(float(result[0].replace('×', '')))
                diameter_list.append(float(result[1]))
        else:
            text = record['最大径']
            if text == text:
                result = regex(text, r'(\d+.?\d*)')
                if len(result) > 0:
                    diameter_list.append(float(result[0]))
                else:
                    diameter_list.append(-1)
            else:
                diameter_list.append(-1)

            text = record['全長']
            if text == text:
                result = regex(text, r'(\d+.?\d*)')
                if len(result) > 0:
                    length_list.append(float(result[0]))
                else:
                    length_list.append(-1)
            else:
                length_list.append(-1)
    df['overall_diameter'] = diameter_list
    df['overall_length'] = length_list
    del df['大きさ']
    del df['最大径']
    del df['全長']

    weight_list: List[float] = []
    for record in df.to_dict(orient='records'):
        text = record['質量']
        if text != text:
            text = record['重さ']
        result = regex(text, r'(\d+.?\d*)g')
        if len(result) == 0:
            weight_list.append(-1)
            continue
        weight_list.append(int(float(result[0]) + 0.5))
    df['weight'] = weight_list
    del df['質量']
    del df['重さ']
    return df
Exemplo n.º 4
0
def get_olympus_lens_list(scraping: IScrapingService) -> DataFrame:
    # レンズのURL一覧を取得する
    page = scraping.get_page(
        'https://www.olympus-imaging.jp/product/dslr/mlens/index.html',
        cache=False)
    lens_list: List[Tuple[str, str]] = []
    for a_element in page.find_all('h2.productName > a'):
        lens_name = a_element.text.split('/')[0].replace('\n', '')
        if 'M.ZUIKO' not in lens_name:
            continue
        lens_product_number = a_element.attrs['href'].replace(
            '/product/dslr/mlens/', '').replace('/index.html', '')
        lens_list.append((lens_name, lens_product_number))

    page = scraping.get_page(
        'https://www.olympus-imaging.jp/product/dslr/record/index.html',
        cache=False)
    for a_element in page.find_all('div.section'):
        div_element = a_element.find('div.mb15 > h2')
        a_element2 = a_element.find('li > a')
        if div_element is None or a_element2 is None:
            continue
        lens_name = div_element.text
        if 'M.ZUIKO DIGITAL' not in lens_name:
            continue
        lens_product_number = a_element2.attrs['href'].replace(
            '/product/dslr/mlens/', '').replace('/index.html', '')
        lens_list.append((lens_name, lens_product_number))

    # レンズごとに情報を取得する
    lens_data_list: List[Dict[str, str]] = []
    for lens_name, lens_product_number in lens_list:
        # 詳細ページから情報を取得する
        if lens_product_number != '14-42_35-56':
            spec_url = f'https://www.olympus-imaging.jp/product/dslr/mlens/{lens_product_number}/spec.html'
        else:
            spec_url = f'https://www.olympus-imaging.jp/product/dslr/mlens/{lens_product_number}/spec/index.html'
        page = scraping.get_page(spec_url)
        temp_dict: Dict[str, str] = {}
        for tr_element in page.find('table').find_all('tr'):
            tr_element: DomObject = tr_element

            # th側は、spanで囲まれてたりstrongで囲まれてたりするクソ仕様なので、力技で解決させた
            th_element = tr_element.find('th > span')
            if th_element is None:
                th_element = tr_element.find('th > strong')
            if th_element is None:
                th_element = tr_element.find('th')

            # td側はそのまま
            td_element = tr_element.find('td')

            # 合体
            temp_dict[th_element.text] = td_element.text

        # 製品トップページから情報を取得する
        index_url = f'https://www.olympus-imaging.jp/product/dslr/mlens/{lens_product_number}/index.html'
        page = scraping.get_page(index_url)
        temp_dict['URL'] = index_url
        table_element = page.find('table')
        # 詳細ページとはth・tdの拾い方を変えているのは、
        # M.ZUIKO DIGITAL ED 30mm F3.5 Macroの製品トップページの時のみ、
        # 希望小売価格「だけ」が取得できない不具合があったため
        for th_element, td_element in zip(table_element.find_all('th'),
                                          table_element.find_all('td')):
            th_element2 = th_element.find('span')
            if th_element2 is None:
                th_element2 = th_element.find('strong')
            if th_element2 is None:
                th_element2 = th_element
            temp_dict[th_element2.text] = td_element.text

        # 必要な列を追加
        temp_dict['name'] = lens_name.replace(' ', ' ')
        temp_dict['product_number'] = lens_product_number

        # 不要な列を削除
        del_column_list = [
            'レンズ構成',
            'フォーカシング方式',
            'AF方式',
            '特長',
            'マウント規格',
            '画角',
            '最近接撮影範囲',
            '絞り羽枚数',
            '同梱品',
            '主な同梱品',
            '別売りアクセサリー',
            '別売アクセサリー',
            '製品名',
            'JANコード',
            'JAN',
            '発売日',
            'オンラインショップ',
            'フード',
            '最大口径比',
            '最小口径比',
            '最大口径比/最小口径比',
            '35mm判換算最大撮影倍率',
            '最大撮影倍率(35mm判換算)',
            '手ぶれ補正性能',
            'ズーム',
            'ズーム方式',
            '35mm判換算',
            '絞り範囲',
        ]
        for column in del_column_list:
            if column in temp_dict:
                del temp_dict[column]

        # 一部列だけ列名を変更しないと結合できないので対処
        if '大きさ 最大径×長さ' in temp_dict:
            temp_dict['大きさ 最大径×全長'] = temp_dict['大きさ 最大径×長さ']
            del temp_dict['大きさ 最大径×長さ']
        if '大きさ 最大径 × 全長' in temp_dict:
            temp_dict['大きさ 最大径×全長'] = temp_dict['大きさ 最大径 × 全長']
            del temp_dict['大きさ 最大径 × 全長']
        if '大きさ 最大径×全長' in temp_dict:
            temp_dict['大きさ 最大径×全長'] = temp_dict['大きさ 最大径×全長']
            del temp_dict['大きさ 最大径×全長']
        if '大きさ 最大径 x 全長' in temp_dict:
            temp_dict['大きさ 最大径×全長'] = temp_dict['大きさ 最大径 x 全長']
            del temp_dict['大きさ 最大径 x 全長']
        if '防滴性能 / 防塵機構' in temp_dict:
            temp_dict['防滴処理'] = temp_dict['防滴性能 / 防塵機構']
            del temp_dict['防滴性能 / 防塵機構']
        if '防滴性能/防塵機構' in temp_dict:
            temp_dict['防滴処理'] = temp_dict['防滴性能/防塵機構']
            del temp_dict['防滴性能/防塵機構']
        if '防滴性能 / 防塵機構搭載' in temp_dict:
            temp_dict['防滴処理'] = temp_dict['防滴性能 / 防塵機構搭載']
            del temp_dict['防滴性能 / 防塵機構搭載']
        if '価格' in temp_dict:
            temp_dict['希望小売価格'] = temp_dict['価格']
            del temp_dict['価格']
        lens_data_list.append(temp_dict)

    df = DataFrame.from_records(lens_data_list)

    # 変換用に整形
    df['maker'] = 'OLYMPUS'

    # focal_length
    w, t = extract_numbers(df['焦点距離'], [r'(\d+)-(\d+)mm', r'(\d+) - (\d+)mm'],
                           [r'(\d+)mm'])
    df['wide_focal_length'] = [int(x) * 2 for x in w]
    df['telephoto_focal_length'] = [int(x) * 2 for x in t]
    # M.ZUIKO DIGITAL ED 150-400mm F4.5 TC1.25x IS PROは内蔵テレコンを持つので、その対策
    with warnings.catch_warnings():
        warnings.simplefilter('ignore', SettingWithCopyWarning)
        df.telephoto_focal_length[df.product_number ==
                                  '150-400_45ispro'] = 1000
    del df['焦点距離']

    # f_number
    w, t = extract_numbers(df['name'], [r'F(\d+\.?\d*)-(\d+\.?\d*)'],
                           [r'F(\d+\.?\d*)'])
    df['wide_f_number'] = [float(x) for x in w]
    df['telephoto_f_number'] = [float(x) for x in t]

    # min_focus_distance
    w, t = extract_numbers(df['最短撮影距離'], [
        r'(\d+\.?\d+)m(.+) / (\d+\.?\d+)m(.+)',
        r'(\d+\.?\d+)m \(.+\) / (\d+\.?\d+)m \(.+\)',
        r'(\d+\.?\d+)m.+/(\d+\.?\d+)m.+'
    ], [r'(\d+\.?\d+)m', r'(\d+\.?\d+) m'])
    df['wide_min_focus_distance'] = [int(Decimal(x).scaleb(3)) for x in w]
    df['telephoto_min_focus_distance'] = [int(Decimal(x).scaleb(3)) for x in t]
    del df['最短撮影距離']

    # max_photographing_magnification
    w, t = extract_numbers(df['最大撮影倍率'], [
        r'(\d+\.?\d+)倍 \(Wide\) / (\d+\.?\d+)倍 \(Tele\)',
        r'Wide:(\d+\.?\d+)倍/Tele:(\d+\.?\d+)倍',
        r'(\d+\.?\d+)倍(Wide) / (\d+\.?\d+)倍(Tele)',
        r'Wide:(\d+\.?\d+)倍/Tele:(\d+\.?\d+)倍',
        r'(\d+\.?\d+)倍(Wide)/ (\d+\.?\d+)倍(Tele)'
    ], [
        r'(\d+\.?\d+)倍(35mm判換算\d+\.?\d+倍相当)',
        r'(\d+\.?\d+)倍(35mm判換算 \d+\.?\d+倍相当)', r'(\d+\.?\d+)倍(Wide / Tele)',
        r'(\d+\.?\d+)倍 \(35mm判換算 \d+\.?\d+倍相当\)',
        r'(\d+\.?\d+)倍(35mm判換算 \d+\.?\d+倍)',
        r'(\d+\.?\d+)倍(マクロモード時)(35mm判換算 \d+\.?\d+倍)'
    ])
    m: List[float] = []
    for a, b, text in zip(w, t, df['最大撮影倍率'].values):
        if a == b:
            mm = float(Decimal(a) * 2)
        elif '換算' in text:
            mm = max(float(Decimal(a)), float(Decimal(b)))
        else:
            mm = max(float(Decimal(a)), float(Decimal(b))) * 2
        m.append(mm)
    df['max_photographing_magnification'] = m
    del df['最大撮影倍率']

    # filter_diameter
    filter_diameter: List[float] = []
    for f in df['フィルターサイズ']:
        if f != f:
            filter_diameter.append(-1)
            continue
        result = regex(f, r'(\d+.?\d*)mm')
        if len(result) > 0:
            filter_diameter.append(float(result[0]))
        else:
            filter_diameter.append(-1)
    df['filter_diameter'] = filter_diameter
    del df['フィルターサイズ']

    # is_drip_proof
    df['is_drip_proof'] = df['防滴処理'].map(lambda x: x == x and x != '')
    del df['防滴処理']

    # has_image_stabilization
    df['has_image_stabilization'] = df['name'].map(lambda x: 'IS' in x)
    del df['レンズ内手ぶれ補正機構']

    # is_inner_zoom
    i: List[bool] = []
    for record in df.to_records():
        if record['wide_focal_length'] == record['telephoto_focal_length']:
            i.append(True)
            continue
        if record['product_number'] in [
                '7-14_28pro', '40-150_28pro', '150-400_45ispro'
        ]:
            i.append(True)
            continue
        i.append(False)
    df['is_inner_zoom'] = i

    # overall_diameter, overall_length
    d, le = extract_numbers(df['大きさ 最大径×全長'], [
        r'φ(\d+.?\d*)x(\d+.?\d*)mm', r'Ø(\d+.?\d*)×(\d+.?\d*)mm',
        r'Φ (\d+.?\d*) mm  x (\d+.?\d*) mm', r'⌀(\d+.?\d*) x (\d+.?\d*)mm',
        r'Ø(\d+.?\d*) × (\d+.?\d*)mm', r'Ø(\d+.?\d*) x (\d+.?\d*)mm',
        r'Ø(\d+.?\d*)mm x (\d+.?\d*)mm', r'Ø(\d+.?\d*)x (\d+.?\d*)mm',
        r'Ø(\d+.?\d*)x(\d+.?\d*)mm', r'⌀(\d+.?\d*)×(\d+.?\d*)mm',
        r'Ø(\d+.?\d*)mm × (\d+.?\d*)mm', r'φ(\d+.?\d*)×(\d+.?\d*)mm'
    ], [])
    df['overall_diameter'] = [float(x) for x in d]
    df['overall_length'] = [float(x) for x in le]
    del df['大きさ 最大径×全長']

    # weight
    weight: List[float] = []
    for f in df['質量']:
        result = regex(f, r'([\d,]+)[^\d]*(g|g)')
        if len(result) > 0:
            weight.append(int(result[0].replace(',', '')))
        else:
            weight.append(-1)
    df['weight'] = weight
    del df['質量']

    # price
    price: List[float] = []
    for f in df['希望小売価格']:
        result = regex(f, r'([\d,]+)円')
        if len(result) > 0:
            price.append(int(result[0].replace(',', '')))
        else:
            price.append(-1)
    df['price'] = price
    del df['希望小売価格']

    # mount・url
    df['mount'] = 'マイクロフォーサーズ'
    df['url'] = df['URL']
    del df['URL']
    return df
Exemplo n.º 5
0
def get_panasonic_old_lens_list(scraping: IScrapingService) -> DataFrame:
    # 情報ページを開く
    page = scraping.get_page(
        'https://panasonic.jp/dc/products/g_series_lens.html', cache=False)

    # 情報URLの一覧を取得する
    link_url_set: Set[str] = set()
    for a_element in page.find_all('a'):
        link_url = a_element.attrs['href']
        if len(regex(link_url,
                     r'(http://panasonic\.jp/dc/p-db/.+\.html)')) > 0:
            link_url_set.add(link_url)

    # 順番に取得する
    temp_list: List[Dict[str, any]] = []
    for link_url in link_url_set:
        page = scraping.get_page(link_url.replace('.html', '_spec.html'))
        table_element = page.find('table')
        temp_dict: Dict[str, any] = {}
        temp_dict['リンク'] = link_url
        temp_dict['型番'] = regex(link_url,
                                r'http://panasonic\.jp/dc/p-db/(.+)\.html')[0]
        for th_element, td_element in zip(table_element.find_all('th'),
                                          table_element.find_all('td')):
            temp_dict[th_element.text] = td_element.text
        temp_list.append(temp_dict)
    df = DataFrame.from_records(temp_list)

    # 変換用に整形
    df['maker'] = 'Panasonic'

    df['name'] = df['レンズ名称']
    del df['レンズ名称']

    df['product_number'] = df['型番']
    del df['型番']

    w, t = extract_numbers(df['焦点距離'], [r'(\d+)mm~(\d+)mm', r'(\d+)-(\d+)mm'],
                           [r'(\d+)mm'])
    df['wide_focal_length'] = [int(x) * 2 for x in w]
    df['telephoto_focal_length'] = [int(x) * 2 for x in t]
    del df['焦点距離']

    w, t = extract_numbers(df['name'], [r'F(\d+\.?\d*)-(\d+\.?\d*)'],
                           [r'F(\d+\.?\d*)'])
    df['wide_f_number'] = [float(x) for x in w]
    df['telephoto_f_number'] = [float(x) for x in t]
    del df['開放絞り']
    del df['絞り形式']
    del df['最小絞り']

    w, t = extract_numbers(
        df['最短撮影距離'],
        [r'(\d+\.?\d+)m / (\d+\.?\d+)m', r'(\d+\.?\d+)m~∞.*(\d+\.?\d+)m~∞'],
        [r'(\d+\.?\d+)m', r'(\d+\.?\d+)m~∞'])
    df['wide_min_focus_distance'] = [int(Decimal(x).scaleb(3)) for x in w]
    df['telephoto_min_focus_distance'] = [int(Decimal(x).scaleb(3)) for x in t]
    del df['最短撮影距離']

    df['max_photographing_magnification'] = 0.0  # なぜか記載がなかったので

    filter_diameter: List[float] = []
    for f in df['フィルター径']:
        result = regex(f, r'(\d+.?\d*)mm')
        if len(result) > 0:
            filter_diameter.append(float(result[0]))
        else:
            filter_diameter.append(-1)
    df['filter_diameter'] = filter_diameter
    del df['フィルター径']

    df['is_drip_proof'] = False  # なぜか記載がなかったので

    df['has_image_stabilization'] = df['name'].map(lambda x: 'O.I.S.' in x)

    i: List[bool] = []
    for record in df.to_records():
        if record['wide_focal_length'] == record['telephoto_focal_length']:
            i.append(True)
            continue
        if record['product_number'] in [
                'H-F007014', 'H-E08018', 'H-PS45175', 'S-E70200', 'S-R70200'
        ]:
            i.append(True)
            continue
        i.append(False)
    df['is_inner_zoom'] = i

    d, le = extract_numbers(df['外形寸法'], [r'(\d+\.?\d*)mm[^\d]*(\d+\.?\d*)mm'],
                            [])
    df['overall_diameter'] = [float(x) for x in d]
    df['overall_length'] = [float(x) for x in le]
    del df['外形寸法']

    weight: List[float] = []
    for f in df['質量']:
        result = regex(f, r'([\d,]+)g')
        if len(result) > 0:
            weight.append(int(result[0].replace(',', '')))
        else:
            weight.append(-1)
    df['weight'] = weight
    del df['質量']

    df['price'] = 0  # なぜか記載がなかったので

    df['mount'] = 'マイクロフォーサーズ'
    del df['レンズ構成']
    del df['マウント']
    df['url'] = df['リンク']
    del df['リンク']

    return df
Exemplo n.º 6
0
def get_panasonic_lens_list(scraping: IScrapingService) -> DataFrame:
    # 情報ページを開く
    page = scraping.get_page('https://panasonic.jp/dc/comparison.html',
                             cache=False)

    # tableタグからデータを収集する
    df1 = DataFrame()
    for table_element in page.find_all('table'):
        if 'LUMIX G' not in table_element.full_text:
            continue
        df1['レンズ名'] = [
            cleansing(x.text) for x in table_element.find_all('th p')
        ]
        df1['URL'] = [
            'https://panasonic.jp' + x.attrs['href']
            for x in table_element.find_all('th a')
        ]
        for tr_element in table_element.find_all('tbody > tr'):
            key = cleansing(tr_element.find('th').text)
            value = [cleansing(x.text) for x in tr_element.find_all('td')]
            df1[key] = value
        break

    df2 = DataFrame()
    for table_element in page.find_all('table'):
        if 'LUMIX S' not in table_element.full_text:
            continue
        df2['レンズ名'] = [
            cleansing(x.text) for x in table_element.find_all('th p')
        ]
        df2['URL'] = [
            'https://panasonic.jp' + x.attrs['href']
            for x in table_element.find_all('th a')
        ]
        for tr_element in table_element.find_all('tbody > tr'):
            if tr_element.find('th') is None:
                continue
            key = cleansing(tr_element.find('th').text)
            value = [cleansing(x.text) for x in tr_element.find_all('td')]
            df2[key] = value
        # なぜか、「最大径×全長」だけ記述位置が異なるので対策
        key = cleansing(table_element.find('tbody > th').text)
        value = [
            cleansing(x.text) for x in table_element.find_all('tbody > td')
        ]
        df2[key] = value
        break

    # データを加工し、結合できるように整える
    df1 = convert_columns(
        df1, {
            'レンズ名': 'name',
            'URL': 'url',
            '品番': 'product_number',
            '35mm判換算焦点距離': 'focal_length',
            '最短撮影距離': 'min_focus_distance',
            '最大撮影倍率': 'max_photographing_magnification',
            '手ブレ補正': 'has_image_stabilization',
            'フィルターサイズ': 'filter_diameter',
            '最大径×全長': 'overall_size',
            '質量': 'weight',
            '防塵・防滴': 'is_drip_proof',
            'メーカー希望小売価格': 'price',
        }, [
            'レンズ構成',
            '絞り羽根 / 形状',
            '最小絞り値',
            'レンズコーティング',
            '対角線画角',
            'レンズキャップ',
        ])
    df1['mount'] = 'マイクロフォーサーズ'

    df2 = convert_columns(
        df2, {
            'レンズ名': 'name',
            'URL': 'url',
            '品番': 'product_number',
            '焦点距離': 'focal_length',
            '撮影距離範囲': 'min_focus_distance',
            '手ブレ補正': 'has_image_stabilization',
            'フィルター径': 'filter_diameter',
            '防塵・防滴': 'is_drip_proof',
            '最大撮影倍率': 'max_photographing_magnification',
            '最大径×全長': 'overall_size',
            '質量': 'weight',
            'メーカー希望小売価格': 'price'
        }, [
            'レンズ構成',
            'マウント',
            '絞り羽根 / 形状',
            '開放絞り',
            '最小絞り',
        ])
    df2['mount'] = 'ライカL'

    # 結合
    df = pandas.concat([df1, df2])

    # 変換用に整形
    df['maker'] = 'Panasonic'

    # focal_length
    w, t = extract_numbers(df['focal_length'],
                           [r'(\d+)mm~(\d+)mm', r'(\d+)-(\d+)mm'],
                           [r'(\d+)mm'])
    df['wide_focal_length'] = [int(x) for x in w]
    df['telephoto_focal_length'] = [int(x) for x in t]
    del df['focal_length']

    # f_number
    w, t = extract_numbers(df['name'], [r'F(\d+\.?\d*)-(\d+\.?\d*)'],
                           [r'F(\d+\.?\d*)'])
    df['wide_f_number'] = [float(x) for x in w]
    df['telephoto_f_number'] = [float(x) for x in t]

    # min_focus_distance
    w, t = extract_numbers(
        df['min_focus_distance'],
        [r'(\d+\.?\d+)m / (\d+\.?\d+)m', r'(\d+\.?\d+)m~∞.*(\d+\.?\d+)m~∞'],
        [r'(\d+\.?\d+)m', r'(\d+\.?\d+)m~∞'])
    df['wide_min_focus_distance'] = [int(Decimal(x).scaleb(3)) for x in w]
    df['telephoto_min_focus_distance'] = [int(Decimal(x).scaleb(3)) for x in t]
    del df['min_focus_distance']

    # max_photographing_magnification
    m: List[float] = []
    for record in df.to_records():
        temp = record['max_photographing_magnification'].replace('倍', '')
        if record['mount'] == 'マイクロフォーサーズ':
            m.append(float(Decimal(temp) * 2))
        else:
            m.append(float(temp))
    df['max_photographing_magnification'] = m

    # filter_diameter
    filter_diameter: List[float] = []
    for f in df['filter_diameter']:
        result = regex(f, r'(\d+.?\d*)mm')
        if len(result) > 0:
            filter_diameter.append(float(result[0]))
        else:
            filter_diameter.append(-1)
    df['filter_diameter'] = filter_diameter

    # is_drip_proof
    df['is_drip_proof'] = df['is_drip_proof'].map(lambda x: x == '○')

    # has_image_stabilization
    df['has_image_stabilization'] = df['has_image_stabilization'].map(
        lambda x: x != '-')

    # is_inner_zoom
    i: List[bool] = []
    for record in df.to_records():
        if record['wide_focal_length'] == record['telephoto_focal_length']:
            i.append(True)
            continue
        if record['product_number'] in [
                'H-F007014', 'H-E08018', 'H-PS45175', 'S-E70200', 'S-R70200'
        ]:
            i.append(True)
            continue
        i.append(False)
    df['is_inner_zoom'] = i

    # overall_diameter, overall_length
    d, le = extract_numbers(df['overall_size'],
                            [r'(\d+\.?\d*)mm[^\d]*(\d+\.?\d*)mm'], [])
    df['overall_diameter'] = [float(x) for x in d]
    df['overall_length'] = [float(x) for x in le]
    del df['overall_size']

    # weight
    weight: List[float] = []
    for f in df['weight']:
        result = regex(f, r'([\d,]+)g')
        if len(result) > 0:
            weight.append(int(result[0].replace(',', '')))
        else:
            weight.append(-1)
    df['weight'] = weight

    # price
    price: List[float] = []
    for f in df['price']:
        result = regex(f, r'([\d,]+) *円')
        if len(result) > 0:
            price.append(int(result[0].replace(',', '')))
        else:
            price.append(-1)
    df['price'] = price
    return df
Exemplo n.º 7
0
def get_leica_lens_list(scraping: IScrapingService) -> DataFrame:
    # レンズのURL一覧を取得する
    lens_list: List[Tuple[str, str]] = []
    page_index = 0
    while True:
        page_url = f'https://leica-camera.com/en-US/photography/lenses/sl?field_pim_categories=&page={page_index}'
        page = scraping.get_page(page_url, cache=False)
        article_elements = page.find_all('article.content-teasers-item')
        if len(article_elements) == 0:
            break
        page_index += 1
        for article_element in article_elements:
            lens_name = article_element.find('div.field--name-external-field-main-product-title'
                                             ).full_text
            if 'SL' not in lens_name:
                continue
            if 'Leica' not in lens_name:
                continue
            if 'hood' in lens_name:
                continue
            lens_url = 'https://leica-camera.com' + article_element.find('a.node-link').attrs['href']
            lens_list.append((lens_name, lens_url))

    # レンズの生情報を取得する
    lens_raw_data_list: List[Dict[str, any]] = []
    for lens_name, lens_url in lens_list:
        page = scraping.get_page(lens_url)
        temp: Dict[str, str] = {'レンズ名': lens_name, 'URL': lens_url}
        for tr_element in page.find_all('tr'):
            th_elements = tr_element.find_all('th')
            td_elements = tr_element.find_all('td')
            if len(td_elements) >= 2:
                temp[td_elements[0].text] = td_elements[1].text
            elif len(th_elements) >= 1 and len(td_elements) >= 1:
                temp[th_elements[0].text] = td_elements[0].text
            else:
                continue
        lens_raw_data_list.append(temp)
    df = DataFrame.from_records(lens_raw_data_list)

    # 変換用に整形
    df['maker'] = 'LEICA'
    df['mount'] = 'ライカL'
    df = convert_columns(df, {
        'レンズ名': 'name',
        'URL': 'url',
        'Order Number': 'Order number',
        'Largest scale': 'Largest reproduction ratio',
        'Filter thread': 'Filter mount',
        'Length': 'Length to bayonet mount',
        'Diameter': 'Largest diameter',
        'Black, anodized': 'Order number',
        'Length to bayonet flange': 'Length to bayonet mount',
        'Focus range': 'Working range',
    }, [
        'Field angle (diagonal, horizontal, vertical)',
        'Number of lenses/groups',
        'Number of asph. surfaces / lenses',
        'Entrance pupil position',
        'Smallest object field',
        'Setting/function',
        'Aperture setting range',
        'Lowest value',
        'Bayonet/sensor format',
        'View angle (diagonal/horizontal/vertical) Full-frame (24 × 36 mm)',
        'Number of lenses/assemblies',
        'Number of aspherical surfaces',
        'Position of the entrance pupil before the bayonet',
        'Setting',
        'Setting/Function',
        'Smallest aperture',
        'Bayonet',
        'Lens hood',
        'Full-frame (24 × 36 mm)',
        'Angle of view (diagonal, horizontal, vertical)',
        'Number of elements/groups',
        'Position of entrance pupil',
        'Smallest value',
        'Number of aspherical lenses',
    ])

    # product_number
    df['product_number'] = df['Order number'].map(lambda x: str(x).replace(' ', ''))
    del df['Order number']

    # wide_focal_length, telephoto_focal_length
    w, t = extract_numbers(df['name'], [r'SL (\d+)-(\d+)mm f', r'SL(\d+)-(\d+) f', r'SL (\d+)-(\d+) f'],
                           [r'SL(\d+) f', r'SL (\d+) f', r'SL 1:\d+\.?\d*/(\d+)'])
    df['wide_focal_length'] = w
    df['telephoto_focal_length'] = t

    # wide_f_number, telephoto_f_number
    w, t = extract_numbers(df['name'], [r'f/(\d+\.?\d*)-(\d+\.?\d*)'],
                           [r'f/(\d+\.?\d*)', r'SL 1:(\d+\.?\d*)/\d+'])
    df['wide_f_number'] = w
    df['telephoto_f_number'] = t

    # wide_min_focus_distance, telephoto_min_focus_distance
    w: List[int] = []
    t: List[int] = []
    for record in list(df['Working range'].values):
        match_result = regex(record, r'(\d+,\d*) m to infinity.+(\d+,\d*) m to infinity')
        if len(match_result) > 0:
            w.append(int(Decimal(match_result[0].replace(',', '.')).scaleb(3)))
            t.append(int(Decimal(match_result[1].replace(',', '.')).scaleb(3)))
            continue
        match_result = regex(record, r'(\d+\.?\d*) m to infinity')
        if len(match_result) > 0:
            w.append(int(Decimal(match_result[0]).scaleb(3)))
            t.append(int(Decimal(match_result[0]).scaleb(3)))
            continue
        match_result = regex(record, r'∞ to (\d+\.?\d*) m')
        if len(match_result) > 0:
            w.append(int(Decimal(match_result[0]).scaleb(3)))
            t.append(int(Decimal(match_result[0]).scaleb(3)))
            continue
        match_result = regex(record, r'(\d+\.?\d*)mm to infinity')
        if len(match_result) > 0:
            w.append(int(match_result[0]))
            t.append(int(match_result[0]))
            continue
        w.append(0)
        t.append(0)
    df['wide_min_focus_distance'] = w
    df['telephoto_min_focus_distance'] = t
    del df['Working range']

    # max_photographing_magnification
    m: List[float] = []
    for record in df.iterrows():
        series: Series = record[1]
        denominator = regex(series['Largest reproduction ratio'].replace(',', '.'), r'1:(\d+\.?\d*)')
        m.append(float((Decimal('1') / Decimal(denominator[0])).quantize(Decimal('0.01'))))
    df['max_photographing_magnification'] = m
    del df['Largest reproduction ratio']

    # filter_diameter
    df['filter_diameter'] = df['Filter mount'].map(lambda x: int(str(x).replace('E', '')))
    del df['Filter mount']

    # is_drip_proof, has_image_stabilization, is_inner_zoom
    is_drip_proof = []
    has_image_stabilization = []
    is_inner_zoom = []
    for record in df.iterrows():
        record = record[1]
        is_drip_proof.append(False)
        if record['O.I.S. Performance as per CIPA']:
            has_image_stabilization.append(True)
        else:
            has_image_stabilization.append(False)
        if record['name'] in ['Leica APO-Vario-Elmarit-SL 90-280 f/2.8-4'] or \
                record['wide_focal_length'] == record['telephoto_focal_length']:
            is_inner_zoom.append(True)
        else:
            is_inner_zoom.append(False)
    df['is_drip_proof'] = is_drip_proof
    df['has_image_stabilization'] = has_image_stabilization
    df['is_inner_zoom'] = is_inner_zoom
    del df['O.I.S. Performance as per CIPA']

    # overall_diameter, overall_length
    overall_diameter = []
    overall_length = []
    for record in df.iterrows():
        record = record[1]
        if '/' in record['Largest diameter']:
            diameter = regex(record['Largest diameter'].replace('\u2009', ' '), r'(\d+\.?\d*)/\d+ mm')
        elif ':' in record['Largest diameter']:
            diameter = regex(record['Largest diameter'].replace('\u2009', ' '), r': (\d+\.?\d*) mm')
        else:
            diameter = regex(record['Largest diameter'].replace('\u2009', ' '), r'(\d+\.?\d*) mm')
        if '/' in record['Length to bayonet mount']:
            length = regex(record['Length to bayonet mount'].replace('\u2009', ' '), r'(\d+\.?\d*)/\d+ mm')
        elif ':' in record['Length to bayonet mount']:
            length = regex(record['Length to bayonet mount'].replace('\u2009', ' '), r': (\d+\.?\d*) mm')
        else:
            length = regex(record['Length to bayonet mount'].replace('\u2009', ' '), r'(\d+\.?\d*) mm')
        overall_diameter.append(float(diameter[0]))
        overall_length.append(float(length[0]))
    df['overall_diameter'] = overall_diameter
    df['overall_length'] = overall_length
    del df['Largest diameter']
    del df['Length to bayonet mount']

    # weight
    weight: List[float] = []
    for i in range(0, len(df)):
        f = df['Weight'].values[i].replace('\u2009', ' ')
        result = regex(f, r'([\d.]+) g')
        if len(result) > 0:
            result2 = regex(f, r'([\d.]+)/[\d.]+ g')
            if len(result2) > 0:
                weight.append(int(result2[0].replace('.', '')))
            else:
                weight.append(int(result[0].replace('.', '')))
        else:
            weight.append(int(f))
    df['weight'] = weight
    del df['Weight']

    df['price'] = 0
    return df
Exemplo n.º 8
0
def get_cosina_lens_list(scraping: IScrapingService) -> DataFrame:
    # レンズのURL一覧を取得する
    lens_list: List[Tuple[str, str]] = []
    page = scraping.get_page(
        'http://www.cosina.co.jp/seihin/voigtlander/mft-mount/index.html',
        encoding='cp932',
        cache=False)
    for a_element in page.find_all('td > a'):
        lens_name = a_element.find('img').attrs['alt']
        lens_url = 'http://www.cosina.co.jp/seihin/voigtlander/mft-mount/' + a_element.attrs[
            'href']
        if 'mm' in lens_name and 'mft' in a_element.attrs['href']:
            if lens_name not in lens_name_table:
                print(lens_name)
                raise Exception('未対応のレンズが含まれています')
            lens_list.append((lens_name_table[lens_name], lens_url))

    # レンズの情報を取得する
    temp_list: List[Dict[str, any]] = []
    for lens_name, lens_url in lens_list:
        page = scraping.get_page(lens_url, encoding='cp932')
        temp: Dict[str, str] = {'レンズ名': lens_name, 'URL': lens_url}
        for tr_element in page.find_all('tr'):
            td_elements = tr_element.find_all('td')
            if len(td_elements) < 2:
                continue
            if 'bgcolor' not in td_elements[0].attrs:
                continue
            if td_elements[0].full_text == '' or td_elements[1].full_text == '':
                continue
            temp[td_elements[0].full_text] = td_elements[1].full_text
        for h2_element in page.find_all('h2'):
            text = h2_element.text
            if '希望小売価格' in text:
                temp['希望小売価格'] = text.replace('\n', '')
        temp_list.append(temp)
    df = DataFrame.from_records(temp_list)

    # 変換用に整形
    df['maker'] = 'COSINA'

    df['name'] = df['レンズ名']
    del df['レンズ名']

    df['product_number'] = ''

    w, t = extract_numbers(df['焦点距離'], [], [r'(\d+\.?\d*)mm'])
    df['wide_focal_length'] = [int(Decimal(x) * 2) for x in w]
    df['telephoto_focal_length'] = [int(Decimal(x) * 2) for x in w]
    del df['焦点距離']
    del df['画角']
    del df['レンズ構成']

    w, t = extract_numbers(df['name'], [], [r'F(\d+\.?\d*)'])
    df['wide_f_number'] = [float(x) for x in w]
    df['telephoto_f_number'] = [float(x) for x in t]
    del df['口径比']
    del df['最小絞り']
    del df['絞り羽根枚数']

    w, t = extract_numbers(df['最短撮影距離'], [], [r'(\d+\.?\d+)m'])
    df['wide_min_focus_distance'] = [int(Decimal(x).scaleb(3)) for x in w]
    df['telephoto_min_focus_distance'] = [int(Decimal(x).scaleb(3)) for x in t]
    del df['最短撮影距離']

    m: List[str] = []
    for record in df['最大撮影倍率']:
        m.append(regex(record, r'1(:|:)(\d+\.?\d*)')[1])
    df['max_photographing_magnification'] = [
        float(str((Decimal(1.0) / Decimal(x)).quantize(Decimal('0.01'))))
        for x in m
    ]
    del df['最大撮影倍率']

    m: List[int] = []
    for record in df['フィルターサイズ']:
        m.append(int(regex(record, r'(\d+)mm')[0]))
    df['filter_diameter'] = m
    del df['フィルターサイズ']

    df['is_drip_proof'] = False
    df['has_image_stabilization'] = False
    df['is_inner_zoom'] = True

    di, le = extract_numbers(df['最大径×全長'], [r'φ(\d+\.?\d*)×(\d+\.?\d*)mm'], [])
    df['overall_diameter'] = di
    df['overall_length'] = le
    del df['最大径×全長']

    weight, _ = extract_numbers(df['重量'], [], [r'(\d+)g'])
    df['weight'] = weight
    del df['重量']

    price, _ = extract_numbers(df['希望小売価格'], [], [r'¥([\d,]+)'])
    df['price'] = [int(x.replace(',', '')) for x in price]
    del df['希望小売価格']

    df['mount'] = 'マイクロフォーサーズ'
    df['url'] = df['URL']
    del df['レンズフード']
    del df['その他:']
    del df['URL']

    return df