예제 #1
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=Touchscreen+Monitors' + \
          '&categoryId=514&storeId=1&nodeId=7721&parentCategoryId=19' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('touchscreenmonitor', df)
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=Keyboards' + \
          '&categoryId=63&storeId=1&nodeId=8650&parentCategoryId=234' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('keyboards', df)
예제 #3
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=Desktop+Memory&' + \
          'categoryId=147&storeId=1&nodeId=7611&parentCategoryId=17&' + \
          'isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('desktopmemory', df)
예제 #4
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=HDMI+Cables' + \
          '&categoryId=2809&storeId=1&nodeId=16742&parentCategoryId=184' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('hdmicables', df)
예제 #5
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'https://m.newegg.com/ProductList?description=FHLEsA70dKzKoDqR2lBeblLSvjuwGkdtB%252fdjJTC%252f8VU%253d&storeid=1&categoryid=-1&nodeid=7627&storetype=2&subcategoryid=280&brandid=-1&nvalue=100007627&showseealldeals=False&itemcount=0&issubcategory=true&level=3' 

headers = {'User-Agent': 'Mozilla/5.0 (Linux; U; Android 4.0.3; ko-kr; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30'}

pg1 = requests.get(baseurl, headers=headers).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('intelmotherboard', df)
예제 #6
0
파일: ssd.py 프로젝트: MathieuK57/newegg
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=Internal+SSD&' + \
          'categoryId=636&storeId=1&nodeId=8120&parentCategoryId=119&' + \
          'isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('ssd', df)
예제 #7
0
파일: gpu.py 프로젝트: MathieuK57/newegg
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=' + \
          'Desktop+Graphics+Cards&categoryId=48&storeId=1&nodeId=7709' + \
          '&parentCategoryId=38&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('gpus', df)
예제 #8
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=Mice' + \
          '&categoryId=65&storeId=1&nodeId=8652&parentCategoryId=234' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('mice', df)
예제 #9
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=Web+Cams' + \
          '&categoryId=152&storeId=1&nodeId=7794&parentCategoryId=29' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('webcams', df)
예제 #10
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=' + \
          'Server%2fWorkstation+Systems&categoryId=386&storeId=1' + \
          '&nodeId=7940&parentCategoryId=271&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('serversystems', df)
예제 #11
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=' + \
          'Server%2fWorkstation+Systems&categoryId=386&storeId=1' + \
          '&nodeId=7940&parentCategoryId=271&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('serversystems', df)
예제 #12
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=Xbox+360+Games&' + \
          'categoryId=516&storeId=8&nodeId=8091&parentCategoryId=323&' + \
          'isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('xbox360', df)
예제 #13
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=Firewalls' + \
          '&categoryId=529&storeId=1&nodeId=8177&parentCategoryId=281' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('firewalls', df)
예제 #14
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=LED+TV&categoryId=798' + \
          '&storeId=10&nodeId=9260&parentCategoryId=264&isSubCategory=true' + \
          '&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('ledtv', df)
예제 #15
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=Speakers' + \
          '&categoryId=60&storeId=1&nodeId=7679&parentCategoryId=37' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('speakers', df)
예제 #16
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=Modems' + \
          '&categoryId=18&storeId=1&nodeId=8185&parentCategoryId=281' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('modems', df)
예제 #17
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=Power+Supplies' + \
          '&categoryId=58&storeId=1&nodeId=7657&parentCategoryId=32' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('psu', df)
예제 #18
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=Processors+-+' + \
          'Desktops&categoryId=343&storeId=1&nodeId=7671&' + \
          'parentCategoryId=34&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('cpus', df)
예제 #19
0
파일: mice.py 프로젝트: MathieuK57/newegg
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=Mice' + \
          '&categoryId=65&storeId=1&nodeId=8652&parentCategoryId=234' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('mice', df)
예제 #20
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=Internal+Hard+Drives' + \
          '&categoryId=14&storeId=1&nodeId=7603&parentCategoryId=15' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('hdd', df)
예제 #21
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=Firewalls' + \
          '&categoryId=529&storeId=1&nodeId=8177&parentCategoryId=281' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('firewalls', df)
예제 #22
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=Internal+SSD&' + \
          'categoryId=636&storeId=1&nodeId=8120&parentCategoryId=119&' + \
          'isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('ssd', df)
예제 #23
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=Case+Fans' + \
          '&categoryId=573&storeId=1&nodeId=7998&parentCategoryId=11' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('casefans', df)
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=Computer+Cases' + \
          '&categoryId=7&storeId=1&nodeId=7583&parentCategoryId=9' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('computercases', df)
예제 #25
0
파일: cpu.py 프로젝트: MathieuK57/newegg
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=Processors+-+' + \
          'Desktops&categoryId=343&storeId=1&nodeId=7671&' + \
          'parentCategoryId=34&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('cpus', df)
예제 #26
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=' + \
          'Desktop+Graphics+Cards&categoryId=48&storeId=1&nodeId=7709' + \
          '&parentCategoryId=38&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('gpus', df)
예제 #27
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=LCD+TV&categoryId=411' + \
          '&storeId=10&nodeId=7719&parentCategoryId=264&isSubCategory=true&' + \
          'categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('lcdtv', df)
예제 #28
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = (
    "http://m.newegg.com/ProductList?description=Wireless+Adapters"
    + "&categoryId=31&storeId=1&nodeId=10074&parentCategoryId=41"
    + "&isSubCategory=true&categoryType=1"
)

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ["%s&Page=%s" % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData("wirelessadapters", df)
예제 #29
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=' + \
          'External+Hard+Drives&categoryId=414&storeId=1&nodeId=7601' + \
          '&parentCategoryId=15&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('externalhdd', df)
예제 #30
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=Plasma+TV&' + \
          'categoryId=385&storeId=10&nodeId=7713&parentCategoryId=264&' + \
          'isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('plasmatv', df)
예제 #31
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=Mac+Memory' + \
          '&categoryId=551&storeId=1&nodeId=7968&parentCategoryId=17' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('macmemory', df)
예제 #32
0
파일: ps3.py 프로젝트: MathieuK57/newegg
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=PS3%20Games' + \
          '&categoryId=545&storeId=8&nodeId=8103&parentCategoryId=282' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('ps3', df)
예제 #33
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=' + \
          'CPU+Fans+%26+Heatsinks&categoryId=574&storeId=1&nodeId=8000' + \
          '&parentCategoryId=11&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('cpufans', df)
예제 #34
0
파일: lcdtv.py 프로젝트: MathieuK57/newegg
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=LCD+TV&categoryId=411' + \
          '&storeId=10&nodeId=7719&parentCategoryId=264&isSubCategory=true&' + \
          'categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('lcdtv', df)
예제 #35
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=Intel+Motherboards' + \
          '&categoryId=280&storeId=1&nodeId=7627&parentCategoryId=20' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('intelmotherboard', df)
예제 #36
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=' + \
          'CPU+Fans+%26+Heatsinks&categoryId=574&storeId=1&nodeId=8000' + \
          '&parentCategoryId=11&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('cpufans', df)
예제 #37
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=Desktop+Memory&' + \
          'categoryId=147&storeId=1&nodeId=7611&parentCategoryId=17&' + \
          'isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('desktopmemory', df)
예제 #38
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=Web+Cams' + \
          '&categoryId=152&storeId=1&nodeId=7794&parentCategoryId=29' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('webcams', df)
예제 #39
0
파일: laptop.py 프로젝트: MathieuK57/newegg
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=Laptops+%2f+' + \
          'Notebooks&categoryId=32&storeId=3&nodeId=6740&' + \
          'parentCategoryId=223&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('laptop', df)
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=HDMI+Cables' + \
          '&categoryId=2809&storeId=1&nodeId=16742&parentCategoryId=184' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('hdmicables', df)
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=USB+Flash+Drives' + \
          '&categoryId=522&storeId=1&nodeId=7960&parentCategoryId=324' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('usbflashdrives', df)
예제 #42
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=Wireless+Adapters' + \
          '&categoryId=31&storeId=1&nodeId=10074&parentCategoryId=41' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('wirelessadapters', df)
예제 #43
0
파일: psu.py 프로젝트: econpy/newegg
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = (
    "http://m.newegg.com/ProductList?description=Power+Supplies"
    + "&categoryId=58&storeId=1&nodeId=7657&parentCategoryId=32"
    + "&isSubCategory=true&categoryType=1"
)

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ["%s&Page=%s" % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData("psu", df)
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=AMD+Motherboards' + \
          '&categoryId=22&storeId=1&nodeId=7625&parentCategoryId=20' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('amdmotherboard', df)
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=LCD+Monitors' + \
          '&categoryId=20&storeId=1&nodeId=7617&parentCategoryId=19' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('lcdmonitor', df)
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=Internal+Hard+Drives' + \
          '&categoryId=14&storeId=1&nodeId=7603&parentCategoryId=15' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('hdd', df)
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=Desktop+NAS' + \
          '&categoryId=124&storeId=1&nodeId=8028&parentCategoryId=241' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('desktopnas', df)
예제 #48
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'https://m.newegg.com/productlist?description=nVjkbn88TPpxbYCO44j7AJ7lKvYiECglej4lhl3FXQM%253d&storeid=3&categoryid=-1&nodeid=6740&storetype=2&subcategoryid=32&brandid=-1&nvalue=100006740&showseealldeals=False&itemcount=0&issubcategory=true&level=3'

headers = {
    'User-Agent':
    'Mozilla/5.0 (Linux; U; Android 4.0.3; ko-kr; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30'
}
pg1 = requests.get(baseurl, headers=headers).content

root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('laptop', df)
예제 #49
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=Mac+Memory' + \
          '&categoryId=551&storeId=1&nodeId=7968&parentCategoryId=17' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('macmemory', df)
예제 #50
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=Plasma+TV&' + \
          'categoryId=385&storeId=10&nodeId=7713&parentCategoryId=264&' + \
          'isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('plasmatv', df)
예제 #51
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages,BoolToInt,getPIDS,getData,insertData

baseurl = 'http://m.newegg.com/ProductList?description=Laptop+Hard+Drives' + \
          '&categoryId=380&storeId=1&nodeId=7605&parentCategoryId=15' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('laptophdd', df)
예제 #52
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = (
    "http://m.newegg.com/ProductList?description=Computer+Cases"
    + "&categoryId=7&storeId=1&nodeId=7583&parentCategoryId=9"
    + "&isSubCategory=true&categoryType=1"
)

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ["%s&Page=%s" % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData("computercases", df)
예제 #53
0
import lxml.html
import requests
from neweggpy.nefuncs import IterPages, BoolToInt, getPIDS, getData, insertData

baseurl = 'http://m.newegg.com/ProductList?description=PS3%20Games' + \
          '&categoryId=545&storeId=8&nodeId=8103&parentCategoryId=282' + \
          '&isSubCategory=true&categoryType=1'

pg1 = requests.get(baseurl).content
root1 = lxml.html.fromstring(pg1)
page_count = IterPages(root1)
URLs = ['%s&Page=%s' % (baseurl, pgnum) for pgnum in range(1, page_count + 1)]

# FETCH AND PARSE THE DATA
pids = getPIDS(URLs, root1)
df = getData(pids)

# PUT DATA IN DATABASE
insertData('ps3', df)