def test_ca_tokens(self):
results = API("ca").retrieve_all()
for part, part_data in results.items():
for p in part_data:
self.assertIsInstance(p, part_classes[part])
self.assertIsNotNone(p.brand)
self.assertIsNotNone(results.to_json())
def __init__(self):
self.api = API()
self.raw_data = [
self.api.retrieve("cpu"),
self.api.retrieve("video-card"),
self.api.retrieve("power-supply"),
self.api.retrieve("memory"),
self.api.retrieve("motherboard"),
self.api.retrieve("internal-hard-drive"),
self.api.retrieve("case")
]
def test_api_default_init(self):
api = API()
self.assertEqual(api.region, "us")
self.assertEqual(api.supported_parts, {"cpu", "cpu-cooler", "motherboard", "memory", "internal-hard-drive",
"video-card", "power-supply", "case", "case-fan", "fan-controller",
"thermal-paste", "optical-drive", "sound-card", "wired-network-card",
"wireless-network-card", "monitor", "external-hard-drive", "headphones",
"keyboard", "mouse", "speakers", "ups"})
self.assertEqual(api.supported_regions, {"au", "be", "ca", "de", "es", "fr", "se",
"in", "ie", "it", "nz", "uk", "us"})
self.assertEqual(api.region, 'us')
def test_api_modify_multithreading(self):
api = API()
self.assertTrue(api.multithreading)
self.assertTrue(api._handler._multithreading)
api.set_multithreading(False)
self.assertFalse(api._handler._multithreading)
self.assertFalse(api.multithreading)
api.set_multithreading(True)
self.assertTrue(api.multithreading)
self.assertTrue(api._handler._multithreading)
class DataHandler:
def __init__(self):
self.api = API()
self.raw_data = [
self.api.retrieve("cpu"),
self.api.retrieve("video-card"),
self.api.retrieve("power-supply"),
self.api.retrieve("memory"),
self.api.retrieve("motherboard"),
self.api.retrieve("internal-hard-drive"),
self.api.retrieve("case")
]
def format_data(self):
"""Convert self.raw_data into json, get the data we need, and return a list with the data on various parts"""
json_data = [None, None, None, None, None, None, None]
for i in range(len(self.raw_data)):
top_level_key = next(self.raw_data[i].keys())
json_data[i] = json.loads(
self.raw_data[i].to_json())[top_level_key]
formatted_data = {
"cpu": [],
"gpu": [],
"psu": [],
"ram": [],
"mobo": [],
"hdd": [],
"case": []
}
key = next(formatted_data).key()
for i in range(0, len(json_data)):
formatted_data[key].append({
f"{json_data[i]['brand']} {json_data[i]['model']}":
f"${json_data[i]['price'][1]}"
})
return formatted_data
def get_part_data(self, part):
"""Retrieve the data on a particular part out of the mess that is the return of format_data
Parts must be cpu, gpu, psu, ram, mobo, hdd, or case"""
try:
return self.format_data()[part]
except KeyError:
raise InvalidPart(
"User entered a part that isn\'t supported/doesn\'t exist")
def test_api_init_exception(self):
with self.assertRaises(UnsupportedRegion) as excinfo:
_ = API('oc')
assert 'Region \'oc\' is not supported for this API!' in str(
excinfo.exception)
def test_api_region_init(self):
api = API('de')
self.assertEqual(api.region, 'de')
mode = "both" # cpu, gpu, both
priceSource = "ul" # pcpp, ul
cpuBlacklists = re.compile(r'i[3-9]-[2-9]|i[3-9]-\d{3}$|FX-|Xeon')
gpuBlacklists = re.compile(
r'Quadro|GT |HD |GTX [3-9]|R[5-9]|Titan|GTX 10|RX 4')
colorMap = {
'AMD': 'rgb(255, 65, 54)',
'Intel': 'rgb(93, 164, 214)',
'Nvidia': 'rgb(44, 160, 101)'
}
os.chdir(os.path.dirname(os.path.realpath(__file__)))
print(os.path.abspath("."))
api = API()
date = datetime.today().strftime('%Y-%m-%d')
if mode != "gpu":
if scrapeWeb:
try:
os.remove('ulCPU.json')
except OSError:
pass
os.system('scrapy crawl ulCPU -o ulCPU.json')
if priceSource == "pcpp":
cpu_data = api.retrieve("cpu")
pcppCPUs = {}
print(len(cpu_data["cpu"]), "pcpartpicker cpus")
print(cpu_data["cpu"][0])
class PCParts:
log.basicConfig(filename='debug.log', level=log.DEBUG)
def __init__(self, debug=False, region='us'):
self.pcpp_api = API(region)
self.parts = {}
self.path = './Parts_Scraper/pickle/'
log.debug("PCParts object init")
#self.refresh_data()
def set_region(self, new_region):
'''
Set region per pcpartspicker API requirements.
'''
if new_region in self.pcpp_api.supported_regions:
self.pcpp_api.set_region(new_region)
return
def refresh_data(self):
'''
Refresh temp storage cache
'''
self.__load_part_data(refresh=True)
def get_JSON(self, part='cpu'):
if part not in self.pcpp_api.supported_parts:
raise Exception("Argument 'part' is not a supported part.")
return
if not self.parts: self.__load_part_data()
js = self.parts[part].to_json(orient='split')
return js
def get_components(self):
'''
Returns dictionary of parts and their components as an array
'''
if not self.parts:
log.debug("Loading data for all parts...")
self.__load_part_data("all")
log.debug("Getting part components...")
components = {}
for part, df in self.parts.items():
components[part] = [c for c in df.columns]
return components
def get_quantities(self):
'''
List out all parts and there quantities
'''
if not self.parts: self.__load_part_data("all")
log.debug("Getting part quantities...")
num_parts = 0
quantities = {}
for part, df in self.parts.items():
quantities[part] = len(df.index)
num_parts += len(df.index)
log.debug("Quantity of parts: " + str(num_parts))
return quantities
def __load_part_data(self, refresh=False):
'''
Loads all part data. Refresh=true will refresh temp data storage cache
'''
if refresh: load_type = "Refreshing"
else: load_type = "Loading"
log.debug("{0} data...".format(load_type))
for part in self.pcpp_api.supported_parts:
self.__add_part_df_to_dict(part, refresh)
log.debug("{0} complete.".format(load_type))
return
def __add_part_df_to_dict(self, part="all", refresh=False):
'''
Convert part objects retrieved from PcPartsPicker API to dataframe
and save as temp data storage(pickle).
If refresh is specified, overwrite pickle storage files
'''
path = self.path + part + '.pkl'
if os.path.exists(path) and not refresh:
df = pd.read_pickle(path)
else:
parts_dict = defaultdict(list)
retrieved_parts = self.pcpp_api.retrieve(part)[part]
for p in retrieved_parts:
for key, comp in vars(p).items():
parts_dict[key].append(comp)
df = pd.DataFrame.from_dict(parts_dict)
df.to_pickle(path)
self.parts[part] = df
return
def test_api_multithreading_kwd(self):
api = API(multithreading=False)
self.assertFalse(api.multithreading)
self.assertFalse(api._handler.multithreading)
from pcpartpicker import API
import json
from PCDatabase import PCDatabase
api = API('us')
data = {}
dataEntity = {
'cpu': ['brand', 'model', 'cores', 'tdp', 'integrated_graphics'],
'motherboard':
['brand', 'model', 'socket', 'form_factor', 'ram_slots', 'max_ram'],
'internal-hard-drive':
['brand', 'model', 'capacity', 'storage_type', 'form_factor'],
'video-card': ['brand', 'model', 'chipset', 'vram'],
'power-supply': ['brand', 'model', 'form_factor', 'wattage'],
'monitor':
['brand', 'model', 'size', 'resolution', 'refresh_rate', 'panel_type'],
'case':
['brand', 'model', 'form_factor', 'external_bays', 'internal_bays'],
'memory': ['brand', 'model', 'module_type', 'speed']
}
tableName = {
'cpu': 'cpus',
'motherboard': 'motherboards',
'internal-hard-drive': 'internalharddrives',
'video-card': 'videocards',
'power-supply': 'powersupplies',
'monitor': 'monitors',
'case': 'casepcs',
'memory': 'memories'
}
from pcpartpicker import API
import pandas as pd
import numpy as np
api = API(multithreading=False)
api.set_region('in')
df = api.retrieve('video-card')['video-card']
new = []
for obj in df:
new.append({
'brand':
obj.brand if obj.brand else np.nan,
'model':
obj.model if obj.model else np.nan,
'chipset':
obj.chipset if obj.chipset else np.nan,
'vram':
obj.vram.total if obj.vram else np.nan,
'core_clock':
obj.core_clock.cycles if obj.core_clock else np.nan,
'boost_clock':
obj.boost_clock.cycles if obj.boost_clock else np.nan,
'interface':
obj.interface if obj.interface else np.nan,
'color':
obj.color if obj.color else np.nan
})
new_df = pd.DataFrame(new)
def __init__(self, database: str):
self._connection = sqlite3.connect(database)
self._api = API()
from pcpartpicker import API
import pickle, random
PARTS = None
api = API()
INTEL = 'Intel'
AMD = 'AMD'
CPU_BRANDS = set((AMD, INTEL))
HZ_IN_GHZ = 1_000_000_000
HZ_IN_MHZ = 1_000_000
BYTES_IN_GB = 1_000_000_000
try:
PARTS = pickle.load(open('parts.pkl', 'rb'))
print('parts successfully loaded from local pickle')
except FileNotFoundError:
print('Couldn\'t find parts pickle, downloading...')
PARTS = api.retrieve_all()
pickle.dump(PARTS, open('parts.pkl', 'wb'))
class Computer:
def __init__(self):
self.cpu = None
self.gpu = None
self.mobo = None
self.case = None
self.psu = None
self.memory = {}
self.storage = {}
def test_api_set_region(self):
api = API()
self.assertEqual(api.region, 'us')
api.set_region('nz')
self.assertEqual(api.region, 'nz')
def test_api_set_region_incorrect_region(self):
with self.assertRaises(UnsupportedRegion) as excinfo:
api = API()
api.set_region('oc')
assert 'Region \'oc\' is not supported for this API!' in str(
excinfo.exception)
def parserTest(cls):
base_api = API()
cls.test_data = {}
for region in base_api.supported_regions:
api = API(region)
cls.test_data.update({region: api.retrieve_all()})
from pcpartpicker import API
import pandas as pd
import numpy as np
api = API(multithreading=False)
api.set_region('in')
df = api.retrieve('cpu')['cpu']
new = []
for obj in df:
new.append({'brand': obj.brand if obj.brand else np.nan,
'model': obj.model if obj.model else np.nan,
'cores': obj.cores if obj.cores else np.nan,
'base_clock': obj.base_clock.cycles if obj.base_clock else np.nan,
'boost_clock': obj.base_clock.cycles if obj.boost_clock else np.nan,
'tdp': obj.tdp if obj.tdp else np.nan,
'integrated_graphics': np.nan if obj.integrated_graphics is None else obj.integrated_graphics,
'multithreading': np.nan if obj.multithreading is None else obj.multithreading})
new_df = pd.DataFrame(new)
new_df.to_csv('pcpp-cpu.csv', index=False)
def __init__(self, debug=False, region='us'):
self.pcpp_api = API(region)
self.parts = {}
self.path = './Parts_Scraper/pickle/'
log.debug("PCParts object init")