def create_notebooks_config():
all_notebook_parameters = set()
for notebook_file in [
nbf for nbf in os.listdir('../../data/notebooks')
if nbf.endswith('.pkl')
]:
notebook = sl.pickle_load('../../data/notebooks/' + notebook_file)
for key in notebook.keys():
all_notebook_parameters.add(key.decode('utf-8'))
import codecs
notebookconf = codecs.open('values', 'wb')
computerconf = codecs.open('config/computers.config',
'rb',
encoding='utf-8')
sorted_notebook_parameters = []
for line in computerconf.readlines():
print line.split('|')[0].replace(
'_', '.').strip() in all_notebook_parameters
if line.split('|')[0].replace('_',
'.').strip() in all_notebook_parameters:
sorted_notebook_parameters.append(line)
# print list(all_notebook_parameters)[0].decode('utf-8')
sorted_notebook_parameters.sort()
print sorted_notebook_parameters
notebookconf.write(''.join(sorted_notebook_parameters))
notebookconf.close()
computerconf.close()
def conccomputers(pkl_folder, shops_key, usd_key, grn_key):
computers = [sl.pickle_load(os.path.join(pkl_folder, pkl))
for pkl in [pkl for pkl in os.listdir(pkl_folder) if pkl.endswith('.pkl')]]
pairs = []
for c in computers:
url = c['Устройство.url']
for shop, usd, grn in zip(c[shops_key], c[usd_key], c[grn_key]):
pair = {}
pair['url'], pair['price_usd'], pair['price_grn'], pair['shop'] = url, usd, grn, shop
pairs.append(pair)
return pairs
def save_sqlconf_file(devices_folders, output):
device_files = []
for devices_folder in devices_folders:
device_files += [
os.path.join(devices_folder, file)
for file in os.listdir(devices_folder)
if os.path.isfile(os.path.join(devices_folder, file))
]
devices = [sl.pickle_load(file) for file in device_files]
for device in devices:
print ', '.join([key for key in device.keys() if '3' in key])
def computers(pkl_folder, config):
'''
pkl_path - path to folder with .pkl path; config - path to .config file
Retunrs dicts for each computer
'''
computers = []
config_lines = _config_lines(config)
for pkl in [pkl for pkl in os.listdir(pkl_folder) if pkl.endswith('.pkl')]:
computer = _computer(sl.pickle_load(os.path.join(pkl_folder, pkl)), config_lines)
computer['id'] = pkl.split('.')[0]
computers.append(computer)
return computers
def computers(pkl_folder, config):
'''
pkl_path - path to folder with .pkl path; config - path to .config file
Retunrs dicts for each computer
'''
computers = []
config_lines = _config_lines(config)
for pkl in [pkl for pkl in os.listdir(pkl_folder) if pkl.endswith('.pkl')]:
computer = _computer(sl.pickle_load(os.path.join(pkl_folder, pkl)),
config_lines)
computer['id'] = pkl.split('.')[0]
computers.append(computer)
return computers
def conccomputers(pkl_folder, shops_key, usd_key, grn_key):
computers = [
sl.pickle_load(os.path.join(pkl_folder, pkl)) for pkl in
[pkl for pkl in os.listdir(pkl_folder) if pkl.endswith('.pkl')]
]
pairs = []
for c in computers:
url = c['Устройство.url']
for shop, usd, grn in zip(c[shops_key], c[usd_key], c[grn_key]):
pair = {}
pair['url'], pair['price_usd'], pair['price_grn'], pair[
'shop'] = url, usd, grn, shop
pairs.append(pair)
return pairs
def shops(pkl_folder, shops_key):
shops = set()
for pkl in [pkl for pkl in os.listdir(pkl_folder) if pkl.endswith('.pkl')]:
for shop in sl.pickle_load(os.path.join(pkl_folder, pkl))[shops_key]:
shops.add(shop)
return shops
def get_cached(name):
path = _picke_path(name)
if os.path.exists(path):
data = pickle_load(path)
if data['time'] > datetime.now() - timedelta(days=30):
return data['data']
def get_cached(name):
path = _picke_path(name)
if os.path.exists(path):
data = pickle_load(path)
if data['time'] > datetime.now() - timedelta(days=30):
return data['data']
def shops(pkl_folder, shops_key):
shops = set()
for pkl in [pkl for pkl in os.listdir(pkl_folder) if pkl.endswith('.pkl')]:
for shop in sl.pickle_load(os.path.join(pkl_folder, pkl))[shops_key]:
shops.add(shop)
return shops