def crawl_detail_page_save_as_mongo():
localtime = time.localtime(time.time())
#driver = webdriver.Chrome()
for retry in range(0, 5):
if retry == 0:
links = readFile('./IDS')
#links = ['6600258', '8992531', '5911916']
#links = ['2868435', '100000287145', '100000287113', '100000177784', '2868393', '100000177750', '100000287133']
#links = ['6600258']
else:
links = readFile('./IDS_REST')
print('retry times :' + str(retry))
print(links)
not_found_list.clear()
for x in links:
try:
date = datetime.datetime.now().strftime('%Y-%m-%d')
if db['phone'].find_one({'id': 'jd' + x}, {'id': 1}) is None:
response = requests.get(detail_page_head + x +
detail_page_tail,
headers=headers)
#print(driver.page_source)
selector = Selector(text=response.text)
phone = parse_as_phone(x, selector)
print(phone.__dict__)
db['phone'].insert(phone.__dict__)
price = parse_as_price(x, date)
print(price.__dict__)
save_to_mongo_as_a_array('prices', price._id, price.id,
'prices', price.price)
sale_promotion = parse_as_sale_promotion(x, date)
print(sale_promotion.__dict__)
save_to_mongo_as_a_array('sale_promotions', sale_promotion._id,
sale_promotion.id, 'sale_promotion',
sale_promotion.sale_promotion)
if localtime.tm_wday in crawl_day_in_week:
res = parse_as_opinion_and_impress(x, date)
print(res['opinion'].__dict__)
save_to_mongo_as_a_array('opinions', res['opinion']._id,
res['opinion'].id,
'opinion_value',
res['opinion'].opinion)
print('{\'id\': \'' + str(res['impress'].id) +
'\', \'impress\': ' + str(res['impress'].impress) +
'}')
save_to_mongo_as_a_array('impresses', res['opinion']._id,
res['impress'].id,
'impress_value',
res['impress'].impress)
res['impress'].impress.clear()
except Exception as e:
not_found_list.append(x)
print(str(x) + ' : ' + str(e))
writeFile('./IDS_REST', not_found_list)
#from tkinter import *
from aStar import aStar
#from drawResult import *
from file import readFile, writeFile
inputFileName = raw_input(
"Enter the input file name (press Enter to default to \"input.txt\"): "
) or "input.txt"
outputFileName = raw_input(
"Enter the output file name (press Enter to default to \"result.txt\"): "
) or "result.txt"
enviro, startList, rendev = readFile(inputFileName)
print("Loaded data from {0}.".format(inputFileName))
solutions, visited = aStar(enviro, startList, rendev)
print("Successfully computed solutions.")
writeFile("result.txt", rendev, solutions)
print("Wrote solutions to {0}.".format(outputFileName))
print("Exiting...")
print(visited)
pic = Tk()
pic = frame_grid(visited, len(enviro), len(enviro[0]))
print(dict['name'])
print(dict['age'])
floatx = float(1)
print(dict)
print(floatx)
# len of string
print(len("hello"))
str = "hello"
print(str[1:3])
# hello()
writeFile("hello")
readFile()
doSomething()
logger = logging.getLogger(__name__)
logger.exception("e")
str = "hello"
print(str[4])
print(str[1])
print(str[1:2])
for i in list:
print(i)
for letter in "pythfdsfdsfdsfsfdson":
print(letter)
def start():
global fig_canvas_agg
global toolbar
global df
global df_Post
global data
global t
global i
global f
global Imag
global Imagfilt
global ifilt
global ienv
global int_ienv
global ienv_filtered
global harm_one
global harm_two
global harm_three
global harm_four
global harm_five
window.find_element('PPM').Update('')
# this is for the Layout Design of the Window
layout2 = [
[sg.Text('Loading')],
[
sg.ProgressBar(1,
orientation='h',
size=(20, 20),
key='progress',
style='winnative')
],
]
# This Creates the Physical Window
window2 = sg.Window('File Load', layout2).Finalize()
progress_bar = window2.FindElement('progress')
# This Updates the Window
# progress_bar.UpdateBar(Current Value to show, Maximum Value to show)
progress_bar.UpdateBar(0, 5)
# adding time.sleep(length in Seconds) has been used to Simulate adding your script in between Bar Updates
time.sleep(.5)
enable_harmonics()
# if default radio button is clicked, returns true for precalc
if TYPE == 'RAW':
if len(data.columns) == 2:
# Column 1: Voltage
v = data.iloc[:, 0].values
i = data.iloc[:, 1].values
#v, i = maths.blanking_first_samples(8000, v, i) #changed to 8000
f, t = maths.get_time_values(i, float(parameters['sample_rate']))
Imag = maths.magnitude_of_current(i, i.size)
window.find_element('Define baseline').Update(disabled=False)
harm_one = maths.filter_ienv(
maths.get_ienv(i, int(parameters['freq_pert']), 1,
int(parameters['bandwith_window']),
float(parameters['sample_rate']),
int(parameters['lpf_bw']), t), 200)
progress_bar.UpdateBar(1, 5)
harm_two = maths.filter_ienv(
maths.get_ienv(i, int(parameters['freq_pert']), 2,
int(parameters['bandwith_window']),
float(parameters['sample_rate']),
int(parameters['lpf_bw']), t), 200)
progress_bar.UpdateBar(2, 5)
time.sleep(.5)
harm_three = maths.filter_ienv(
maths.get_ienv(i, int(parameters['freq_pert']), 3,
int(parameters['bandwith_window']),
float(parameters['sample_rate']),
int(parameters['lpf_bw']), t), 200)
progress_bar.UpdateBar(3, 5)
harm_four = maths.filter_ienv(
maths.get_ienv(i, int(parameters['freq_pert']), 4,
int(parameters['bandwith_window']),
float(parameters['sample_rate']),
int(parameters['lpf_bw']), t), 200)
progress_bar.UpdateBar(4, 5)
harm_five = maths.filter_ienv(
maths.get_ienv(i, int(parameters['freq_pert']), 5,
int(parameters['bandwith_window']),
float(parameters['sample_rate']),
int(parameters['lpf_bw']), t), 200)
time.sleep(0.5)
# This will Close The Window
window2.Close()
int_ienv = maths.cumulative_sum_ienv(harm_two)
ienv_filtered = maths.filter_ienv(harm_two, 200)
# open harmonic with 2nd selected
window.find_element("r2").Update(value=True)
show_harmonics_graph()
window.find_element('Harmonics').Update(disabled=False, value=True)
window.find_element('Time Domain').Update(disabled=False)
window.find_element('Freq Domain').Update(disabled=False)
window.find_element('Cumulative Sum').Update(disabled=False)
window.find_element('Envelope').Update(disabled=False)
d = {
't': t,
'i': i,
'f': f,
'Imag': Imag,
'harm_one': harm_one,
'harm_two': harm_two,
'harm_three': harm_three,
'harm_four': harm_four,
'harm_five': harm_five,
'int_ienv': int_ienv,
'ienv_filtered': ienv_filtered
}
df_Post = pd.DataFrame(d)
window.find_element('Harmonic Container').Update(visible=True)
if df_Post is not None:
#if password_attempt == PASSWORD:
# window.find_element('Save Raw Data').Update(disabled=False)
window.find_element('Save Processed Data').Update(
disabled=False)
elif len(data.columns) == 11:
sg.popup_error(
'Error: Select Post Calculation to Load Calculated Data Files')
else:
sg.popup_error('Error: Incompatible Data File')
elif TYPE == 'PROCESSED':
df_Post = file.readFile(fname, 1)
print(df_Post)
print(len(df_Post.columns))
if len(df_Post.columns) == 11:
t = df_Post['t']
i = df_Post['i']
f = df_Post['f']
Imag = df_Post['Imag']
harm_one = df_Post['harm_one']
harm_two = df_Post['harm_two']
harm_three = df_Post['harm_three']
harm_four = df_Post['harm_four']
harm_five = df_Post['harm_five']
int_ienv = df_Post['int_ienv']
ienv_filtered = df_Post['ienv_filtered']
# open harmonic with 2nd selected
window.find_element("r2").Update(value=True)
show_harmonics_graph()
window.find_element('Harmonics').Update(disabled=False, value=True)
window.find_element('Time Domain').Update(disabled=False)
window.find_element('Freq Domain').Update(disabled=False)
window.find_element('Cumulative Sum').Update(disabled=False)
window.find_element('Envelope').Update(disabled=False)
time.sleep(0.1)
progress_bar.UpdateBar(1, 5)
progress_bar.UpdateBar(2, 5)
time.sleep(0.2)
progress_bar.UpdateBar(3, 5)
progress_bar.UpdateBar(4, 5)
progress_bar.UpdateBar(5, 5)
time.sleep(0.5)
# This will Close The Window
window2.Close()
if df_Post is not None:
window.find_element('Save Processed Data').Update(
disabled=False)
elif len(df_Post.columns) == 2:
sg.popup_error('Error: Select Raw Data to Load Raw Data files')
else:
sg.popup_error('Error: Incompatible Data File')
window.TKroot.title('BeerBerry')
start()
# if window.find_element("=autosave-") == True:
# outFile = values['Save Processed Data']
# fileName = exc_parameters['name']
# file.writeFile(outFile, df_Post, 2)
elif event == 'Select Data File':
if fname != values['Select Data File']:
fname = values['Select Data File']
if fname:
# determine file type
success = False
temp = file.readFile(fname, 0)
if len(temp.columns) == 2:
if (password_attempt == PASSWORD):
TYPE = 'RAW'
success = True
data = file.readFile(fname, 0)
else:
sg.popup_error(
'Error: Please login to load Raw data files')
elif len(temp.columns) == 11:
TYPE = 'PROCESSED'
success = True
else:
sg.popup_error('Error: Incompatible Data File')
def crawl_detail_page_save_as_mongo():
# 获得当前时间
# 结构 time.struct_time(tm_year=2019, tm_mon=1, tm_mday=8, tm_hour=15, tm_min=50, tm_sec=27, tm_wday=1, tm_yday=8, tm_isdst=0) wday ： 0到 6
localtime = time.localtime(time.time())
# 获得当前的时间，并将它转换成这种格式 ： 2019-01-08
date = datetime.datetime.now().strftime('%Y-%m-%d')
# 重试5次
for retry in range(0, 5):
if retry == 0:
# 如果是重试的第一次，那么从IDS_JD文件中获得IDS并赋值到links
# links = readFile('./IDS_JD')
# 如果是重试的第一次，那么数据库中获得IDS并赋值到links
cursor.execute('select skuid from t_id_pool where source = %s;',
['京东'])
# 读出来的是一个二维数组，需要把二维数组中每个元素的第一个元素拿出，就形成id了。
links = [x[0] for x in cursor.fetchall()]
else:
# 如果是不是重试的第一次，那么从IDS_JD_REST文件中获得IDS并赋值到links
links = readFile('./IDS_JD_REST')
# 打印重试的次数
print('retry times :' + str(retry))
# 打印IDS数组
print(links)
# 清空没有找到的IDS
not_found_list.clear()
# 循环从IDS中获得ID，并进行相应的操作
for x in links:
try:
# 如果jd数据库中没有找到这个ID的商品那么，就进行下面的操作
# if db['phone'].find_one({'id': 'jd' + x}, {'id': 1}) is None:
# 将详情页的头和尾和ID进行拼接，并设置请求头，发送请求并获得响应
response = requests.get(detail_page_head + x +
detail_page_tail,
headers=headers)
#print(driver.page_source) # 打印网页源代码
# 将响应的文本传入，获得一个选择器对象。
selector = Selector(text=response.text)
# 传入手机的ID和选择器对象，对详情页的手机信息进行解析，获得一个手机对象。
phone = parse_as_phone(x, selector)
# 打印手机对象
print(phone.__dict__)
# 执行sql语句，从mysql group表中按型号、内存、硬盘来查找是否有结果
cursor.execute(
'select * from t_group where ram = %s and rom = %s and model = %s;',
[phone.ram, phone.rom, phone.model])
# 有结果的话（结果为1），说明已经分配到组了，
ret1 = cursor.rowcount
# print(ret1)
# 数据库中没有该组：
if ret1 is 0:
# print('没有组' + phone.id)
# 插入组，default_id是第一个进组的item
cursor.execute(
'insert into t_group (name, ram, rom, model, brand, default_id) values( "%s", "%s" , "%s" , "%s" , "%s" , "%s" )'
% (phone.name, phone.ram, phone.rom, phone.model,
phone.brand, phone.id))
# 提交到数据库执行
mysql_db.commit()
# 按道理来讲，mysql和mongodb的组是一致的，保存该组到mongodb
db['phone'].insert(phone.__dict__)
# 有没有组都需要对item表进行插入
# 从group表中获得组的ID
cursor.execute(
'select id from t_group where ram = %s and rom = %s and model = %s;',
[phone.ram, phone.rom, phone.model])
group_id = cursor.fetchone()[0]
# print(group_id)
# 从item表中按ID获得结果
cursor.execute('select id from t_item where id = %s;',
[phone.id])
ret2 = cursor.rowcount
# 结果不为0，说明item已经插入过了
# 结果为0，说明item没有插入
if ret2 is 0:
# 插入item
cursor.execute(
'insert into t_item (id, name, url, source , group_id) values ( "%s", "%s" , "%s" , "%s" , "%s")'
% (phone.id, phone.name, phone.url, '京东', group_id))
mysql_db.commit()
# 京东价格爬虫
price = parse_as_price(x, date)
print(price.__dict__)
# tableName : prices , _id : _id , id : id , arrayName : prices , arrayValue: price.price
save_to_mongo_as_a_array('prices', price._id, price.id,
'prices', price.price)
# 京东活动爬虫
sale_promotion = parse_as_sale_promotion(x, date)
print(sale_promotion.__dict__)
save_to_mongo_as_a_array('sale_promotions', sale_promotion._id,
sale_promotion.id, 'sale_promotion',
sale_promotion.sale_promotion)
except Exception as e:
# 如果发生错误则回滚
mysql_db.rollback()
not_found_list.append(x)
print(str(x) + ' : ' + str(e))
writeFile('./IDS_JD_REST', not_found_list)
# parse_as_opinion_and_impress()
# 印象标签和好评率就不再retry了
# cursor.execute('select skuid from t_id_pool where source = %s;'
# , ['京东'])
# links = [x[0] for x in cursor.fetchall()]
# 从group表中获得所有以jd开头的skuid
cursor.execute('select default_id from t_group where default_id like %s;',
['%jd%'])
# 读出来的是一个二维数组，需要把二维数组中每个元素的第一个元素拿出，就形成id了。
links = [x[0].replace('jd', '') for x in cursor.fetchall()]
print(links)
for x in links:
res = parse_as_opinion_and_impress(x, date)
print(res['opinion'].__dict__)
save_to_mongo_as_a_array('opinions', res['opinion']._id,
res['opinion'].id, 'opinion_value',
res['opinion'].opinion)
print('{\'id\': \'' + str(res['impress'].id) + '\', \'impress\': ' +
str(res['impress'].impress) + '}')
save_to_mongo_as_a_array('impresses', res['opinion']._id,
res['impress'].id, 'impress_value',
res['impress'].impress)
res['impress'].impress.clear()
plt.plot(x, y)
plt.show()
def updateNeuron(neuron, x1, x2, tag):
newBiais = neuron.get('biais') + step * (tag * neuron.get('out')) * -0.5
newW1 = neuron.get('weights')[0] + step * (tag * neuron.get('out')) * x1
newW2 = neuron.get('weights')[1] + step * (tag * neuron.get('out')) * x2
neuron['biais'] = newBiais
neuron['weights'] = [newW1, newW2]
return neuron
if __name__ == '__main__':
size = 100
step = 0.01
random.seed(15)
x2BiggerThan0_5 = False
# A mettre a true si x2 >0.5
writeFile(size, x2BiggerThan0_5)
lines = readFile()
trainNeuron(5, True)
trainNeuron(10, True)
trainNeuron(50, True)
trainNeuron(100, True)
printHead()
student.printStu(f)
elif choice == 4:
name = input('请输入要搜索的姓名:')
f = student.equalstu(stu, 'name', name)
if f == []:
print('不存在')
else:
printHead()
student.printStu(f)
elif choice == 0:
break
if __name__ == '__main__':
stu = file.readFile()
while True:
MainInfo()
choice = int(input('Choose one option youo what to do:\n'))
if choice == 1:
printHead()
student.sortStu(stu, 'num')
student.printStu(stu)
elif choice == 2:
changeinfoManage(stu)
elif choice == 3:
menurollManage(stu)
elif choice == 4:
searchManage(stu)
elif choice == 0:
break