def run(self):
count = 0
while not self.thread_stop:
count += 1
cnt = "thread object(%d),Time:%s\n" %(self.thread_num,time.ctime())
writeFile(self.file,cnt)
time.sleep(self.interval)
if(count>5):
self.thread_stop = True
def run(self):
count = 0
while not self.thread_stop:
count += 1
cnt = "thread object(%d),Time:%s\n" % (self.thread_num,
time.ctime())
writeFile(self.file, cnt)
time.sleep(self.interval)
if (count > 5):
self.thread_stop = True
def crawl_detail_page_save_as_mongo():
localtime = time.localtime(time.time())
#driver = webdriver.Chrome()
for retry in range(0, 5):
if retry == 0:
links = readFile('./IDS')
#links = ['6600258', '8992531', '5911916']
#links = ['2868435', '100000287145', '100000287113', '100000177784', '2868393', '100000177750', '100000287133']
#links = ['6600258']
else:
links = readFile('./IDS_REST')
print('retry times :' + str(retry))
print(links)
not_found_list.clear()
for x in links:
try:
date = datetime.datetime.now().strftime('%Y-%m-%d')
if db['phone'].find_one({'id': 'jd' + x}, {'id': 1}) is None:
response = requests.get(detail_page_head + x +
detail_page_tail,
headers=headers)
#print(driver.page_source)
selector = Selector(text=response.text)
phone = parse_as_phone(x, selector)
print(phone.__dict__)
db['phone'].insert(phone.__dict__)
price = parse_as_price(x, date)
print(price.__dict__)
save_to_mongo_as_a_array('prices', price._id, price.id,
'prices', price.price)
sale_promotion = parse_as_sale_promotion(x, date)
print(sale_promotion.__dict__)
save_to_mongo_as_a_array('sale_promotions', sale_promotion._id,
sale_promotion.id, 'sale_promotion',
sale_promotion.sale_promotion)
if localtime.tm_wday in crawl_day_in_week:
res = parse_as_opinion_and_impress(x, date)
print(res['opinion'].__dict__)
save_to_mongo_as_a_array('opinions', res['opinion']._id,
res['opinion'].id,
'opinion_value',
res['opinion'].opinion)
print('{\'id\': \'' + str(res['impress'].id) +
'\', \'impress\': ' + str(res['impress'].impress) +
'}')
save_to_mongo_as_a_array('impresses', res['opinion']._id,
res['impress'].id,
'impress_value',
res['impress'].impress)
res['impress'].impress.clear()
except Exception as e:
not_found_list.append(x)
print(str(x) + ' : ' + str(e))
writeFile('./IDS_REST', not_found_list)
def main():
#Membaca file input
Nb = f.openFile(sys.argv[1])
#Mengubah string menjadi list of char
listNb = Nb.split(' ')
#Mengubah list of char menjadi list of integer sesuai nilai kartu
listNb = engine.cardsToNb(listNb)
strSolusi = ''.join(engine.solve(listNb, listOp))
#Validasi apakah solusi bernilai 24
if eval(strSolusi) == 24:
print(strSolusi)
else:
print('Solusi tidak ditemukan.')
#Mengeluarkan file output
f.writeFile(sys.argv[2], strSolusi)
return 0
#from tkinter import *
from aStar import aStar
#from drawResult import *
from file import readFile, writeFile
inputFileName = raw_input(
"Enter the input file name (press Enter to default to \"input.txt\"): "
) or "input.txt"
outputFileName = raw_input(
"Enter the output file name (press Enter to default to \"result.txt\"): "
) or "result.txt"
enviro, startList, rendev = readFile(inputFileName)
print("Loaded data from {0}.".format(inputFileName))
solutions, visited = aStar(enviro, startList, rendev)
print("Successfully computed solutions.")
writeFile("result.txt", rendev, solutions)
print("Wrote solutions to {0}.".format(outputFileName))
print("Exiting...")
print(visited)
pic = Tk()
pic = frame_grid(visited, len(enviro), len(enviro[0]))
from file import writeFile
writeFile('dsds')
print(r'hello \nwork')
print('the sum 1+2={0} and 3+5={1}'.format(3, 4))
dict['name'] = "dsfdsfds"
dict = {"name": "vanhung", 'age': "1"}
print(dict['name'])
print(dict['age'])
floatx = float(1)
print(dict)
print(floatx)
# len of string
print(len("hello"))
str = "hello"
print(str[1:3])
# hello()
writeFile("hello")
readFile()
doSomething()
logger = logging.getLogger(__name__)
logger.exception("e")
str = "hello"
print(str[4])
print(str[1])
print(str[1:2])
for i in list:
print(i)
for letter in "pythfdsfdsfdsfsfdson":
success = True
else:
sg.popup_error('Error: Incompatible Data File')
if success:
# window.find_element('Filename').Update(fname)
window.TKroot.title('BeerBerry - ' + fname)
start()
# elif event == 'Save Raw Data':
# outFile = values['Save Raw Data']
# file.writeFile(outFile, data, 0)
elif event == 'Save Processed Data':
outFile = values['Save Processed Data']
file.writeFile(outFile, df_Post, 1)
elif event == 'Save Figure':
outFile = values['Save Figure']
file.writeFile(outFile, data, 0)
elif event == 'Copy Height':
pyperclip.copy(str(height))
elif event == 'Copy Area':
pyperclip.copy(str(area))
elif event == 'Authenticate':
# logout
if password_attempt == PASSWORD:
password_attempt = None
def crawl_detail_page_save_as_mongo():
# 获得当前时间
# 结构 time.struct_time(tm_year=2019, tm_mon=1, tm_mday=8, tm_hour=15, tm_min=50, tm_sec=27, tm_wday=1, tm_yday=8, tm_isdst=0) wday ： 0到 6
localtime = time.localtime(time.time())
# 获得当前的时间，并将它转换成这种格式 ： 2019-01-08
date = datetime.datetime.now().strftime('%Y-%m-%d')
# 重试5次
for retry in range(0, 5):
if retry == 0:
# 如果是重试的第一次，那么从IDS_JD文件中获得IDS并赋值到links
# links = readFile('./IDS_JD')
# 如果是重试的第一次，那么数据库中获得IDS并赋值到links
cursor.execute('select skuid from t_id_pool where source = %s;',
['京东'])
# 读出来的是一个二维数组，需要把二维数组中每个元素的第一个元素拿出，就形成id了。
links = [x[0] for x in cursor.fetchall()]
else:
# 如果是不是重试的第一次，那么从IDS_JD_REST文件中获得IDS并赋值到links
links = readFile('./IDS_JD_REST')
# 打印重试的次数
print('retry times :' + str(retry))
# 打印IDS数组
print(links)
# 清空没有找到的IDS
not_found_list.clear()
# 循环从IDS中获得ID，并进行相应的操作
for x in links:
try:
# 如果jd数据库中没有找到这个ID的商品那么，就进行下面的操作
# if db['phone'].find_one({'id': 'jd' + x}, {'id': 1}) is None:
# 将详情页的头和尾和ID进行拼接，并设置请求头，发送请求并获得响应
response = requests.get(detail_page_head + x +
detail_page_tail,
headers=headers)
#print(driver.page_source) # 打印网页源代码
# 将响应的文本传入，获得一个选择器对象。
selector = Selector(text=response.text)
# 传入手机的ID和选择器对象，对详情页的手机信息进行解析，获得一个手机对象。
phone = parse_as_phone(x, selector)
# 打印手机对象
print(phone.__dict__)
# 执行sql语句，从mysql group表中按型号、内存、硬盘来查找是否有结果
cursor.execute(
'select * from t_group where ram = %s and rom = %s and model = %s;',
[phone.ram, phone.rom, phone.model])
# 有结果的话（结果为1），说明已经分配到组了，
ret1 = cursor.rowcount
# print(ret1)
# 数据库中没有该组：
if ret1 is 0:
# print('没有组' + phone.id)
# 插入组，default_id是第一个进组的item
cursor.execute(
'insert into t_group (name, ram, rom, model, brand, default_id) values( "%s", "%s" , "%s" , "%s" , "%s" , "%s" )'
% (phone.name, phone.ram, phone.rom, phone.model,
phone.brand, phone.id))
# 提交到数据库执行
mysql_db.commit()
# 按道理来讲，mysql和mongodb的组是一致的，保存该组到mongodb
db['phone'].insert(phone.__dict__)
# 有没有组都需要对item表进行插入
# 从group表中获得组的ID
cursor.execute(
'select id from t_group where ram = %s and rom = %s and model = %s;',
[phone.ram, phone.rom, phone.model])
group_id = cursor.fetchone()[0]
# print(group_id)
# 从item表中按ID获得结果
cursor.execute('select id from t_item where id = %s;',
[phone.id])
ret2 = cursor.rowcount
# 结果不为0，说明item已经插入过了
# 结果为0，说明item没有插入
if ret2 is 0:
# 插入item
cursor.execute(
'insert into t_item (id, name, url, source , group_id) values ( "%s", "%s" , "%s" , "%s" , "%s")'
% (phone.id, phone.name, phone.url, '京东', group_id))
mysql_db.commit()
# 京东价格爬虫
price = parse_as_price(x, date)
print(price.__dict__)
# tableName : prices , _id : _id , id : id , arrayName : prices , arrayValue: price.price
save_to_mongo_as_a_array('prices', price._id, price.id,
'prices', price.price)
# 京东活动爬虫
sale_promotion = parse_as_sale_promotion(x, date)
print(sale_promotion.__dict__)
save_to_mongo_as_a_array('sale_promotions', sale_promotion._id,
sale_promotion.id, 'sale_promotion',
sale_promotion.sale_promotion)
except Exception as e:
# 如果发生错误则回滚
mysql_db.rollback()
not_found_list.append(x)
print(str(x) + ' : ' + str(e))
writeFile('./IDS_JD_REST', not_found_list)
# parse_as_opinion_and_impress()
# 印象标签和好评率就不再retry了
# cursor.execute('select skuid from t_id_pool where source = %s;'
# , ['京东'])
# links = [x[0] for x in cursor.fetchall()]
# 从group表中获得所有以jd开头的skuid
cursor.execute('select default_id from t_group where default_id like %s;',
['%jd%'])
# 读出来的是一个二维数组，需要把二维数组中每个元素的第一个元素拿出，就形成id了。
links = [x[0].replace('jd', '') for x in cursor.fetchall()]
print(links)
for x in links:
res = parse_as_opinion_and_impress(x, date)
print(res['opinion'].__dict__)
save_to_mongo_as_a_array('opinions', res['opinion']._id,
res['opinion'].id, 'opinion_value',
res['opinion'].opinion)
print('{\'id\': \'' + str(res['impress'].id) + '\', \'impress\': ' +
str(res['impress'].impress) + '}')
save_to_mongo_as_a_array('impresses', res['opinion']._id,
res['impress'].id, 'impress_value',
res['impress'].impress)
res['impress'].impress.clear()
plt.plot(x, y)
plt.show()
def updateNeuron(neuron, x1, x2, tag):
newBiais = neuron.get('biais') + step * (tag * neuron.get('out')) * -0.5
newW1 = neuron.get('weights')[0] + step * (tag * neuron.get('out')) * x1
newW2 = neuron.get('weights')[1] + step * (tag * neuron.get('out')) * x2
neuron['biais'] = newBiais
neuron['weights'] = [newW1, newW2]
return neuron
if __name__ == '__main__':
size = 100
step = 0.01
random.seed(15)
x2BiggerThan0_5 = False
# A mettre a true si x2 >0.5
writeFile(size, x2BiggerThan0_5)
lines = readFile()
trainNeuron(5, True)
trainNeuron(10, True)
trainNeuron(50, True)
trainNeuron(100, True)
