def getFlightPrice(driver, url, id, worker_num):
"""
This function send url to remote server and get the result.
Save the result into file.
Input parameter:
dirver: The web driver.
url: type[string] . The url address.
id: type[int] The flight id.
worker_num: type[int] the worker number.
"""
# flight_module_class_name='flight-module.segment.offer-listing'
flight_id=str(id)
re = Recorder(flight_id,recorder.RecorderMode.binary)
if runDriver(driver,url,id)==True:
#Write the fight id.
flight_id="<flight_id>"+flight_id
re.writeN(flight_id)
#Write the url
re.write("<url>")
re.writeN(url)
#Write the search date
t = datetime.datetime.now().strftime("%Y-%m-%d %H %M %S")
search_date = "<search_date>"+t
re.writeN(search_date)
#Write the worker number
re.write("<worker_num>")
re.writeN(str(worker_num))
time.sleep(1)
body_element = driver.find_element_by_tag_name('body')
re.writeN(body_element.text)
time.sleep(1)
re.finish()
else:
print("worker[%d] failed to handle flight_id[%d]" %(worker_num, id))
def getFlightPrice(driver, url, id, worker_num):
"""
This function send url to remote server and get the result.
Save the result into file.
Input parameter:
dirver: The web driver.
url: type[string] . The url address.
id: type[int] The flight id.
worker_num: type[int] the worker number.
"""
# flight_module_class_name='flight-module.segment.offer-listing'
flight_id = str(id)
re = Recorder(flight_id, recorder.RecorderMode.binary)
if runDriver(driver, url, id) == True:
#Write the fight id.
flight_id = "<flight_id>" + flight_id
re.writeN(flight_id)
#Write the url
re.write("<url>")
re.writeN(url)
#Write the search date
t = datetime.datetime.now().strftime("%Y-%m-%d %H %M %S")
search_date = "<search_date>" + t
re.writeN(search_date)
#Write the worker number
re.write("<worker_num>")
re.writeN(str(worker_num))
time.sleep(1)
body_element = driver.find_element_by_tag_name('body')
re.writeN(body_element.text)
time.sleep(1)
re.finish()
else:
print("worker[%d] failed to handle flight_id[%d]" % (worker_num, id))
output = model(image, question, question_length)
loss = criterion(output, answer)
if is_train:
loss.backward()
if args.gradient_clipping:
nn.utils.clip_grad_value_(model.parameters(),
args.gradient_clipping)
optimizer.step()
recorder.batch_end(loss.item(),
output.cpu().detach(), answer.cpu(), types.cpu())
if is_train and (batch_idx % args.log_interval == 0):
recorder.log_batch(batch_idx, batch_size)
recorder.log_epoch()
if not is_train:
if args.lr_reduce:
reduce_scheduler.step(recorder.get_epoch_loss())
if not args.cv_pretrained:
recorder.log_data(image, question, answer, types)
if __name__ == '__main__':
writer = SummaryWriter(args.log)
recorder = Recorder(writer, args, batch_record_idx)
for epoch_idx in range(start_epoch, args.epochs):
epoch(epoch_idx, is_train=True)
epoch(epoch_idx, is_train=False)
save_checkpoint(epoch_idx, model.module if args.multi_gpu else model,
optimizer, args, recorder.batch_record_idx)
recorder.finish()
writer.close()
