def init():
parser = argparse.ArgumentParser()
parser.add_argument('--config_path', default=None, type=str, help='配置文件路径')
args, _ = parser.parse_known_args()
cfg.merge_from_file(args.config_path)
utils.mkdir(cfg.test.output_dir)
def save(self, path):
utils.mkdir(path)
fname = self.get_name(path)
torch.save({
'kwargs': self.get_args(),
'model_state_dict': self.state_dict(),
}, fname)
def main(unused_argv):
if FLAGS.config is None:
raise Exception("Config file must be provided")
json_file = open('model_configs/%s.json' % FLAGS.config).read()
json_dict = json.loads(json_file)
BatchedInput = utils.get_batched_input_class(
json_dict.get("input", "default"))
Model = utils.get_model_class(json_dict.get("model"))
hparams = utils.create_hparams(FLAGS, Model)
hparams.hcopy_path = configs.HCOPY_PATH
hparams.hcopy_config = os.path.join(configs.HCOPY_CONFIG_PATH)
# create directories for logs and saved parameters
utils.mkdir(hparams.summaries_dir)
utils.mkdir(hparams.out_dir)
utils.clear_log(hparams)
random_seed = FLAGS.random_seed
if random_seed is not None and random_seed > 0:
random.seed(random_seed)
np.random.seed(random_seed)
train(Model, BatchedInput, hparams)
def __init__(self,
folder,
file_name=None,
append=None,
time_format=None,
use_pprint=False):
utils.mkdir(folder.strip())
self.folder = folder.strip()
if file_name and file_name.strip()[-1] == '+':
self.file_name = file_name.strip()[:-1]
self.append = False if append is False else True
else:
self.file_name = None
self.append = bool(append)
self.time_format = time_format or '%Y%m%d_%H%M%S.log'
self.path = None
self.file = None
self.init = True
if use_pprint:
from pprint import pprint
self.print_fn = lambda text, stream=sys.stdout: print(text, file=stream) \
if isinstance(text, str) else pprint(text, stream, width=200)
else:
self.print_fn = lambda text, stream=sys.stdout: print(text,
file=stream)
def create_sample_set(data_dir):
"""Create a sample set of the training data for quick experimentation"""
train_dir = os.path.join(data_dir, 'train')
driver_data = get_driver_data(data_dir)
sample_set = set({
img
for (img, dr_id) in driver_data.items()
if dr_id in DRIVER_IDS_TRAIN[:2]
})
valid_set = set({
img
for (img, dr_id) in driver_data.items()
if dr_id in DRIVER_IDS_TRAIN[2:3]
})
train_files = glob(train_dir + '/c?/*.jpg')
sample_files = [
f for f in train_files if os.path.basename(f) in sample_set
]
valid_files = [f for f in train_files if os.path.basename(f) in valid_set]
for train_path in sample_files:
target_path = get_sample_path(train_path)
mkdir(os.path.dirname(target_path))
copyfile(train_path, target_path)
for train_path in valid_files:
target_path = get_valid_path(get_sample_path(train_path))
mkdir(os.path.dirname(target_path))
copyfile(train_path, target_path)
def setup_train_data(data_dir):
labels_file = join(data_dir, 'labels.csv')
df_train = pd.read_csv(labels_file)
df_grouped = df_train.groupby('breed').aggregate(lambda x: list(x))
for breed in df_grouped.index:
ids = df_grouped.id[breed]
breed_len = len(ids)
breed_len_train = np.ceil(breed_len * TRAIN_AMT).astype('int')
train_ids = ids[:breed_len_train]
valid_ids = ids[breed_len_train:]
train_dir = join(data_dir, 'train')
target_train_dir = join(train_dir, breed)
target_valid_dir = join(data_dir, 'valid', breed)
mkdir(target_train_dir)
mkdir(target_valid_dir)
for tid in train_ids:
fpath = '%s.jpg' % tid
src_path = join(train_dir, fpath)
dst_path = join(target_train_dir, fpath)
os.rename(src_path, dst_path)
for vid in valid_ids:
fpath = '%s.jpg' % vid
src_path = join(train_dir, fpath)
dst_path = join(target_valid_dir, fpath)
os.rename(src_path, dst_path)
def __init__(self,
image_set,
path,
use_difficult=False,
name=None,
training=True):
super(PascalVoc, self).__init__()
self.training = training # super
assert image_set in ('train', 'trainval', 'val', 'test')
self.image_set = image_set
self.use_difficult = use_difficult
self.name = (name or 'pascal_voc') + '_' + self.image_set + \
('_no_hard', '_use_hard')[self.use_difficult] + \
(('_no_flip', '_use_flip')[cfg.flip] if self.training else '') # super
self.path = path
self.image_path = os.path.join(self.path, 'JPEGImages', '{}.jpg')
self.xml_path = os.path.join(self.path, 'Annotations', '{}.xml')
utils.mkdir(cfg.cache_path)
self.cache_path = os.path.join(
cfg.cache_path, '{}_annotations.pkl'.format(self.name)) # super
self.image_set_path = ''
self.num_class = len(CLASSES)
self.class_to_index = dict(zip(CLASSES, range(
self.num_class))) # {'cls': id, ...}
self.image_ids = self._load_image_id() # super
self.all_annotations = None # super
def _export_tabular_model(self, domain_id, task_id):
domain_folder = self.get_models_folder(domain_id)
utils.mkdir(domain_folder)
model_path = self._get_tabular_model_path(domain_folder, task_id)
q_table = self._get_q_function(domain_id, task_id)
np.save(model_path, q_table)
def extract_audio(self, args):
if (len(args) != 2):
self.raise_err("extract_audio")
return
src_dir, to_dir = args
print('>>> Extract Audio. <<<\n')
# walk src_dir, do 'Extract Audio'.
for root, subdirs, files in os.walk(src_dir):
if not subdirs:
print('Extract Audio: {}'.format(root))
pid = root.split('/video/mpg_6000')[0]
pid = pid.split('/')[-1]
for f in files:
if (not (f.endswith('.mp4') or f.endswith('.mpg'))):
continue
# s2/lgbm3p.mpg
movie_id = f.split('.')[0]
src_movie_file = os.path.join(root, f)
to_extract_dir = os.path.join(to_dir, pid, movie_id)
utils.mkdir(to_extract_dir)
if (os.path.exists(
os.path.join(to_extract_dir, 'audio.wav'))):
continue
cmd = 'ffmpeg -i {} -vn {}'.format(
src_movie_file,
os.path.join(to_extract_dir, 'audio.wav'))
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
try:
buff = proc.communicate()
except Exception as e:
print('Extract Audio FAILED.\n')
sys.exit(1)
proc.stdout.close()
def parse_args(self):
opt = self.parser.parse_args()
str_ids = str(opt.gpu_ids)
opt.gpu_ids = []
for str_id in str_ids.split(','):
opt.gpu_ids.append(int(str_id))
if len(opt.gpu_ids) > 0:
torch.cuda.set_device(opt.gpu_ids[0])
args = vars(opt)
print('-------- [INFO] Options --------')
for k, v in sorted(args.items()):
print('%s: %s' % (str(k), str(v)))
expr_dir = os.path.join(opt.ckpt_dir, opt.model)
utils.mkdir(expr_dir)
test_dir = os.path.join(opt.test_dir, opt.model)
utils.mkdir(test_dir)
file_name = os.path.join(expr_dir, 'opt.txt')
with open(file_name, 'wt') as opt_file:
opt_file.write(' [INFO] Options\n')
for k, v in sorted(args.items()):
opt_file.write('%s: %s\n' % (str(k), str(v)))
print('------------- END -------------')
return opt
def parse_args(self):
opt = self.parser.parse_args()
str_ids = str(opt.gpu_ids)
opt.gpu_ids = []
for str_id in str_ids.split(','):
opt.gpu_ids.append(int(str_id))
if len(opt.gpu_ids) > 0:
torch.cuda.set_device(opt.gpu_ids[0])
args = vars(opt)
print('-------- [INFO] Options --------')
for k, v in sorted(args.items()):
print('%s: %s' % (str(k), str(v)))
expr_dir = os.path.join(opt.ckpt_dir, opt.model)
utils.mkdir(expr_dir)
test_dir = os.path.join(opt.test_dir, opt.model)
utils.mkdir(test_dir)
file_name = os.path.join(expr_dir, 'opt.txt')
with open(file_name, 'wt') as opt_file:
opt_file.write(' [INFO] Options\n')
for k, v in sorted(args.items()):
opt_file.write('%s: %s\n' % (str(k), str(v)))
print('------------- END -------------')
return opt
def _export_dqn_model(self, domain_id, task_id, automaton, automaton_state):
domain_folder = self.get_models_folder(domain_id)
utils.mkdir(domain_folder)
model_path = self._get_dqn_model_path(domain_folder, task_id, automaton_state)
model = self._get_q_function(domain_id, task_id)[automaton.get_state_id(automaton_state)]
torch.save(model.state_dict(), model_path)
def run(self):
# Data loading code
print("Loading data")
test_dir = self.config['Dataset']['TestPath']
pred_result_dir = os.path.join(test_dir, 'pred_result_info')
utils.mkdir(pred_result_dir)
dataset_test = torchvision.datasets.ImageFolder(
test_dir, transforms.Compose([transforms.ToTensor()]))
test_sampler = torch.utils.data.SequentialSampler(dataset_test)
data_loader_test = torch.utils.data.DataLoader(dataset_test,
batch_size=1,
sampler=test_sampler,
num_workers=1,
pin_memory=True)
print("Creating model")
model = getModels(model_name=self.config['Model']['Name'],
num_classes=self.config['Model']['NumClass'])
model.to(self.device)
checkpoint = torch.load(os.path.join(
self.output_model_path, self.config['Misc']['BestModelName']),
map_location='cpu')
model.load_state_dict(checkpoint['model'])
correct_1 = 0.0
model.eval()
print(dataset_test.class_to_idx)
with torch.no_grad():
for i, (image, label) in enumerate(data_loader_test):
image = image.to(self.device, non_blocking=True)
label = label.to(self.device, non_blocking=True)
output = model(image)
_, pred = output.topk(1, 1, largest=True, sorted=True)
label = label.view(label.size(0), -1).expand_as(pred)
correct = pred.eq(label)
correct_1 += correct[:, :1].sum()
for key in dataset_test.class_to_idx:
if dataset_test.class_to_idx[key] == label:
label_info = key
if dataset_test.class_to_idx[key] == pred:
pred_info = key
src_img = cv2.imread(dataset_test.imgs[i][0], 1)
cv2.putText(src_img, 'label: {}'.format(label_info), (20, 30),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 1)
cv2.putText(src_img, 'pred: {}'.format(pred_info), (20, 60),
cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 20), 1)
if label_info != pred_info:
shutil.move(dataset_test.imgs[i][0], pred_result_dir)
# cv2.imwrite(os.path.join(pred_result_dir, '{}.png'.format(i)), src_img)
cv2.imshow('img', src_img)
cv2.waitKey(33)
print("acc: ", correct_1 / len(data_loader_test.dataset))
def _get_voc_results_file_template():
"""获取检测结果文件路径模板"""
# {output_dir}/detection_result/{dataset_name}_{image_set}_{cls_name}.txt
path = os.path.join(cfg.test.output_dir, 'detection_result')
utils.mkdir(path)
file_name = '_'.join([cfg.dataset_name[0], cfg.image_set[0], '{}.txt'])
file_path = os.path.join(path, file_name)
return file_path
def initialize(self, opt, **kwargs):
self.gpu_ids = opt.gpu_ids
if len(self.gpu_ids) < 1:
self.use_cuda = False
self.isTrain = opt.isTrain
self.ImgTensor = torch.cuda.FloatTensor if self.gpu_ids else torch.Tensor
self.LblTensor = torch.cuda.FloatTensor if self.gpu_ids else torch.Tensor
self.save_dir = opt.save_dir
mkdir(self.save_dir)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--data", type=str, required=True, help="Path to the dataset directory.")
parser.add_argument("--degree", type=int, default=5, help="Degree of the bezier curves.")
parser.add_argument("--ckpt", type=str, required=True, help="Path to the checkpoint file.")
parser.add_argument("--gpu_ids", type=str, default='', help="Specify the GPU ids.")
parser.add_argument("--seed", type=int, default=0, help="Random seed.")
parser.add_argument("--batch_size", type=int, default=256, help="Batch size.")
parser.add_argument("--num_workers", type=int, default=4, help="Number of workers.")
parser.add_argument("--input_size", type=int, nargs=2, required=True, help="Size of the input image (w, h).")
parser.add_argument("--max_lane", type=int, default=4, help="Maximum number of lanes.")
parser.add_argument("--num_points", type=int, default=72, help="Number of points sampled from each lane.")
parser.add_argument("--feat_dim", type=int, default=384, help="The output feature dimension of the backbone.")
parser.add_argument("--save_name", type=str, default="./video/output.avi", help="Path to save the video.")
args = parser.parse_args()
print(args)
for s in args.gpu_ids:
try:
int(s)
except ValueError as e:
print("Invalid gpu id: {}".format(s))
raise ValueError
os.environ["CUDA_VISIBLE_DEVICES"] = ','.join(args.gpu_ids)
if args.gpu_ids:
if torch.cuda.is_available():
use_gpu = True
cudnn.benchmark = True
torch.cuda.manual_seed(args.seed)
else:
use_gpu = False
else:
use_gpu = False
train_loader, val_loader, test_loader, num_fc_nodes = build_dataloader(args.data,
args.batch_size,
tuple(args.input_size),
args.degree,
args.num_points,
args.max_lane, use_gpu,
args.num_workers)
model = CustomResnet(args.feat_dim, '', args.max_lane, num_fc_nodes)
if os.path.exists(args.ckpt):
model.load_state_dict(torch.load(args.ckpt))
if use_gpu:
model = model.cuda()
folder, _ = os.path.split(args.save_name)
mkdir(folder)
evaluate(model, test_loader, args.degree, args.save_name, use_gpu)
def create_validation_set(data_dir):
"""Create a validation set from a subset of the training files"""
files = get_rand_train_files(data_dir)
mkdir(data_dir + '/valid')
for i in range(2000):
file_name = os.path.basename(files[i])
os.rename(files[i], data_dir + '/valid/' + file_name)
def create_test_set(data_dir):
"""Create a single unknown class for the test set"""
files = glob(data_dir + '/test/*.jpg')
mkdir(data_dir + '/test/unknown')
for f in files:
os.rename(f, data_dir + '/test/unknown/' + os.path.basename(f))
def mxnet_sat_alignment(self, args):
if (len(args) != 3):
self.raise_err("mxnet SAT alignment")
return
src_dir, to_dir, model_dir = args
# walk movie dir, do 'face crop' and 'pose estimate'.
for root, subdirs, files in os.walk(src_dir):
if not subdirs:
pid = root.split('/video/mpg_6000')[0]
pid = pid.split('/')[-1]
for f in files:
if (not (f.endswith('.mp4') or f.endswith('.mpg'))):
continue
movie_id = f.split('.')[0]
src_movie_file = os.path.join(root, f)
to_crop_dir = os.path.join(to_dir, pid, movie_id)
utils.mkdir(to_crop_dir)
print('mxnet SAT: {}'.format(to_crop_dir))
if (os.path.exists(
os.path.join(to_crop_dir, "landmark.txt"))):
continue
labelfile = open(os.path.join(to_crop_dir, 'landmark.txt'),
'w')
cap = cv2.VideoCapture(src_movie_file)
if (cap.isOpened()):
idx = 0
success, image = cap.read()
while (success):
[h, w, c] = image.shape
if c > 3:
image = image[:, :, :3]
lmk_results = utils.get_mxnet_sat_alignment(
model_dir, image)
if (lmk_results is None):
os.system('rm -rf {}'.format(to_crop_dir))
break
_, _, img_cropped, lmk_cropped, center_x, center_y, ratio = lmk_results
labelfile.write('{}\n'.format(','.join(
['{:.2f}'.format(v) for v in lmk_cropped])))
cv2.imwrite(
os.path.join(to_crop_dir,
'{}.jpg'.format(idx)),
img_cropped,
[int(cv2.IMWRITE_JPEG_QUALITY), 100])
idx += 1
success, image = cap.read()
labelfile.close()
cap.release()
def save_model(data_dir, model):
json_data = model.to_json()
cache_dir = os.path.join(data_dir, 'cache')
t = strftime('%Y-%m-%d-%H')
architecture_file = 'architecture-%s.json' % t
weights_file = 'model-weights-%s.json' % t
mkdir(cache_dir)
open(os.path.join(cache_dir, architecture_file), 'w').write(json_data)
model.save_weights(os.path.join(cache_dir, weights_file), overwrite=True)
def _solve_ilasp_task(self, domain_id):
utils.mkdir(self.get_automaton_solution_folder(domain_id))
ilasp_task_filename = os.path.join(self.get_automaton_task_folder(domain_id),
ISAAlgorithmBase.AUTOMATON_TASK_FILENAME % self.automaton_counters[domain_id])
ilasp_solution_filename = os.path.join(self.get_automaton_solution_folder(domain_id),
ISAAlgorithmBase.AUTOMATON_SOLUTION_FILENAME % self.automaton_counters[domain_id])
return solve_ilasp_task(ilasp_task_filename, ilasp_solution_filename, timeout=self.ilasp_timeout,
version=self.ilasp_version, max_body_literals=self.max_body_literals,
binary_folder_name=self.binary_folder_name, compute_minimal=self.ilasp_compute_minimal)
def create_label_dirs(parent_dir):
cats = glob(parent_dir + '/cat.*.jpg')
dogs = glob(parent_dir + '/dog.*.jpg')
mkdir(parent_dir + '/cats')
mkdir(parent_dir + '/dogs')
for cat in cats:
os.rename(cat, parent_dir + '/cats/' + os.path.basename(cat))
for dog in dogs:
os.rename(dog, parent_dir + '/dogs/' + os.path.basename(dog))
def save_network(self, network, network_label, iter_step):
util.mkdir(self.opt['path']['checkpoints']['models'])
save_filename = '{}_{}.pth'.format(iter_step, network_label)
save_path = os.path.join(self.opt['path']['checkpoints']['models'],
save_filename)
if isinstance(network, nn.DataParallel):
network = network.module
state_dict = network.state_dict()
for key, param in state_dict.items():
state_dict[key] = param.cpu()
torch.save(state_dict, save_path)
def save_model(path, model, optimizer=None, scheduler=None, iteration=None):
"""保存模型"""
folder = os.path.split(path)[0]
utils.mkdir(folder)
state = dict()
state['model'] = model.state_dict()
if optimizer is not None:
state['optimizer'] = optimizer.state_dict()
if scheduler is not None:
state['scheduler'] = scheduler.state_dict()
if iteration is not None:
state['iteration'] = iteration
torch.save(state, path)
def create_validation_set(data_dir):
"""Move the validation files to a separate directory"""
train_dir = os.path.join(data_dir, 'train')
driver_data = get_driver_data(data_dir)
valid_set = set({
img
for (img, dr_id) in driver_data.items() if dr_id in DRIVER_IDS_VALID
})
train_files = glob(train_dir + '/c?/*.jpg')
valid_files = [f for f in train_files if os.path.basename(f) in valid_set]
for train_path in valid_files:
valid_path = get_valid_path(train_path)
mkdir(os.path.dirname(valid_path))
os.rename(train_path, valid_path)
def crop_images(self):
# To generate crop car images.
image_dir = os.path.join(self.root_dir, 'car_ims_cropped')
print("Cropping Images...")
utils.mkdir(image_dir)
for row in tqdm(self.index_table):
name = row[0][0]
image = Image.open(os.path.join(self.root_dir, name))
image = image.crop((
row[1][0][0],
row[2][0][0],
row[3][0][0],
row[4][0][0],
))
image.save(os.path.join(image_dir, name.split('/')[-1]))
def create_sample_set(data_dir):
"""Create a validation set from a subset of the training files"""
files_train = get_rand_train_files(data_dir)
files_valid = get_rand_train_files(data_dir)
mkdir(data_dir + '/sample/train')
mkdir(data_dir + '/sample/valid')
for i in range(200):
file_name = os.path.basename(files_train[i])
copyfile(files_train[i], data_dir + '/sample/train/' + file_name)
for i in range(50):
file_name = os.path.basename(files_valid[i])
copyfile(files_valid[i], data_dir + '/sample/valid/' + file_name)
def validate(self, val_batch, current_step):
avg_psnr = 0.0
avg_ssim = 0.0
idx = 0
for _, val_data in enumerate(val_batch):
idx += 1
img_name = os.path.splitext(
os.path.basename(val_data['LR_path'][0]))[0]
img_dir = os.path.join(
self.opt['path']['checkpoints']['val_image_dir'], img_name)
util.mkdir(img_dir)
self.val_lr = val_data['LR'].to(self.device)
self.val_hr = val_data['HR'].to(self.device)
self.G.eval()
with torch.no_grad():
self.val_sr = self.G(self.val_lr)
self.G.train()
val_LR = self.val_lr.detach()[0].float().cpu()
val_SR = self.val_sr.detach()[0].float().cpu()
val_HR = self.val_hr.detach()[0].float().cpu()
sr_img = util.tensor2img(val_SR) # uint8
gt_img = util.tensor2img(val_HR) # uint8
# Save SR images for reference
save_img_path = os.path.join(
img_dir, '{:s}_{:d}.png'.format(img_name, current_step))
cv2.imwrite(save_img_path, sr_img)
# calculate PSNR
crop_size = 4
gt_img = gt_img / 255.
sr_img = sr_img / 255.
cropped_sr_img = sr_img[crop_size:-crop_size,
crop_size:-crop_size, :]
cropped_gt_img = gt_img[crop_size:-crop_size,
crop_size:-crop_size, :]
avg_psnr += PSNR(cropped_sr_img * 255, cropped_gt_img * 255)
avg_ssim += SSIM(cropped_sr_img * 255, cropped_gt_img * 255)
avg_psnr = avg_psnr / idx
avg_ssim = avg_ssim / idx
return avg_psnr, avg_ssim
def save_training_state(self, epoch, iter_step):
'''Saves training state during training, which will be used for resuming'''
state = {
'epoch': epoch,
'iter': iter_step,
'schedulers': [],
'optimizers': []
}
for s in self.schedulers:
state['schedulers'].append(s.state_dict())
for o in self.optimizers:
state['optimizers'].append(o.state_dict())
save_filename = '{}.state'.format(iter_step)
util.mkdir(self.opt['path']['checkpoints']['states'])
save_path = os.path.join(self.opt['path']['checkpoints']['states'],
save_filename)
torch.save(state, save_path)
def _generate_ilasp_task(self, task, domain_id):
utils.mkdir(self.get_automaton_task_folder(domain_id))
ilasp_task_filename = ISAAlgorithmBase.AUTOMATON_TASK_FILENAME % self.automaton_counters[domain_id]
observables = task.get_observables()
if self.use_restricted_observables:
observables = task.get_restricted_observables()
# the sets of examples are sorted to make sure that ILASP produces the same solution for the same sets (ILASP
# can produce different hypothesis for the same set of examples but given in different order)
generate_ilasp_task(self.num_automaton_states[domain_id], ISAAlgorithmBase.ACCEPTING_STATE_NAME,
ISAAlgorithmBase.REJECTING_STATE_NAME, observables, sorted(self.goal_examples[domain_id]),
sorted(self.dend_examples[domain_id]), sorted(self.inc_examples[domain_id]),
self.get_automaton_task_folder(domain_id), ilasp_task_filename, self.symmetry_breaking_method,
self.max_disjunction_size, self.learn_acyclic_graph, self.use_compressed_traces,
self.avoid_learning_only_negative, self.prioritize_optimal_solutions, self.binary_folder_name)
def __init__(self):
# 加载配置文件
self.config = utils.loadYaml('../config/config.yaml')
# 训练结果保存路径
self.output_model_path = os.path.join(
'../output/', self.config['Misc']['OutputFolderName'])
if self.config['Misc']['OutputFolderName']:
utils.mkdir(self.output_model_path)
else:
raise IOError('请输入训练结果保存路径...')
# gpu使用
self.device = utils.set_gpu(self.config)
self.summaryWriter = SummaryWriter(log_dir=self.output_model_path)
# logger日志
setup_logger(self.output_model_path)
self.logger = get_logger()
def get_one_page(utils):
time_one_page = time()
news_api = AppleNews(html_r(), 'http://www.appledaily.com.tw/')
api = SaveAppleNewsToHtml(news_api.home_url)
api.store_in = "one_page/"
utils.mkdir(api.store_in, True)
utils.mkdir(api.store_in + '/img', True)
f = open(api.store_in + 'apple.html', 'w')
PageContent = utils.GetPage(opt.page)
PageContent = news_api.page_parser(PageContent)
api.PastHeader(f, "")
PageContent = api.page_compose(PageContent)
api.PastEntry(f, "", "", ''.join(PageContent), "")
api.PastTail(f)
logger.info("get one page spend %d sec", time() - time_one_page)
sys.exit()
setdefaulttimeout(timeout)
if opt.page:
get_one_page(utils)
time_get_list = time()
news_api = AppleNews(html_r(), 'http://www.appledaily.com.tw/')
news_api.get_list('appledaily/todayapple')
time_end_get_list = time() - time_get_list
logger.info("get list spend %d sec", time_end_get_list)
api = SaveAppleNewsToHtml(news_api.home_url)
#Folder Prepare
api.store_in = opt.folder + "/" + strftime("%Y-%m-%d", localtime())
utils.mkdir(opt.folder, False)
utils.mkdir(api.store_in, False)
utils.mkdir(api.store_in + '/img', False)
blacklist = open('blacklist.txt', 'r').readlines()
if blacklist:
blf = open(api.store_in + '/blacklist.html', 'w')
api.PastHeader(blf, 'BalckList')
for Classify in news_api.news_lists:
if NameMap.get(Classify):
f = open(api.store_in + "/" + NameMap.get(Classify) + '.html', 'w')
else:
logger.info('%s, not support' % (Classify))
continue
