class TaskLoader(BaseTask):
def __init__(self,logger,project_input_dir,output_dir, process_id, core_call_type,comp_type,max_error_threshold):
self.logger = logger
self.project_input_dir = project_input_dir
self.output_dir = output_dir
self.config = Config ('core/config.ini')
project_comparator_obj = ProjectComparator.objects.get(id=process_id)
# print os.path.join(self.project_input_dir, project_comparator_obj.project_platform_import_file)
self.process_id = process_id
if core_call_type == 'core':
#get input files
self.platform_import_file = self.config.get_config_value('input_files','platform_import_file')
self.url_to_scrape_file = self.config.get_config_value('input_files','url_to_scrape_file')
self.field_to_xpath_file = self.config.get_config_value('input_files','field_to_xpath_file')
#get output files
self.scrapped_data_file = self.config.get_config_value('output_files','scrapped_output_csv')
self.comparision_report_file = self.config.get_config_value('output_files','comparison_report')
elif core_call_type == 'db':
try:
project_comparator_obj = ProjectComparator.objects.get(id=process_id)
#get input files
# print project_comparator_obj
self.platform_import_file = os.path.join(self.project_input_dir, project_comparator_obj.project_platform_import_file)
self.url_to_scrape_file = os.path.join(self.project_input_dir, project_comparator_obj.project_url_file)
self.field_to_xpath_file = os.path.join(self.project_input_dir, project_comparator_obj.project_xpath_file)
#get output files
self.scrapped_data_file = project_comparator_obj.project_scrapper_output_file
self.comparision_report_file = project_comparator_obj.project_report_file
self.comp_type = comp_type
self.max_error_threshold = max_error_threshold
except Exception:
# print '>>>>>>More than one record exists for %s' % process_id, '>>>>>>
pass
def start_task(self):
#Start Scrapping Task
if (self.comp_type == 1):
# Immediate comparison Task
self.reporter = Reporter(self.process_id,self.url_to_scrape_file,self.field_to_xpath_file,self.platform_import_file,self.scrapped_data_file,self.comparision_report_file,self.logger,self.output_dir,self.comp_type,self.max_error_threshold)
self.reporter.start_task()
self.reporter.stop_task()
self.total_error_count = self.max_error_threshold
else:
# Bulk comparison Task
self.scrapper = Scrapper(self.process_id,self.url_to_scrape_file,self.field_to_xpath_file,self.scrapped_data_file,self.logger,self.output_dir,self.comp_type)
self.logger.info('*****************************')
self.scrapper.start_task()
self.scrapper.stop_task()
self.logger.info('*****************************')
#Comparison task
if (os.path.exists(os.path.join(self.output_dir,self.scrapped_data_file))):
self.comparator = Comparator(self.process_id,self.platform_import_file,self.scrapped_data_file,self.comparision_report_file,self.logger,self.output_dir,self.scrapper)
self.logger.info( '*****************************')
self.comparator.start_task()
self.total_error_count = self.comparator.stop_task()
self.logger.info('*****************************')
else:
self.logger.info('ComparisonId = {} : Output csv missing. Comparison Not performed!!'.format(self.process_id))
def stop_task(self):
self.logger.info('*****************************')
self.logger.info('ComparisonId = {} : Closing Comparator!!!!'.format(self.process_id))
self.logger.info('*****************************')
return self.total_error_count
def main():
global cfg, best_PSNR
args = parse_args()
cfg = Config.from_file(args.config)
os.environ["CUDA_VISIBLE_DEVICES"] = ','.join(
str(gpu) for gpu in cfg.device)
# os.environ["CUDA_VISIBLE_DEVICES"] = '0'
cudnn.benchmark = True
cudnn.fastest = True
if hasattr(datasets, cfg.dataset):
ds = getattr(datasets, cfg.dataset)
else:
raise ValueError('Unknown dataset ' + cfg.dataset)
model = getattr(models, cfg.model.name)(cfg.model).cuda()
cfg.train.input_mean = model.input_mean
cfg.train.input_std = model.input_std
cfg.test.input_mean = model.input_mean
cfg.test.input_std = model.input_std
# Data loading code
# train_loader = torch.utils.data.DataLoader(
# ds(cfg.train),
# batch_size=cfg.train.batch_size,
# shuffle=True,
# num_workers=0, #32,
# pin_memory=True,
# drop_last=True)
val_loader = torch.utils.data.DataLoader(
datasets.UCF101(cfg.test, False),
batch_size=cfg.test.batch_size,
shuffle=False,
num_workers=0, #32,
pin_memory=True)
cfg.train.optimizer.args.max_iter = (
cfg.train.optimizer.args.max_epoch * len(train_loader))
policies = model.get_optim_policies()
for group in policies:
print(('group: {} has {} params, lr_mult: {}, decay_mult: {}'.format(
group['name'],
len(group['params']), group['lr_mult'], group['decay_mult'])))
optimizer = Optim(policies, cfg.train.optimizer)
if cfg.resume or cfg.weight:
checkpoint_path = cfg.resume if cfg.resume else cfg.weight
if os.path.isfile(checkpoint_path):
print(("=> loading checkpoint '{}'".format(checkpoint_path)))
checkpoint = torch.load(checkpoint_path)
model.load_state_dict(checkpoint['state_dict'], False)
if cfg.resume:
optimizer.load_state_dict(checkpoint['grad_dict'])
else:
print(("=> no checkpoint found at '{}'".format(checkpoint_path)))
model = DataParallelwithSyncBN(
model, device_ids=range(len(cfg.device))).cuda()
# define loss function (criterion) optimizer and evaluator
criterion = torch.nn.MSELoss().cuda()
evaluator = EvalPSNR(255.0 / np.mean(cfg.test.input_std))
# PSNR = validate(val_loader, model, optimizer, criterion, evaluator)
# return
for epoch in range(cfg.train.optimizer.args.max_epoch):
# train for one epoch
# train(train_loader, model, optimizer, criterion, epoch)
# evaluate on validation set
if ((epoch + 1) % cfg.logging.eval_freq == 0
or epoch == cfg.train.optimizer.args.max_epoch - 1):
PSNR = validate(val_loader, model, optimizer, criterion, evaluator)
# remember best PSNR and save checkpoint
is_best = PSNR > best_PSNR
best_PSNR = max(PSNR, best_PSNR)
save_checkpoint({
'epoch': epoch + 1,
'arch': dict(cfg),
'state_dict': model.module.state_dict(),
'grad_dict': optimizer.state_dict(),
'best_PSNR': best_PSNR,
}, is_best)
def __init__(self, logger):
self.config = Config()
self.sessions = {}
self.parent_logger = logger
self.logger = logger.get_child('session_manager')