def refresh_libru(): print 'scanning what should be refreshed...' refreshes = Refresh.objects.all() links = [] ref_dm = WaitingDM() for refresh in refreshes: headers, page = ref_dm.download_headers(refresh.link) print refresh.link, field = LAST_MODIFIED if field in headers: new_date = headers[field] if refresh.check_refreshable(new_date): links.append( (refresh.link, new_date) ) print '--> REFRESH', else: print ' no', field, 'in headers!', print if not links: print 'nothing to refresh. Everything is up-to-date' parser = LibRu() parser_name = parser.get_filename()+'_refresh' storage = TaskStorageDB( parser_name, [],Q(parser_name=parser_name, good=True) ) for link, date in links: refresh = Refresh.objects.get(link=link) refresh.delete() #TODO don't forget about make last-modified links actual # now actual is when it'll be updated by again scanning # refresh.last_modified = date task = DirPage(link) print 'adding task for refresh:', task storage.accept_new_tasks( [task] ) dm = DM_LibRu_AddRefreshLinks() tm = TM_LibRuRefresh(storage, dm) tm.run() print 'refreshing is finished'