def refresh_libru():
print 'scanning what should be refreshed...'
refreshes = Refresh.objects.all()
links = []
ref_dm = WaitingDM()
for refresh in refreshes:
headers, page = ref_dm.download_headers(refresh.link)
print refresh.link,
field = LAST_MODIFIED
if field in headers:
new_date = headers[field]
if refresh.check_refreshable(new_date):
links.append( (refresh.link, new_date) )
print '--> REFRESH',
else:
print ' no', field, 'in headers!',
print
if not links:
print 'nothing to refresh. Everything is up-to-date'
parser = LibRu()
parser_name = parser.get_filename()+'_refresh'
storage = TaskStorageDB( parser_name, [],Q(parser_name=parser_name, good=True) )
for link, date in links:
refresh = Refresh.objects.get(link=link)
refresh.delete()
#TODO don't forget about make last-modified links actual
# now actual is when it'll be updated by again scanning
# refresh.last_modified = date
task = DirPage(link)
print 'adding task for refresh:', task
storage.accept_new_tasks( [task] )
dm = DM_LibRu_AddRefreshLinks()
tm = TM_LibRuRefresh(storage, dm)
tm.run()
print 'refreshing is finished'
