def update_rate(name, count=None):
info = Work.get_info('rate', name)
if not info:
info = {'begin': datetime.utcnow(), 'count': 0}
elif count == -1:
delta = datetime.utcnow() - info['begin']
logger.info('reached %s rate limit after %s seconds', name, delta.seconds)
info['count'] = count or info['count'] + 1
Work.set_info('rate', name, info)
def _validate_rate(name, limit, minutes):
info = Work.get_info('rate', name)
if info:
if datetime.utcnow() > info['begin'] + timedelta(minutes=minutes):
Work.set_info('rate', name, None)
elif info['count'] >= limit:
update_rate(name, count=-1)
return False
elif info['count'] == -1:
return False
return True
def validate_update_path():
if datetime.now().hour not in range(*TIME_RANGE):
return False
res = Work.get_info(NAME, 'updated')
if res and res > datetime.utcnow() - DELTA_UPDATE:
return False
return True
def update_path():
paths = Settings.get_settings('paths')
excl = paths['media_root_exclude']
re_excl = re.compile(r'^(%s)/' % '|'.join([re.escape(p.rstrip('/')) for p in excl]))
for file in iter_files(str(paths['media_root'])):
if not re_excl.search(file):
Media.add_file(file)
time.sleep(.05)
for media in Media.find({'files': {'$exists': True}}, timeout=False):
files_orig = media['files'][:]
for file in files_orig:
if not os.path.exists(file) or re_excl.search(file):
media['files'].remove(file)
if not media['files'] and not media.get('urls'):
Media.remove({'_id': media['_id']}, safe=True)
elif media['files'] != files_orig:
Media.save(media, safe=True)
Work.set_info(NAME, 'updated', datetime.utcnow())
def import_releases(type):
res = Work.get_info(NAME, type)
if not res or res < datetime.utcnow() - DELTA_IMPORT:
globals().get('_import_%s' % type)()
Work.set_info(NAME, type, datetime.utcnow())
