def update_project(project_id):
""" Check for updates on the specified project. """
session = db.Session()
project = db.Project.by_id(session, project_id)
with blacklist_lock:
if project.backend in blacklist_dict:
LOG.info(
"Backend is blacklisted. Rescheduling to {}".format(
blacklist_dict[project.backend]
)
)
project.next_check = blacklist_dict[project.backend]
session.add(project)
session.commit()
blacklist_lock.release()
return
try:
utilities.check_project_release(project, session),
except anitya.lib.exceptions.RateLimitException as err:
with blacklist_lock:
LOG.debug("Rate limit threshold reached. Adding {} to blacklist.".format(
project.backend
))
blacklist_dict[project.backend] = err.reset_time.to('utc').datetime
except anitya.lib.exceptions.AnityaException as err:
LOG.info(err)
def update_project(self, project_id: int) -> None:
"""
Check for updates on the specified project.
Args:
project_id: Id of project to check
"""
session = db.Session()
project = db.Project.query.filter(db.Project.id == project_id).one()
if project.backend in self.blacklist_dict:
self.blacklist_project(project, self.blacklist_dict[project.backend])
_log.info(
"Backend is blacklisted. Rescheduling to {}".format(
self.blacklist_dict[project.backend]
)
)
project.next_check = self.blacklist_dict[project.backend]
session.add(project)
session.commit()
return
try:
_log.debug(f"Checking project {project.name}")
utilities.check_project_release(project, session)
except RateLimitException as err:
self.blacklist_project(project, err.reset_time)
return
except AnityaException as err:
_log.info(f"{project.name} : {str(err)}")
with self.error_counter_lock:
self.error_counter += 1
return
with self.success_counter_lock:
self.success_counter += 1
def main(debug, feed):
''' Retrieve all the packages and for each of them update the release
version.
'''
time = arrow.utcnow().datetime
db.initialize(config)
session = db.Session()
run = db.Run(status='started')
session.add(run)
session.commit()
LOG.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
if debug:
# Console handler
chand = logging.StreamHandler()
chand.setLevel(logging.INFO)
chand.setFormatter(formatter)
LOG.addHandler(chand)
# Save the logs in a file
fhand = logging.FileHandler('/var/tmp/anitya_cron.log')
fhand.setLevel(logging.INFO)
fhand.setFormatter(formatter)
LOG.addHandler(fhand)
if feed:
projects = list(projects_by_feed(session))
session.commit()
else:
# Get all projects, that are ready for check
projects = db.Project.query.order_by(
sa.func.lower(db.Project.name)
).filter(db.Project.next_check < time).all()
project_ids = [project.id for project in projects]
N = config.get('CRON_POOL', 10)
LOG.info("Launching pool (%i) to update %i projects", N, len(project_ids))
p = multiprocessing.Pool(N)
p.map(update_project, project_ids)
run = db.Run(status='ended')
session.add(run)
session.commit()
def run(self):
"""
Start the check run, the run is made of three stages:
1. Preparation - get current date, clear counters, prepare queue
of project
2. Execution - process every project in the queue
3. Finalize - create `db.Run` entry with counters and time
"""
# 1. Preparation phase
# We must convert it to datetime for comparison with sqlalchemy TIMESTAMP column
session = db.Session()
time = arrow.utcnow().datetime
self.clear_counters()
queue = self.construct_queue(time)
total_count = len(queue)
projects_left = len(queue)
projects_iter = iter(queue)
if not queue:
return
# 2. Execution
_log.info(
"Starting check on {} for total of {} projects".format(time, total_count)
)
futures = {}
pool_size = config.get("CRON_POOL")
timeout = config.get("CHECK_TIMEOUT")
with ThreadPoolExecutor(pool_size) as pool:
# Wait till every project in queue is checked
while projects_left:
for project in projects_iter:
future = pool.submit(self.update_project, project)
futures[future] = project
if len(futures) > pool_size:
break # limit job submissions
# Wait for jobs that aren't completed yet
try:
for future in as_completed(futures, timeout=timeout):
projects_left -= 1 # one project down
# log any exception
if future.exception():
try:
future.result()
except Exception as e:
_log.exception(e)
del futures[future]
break # give a chance to add more jobs
except TimeoutError:
projects_left -= 1
_log.info(f"Thread was killed because the execution took too long.")
with self.error_counter_lock:
self.error_counter += 1
# 3. Finalize
_log.info(
"Check done. Checked ({}): error ({}), success ({}), limit ({})".format(
total_count,
self.error_counter,
self.success_counter,
self.ratelimit_counter,
)
)
run = db.Run(
created_on=time,
total_count=total_count,
error_count=self.error_counter,
ratelimit_count=self.ratelimit_counter,
success_count=self.success_counter,
)
session.add(run)
session.commit()