def main(debug, feed):
''' Retrieve all the packages and for each of them update the release
version.
'''
time = arrow.utcnow().datetime
db.initialize(config)
session = db.Session()
run = db.Run(status='started')
session.add(run)
session.commit()
LOG.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
if debug:
# Console handler
chand = logging.StreamHandler()
chand.setLevel(logging.INFO)
chand.setFormatter(formatter)
LOG.addHandler(chand)
# Save the logs in a file
fhand = logging.FileHandler('/var/tmp/anitya_cron.log')
fhand.setLevel(logging.INFO)
fhand.setFormatter(formatter)
LOG.addHandler(fhand)
if feed:
projects = list(projects_by_feed(session))
session.commit()
else:
# Get all projects, that are ready for check
projects = db.Project.query.order_by(
sa.func.lower(db.Project.name)
).filter(db.Project.next_check < time).all()
project_ids = [project.id for project in projects]
N = config.get('CRON_POOL', 10)
LOG.info("Launching pool (%i) to update %i projects", N, len(project_ids))
p = multiprocessing.Pool(N)
p.map(update_project, project_ids)
run = db.Run(status='ended')
session.add(run)
session.commit()
def run(self):
"""
Start the check run, the run is made of three stages:
1. Preparation - get current date, clear counters, prepare queue
of project
2. Execution - process every project in the queue
3. Finalize - create `db.Run` entry with counters and time
"""
# 1. Preparation phase
# We must convert it to datetime for comparison with sqlalchemy TIMESTAMP column
session = db.Session()
time = arrow.utcnow().datetime
self.clear_counters()
queue = self.construct_queue(time)
total_count = len(queue)
projects_left = len(queue)
projects_iter = iter(queue)
if not queue:
return
# 2. Execution
_log.info(
"Starting check on {} for total of {} projects".format(time, total_count)
)
futures = {}
pool_size = config.get("CRON_POOL")
timeout = config.get("CHECK_TIMEOUT")
with ThreadPoolExecutor(pool_size) as pool:
# Wait till every project in queue is checked
while projects_left:
for project in projects_iter:
future = pool.submit(self.update_project, project)
futures[future] = project
if len(futures) > pool_size:
break # limit job submissions
# Wait for jobs that aren't completed yet
try:
for future in as_completed(futures, timeout=timeout):
projects_left -= 1 # one project down
# log any exception
if future.exception():
try:
future.result()
except Exception as e:
_log.exception(e)
del futures[future]
break # give a chance to add more jobs
except TimeoutError:
projects_left -= 1
_log.info(f"Thread was killed because the execution took too long.")
with self.error_counter_lock:
self.error_counter += 1
# 3. Finalize
_log.info(
"Check done. Checked ({}): error ({}), success ({}), limit ({})".format(
total_count,
self.error_counter,
self.success_counter,
self.ratelimit_counter,
)
)
run = db.Run(
created_on=time,
total_count=total_count,
error_count=self.error_counter,
ratelimit_count=self.ratelimit_counter,
success_count=self.success_counter,
)
session.add(run)
session.commit()