def clean_targets(job_list, db): # print('clean_targets (%r)' % job_list) job_list = set(job_list) # now we need to delete the definition closure from compmake.jobs.queries import definition_closure closure = definition_closure(job_list, db) basic = job_list - closure from compmake.jobs.queries import parents other_clean = set() for job_id in job_list: other_clean.update(parents(job_id, db)) other_clean = other_clean - closure # # print('deleting: %r' % closure) # print('only cleaning: %r' % basic) # print('other cleaning: %r' % other_clean) # for job_id in closure | basic | other_clean: clean_cache_relations(job_id, db) # delete all in closure for job_id in closure: from compmake.jobs.storage import delete_all_job_data delete_all_job_data(job_id, db) # just remove cache in basic for job_id in basic: # Cleans associated objects if job_cache_exists(job_id, db): delete_job_cache(job_id, db)
def details_why_one(job_id, context, cq): db = context.get_compmake_db() if job_cache_exists(job_id, db): cache = get_job_cache(job_id, db) why = str(cache.exception) lines = why.split('\n') one = lines[0] if len(lines) > 1: one += ' ... +%d lines' % (len(lines)-1) print('%20s: %s' %(job_id, one))
def go(path): db = StorageFilesystem(path, compress=True) args = ['failed'] cq = CacheQueryDB(db) context = Context(db) if not list(db.keys()): msg = 'Compmake DB is empty' logger.error(msg) else: job_list = parse_job_list(args, context=context, cq=cq) s = "" if job_list: job_list = job_list[:2] s += 'Running on host: %s' % hostname s += "\nJob failed in path %s" % path for job_id in job_list: if job_cache_exists(job_id, db): cache = get_job_cache(job_id, db) status = Cache.state2desc[cache.state] s += "\nFailure of job %s" % job_id if cache.state in [Cache.FAILED, Cache.BLOCKED]: why = str(cache.exception).strip() else: why = 'No why for job done.' s += '\n' + "```\n" + why + "\n```" s += '\n\n' else: logger.warning('no cache for %s' % job_id) s += '\n@censi' s += '\n@jacopo' s += '\n@paull' s += '\n@walter' s += '\n@daniele' print(s) slack.chat.post_message(channel, s, link_names=1) else: s = 'Everything is fine' # slack.chat.post_message(channel, s) logger.info('No jobs found')
def get_sizes(job_id, db): """ Returns byte sizes for jobs pieces. Returns dict with keys 'args','cache','result','total'. """ res = {} res['args'] = job_args_sizeof(job_id, db) if job_cache_exists(job_id, db): res['cache'] = job_cache_sizeof(job_id, db) else: res['cache'] = 0 if job_userobject_exists(job_id, db): res['result'] = job_userobject_sizeof(job_id, db) else: res['result'] = 0 res['total'] = res['cache'] + res['args'] + res['result'] return res
def details_why_one(job_id, context, cq): # @UnusedVariable db = context.get_compmake_db() lines = [] if job_cache_exists(job_id, db): cache = get_job_cache(job_id, db) status = Cache.state2desc[cache.state] if cache.state in [Cache.FAILED, Cache.BLOCKED]: why = cache.exception why = why.strip() lines = why.split('\n') one = lines[0] if len(lines) > 1: one += ' [+%d lines] ' % (len(lines)-1) details = (job_id, status, one) return details return None