def action_group(user, categories, action, dmfilestat_dict, user_comment, backup_directory=None, confirmed=False): '''Single task to group multiple results' actions status. Cycle through the dmfilestat objects per result_id, then per category. DELETE action is performed immediately ARCHIVE/EXPORT change action state to Pending, actual action will be launched by data_management.py periodic task. TEST action prints selected files to log. ''' project_msg = {} for result_pk, DMFileStats in dmfilestat_dict.iteritems(): logger.debug("result_pk: %s" % result_pk, extra=logid) logger.debug("%s contains %d" % (type(DMFileStats), DMFileStats.count()), extra=logid) msg_dict = {} for selection_id in categories: logger.debug("category: %s" % selection_id, extra=logid) for dmfilestat in DMFileStats.filter(dmfileset__type=selection_id): try: if action == dmactions.TEST: test_action(user, user_comment, dmfilestat) status = "success" elif action == dmactions.DELETE: delete_action(user, user_comment, dmfilestat, confirmed=confirmed) status = "success" elif action == dmactions.ARCHIVE or action == dmactions.EXPORT: status = dmactions.set_action_pending( user, user_comment, action, dmfilestat, backup_directory) else: status = "error, unknown action POSTed: '%s'" % action logger.error(status, extra=logid) except Exception as inst: msg_dict[selection_id] = "Error: %s" % str(inst) logger.error("%s - %s" % (selection_id, msg_dict[selection_id]), extra=logid) logger.error(traceback.format_exc(), extra=logid) EventLog.objects.add_entry(dmfilestat.result, "%s - %s. User Comment: %s" % ( selection_id, msg_dict[selection_id], user_comment), username=user) else: msg_dict[selection_id] = status logger.debug("%s - %s" % (selection_id, msg_dict[selection_id]), extra=logid) # Generates message per result logger.debug("%s" % msg_dict, extra=logid) project_msg[result_pk] = msg_dict logger.debug(project_msg, extra=logid) # Generate a status message per group of results? project_msg_banner(user, project_msg, action)
def _process_task(pfilename): ''' Recursive celery task. To trigger an orphaned task: python -c "from iondb.bin import djangoinit; from iondb.rundb.data import dmactions; dmactions._process_task.(<filename>)" where <filename> is full path to the data file found in /var/spool/ion ''' logid = {'logid': "%s" % ('dmactions')} from datetime import datetime from datetime import timedelta logger.debug("Function: %s()" % sys._getframe().f_code.co_name, extra=logid) logger.debug("Task ID: %s" % _process_task.request.id, extra=logid) # catch all unhandled exceptions and clean up try: try: list_of_file_dict = get_action_param_var(pfilename) except Exception as e: logger.error("Error accessing file: %s. Cannot continue the DM action!" % (pfilename), extra=logid) # parse the filename to extract the dmfilestat pk and retrieve dmfilestat object. dmfilestat_pk = os.path.basename(pfilename).split("_")[1] dmfilestat = DMFileStat.objects.get(id=dmfilestat_pk) raise e dmfilestat = DMFileStat.objects.get(id=list_of_file_dict[0]['pk']) terminate = True # flag to indicate recursion termination total_processed = 0 fstatus = "Success" start_time = datetime.now() max_time_delta = timedelta(seconds=10) # list_of_file_dict contains zero, one, or two dictionary variables to iterate over. for d_cnt, mydict in enumerate(list_of_file_dict): logid = {'logid': "%s" % (mydict.get('lockfile', '_process_task'))} # The dictionary contains an element named 'to_process' which is a list variable to iterate over logger.debug("%d, start_dir: %s" % (d_cnt, mydict['start_dir']), extra=logid) logger.info("%6d %s %s" % (len(mydict['to_process']), dmfilestat.dmfileset.type, dmfilestat.result.resultsName), extra=logid) while (datetime.now() - start_time) < max_time_delta: # If there are no files left to process, (all to_process lists are empty), the recursion ends if len(mydict['to_process']) > 0: terminate = False try: # process one file and remove entry from the list path = mydict['to_process'].pop(0) j = mydict['processed_cnt'] + 1 this_file_size = 0 if not os.path.islink(path): this_file_size = os.lstat(path)[6] if _process(path, mydict['action'], mydict['archivepath'], mydict['start_dir'], mydict['to_keep']): mydict['processed_cnt'] = j mydict['total_size'] += this_file_size logger.debug("%04d/%04d %s %10d %s" % ( j, mydict['total_cnt'], mydict['action'], mydict['total_size'], path), extra=logid) except (OSError, IOError) as e: # IOError: [Errno 28] No space left on device: if e.errno == errno.ENOSPC: raise elif e.errno == errno.ENOENT or e.errno == errno.ESTALE: logger.warn("%04d No longer exists %s" % (j, path), extra=logid) continue else: raise except (DMExceptions.RsyncError, DMExceptions.MediaNotAvailable): raise except: errmsg = "%04d/%04d %s %10d %s" % ( j, mydict['total_cnt'], mydict['action'], mydict['total_size'], path) logger.error(errmsg, extra=logid) logger.error(traceback.format_exc(), extra=logid) if not mydict['action'] in [EXPORT, TEST] and dmfilestat.dmfileset.del_empty_dir: thisdir = os.path.dirname(path) try: if len(os.listdir(thisdir)) == 0: if not "plugin_out" in thisdir: try: os.rmdir(thisdir) logger.debug("Removed empty directory: %s" % thisdir, extra=logid) except Exception as e: logger.warn("rmdir [%d] %s: %s" % ( e.errno, e.strerror, thisdir), extra=logid) except OSError as e: if e.errno == errno.ENOENT: logger.warn("del_empty_dir Does not exist %s" % (path), extra=logid) continue else: raise e else: break # only expect to execute this line when no files to process total_processed += mydict['total_size'] except Exception as e: fstatus = "Error" terminate = True dmfilestat.setactionstate('E') logger.error("DM Action failure on %s for %s report." % (dmfilestat.dmfileset.type, dmfilestat.result.resultsName), extra=logid) logger.error("This %s action will need to be manually completed." % (mydict['action']), extra=logid) logger.error("The following is the exception error:\n" + traceback.format_exc(), extra=logid) EventLog.objects.add_entry( dmfilestat.result, "%s - %s. Action not completed. User intervention required." % (fstatus, e), username='******') # Release the task lock try: applock = TaskLock(mydict['lockfile']) applock.unlock() except: logger.error(traceback.format_exc(), extra=logid) # Do the user notification try: # pop up a message banner if mydict['msg_banner']: dmfileset = dmfilestat.dmfileset project_msg = {} msg_dict = {} msg_dict[dmfileset.type] = fstatus project_msg[dmfilestat.result_id] = msg_dict project_msg_banner('', project_msg, mydict['action']) except: logger.error(traceback.format_exc(), extra=logid) # ==================================================================== # Exit function here on error # ==================================================================== return # Remove the data file here, no earlier. In case the task is clobbered, celery # will relaunch the task, access the data file and continue the action. try: os.unlink(pfilename) except: pass if not terminate: # ==================================================================== # Launch next task # ==================================================================== try: mydict.get('action', 'unk') pfilename = set_action_param_var(list_of_file_dict) _process_task.delay(pfilename) except: logger.error(traceback.format_exc(), extra=logid) else: # ==================================================================== # No more files to process. Clean up and exit. # ==================================================================== try: dmfilestat.diskspace = float(total_processed) / (1024 * 1024) dmfilestat.save() logger.info("%0.1f MB %s processed" % (dmfilestat.diskspace, dmfilestat.dmfileset.type), extra=logid) if mydict['action'] in [ARCHIVE, DELETE]: _brokenlinks_delete([dmfilestat.result.get_report_dir(), dmfilestat.result.experiment.expDir]) _emptydir_delete([dmfilestat.result.get_report_dir(), dmfilestat.result.experiment.expDir]) except: logger.error(traceback.format_exc(), extra=logid) # Do the user notification try: _action_complete_update(mydict['user'], mydict['user_comment'], dmfilestat, mydict['action']) # pop up a message banner if mydict['msg_banner']: dmfileset = dmfilestat.dmfileset project_msg = {} msg_dict = {} msg_dict[dmfileset.type] = fstatus project_msg[dmfilestat.result_id] = msg_dict project_msg_banner(mydict['user'], project_msg, mydict['action']) except: logger.error(traceback.format_exc(), extra=logid) # Release the task lock try: applock = TaskLock(mydict['lockfile']) applock.unlock() except: logger.error(traceback.format_exc(), extra=logid) return
def action_group(user, categories, action, dmfilestat_dict, user_comment, backup_directory=None, confirmed=False): '''Single task to group multiple results' actions status. Cycle through the dmfilestat objects per result_id, then per category. DELETE action is performed immediately ARCHIVE/EXPORT change action state to Pending, actual action will be launched by data_management.py periodic task. TEST action prints selected files to log. ''' project_msg = {} for result_pk, DMFileStats in dmfilestat_dict.iteritems(): logger.debug("result_pk: %s" % result_pk, extra=logid) logger.debug("%s contains %d" % (type(DMFileStats), DMFileStats.count()), extra=logid) msg_dict = {} for selection_id in categories: logger.debug("category: %s" % selection_id, extra=logid) for dmfilestat in DMFileStats.filter(dmfileset__type=selection_id): try: if action == dmactions.TEST: test_action(user, user_comment, dmfilestat) status = "success" elif action == dmactions.DELETE: delete_action(user, user_comment, dmfilestat, confirmed=confirmed) status = "success" elif action == dmactions.ARCHIVE or action == dmactions.EXPORT: status = dmactions.set_action_pending( user, user_comment, action, dmfilestat, backup_directory) else: status = "error, unknown action POSTed: '%s'" % action logger.error(status, extra=logid) except Exception as inst: msg_dict[selection_id] = "Error: %s" % str(inst) logger.error("%s - %s" % (selection_id, msg_dict[selection_id]), extra=logid) logger.error(traceback.format_exc(), extra=logid) EventLog.objects.add_entry( dmfilestat.result, "%s - %s. User Comment: %s" % (selection_id, msg_dict[selection_id], user_comment), username=user) else: msg_dict[selection_id] = status logger.debug("%s - %s" % (selection_id, msg_dict[selection_id]), extra=logid) # Generates message per result logger.debug("%s" % msg_dict, extra=logid) project_msg[result_pk] = msg_dict logger.debug(project_msg, extra=logid) # Generate a status message per group of results? project_msg_banner(user, project_msg, action)
def _process_task(pfilename): ''' Recursive celery task ''' from datetime import datetime from datetime import timedelta logger.debug("Function: %s()" % sys._getframe().f_code.co_name) logger.debug("Task ID: %s" % _process_task.request.id) #catch all unhandled exceptions and clean up try: list_of_file_dict = get_action_param_file(pfilename) os.unlink(pfilename) dmfilestat = DMFileStat.objects.get(id=list_of_file_dict[0]['pk']) terminate = True # flag to indicate recursion termination total_processed = 0 start_time = datetime.now() max_time_delta = timedelta(seconds=10) # list_of_file_dict contains zero, one, or two dictionary variables to iterate over. for q,dict in enumerate(list_of_file_dict): # The dictionary contains an element named 'to_process' which is a list variable to iterate over logger.debug("%d, start_dir: %s" % (q,dict['start_dir'])) while (datetime.now() - start_time) < max_time_delta: # If there are no files left to process, (all to_process lists are empty), the recursion ends if len(dict['to_process']) > 0: terminate = False try: # process one file and remove entry from the list path = dict['to_process'].pop(0) j = dict['processed_cnt'] + 1 this_file_size = 0 if not os.path.islink(path): this_file_size = os.lstat(path)[6] if _process(path, dict['action'], dict['archivepath'], dict['start_dir'], dict['to_keep']): dict['processed_cnt'] = j dict['total_size'] += this_file_size logger.info("%04d/%04d %s %10d %s" % (j, dict['total_cnt'], dict['action'], dict['total_size'], path)) except (OSError,IOError) as e: #IOError: [Errno 28] No space left on device: if e.errno == errno.ENOSPC: raise elif e.errno == errno.ENOENT: logger.warn("%04d No longer exists %s" % (j,path)) continue except: errmsg = "%04d/%04d %s %10d %s" % (j, dict['total_cnt'], dict['action'], dict['total_size'], path) logger.error(errmsg) logger.error(traceback.format_exc()) if not dict['action'] in [EXPORT,TEST] and dmfilestat.dmfileset.del_empty_dir: dir = os.path.dirname(path) try: if len(os.listdir(dir)) == 0: if not "plugin_out" in dir: try: os.rmdir(dir) logger.debug("Removed empty directory: %s" % dir) except Exception as e: logger.warn("rmdir [%d] %s: %s" % (e.errno,e.strerror,dir)) except OSError as e: if e.errno == errno.ENOENT: logger.warn("del_empty_dir Does not exist %s" % (path)) continue else: raise e else: break # only expect to execute this line when no files to process total_processed += dict['total_size'] except: dmfilestat.setactionstate('E') logger.error("DM Action failure on %s for %s report." % (dmfilestat.dmfileset.type,dmfilestat.result.resultsName)) logger.error("This %s action will need to be manually completed." % (dict['action'])) logger.error("The following is the exception error:\n"+traceback.format_exc()) EventLog.objects.add_entry(dmfilestat.result,"%s - %s" % (dmfilestat.dmfileset.type, msg),username='******') if dict['lockfile']: applock = TaskLock(dict['lockfile']) applock.unlock() return #logger.debug("Sleep for 1") #import time #time.sleep(1) if terminate: try: # No more files to process. Do the clean up. dmfilestat.diskspace = float(total_processed)/(1024*1024) dmfilestat.save() logger.info("%0.1f MB %s processed" % (dmfilestat.diskspace, dmfilestat.dmfileset.type)) if dict['action'] in [ARCHIVE, DELETE]: _emptydir_delete(dmfilestat) _action_complete_update(dict['user'], dict['user_comment'], dmfilestat, dict['action']) # pop up a message banner if dict['msg_banner']: dmfileset = dmfilestat.dmfileset project_msg = {} msg_dict = {} msg_dict[dmfileset.type] = "Success" project_msg[dmfilestat.result_id] = msg_dict project_msg_banner('', project_msg, dict['action']) if dict['lockfile']: applock = TaskLock(dict['lockfile']) applock.unlock() except: logger.exception(traceback.format_exc()) else: # Launch next task try: pfilename = set_action_param_file(list_of_file_dict) celery_result = _process_task.delay(pfilename) except: logger.error(traceback.format_exc()) return
def _process_task(pfilename): ''' Recursive celery task ''' from datetime import datetime from datetime import timedelta logger.debug("Function: %s()" % sys._getframe().f_code.co_name) logger.debug("Task ID: %s" % _process_task.request.id) #catch all unhandled exceptions and clean up try: list_of_file_dict = get_action_param_file(pfilename) os.unlink(pfilename) dmfilestat = DMFileStat.objects.get(id=list_of_file_dict[0]['pk']) terminate = True # flag to indicate recursion termination total_processed = 0 start_time = datetime.now() max_time_delta = timedelta(seconds=10) # list_of_file_dict contains zero, one, or two dictionary variables to iterate over. for q, dict in enumerate(list_of_file_dict): # The dictionary contains an element named 'to_process' which is a list variable to iterate over logger.debug("%d, start_dir: %s" % (q, dict['start_dir'])) while (datetime.now() - start_time) < max_time_delta: # If there are no files left to process, (all to_process lists are empty), the recursion ends if len(dict['to_process']) > 0: terminate = False try: # process one file and remove entry from the list path = dict['to_process'].pop(0) j = dict['processed_cnt'] + 1 this_file_size = 0 if not os.path.islink(path): this_file_size = os.lstat(path)[6] if _process(path, dict['action'], dict['archivepath'], dict['start_dir'], dict['to_keep']): dict['processed_cnt'] = j dict['total_size'] += this_file_size logger.info("%04d/%04d %s %10d %s" % (j, dict['total_cnt'], dict['action'], dict['total_size'], path)) except (OSError, IOError) as e: #IOError: [Errno 28] No space left on device: if e.errno == errno.ENOSPC: raise elif e.errno == errno.ENOENT: logger.warn("%04d No longer exists %s" % (j, path)) continue except: errmsg = "%04d/%04d %s %10d %s" % ( j, dict['total_cnt'], dict['action'], dict['total_size'], path) logger.error(errmsg) logger.error(traceback.format_exc()) if not dict['action'] in [ EXPORT, TEST ] and dmfilestat.dmfileset.del_empty_dir: dir = os.path.dirname(path) try: if len(os.listdir(dir)) == 0: if not "plugin_out" in dir: try: os.rmdir(dir) logger.debug( "Removed empty directory: %s" % dir) except Exception as e: logger.warn("rmdir [%d] %s: %s" % (e.errno, e.strerror, dir)) except OSError as e: if e.errno == errno.ENOENT: logger.warn("del_empty_dir Does not exist %s" % (path)) continue else: raise e else: break # only expect to execute this line when no files to process total_processed += dict['total_size'] except: dmfilestat.setactionstate('E') logger.error( "DM Action failure on %s for %s report." % (dmfilestat.dmfileset.type, dmfilestat.result.resultsName)) logger.error("This %s action will need to be manually completed." % (dict['action'])) logger.error("The following is the exception error:\n" + traceback.format_exc()) EventLog.objects.add_entry(dmfilestat.result, "%s - %s" % (dmfilestat.dmfileset.type, msg), username='******') if dict['lockfile']: applock = TaskLock(dict['lockfile']) applock.unlock() return #logger.debug("Sleep for 1") #import time #time.sleep(1) if terminate: try: # No more files to process. Do the clean up. dmfilestat.diskspace = float(total_processed) / (1024 * 1024) dmfilestat.save() logger.info("%0.1f MB %s processed" % (dmfilestat.diskspace, dmfilestat.dmfileset.type)) if dict['action'] in [ARCHIVE, DELETE]: _emptydir_delete(dmfilestat) _action_complete_update(dict['user'], dict['user_comment'], dmfilestat, dict['action']) # pop up a message banner if dict['msg_banner']: dmfileset = dmfilestat.dmfileset project_msg = {} msg_dict = {} msg_dict[dmfileset.type] = "Success" project_msg[dmfilestat.result_id] = msg_dict project_msg_banner('', project_msg, dict['action']) if dict['lockfile']: applock = TaskLock(dict['lockfile']) applock.unlock() except: logger.exception(traceback.format_exc()) else: # Launch next task try: pfilename = set_action_param_file(list_of_file_dict) celery_result = _process_task.delay(pfilename) except: logger.error(traceback.format_exc()) return
def _process_task(pfilename): ''' Recursive celery task. To trigger an orphaned task: python -c "from iondb.bin import djangoinit; from iondb.rundb.data import dmactions; dmactions._process_task.(<filename>)" where <filename> is full path to the data file found in /var/spool/ion ''' logid = {'logid':"%s" % ('dmactions')} from datetime import datetime from datetime import timedelta logger.debug("Function: %s()" % sys._getframe().f_code.co_name, extra = logid) logger.debug("Task ID: %s" % _process_task.request.id, extra = logid) #catch all unhandled exceptions and clean up try: try: list_of_file_dict = get_action_param_var(pfilename) except Exception as e: logger.error("Error accessing file: %s. Cannot continue the DM action!" % (pfilename), extra = logid) # parse the filename to extract the dmfilestat pk and retrieve dmfilestat object. dmfilestat_pk = os.path.basename(pfilename).split("_")[1] dmfilestat = DMFileStat.objects.get(id=dmfilestat_pk) raise e dmfilestat = DMFileStat.objects.get(id=list_of_file_dict[0]['pk']) terminate = True # flag to indicate recursion termination total_processed = 0 fstatus = "Success" start_time = datetime.now() max_time_delta = timedelta(seconds=10) # list_of_file_dict contains zero, one, or two dictionary variables to iterate over. for d_cnt, dict in enumerate(list_of_file_dict): logid = {'logid':"%s" % (dict.get('lockfile','_process_task'))} # The dictionary contains an element named 'to_process' which is a list variable to iterate over logger.debug("%d, start_dir: %s" % (d_cnt, dict['start_dir']), extra = logid) logger.info("%6d %s %s" %(len(dict['to_process']), dmfilestat.dmfileset.type, dmfilestat.result.resultsName), extra = logid) while (datetime.now() - start_time) < max_time_delta: # If there are no files left to process, (all to_process lists are empty), the recursion ends if len(dict['to_process']) > 0: terminate = False try: # process one file and remove entry from the list path = dict['to_process'].pop(0) j = dict['processed_cnt'] + 1 this_file_size = 0 if not os.path.islink(path): this_file_size = os.lstat(path)[6] if _process(path, dict['action'], dict['archivepath'], dict['start_dir'], dict['to_keep']): dict['processed_cnt'] = j dict['total_size'] += this_file_size logger.debug("%04d/%04d %s %10d %s" % (j, dict['total_cnt'], dict['action'], dict['total_size'], path), extra = logid) except (OSError, IOError) as e: #IOError: [Errno 28] No space left on device: if e.errno == errno.ENOSPC: raise elif e.errno == errno.ENOENT or e.errno == errno.ESTALE: logger.warn("%04d No longer exists %s" % (j, path), extra = logid) continue else: raise except (DMExceptions.RsyncError, DMExceptions.MediaNotAvailable): raise except: errmsg = "%04d/%04d %s %10d %s" % (j, dict['total_cnt'], dict['action'], dict['total_size'], path) logger.error(errmsg, extra = logid) logger.error(traceback.format_exc(), extra = logid) if not dict['action'] in [EXPORT, TEST] and dmfilestat.dmfileset.del_empty_dir: dir = os.path.dirname(path) try: if len(os.listdir(dir)) == 0: if not "plugin_out" in dir: try: os.rmdir(dir) logger.debug("Removed empty directory: %s" % dir, extra = logid) except Exception as e: logger.warn("rmdir [%d] %s: %s" % (e.errno, e.strerror, dir), extra = logid) except OSError as e: if e.errno == errno.ENOENT: logger.warn("del_empty_dir Does not exist %s" % (path), extra = logid) continue else: raise e else: break # only expect to execute this line when no files to process total_processed += dict['total_size'] except Exception as e: fstatus = "Error" terminate = True dmfilestat.setactionstate('E') logger.error("DM Action failure on %s for %s report." % (dmfilestat.dmfileset.type, dmfilestat.result.resultsName), extra = logid) logger.error("This %s action will need to be manually completed." % (dict['action']), extra = logid) logger.error("The following is the exception error:\n"+traceback.format_exc(), extra = logid) EventLog.objects.add_entry(dmfilestat.result,"%s - %s. Action not completed. User intervention required." % (fstatus, e), username='******') # Release the task lock try: applock = TaskLock(dict['lockfile']) applock.unlock() except: logger.error(traceback.format_exc(), extra = logid) # Do the user notification try: # pop up a message banner if dict['msg_banner']: dmfileset = dmfilestat.dmfileset project_msg = {} msg_dict = {} msg_dict[dmfileset.type] = fstatus project_msg[dmfilestat.result_id] = msg_dict project_msg_banner('', project_msg, dict['action']) except: logger.error(traceback.format_exc(), extra = logid) # ==================================================================== # Exit function here on error # ==================================================================== return # Remove the data file here, no earlier. In case the task is clobbered, celery # will relaunch the task, access the data file and continue the action. try: os.unlink(pfilename) except: pass if not terminate: # ==================================================================== # Launch next task # ==================================================================== try: dict.get('action','unk') pfilename = set_action_param_var(list_of_file_dict) _process_task.delay(pfilename) except: logger.error(traceback.format_exc(), extra = logid) else: # ==================================================================== # No more files to process. Clean up and exit. # ==================================================================== try: dmfilestat.diskspace = float(total_processed)/(1024*1024) dmfilestat.save() logger.info("%0.1f MB %s processed" % (dmfilestat.diskspace, dmfilestat.dmfileset.type), extra = logid) if dict['action'] in [ARCHIVE, DELETE]: _brokenlinks_delete([dmfilestat.result.get_report_dir(), dmfilestat.result.experiment.expDir]) _emptydir_delete([dmfilestat.result.get_report_dir(), dmfilestat.result.experiment.expDir]) except: logger.error(traceback.format_exc(), extra = logid) # Do the user notification try: _action_complete_update(dict['user'], dict['user_comment'], dmfilestat, dict['action']) # pop up a message banner if dict['msg_banner']: dmfileset = dmfilestat.dmfileset project_msg = {} msg_dict = {} msg_dict[dmfileset.type] = fstatus project_msg[dmfilestat.result_id] = msg_dict project_msg_banner(dict['user'], project_msg, dict['action']) except: logger.error(traceback.format_exc(), extra = logid) # Release the task lock try: applock = TaskLock(dict['lockfile']) applock.unlock() except: logger.error(traceback.format_exc(), extra = logid) return