def __init__(self, model="local", **kwargs): kwargs = defaultdict(lambda: None, kwargs) self.model = model if self.model == "sge": self.run = Qsub(queue_parameter=kwargs['queue_parameter']) elif self.model == "local": self.run = Local(count=kwargs['count']) else: raise "model %s not support !" % self.model
class Run(object): def __init__(self, model="local", **kwargs): kwargs = defaultdict(lambda: None, kwargs) self.model = model if self.model == "sge": self.run = Qsub(queue_parameter=kwargs['queue_parameter']) elif self.model == "local": self.run = Local(count=kwargs['count']) else: raise "model %s not support !" % self.model def start(self, scripts, hold_jid=None): if self.model == "sge": orders = dict() for (s, m) in sorted(scripts, key=lambda x: self.order_s(x[0])): if not os.path.isfile(s): continue err = re.sub("\.sh$", ".err", s) log = re.sub("\.sh$", ".out", s) num = self.order_s(s) try: hold = ",".join([str(i) for i, j in orders.iteritems() if int(j) < num]) if len(orders) else str() hold += ",%s" % hold_jid if hold_jid else "" ids, jobs = self.run.run(s, mem=m, err=err, log=log, hold=hold) orders[ids] = num print "ID [ {0} ] : Task [ {1} ] has been successfully delivered !".format(ids, jobs) except IOError as e: self.run.close() raise IOError(e) return orders elif self.model == "local": orders = dict() for k, g in groupby(scripts, key=lambda x: self.order_s(x)): orders[k] = list(g) for s, m in sorted(orders.iteritems(), key=lambda x: x[0]): for j in m: err = re.sub("\.sh$", ".err", j) log = re.sub("\.sh$", ".out", j) self.run.run(j, err, log) self.run.close() return orders else: pass @staticmethod def order_s(name): name = os.path.basename(name) order = re.compile("step([0-9]+)\..*\.sh$", re.I).match(name) if order: return int(order.group(1)) else: return 0