Exemple #1
0
	def __init__(self, model="local", **kwargs):
		kwargs = defaultdict(lambda: None, kwargs)
		self.model = model
		if self.model == "sge":
			self.run = Qsub(queue_parameter=kwargs['queue_parameter'])
		elif self.model == "local":
			self.run = Local(count=kwargs['count'])
		else:
			raise "model %s not support !" % self.model
Exemple #2
0
class Run(object):
	def __init__(self, model="local", **kwargs):
		kwargs = defaultdict(lambda: None, kwargs)
		self.model = model
		if self.model == "sge":
			self.run = Qsub(queue_parameter=kwargs['queue_parameter'])
		elif self.model == "local":
			self.run = Local(count=kwargs['count'])
		else:
			raise "model %s not support !" % self.model

	def start(self, scripts, hold_jid=None):
		if self.model == "sge":
			orders = dict()
			for (s, m) in sorted(scripts, key=lambda x: self.order_s(x[0])):
				if not os.path.isfile(s):
					continue
				err = re.sub("\.sh$", ".err", s)
				log = re.sub("\.sh$", ".out", s)
				num = self.order_s(s)
				try:
					hold = ",".join([str(i) for i, j in orders.iteritems() if int(j) < num]) if len(orders) else str()
					hold += ",%s" % hold_jid if hold_jid else ""
					ids, jobs = self.run.run(s, mem=m, err=err, log=log, hold=hold)
					orders[ids] = num
					print "ID [ {0} ] : Task [ {1} ] has been successfully delivered !".format(ids, jobs)
				except IOError as e:
					self.run.close()
					raise IOError(e)
			return orders
		elif self.model == "local":
			orders = dict()
			for k, g in groupby(scripts, key=lambda x: self.order_s(x)):
				orders[k] = list(g)
			for s, m in sorted(orders.iteritems(), key=lambda x: x[0]):
				for j in m:
					err = re.sub("\.sh$", ".err", j)
					log = re.sub("\.sh$", ".out", j)
					self.run.run(j, err, log)
				self.run.close()
			return orders
		else:
			pass

	@staticmethod
	def order_s(name):
		name = os.path.basename(name)
		order = re.compile("step([0-9]+)\..*\.sh$", re.I).match(name)
		if order:
			return int(order.group(1))
		else:
			return 0