def initModules(self):
if self._initModules:
return
self._initModules = True
names = defaultdict(int)
for module in self.wfData.modules:
md = ModuleData(module)
require, next, find = self.wfData.getConnectModules(md.flowId)
if not find:
continue
m = modulefactory.createModule(md.moduleClass, md.flowId, require, next, self, md)
if m:
if names[md.name] > 0:
m.outdirname = '%s%d' % (md.name, names[md.name])
else:
m.outdirname = md.name
names[md.name] += 1
err = self.addModule(m)
config.Log.info('%s add module:%s:%d' % (self.getTaskId(), m, err))
else:
config.Log.info('%s create module failed : %s' % (self.getTaskId(), md.flowId))
for mname, module in self.modules.items():
for nm in module.next:
if nm not in self.modules:
raise MsgException(ErrorCode.ConnectModuleError, u'初始化模块时，模块连接出错')
for pm in module.require:
if pm not in self.modules:
raise MsgException(ErrorCode.ConnectModuleError, u'初始化模块时，模块连接出错')
def run(self):
createDir(self.workflow.getModuleJsonDir())
self.moduleData.tmpDir = os.path.join(self.workflow.getTempDir(), self.outdirname)
createDir(self.moduleData.tmpDir)
createDir(os.path.join(self.workflow.getOutputDir(), self.outdirname))
filename = self._getFileName()
config.Log.info('write module json to file: %s' % filename)
self.moduleData.saveToFile(filename)
num = len(self.require)
# while True:
# i = 0
# for require in self.require:
# if self.workflow.modules[require].uploading == False:
# i = i+1
# if i == num:
# break
# else:
# time.sleep(5)
try:
rep = requests.post(self.moduleData.start_url,data={
'module_info': filename
})
startcontent = json.loads(rep.content)
config.Log.info(rep.content)
self.restmoduleId = startcontent['instance_id']
config.Log.info('start restmodule %s' % self.restmoduleId)
except:
self.error = ErrorCode.StartRestModuleError
raise MsgException(ErrorCode.StartRestModuleError, u'start restmodule error')
self.err = ErrorCode.NoError
def __init__(self, obj = None, file = None):
assert obj or file, u'ModuleData InValid argument.'
if not obj:
try:
obj = json.load(file)
except:
config.Log.info(traceback.format_exc())
raise MsgException(ErrorCode.LoadJsonError, u'读取json文件错误')
self.__obj = obj
config.Log.info('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#####%s' % obj)
def _prepareOutput(self):
def __getBaseName():
for inp in self.moduleData.inputFiles:
if inp.get('value') and os.path.exists(inp.get('value')):
return inp.get('value')
return self.name
try:
basefile = __getBaseName()
if basefile:
basename = os.path.splitext(os.path.basename(basefile))[0]
outputDir = os.path.join(self.workflow.getOutputDir(),
self.outdirname)
for outputfile in self.moduleData.outputFiles:
outputfile['value'] = os.path.join(
outputDir, basename + outputfile['suffix'])
except:
config.Log.info(traceback.format_exc())
raise MsgException(ErrorCode.PrepareOutputsError,
u'prepare outputs error.')
def _prepareInput(self):
multis = []
try:
for inputfile in self.moduleData.inputFiles:
if inputfile.get('multi', False):
multis.append(inputfile['name'])
else:
inputfile['value'] = self._getInputValueFromInputMap(
inputfile['name'], self.index)
for multi in multis:
inputs = self._getInputsFromInputMap(multi)
if inputs:
for inp in inputs:
inp['name'] = multi
self.moduleData.setMultiInputs(inputs)
except:
config.Log.info(traceback.format_exc())
raise MsgException(ErrorCode.PrepareInputsError,
u'prepare inputs error.')
def _createLocalFiles(dirId, fsRoot, createFile = True):
fs.createDir(fsRoot)
dirs = webos.listDir(userid, dirId)
config.Log.info(dirs)
for d in dirs:
fpath = os.path.join(fsRoot, d['path'])
if d['type'] == 'dir':
_createLocalFiles(d['id'], fpath)
elif createFile:
ft = webos.checkFileType(fpath)
if ft == webos.FileType.NORMAL_FILE:
dataserver.downloadFile(d['metadataid'], fpath)
self.taskData.replaceArgs(d['metadataid'], fpath)
elif ft == webos.FileType.TIFF_FILE:
url = tiffserver.queryDataUrl(d['metadataid'], userid)
if not url:
raise MsgException(ErrorCode.GetNetimgUrlFaild, u'get netimg url faild. metadataid = %s' % d['metadataid'])
self.taskData.replaceArgs(d['metadataid'], url)
else:
# TODO url for shp
pass
def saveToFile(self, filename):
with open(filename, 'w') as f:
try:
json.dump(self.__obj, f, ensure_ascii=False)
except:
raise MsgException(ErrorCode.WriteModuleJsonError, 'write flow file error: %s' % traceback.format_exc())