def shellProcess(self):
"""
Invoke the subprocess command to run the process on the operating system
:return: None
"""
modUtils.printAndLog("Shelling process: %s" % "\n" +
" \\\n\t".join(self.commands))
with Popen(self.commands, stdout=PIPE, stderr=STDOUT,
bufsize=1) as process:
for line in process.stdout: # b'\n'-separated lines
sys.stdout.buffer.write(line) # pass bytes as is
logging.debug(line)
if process.returncode != 0:
raise RuntimeError("%s command failed with exit code %s" %
(str(self), str(process.returncode)))
def createTasks(self):
"""
Determine what tasks to run based on what files are available in the Google bucket
:return: None
"""
self.createPretrainingDataTasks = []
for i, tensorflowRecordFilePath in enumerate(
modConfig.tensorflowRecordFilePaths):
if not modUtils.bucketFileExists(tensorflowRecordFilePath):
self.createPretrainingDataTasks.append(
clsCreatePretrainingDataTask(fileCounter=i))
modUtils.printAndLog(
"Tensorflow record file path not found %s, adding %s" %
(tensorflowRecordFilePath,
str(self.createPretrainingDataTasks[-1])))
self.runPretrainingTask = clsRunPretrainingTask()
def healthCheck(self):
"""
Method to determine if all files exist required to run the task
:return: None
"""
modUtils.printAndLog("Checking for input text file")
if not modUtils.bucketFileExists(
modConfig.inputTextFilePaths[self.fileCounter]):
raise FileNotFoundError(
"Missing file %s" %
modConfig.inputTextFilePaths[self.fileCounter])
modUtils.printAndLog("Checking for bert vocab file")
if not modUtils.bucketFileExists(modConfig.bertVocabFilePath):
raise FileNotFoundError("Missing file %s" %
modConfig.bertVocabFilePath)
def runTasks(self):
"""
Run tasks in sequence
:return: None
"""
modUtils.printAndLog("Running tasks")
if self.createPretrainingDataTasks:
for createPretrainingDataTask in self.createPretrainingDataTasks:
modUtils.printAndLog("Executing %s" %
str(createPretrainingDataTask))
createPretrainingDataTask.execute()
modUtils.printAndLog("Executing %s" % str(self.runPretrainingTask))
self.runPretrainingTask.execute()
modUtils.printAndLog("All tasks complete")
def healthCheck(self):
"""
Method to determine if all files exist required to run the task
:return: None
"""
modUtils.printAndLog("Checking for tensorflow record files")
for tensorflowRecordFilePath in modConfig.tensorflowRecordFilePaths:
if not modUtils.bucketFileExists(tensorflowRecordFilePath):
raise FileNotFoundError("Missing a file %s" %
tensorflowRecordFilePath)
modUtils.printAndLog("Checking for bert config file")
if not modUtils.bucketFileExists(modConfig.bertConfigFilePath):
raise FileNotFoundError("Missing file %s" %
modConfig.bertConfigFilePath)
modUtils.printAndLog("Checking for bert model checkpoint file")
if not modUtils.bucketFileExists(
modConfig.bertModelCheckpointFilePath + ".index"):
raise FileNotFoundError("Missing file %s" %
modConfig.bertModelCheckpointFilePath +
".index")