def merge(self):
for source in self.sources:
if not source.fields_map:
self.alarm('存在映射为空的源:{},请检查！'.format(str(source)))
return
task = Task()
task.add_sources(self.sources)
task.merge()
info = '\n{}:\n'.format(datetime.datetime.utcnow())
for x in task.stats:
info += '数据源:{}, 标签:{}, 抽取数目:{}, 去重丢弃:{}\n'.format(
x[0], x[1], x[2], x[3])
orig_text = self.textEdit_2.toPlainText()
self.textEdit_2.setText(orig_text + info)
self.table_info, self.tag_info = task.merge_completed()
parser.add_argument(
"--path",
"-f",
required=True,
help="The path the the file that contains a command per line")
parser.add_argument("--process",
"-p",
required=True,
help="The number of parallel process",
type=int)
parser.add_argument("--timeout",
"-t",
help="The maximum execution time (in sec)",
type=int,
default=None)
parser.add_argument("--gpu",
help="The processes are using a CPU",
action='store_true',
default=False)
args = parser.parse_args()
tasks = []
with open(args.path, 'r', encoding="utf8") as fd:
content = fd.read()
for line in content.split("\n"):
if len(line.strip()) == 0:
continue
tasks.append(Task(line))
runner = TaskRunner(tasks, args)
runner.execute()
