def process(self,datas): if self.connector.TypeName == 'MongoDBConnector': etype = self.ExecuteType; table = self.Table; work = {'OnlyInsert': lambda d: table.save(d),'InsertOrUpdate':lambda d: table.save(d)}; for data in datas: work[etype](data); yield data; else: if self.filetype in ['csv', 'txt']: field = extends.getkeys(datas); self.writer = csv.DictWriter(self.file, field, delimiter=sp, lineterminator='\n') self.writer.writeheader() for data in datas: self.writer.writerow(data); yield data; elif self.filetype == 'json': self.file.write('[') for data in datas: json.dump(data, self.file, ensure_ascii=False) self.file.write(','); yield data; self.file.write(']') self.file.close();
def FileConnect(etl, datas, type): path = etl.NewTableName filetype = path.split('.')[-1].lower() encode = 'utf-8' if not etl.Init: file = open(path, type, encoding=encode) etl.file = file etl.filetype = filetype if filetype in ['csv', 'txt']: sp = ',' if filetype == 'csv' else '\t' if not etl.Init: import csv if type == 'w': field = extends.getkeys(datas) etl.writer = csv.DictWriter(file, field, delimiter=sp, lineterminator='\n') etl.writer.writeheader() etl.Init = True if type == 'r': reader = csv.DictReader(file, delimiter=sp) for r in reader: yield r else: for data in datas: etl.writer.writerow(data) yield data elif filetype == 'json': if type == 'r': items = json.load(file) for r in items: yield r else: file.write('[') for data in datas: json.dump(data, file, ensure_ascii=False) file.write(',') yield data etl.Init = True
def FileConnect(etl, datas, type): path = etl.NewTableName; filetype = path.split('.')[-1].lower(); encode = 'utf-8'; if not etl.Init: file=open(path, type, encoding=encode) etl.file = file; etl.filetype = filetype; if filetype in ['csv', 'txt']: sp = ',' if filetype == 'csv' else '\t'; if not etl.Init: import csv if type == 'w': field = extends.getkeys(datas); etl.writer = csv.DictWriter(file, field, delimiter=sp, lineterminator='\n') etl.writer.writeheader() etl.Init = True; if type == 'r': reader = csv.DictReader(file, delimiter=sp) for r in reader: yield r; else: for data in datas: etl.writer.writerow(data); yield data; elif filetype == 'json': if type == 'r': items = json.load(file); for r in items: yield r; else: file.write('[') for data in datas: json.dump(data, file, ensure_ascii=False) file.write(','); yield data; etl.Init = True