def process(self,datas):
if self.connector.TypeName == 'MongoDBConnector':
etype = self.ExecuteType;
table = self.Table;
work = {'OnlyInsert': lambda d: table.save(d),'InsertOrUpdate':lambda d: table.save(d)};
for data in datas:
work[etype](data);
yield data;
else:
if self.filetype in ['csv', 'txt']:
field = extends.getkeys(datas);
self.writer = csv.DictWriter(self.file, field, delimiter=sp, lineterminator='\n')
self.writer.writeheader()
for data in datas:
self.writer.writerow(data);
yield data;
elif self.filetype == 'json':
self.file.write('[')
for data in datas:
json.dump(data, self.file, ensure_ascii=False)
self.file.write(',');
yield data;
self.file.write(']')
self.file.close();
def process(self,datas):
if self.connector.TypeName == 'MongoDBConnector':
etype = self.ExecuteType;
table = self.Table;
work = {'OnlyInsert': lambda d: table.save(d),'InsertOrUpdate':lambda d: table.save(d)};
for data in datas:
work[etype](data);
yield data;
else:
if self.filetype in ['csv', 'txt']:
field = extends.getkeys(datas);
self.writer = csv.DictWriter(self.file, field, delimiter=sp, lineterminator='\n')
self.writer.writeheader()
for data in datas:
self.writer.writerow(data);
yield data;
elif self.filetype == 'json':
self.file.write('[')
for data in datas:
json.dump(data, self.file, ensure_ascii=False)
self.file.write(',');
yield data;
self.file.write(']')
self.file.close();
def FileConnect(etl, datas, type):
path = etl.NewTableName
filetype = path.split('.')[-1].lower()
encode = 'utf-8'
if not etl.Init:
file = open(path, type, encoding=encode)
etl.file = file
etl.filetype = filetype
if filetype in ['csv', 'txt']:
sp = ',' if filetype == 'csv' else '\t'
if not etl.Init:
import csv
if type == 'w':
field = extends.getkeys(datas)
etl.writer = csv.DictWriter(file,
field,
delimiter=sp,
lineterminator='\n')
etl.writer.writeheader()
etl.Init = True
if type == 'r':
reader = csv.DictReader(file, delimiter=sp)
for r in reader:
yield r
else:
for data in datas:
etl.writer.writerow(data)
yield data
elif filetype == 'json':
if type == 'r':
items = json.load(file)
for r in items:
yield r
else:
file.write('[')
for data in datas:
json.dump(data, file, ensure_ascii=False)
file.write(',')
yield data
etl.Init = True
def FileConnect(etl, datas, type):
path = etl.NewTableName;
filetype = path.split('.')[-1].lower();
encode = 'utf-8';
if not etl.Init:
file=open(path, type, encoding=encode)
etl.file = file;
etl.filetype = filetype;
if filetype in ['csv', 'txt']:
sp = ',' if filetype == 'csv' else '\t';
if not etl.Init:
import csv
if type == 'w':
field = extends.getkeys(datas);
etl.writer = csv.DictWriter(file, field, delimiter=sp, lineterminator='\n')
etl.writer.writeheader()
etl.Init = True;
if type == 'r':
reader = csv.DictReader(file, delimiter=sp)
for r in reader:
yield r;
else:
for data in datas:
etl.writer.writerow(data);
yield data;
elif filetype == 'json':
if type == 'r':
items = json.load(file);
for r in items:
yield r;
else:
file.write('[')
for data in datas:
json.dump(data, file, ensure_ascii=False)
file.write(',');
yield data;
etl.Init = True
