def onCall(self, value, fileStream, fileStream2): uploaded = "Uploaded" uploadDir = "{}/upload/".format(sponge.home) # Single file. if fileStream.hasNext(): IOUtils.readLines(fileStream.inputStream, StandardCharsets.UTF_8) uploaded += " " + fileStream.filename # Multiple files. while fileStream2.hasNext(): fs2 = fileStream2.next() IOUtils.readLines(fs2.inputStream, StandardCharsets.UTF_8) uploaded += " " + fs2.filename return uploaded
def process(self, inputStream, outputStream): # csv_writer=csv.writer(outputStream) self.delimit =self.ff.getAttribute('delimiter') self.escape_char =self.ff.getAttribute('escape_char') fields =self.ff.getAttribute('fields') self.quote_char =self.ff.getAttribute('quote_char') self.encod =self.ff.getAttribute('file_encoding') schema_length =self.ff.getAttribute('schema_length') file_type =self.ff.getAttribute('file_type') if str(self.encod).lower()=='utf8': self.encod=StandardCharsets.UTF_8 elif str(self.encod).lower()=='iso-8859-1': self.encod=StandardCharsets.ISO_8859_1 elif str(self.encod).lower()=='us-ascii': self.encod=StandardCharsets.US_ASCII elif str(self.encod).lower()=='cp1252': self.encod=Charset.forName("windows-1252") file_data='' count=0 for data in IOUtils.readLines(inputStream,self.encod): file_data=file_data+'\n'+data file=StringIO(file_data) str_list=[] if file_type=='Delimited': csv_writer=csv.writer(outputStream,delimiter=str(self.delimit),escapechar=str(self.escape_char),quotechar=str(self.quote_char),quoting=csv.QUOTE_MINIMAL) field_list=json.loads(fields) # IOUtils.write(str(x), outputStream, StandardCharsets.UTF_8) # file = IOUtils.readLines(inputStream, StandardCharsets.UTF_8) data='' csv_reader=csv.reader(file,delimiter=str(self.delimit),escapechar=str(self.escape_char),quotechar=str(self.quote_char)) next(csv_reader) for line in csv_reader: count+=1 row=list(line[x[0]].strip() if ('int','double','float','decimal','long') in x[1]['type'] else line[x[0]] for x in enumerate(field_list['fields'])) row=[str(item).encode('utf8').decode('utf8') for item in row] csv_writer.writerow(row) if file_type=='Fixed Width': field_list=fields.split(')|(') len_list=list(map(lambda x:x.split(',')[1:4],field_list)) len_list=[[i[0],i[i],i[2].replace("'",'')] for i in len_list] csv_writer=csv.writer(outputStream,delimiter=',',quotechar='"',quoting=csv.QUOTE_MINIMAL) for line in file: count=count+1 line=data.splitlines()[0] if int(schema_length.strip()) != int(len(line)) raise Exception('Schema length validation failed for line :'+str(count)+', line length :'+str(len(line))+' Expected length : '+str(schema_length)) row=list([line[int(x[0])-1:(int(x[1])+int(x[0])-1)].strip() if x[2].lower() in ('int','double','float','decimal','long') else line[int(x[0])-1:(int(x[1])+int(x[0])-1)] for x in len_list]) row=[str(item).encode('utf8').decode('utf8') for item in row] csv_writer(row)