def all_docs(): with open(filename, newline='') as doc_file: fields = get_fieldnames(doc_file) dict_reader = csv.DictReader(doc_file, fieldnames=fields) if 'ticket' in doc_type: fields.append("ticket_time") echo('Using the following ' + str(len(fields)) + ' fields:', quiet) for field in fields: echo(field, quiet) i = 0 for row in dict_reader: # Prepare meta info for each indexed document. meta = { 'index': idx_name, 'type': doc_type, } if id_field_idx is not None: meta['id'] = row[fields[int(id_field_idx)]] # Convert tim inteval to an integer in minutes. for k, v in row.items(): if isinstance(v, str) and isperiod(v): row[k] = t2i(v) if 'ticket' in doc_type: row['ticket_time'] = time_interval(row['create_time'], row['close_time'], '%m/%d/%Y %I:%M:%S %p') i += 1 echo('Sending item %s to ES ...' % i, quiet) yield index_op(row, meta)
def test_2_row(self): dict_row = { 'ticket_no': '10002', 'name': 'Zhang Gavin', 'job_desc': 'aaaa_', 'work_title': 'supervisor' } fields = get_fieldnames(self.doc_file) dict_reader = csv.DictReader(self.doc_file, fields) self.assertEqual(next(dict_reader), dict_row)
def test_3_op(self): dict_action = { '_index': 'qd', '_type': 'ticket', '_id': '10002', '_source': { 'ticket_no': '10002', 'name': 'Zhang Gavin', 'job_desc': 'aaaa_', 'work_title': 'supervisor' } } fields = get_fieldnames(self.doc_file) dict_reader = csv.DictReader(self.doc_file, fields) row = next(dict_reader) meta = {'index': 'qd', 'type': 'ticket', 'id': row[fields[0]]} self.assertEqual(index_op(row, meta), dict_action)
def test_3_op(self): dict_action = { '_index': 'qd', '_type': 'ticket', '_id': '10002', '_source': { 'ticket_no': '10002', 'name': 'Zhang Gavin', 'job_desc': 'aaaa_', 'work_title': 'supervisor' } } fields = get_fieldnames(self.doc_file) dict_reader = csv.DictReader(self.doc_file, fields) row = next(dict_reader) meta = { 'index': 'qd', 'type': 'ticket', 'id': row[fields[0]] } self.assertEqual(index_op(row, meta), dict_action)
def cli(filename): with open(filename, newline='') as file_obj: headers = get_fieldnames(file_obj) for item in headers: print(item)
def test_1_fieldnames(self): fieldnames = ['ticket_no', 'name', 'work_title', 'job_desc'] fields = get_fieldnames(self.doc_file) self.assertEqual(len(fields), 4) self.assertEqual(fields, fieldnames)
import argparse from utils import get_fieldnames parser = argparse.ArgumentParser() parser.add_argument('-f', '--file', required=True, help='File to be processed') parser.add_argument('--field', required=True, help='Field name to be inserted') parser.add_argument('-i', '--index', help='Position for the new field') args = parser.parse_args() with open(args.file, newline='') as f: fields = get_fieldnames(f) if args.index: fields.insert(args.index, args.field) else: fields.append(args.field)