def all_docs():
with open(filename, newline='') as doc_file:
fields = get_fieldnames(doc_file)
dict_reader = csv.DictReader(doc_file, fieldnames=fields)
if 'ticket' in doc_type:
fields.append("ticket_time")
echo('Using the following ' + str(len(fields)) + ' fields:',
quiet)
for field in fields:
echo(field, quiet)
i = 0
for row in dict_reader:
# Prepare meta info for each indexed document.
meta = {
'index': idx_name,
'type': doc_type,
}
if id_field_idx is not None:
meta['id'] = row[fields[int(id_field_idx)]]
# Convert tim inteval to an integer in minutes.
for k, v in row.items():
if isinstance(v, str) and isperiod(v):
row[k] = t2i(v)
if 'ticket' in doc_type:
row['ticket_time'] = time_interval(row['create_time'],
row['close_time'],
'%m/%d/%Y %I:%M:%S %p')
i += 1
echo('Sending item %s to ES ...' % i, quiet)
yield index_op(row, meta)
def all_docs():
with open(filename, newline='') as doc_file:
fields = get_fieldnames(doc_file)
dict_reader = csv.DictReader(doc_file, fieldnames=fields)
if 'ticket' in doc_type:
fields.append("ticket_time")
echo('Using the following ' + str(len(fields)) + ' fields:',
quiet)
for field in fields:
echo(field, quiet)
i = 0
for row in dict_reader:
# Prepare meta info for each indexed document.
meta = {
'index': idx_name,
'type': doc_type,
}
if id_field_idx is not None:
meta['id'] = row[fields[int(id_field_idx)]]
# Convert tim inteval to an integer in minutes.
for k, v in row.items():
if isinstance(v, str) and isperiod(v):
row[k] = t2i(v)
if 'ticket' in doc_type:
row['ticket_time'] = time_interval(row['create_time'],
row['close_time'],
'%m/%d/%Y %I:%M:%S %p')
i += 1
echo('Sending item %s to ES ...' % i, quiet)
yield index_op(row, meta)
def test_2_row(self):
dict_row = {
'ticket_no': '10002',
'name': 'Zhang Gavin',
'job_desc': 'aaaa_',
'work_title': 'supervisor'
}
fields = get_fieldnames(self.doc_file)
dict_reader = csv.DictReader(self.doc_file, fields)
self.assertEqual(next(dict_reader), dict_row)
def test_2_row(self):
dict_row = {
'ticket_no': '10002',
'name': 'Zhang Gavin',
'job_desc': 'aaaa_',
'work_title': 'supervisor'
}
fields = get_fieldnames(self.doc_file)
dict_reader = csv.DictReader(self.doc_file, fields)
self.assertEqual(next(dict_reader), dict_row)
def test_3_op(self):
dict_action = {
'_index': 'qd',
'_type': 'ticket',
'_id': '10002',
'_source': {
'ticket_no': '10002',
'name': 'Zhang Gavin',
'job_desc': 'aaaa_',
'work_title': 'supervisor'
}
}
fields = get_fieldnames(self.doc_file)
dict_reader = csv.DictReader(self.doc_file, fields)
row = next(dict_reader)
meta = {'index': 'qd', 'type': 'ticket', 'id': row[fields[0]]}
self.assertEqual(index_op(row, meta), dict_action)
def test_3_op(self):
dict_action = {
'_index': 'qd',
'_type': 'ticket',
'_id': '10002',
'_source': {
'ticket_no': '10002',
'name': 'Zhang Gavin',
'job_desc': 'aaaa_',
'work_title': 'supervisor'
}
}
fields = get_fieldnames(self.doc_file)
dict_reader = csv.DictReader(self.doc_file, fields)
row = next(dict_reader)
meta = {
'index': 'qd',
'type': 'ticket',
'id': row[fields[0]]
}
self.assertEqual(index_op(row, meta), dict_action)
def cli(filename):
with open(filename, newline='') as file_obj:
headers = get_fieldnames(file_obj)
for item in headers:
print(item)
def test_1_fieldnames(self):
fieldnames = ['ticket_no', 'name', 'work_title', 'job_desc']
fields = get_fieldnames(self.doc_file)
self.assertEqual(len(fields), 4)
self.assertEqual(fields, fieldnames)
def cli(filename):
with open(filename, newline='') as file_obj:
headers = get_fieldnames(file_obj)
for item in headers:
print(item)
import argparse
from utils import get_fieldnames
parser = argparse.ArgumentParser()
parser.add_argument('-f', '--file', required=True, help='File to be processed')
parser.add_argument('--field', required=True, help='Field name to be inserted')
parser.add_argument('-i', '--index', help='Position for the new field')
args = parser.parse_args()
with open(args.file, newline='') as f:
fields = get_fieldnames(f)
if args.index:
fields.insert(args.index, args.field)
else:
fields.append(args.field)
def test_1_fieldnames(self):
fieldnames = ['ticket_no', 'name', 'work_title', 'job_desc']
fields = get_fieldnames(self.doc_file)
self.assertEqual(len(fields), 4)
self.assertEqual(fields, fieldnames)
import argparse
from utils import get_fieldnames
parser = argparse.ArgumentParser()
parser.add_argument('-f', '--file', required=True,
help='File to be processed')
parser.add_argument('--field', required=True,
help='Field name to be inserted')
parser.add_argument('-i', '--index',
help='Position for the new field')
args = parser.parse_args()
with open(args.file, newline='') as f:
fields = get_fieldnames(f)
if args.index:
fields.insert(args.index, args.field)
else:
fields.append(args.field)
