def get_item_sink(output_file):
fh = get_file_handle(output_file, 'w')
if output_file.endswith('.csv'):
set_max_field_size_limit()
writer = None
def sink(item):
nonlocal writer
if writer is None:
fields = list(six.iterkeys(item))
writer = csv.DictWriter(fh,
fieldnames=fields,
extrasaction='ignore')
writer.writeheader()
writer.writerow(item)
else:
def sink(item):
fh.write(json.dumps(item) + '\n')
try:
yield sink
finally:
fh.close()
def extract_csv_column(input, output, column):
"""Extracts column from given CSV file. Deprecated - use extract_field."""
set_max_field_size_limit()
with smart_open(input, 'r') as input_file, smart_open(output, 'w') as output_file:
reader = csv.DictReader(input_file)
for row in reader:
output_file.write(row[column] + '\n')
def extract_csv_column_unique(input, output, column):
set_max_field_size_limit()
with smart_open(input, 'r') as input_file, smart_open(output,
'w') as output_file:
reader = csv.DictReader(input_file)
seen = set() # set for fast O(1) amortized lookup
for row in reader:
if row[column] in seen:
continue
seen.add(row[column])
output_file.write(row[column] + '\n')
def get_item_iterable(input_file):
fh = get_file_handle(input_file, 'r')
if input_file.endswith('.csv'):
set_max_field_size_limit()
reader = csv.DictReader(fh)
else:
reader = (json.loads(line) for line in fh)
try:
yield reader
finally:
fh.close()
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import argparse
import csv
from ethereumetl.csv_utils import set_max_field_size_limit
from ethereumetl.file_utils import smart_open
parser = argparse.ArgumentParser(description='Extracts a single column from a given csv file.')
parser.add_argument('-i', '--input', default='-', type=str, help='The input file. If not specified stdin is used.')
parser.add_argument('-o', '--output', default='-', type=str, help='The output file. If not specified stdout is used.')
parser.add_argument('-c', '--column', required=True, type=str, help='The csv column name to extract.')
args = parser.parse_args()
set_max_field_size_limit()
with smart_open(args.input, 'r') as input_file, smart_open(args.output, 'w') as output_file:
reader = csv.DictReader(input_file)
for row in reader:
output_file.write(row[args.column] + '\n')