def to_csv(self): """Export table from json engine to CSV file""" for keys in list(self.script.tables): table_name = self.table_name() header = self.script.tables[keys].get_insert_columns(join=False, create=True) csv_outfile = json2csv(table_name, header_values=header) sort_csv(csv_outfile)
def to_csv(self, sort=True, path=None): # Due to Cyclic imports we can not move this import to the top from retriever.lib.engine_tools import sort_csv for table_name in self.script_table_registry[self.script.name]: csv_file_output = os.path.normpath( os.path.join(path if path else '', table_name[0] + '.csv')) csv_file = open_fw(csv_file_output) csv_writer = open_csvw(csv_file) self.get_cursor() self.set_engine_encoding() self.cursor.execute("SELECT * FROM {};".format(table_name[0])) row = self.cursor.fetchone() column_names = [ u'{}'.format(tuple_i[0]) for tuple_i in self.cursor.description ] csv_writer.writerow(column_names) while row is not None: csv_writer.writerow(row) row = self.cursor.fetchone() csv_file.close() if sort: sort_csv(csv_file_output) self.disconnect()
def to_csv(self): """Export table from xml engine to CSV file.""" for keys in list(self.script.tables): table_name = self.table_name() header = self.script.tables[keys].get_insert_columns(join=False, create=True) csv_outfile = xml2csv(table_name, header_values=header) sort_csv(csv_outfile)
def to_csv(self, sort=True, path=None, select_columns=None): """Export table from json engine to CSV file""" for table_item in self.script_table_registry[self.script.name]: header = table_item[1].get_insert_columns(join=False, create=True) outputfile = os.path.normpath( os.path.join(path if path else '', os.path.splitext(os.path.basename(table_item[0]))[0] + '.csv')) csv_outfile = json2csv(table_item[0], output_file=outputfile, header_values=header) sort_csv(csv_outfile)
def to_csv(self, sort=True, path=None): """Export table from xml engine to CSV file.""" for table_item in self.script_table_registry[self.script.name]: header = table_item[1].get_insert_columns(join=False, create=True) outputfile = os.path.normpath( os.path.join(path if path else '', os.path.splitext(os.path.basename(table_item[0]))[0] + '.csv')) csv_outfile = xml2csv(table_item[0], outputfile=outputfile, header_values=header) sort_csv(csv_outfile)
def to_csv(self): """Export table from json engine to CSV file""" for keys in list(self.script.tables): table_name = self.opts['table_name'].format(db=self.db_name, table=keys) header = self.script.tables[keys].get_insert_columns(join=False, create=True) csv_outfile = json2csv(table_name, header_values=header) sort_csv(csv_outfile)
def to_csv(self, sort=True, path=None, select_columns=None, select_table=None): """Create a CSV file from the a data store. sort flag to create a sorted file, path to write the flag else write to the PWD, select_columns flag is used by large files to select columns data and has SELECT LIMIT 3. """ # Due to Cyclic imports we can not move this import to the top from retriever.lib.engine_tools import sort_csv for table_name in self.script_table_registry[self.script.name]: csv_file_output = os.path.normpath( os.path.join(path if path else '', table_name[0] + '.csv')) self.get_cursor() self.set_engine_encoding() csv_file = open_fw(csv_file_output, encoding=self.encoding) csv_writer = open_csvw(csv_file) limit = "" cols = "*" if select_columns: limit = "LIMIT 3" cols = ",".join(select_columns) sql_query = "SELECT {cols} FROM {tab} {limit};" self.cursor.execute( sql_query.format(cols=cols, tab=table_name[0], limit=limit)) row = self.cursor.fetchone() column_names = [ u'{}'.format(tuple_i[0]) for tuple_i in self.cursor.description ] csv_writer.writerow(column_names) while row is not None: csv_writer.writerow(row) row = self.cursor.fetchone() csv_file.close() if sort: sort_csv(csv_file_output) self.disconnect()
def test_sort_csv(): """Test md5 sum calculation.""" data_file = create_file( ['User,Country,Age', 'Ben,US,24', 'Alex,US,25', 'Alex,PT,25']) out_file = sort_csv(data_file) obs_out = file_2list(out_file) os.remove(out_file) assert obs_out == [ 'User,Country,Age', 'Alex,PT,25', 'Alex,US,25', 'Ben,US,24' ]
def to_csv(self): # Due to Cyclic imports we can not move this import to the top from retriever.lib.engine_tools import sort_csv for _ in list(self.script.urls.keys()): table_name = self.table_name() csv_file_output = os.path.normpath(table_name + '.csv') csv_file = open_fw(csv_file_output) csv_writer = open_csvw(csv_file) self.get_cursor() self.set_engine_encoding() self.cursor.execute("SELECT * FROM {};".format(table_name)) row = self.cursor.fetchone() colnames = [u'{}'.format(tuple_i[0]) for tuple_i in self.cursor.description] csv_writer.writerow(colnames) while row is not None: csv_writer.writerow(row) row = self.cursor.fetchone() csv_file.close() sort_csv(csv_file_output) self.disconnect()
def to_csv(self, sort=True, path=None, select_columns=None): """Export table from xml engine to CSV file.""" for table_item in self.script_table_registry[self.script.name]: header = table_item[1].get_insert_columns(join=False, create=True) outputfile = os.path.normpath( os.path.join( path if path else '', os.path.splitext(os.path.basename(table_item[0]))[0] + '.csv')) empty_rows = 1 if hasattr(self.script, "empty_rows"): empty_rows = self.script.empty_rows input_file = table_item[0] header_values = header csv_outfile = xml2csv_test(input_file, outputfile, header_values, row_tag="row") sort_csv(csv_outfile, encoding=self.encoding)
def to_csv(self, sort=True): # Due to Cyclic imports we can not move this import to the top from retriever.lib.engine_tools import sort_csv for table_n in list(self.script.tables.keys()): table_name = self.table_name(name=table_n) csv_file_output = os.path.normpath(table_name + '.csv') csv_file = open_fw(csv_file_output) csv_writer = open_csvw(csv_file) self.get_cursor() self.set_engine_encoding() self.cursor.execute("SELECT * FROM {};".format(table_name)) row = self.cursor.fetchone() colnames = [u'{}'.format(tuple_i[0]) for tuple_i in self.cursor.description] csv_writer.writerow(colnames) while row is not None: csv_writer.writerow(row) row = self.cursor.fetchone() csv_file.close() if sort: sort_csv(csv_file_output) self.disconnect()
def test_sort_csv(): """Test md5 sum calculation.""" data_file = create_file(['User,Country,Age', 'Ben,US,24', 'Alex,US,25', 'Alex,PT,25']) out_file = sort_csv(data_file) obs_out = file_2list(out_file) os.remove(out_file) assert obs_out == [ 'User,Country,Age', 'Alex,PT,25', 'Alex,US,25', 'Ben,US,24']
def to_csv(self, sort=True, path=None, select_columns=None): """Create a CSV file from the a data store. sort flag to create a sorted file, path to write the flag else write to the PWD, select_columns flag is used by large files to select columns data and has SELECT LIMIT 3. """ # Due to Cyclic imports we can not move this import to the top from retriever.lib.engine_tools import sort_csv for table_name in self.script_table_registry[self.script.name]: csv_file_output = os.path.normpath(os.path.join(path if path else '', table_name[0] + '.csv')) csv_file = open_fw(csv_file_output) csv_writer = open_csvw(csv_file) self.get_cursor() self.set_engine_encoding() limit = "" cols = "*" if select_columns: limit = "LIMIT 3" cols = ",".join(select_columns) sql_query = "SELECT {cols} FROM {tab} {limit};" self.cursor.execute(sql_query.format(cols=cols, tab=table_name[0], limit=limit)) row = self.cursor.fetchone() column_names = [u'{}'.format(tuple_i[0]) for tuple_i in self.cursor.description] csv_writer.writerow(column_names) while row is not None: csv_writer.writerow(row) row = self.cursor.fetchone() csv_file.close() if sort: sort_csv(csv_file_output) self.disconnect()
def to_csv(self, sort=True): """Export table from json engine to CSV file""" for table_item in self.script_table_registry[self.script.name]: header = table_item[1].get_insert_columns(join=False, create=True) csv_outfile = json2csv(table_item[0], header_values=header) sort_csv(csv_outfile)
def to_csv(self, sort=True): """Export sorted version of CSV file""" for table_item in self.script_table_registry[self.script.name]: sort_csv(table_item[0])
def to_csv(self): """Export sorted version of CSV file""" for keys in self.script.tables: table_name = self.opts['table_name'].format(db=self.db_name, table=keys) sort_csv(table_name)
def to_csv(self, sort=True, path=None, select_columns=None): """Export sorted version of CSV file""" for table_item in self.script_table_registry[self.script.name]: sort_csv(table_item[0])
def to_csv(self): """Export sorted version of CSV file""" for keys in self.script.tables: sort_csv(self.table_name())