def dontruntest_big_csvs(syn, project, schedule_for_cleanup):
cols = [
Column(name='name', columnType='STRING', maximumSize=1000),
Column(name='foo',
columnType='STRING',
enumValues=['foo', 'bar', 'bat']),
Column(name='x', columnType='DOUBLE'),
Column(name='n', columnType='INTEGER'),
Column(name='is_bogus', columnType='BOOLEAN')
]
schema1 = syn.store(Schema(name='Big Table', columns=cols, parent=project))
# write rows to CSV file
with tempfile.NamedTemporaryFile(delete=False) as temp:
schedule_for_cleanup(temp.name)
filename = temp.name
with io.open(filename, mode='w', encoding="utf-8", newline='') as temp:
writer = csv.writer(temp,
quoting=csv.QUOTE_NONNUMERIC,
lineterminator=str(os.linesep))
writer.writerow([col.name for col in cols])
for i in range(10):
for j in range(100):
foo = cols[1].enumValues[random.randint(0, 2)]
writer.writerow(
('Robot ' + str(i * 100 + j), foo, random.random() * 200.0,
random.randint(0, 100), random.random() >= 0.5))
# upload CSV
syn._uploadCsv(filepath=temp.name, schema=schema1)
from synapseclient.table import CsvFileTable
CsvFileTable.from_table_query(syn, "select * from %s" % schema1.id)
def dontruntest_big_csvs():
cols = []
cols.append(Column(name='name', columnType='STRING', maximumSize=1000))
cols.append(Column(name='foo', columnType='STRING', enumValues=['foo', 'bar', 'bat']))
cols.append(Column(name='x', columnType='DOUBLE'))
cols.append(Column(name='n', columnType='INTEGER'))
cols.append(Column(name='is_bogus', columnType='BOOLEAN'))
schema1 = syn.store(Schema(name='Big Table', columns=cols, parent=project))
print "Created table:", schema1.id
print "with columns:", schema1.columnIds
## write rows to CSV file
with tempfile.NamedTemporaryFile(delete=False) as temp:
schedule_for_cleanup(temp.name)
writer = csv.writer(temp, quoting=csv.QUOTE_NONNUMERIC, lineterminator=os.linesep)
writer.writerow([col.name for col in cols])
for i in range(10):
for j in range(100):
foo = cols[1].enumValues[random.randint(0,2)]
writer.writerow(('Robot ' + str(i*100 + j), foo, random.random()*200.0, random.randint(0,100), random.random()>=0.5))
print "wrote 100 rows to disk"
## upload CSV
UploadToTableResult = syn._uploadCsv(filepath=temp.name, schema=schema1)
from synapseclient.table import CsvFileTable
results = CsvFileTable.from_table_query(syn, "select * from %s" % schema1.id)
print "etag:", results.etag
print "tableId:", results.tableId
for row in results:
print row
def dontruntest_big_csvs():
cols = []
cols.append(Column(name='name', columnType='STRING', maximumSize=1000))
cols.append(
Column(name='foo',
columnType='STRING',
enumValues=['foo', 'bar', 'bat']))
cols.append(Column(name='x', columnType='DOUBLE'))
cols.append(Column(name='n', columnType='INTEGER'))
cols.append(Column(name='is_bogus', columnType='BOOLEAN'))
schema1 = syn.store(Schema(name='Big Table', columns=cols, parent=project))
print("Created table:", schema1.id)
print("with columns:", schema1.columnIds)
## write rows to CSV file
with tempfile.NamedTemporaryFile(delete=False) as temp:
schedule_for_cleanup(temp.name)
filename = temp.name
with io.open(filename, mode='w', encoding="utf-8", newline='') as temp:
writer = csv.writer(temp,
quoting=csv.QUOTE_NONNUMERIC,
lineterminator=str(os.linesep))
writer.writerow([col.name for col in cols])
for i in range(10):
for j in range(100):
foo = cols[1].enumValues[random.randint(0, 2)]
writer.writerow(
('Robot ' + str(i * 100 + j), foo, random.random() * 200.0,
random.randint(0, 100), random.random() >= 0.5))
print("wrote 100 rows to disk")
## upload CSV
UploadToTableResult = syn._uploadCsv(filepath=temp.name, schema=schema1)
from synapseclient.table import CsvFileTable
results = CsvFileTable.from_table_query(syn,
"select * from %s" % schema1.id)
print("etag:", results.etag)
print("tableId:", results.tableId)
for row in results:
print(row)