def get_fields(cls, file, encoding):
sample_data = defaultdict(OrderedSet)
for f, enc, _ in UploadScript._get_files(file, encoding):
csvf = _open(f, encoding)
reader = csv.DictReader(csvf)
for row in itertools.islice(reader, 0, 5):
for field_name, value in row.items():
if value.strip():
sample_data[field_name].add(value.strip())
# Delete empty data
for values in sample_data.values():
if "" in values:
values.remove("")
# Guess types and destinations
for field_name, values in sorted(sample_data.items(),
key=itemgetter(0)):
filtered_field_name = to_valid_field_name(field_name)
try:
value = next(iter(values))
except StopIteration:
value = None
suggested_destination, suggested_type = guess_destination_and_type(
filtered_field_name, value)
yield ArticleField(field_name,
destination=suggested_destination,
values=list(
itertools.islice(sample_data[field_name], 0,
5)),
suggested_type=suggested_type)
def get_fields(cls, upload):
sample_data = defaultdict(OrderedSet)
for f, _ in UploadScript._get_files(upload):
reader = cls.get_reader(f)
for row in itertools.islice(reader, 0, 5):
for field_name, value in row.items():
if value.strip():
sample_data[field_name].add(value.strip())
# Delete empty data
for values in sample_data.values():
if "" in values:
values.remove("")
# Guess types and destinations
for field_name, values in sorted(sample_data.items(), key=itemgetter(0)):
filtered_field_name = to_valid_field_name(field_name)
try:
value = next(iter(values))
except StopIteration:
value = None
suggested_destination, suggested_type = guess_destination_and_type(filtered_field_name, value)
yield ArticleField(field_name,
suggested_destination=suggested_destination,
values=list(itertools.islice(sample_data[field_name], 0, 5)),
suggested_type=suggested_type)
def todo_test_zip_file(self):
from tempfile import NamedTemporaryFile
from django.core.files import File
# does _get_units perform normally
with NamedTemporaryFile(prefix=u"upload_test", suffix=".txt") as f:
f.write("Test")
f.flush()
s = UploadScript(project=amcattest.create_test_project().id,
file=File(f))
self.assertEqual({u.name for u in s._get_units()}, {f.name})
# does a zip file work?
#handle, fn = mkstemp(suffix=".zip")
with NamedTemporaryFile(suffix=".zip") as f:
with zipfile.ZipFile(f, "w") as zf:
zf.writestr("test.txt", "TEST")
zf.writestr("x/test.txt", "TAST")
s = UploadScript(project=amcattest.create_test_project().id,
file=File(f))
self.assertEqual({f.name
for f in s._get_units()},
{"test.txt", "x/test.txt"})
self.assertEqual({f.read()
for f in s._get_units()}, {"TEST", "TAST"})
def upload_article(request, project):
plugin_type = UploadScript.get_plugin_type()
#scripts = (Datatable(PluginResource, rowlink="./upload-articles/{id}")
# .filter(plugin_type=plugin_type)
# .hide('id', 'class_name', 'plugin_type'))
scripts = (Datatable(PluginResource, rowlink="./upload-articles/{id}")
.filter(plugin_type=plugin_type).hide('id', 'plugin_type'))
can_create_plugin = False#Plugin.can_create(request.user)
return table_view(request, project, scripts, selected='article sets',
template='navigator/project/upload.html', can_create_plugin=can_create_plugin,
plugin_type=plugin_type)
def todo_test_zip_file(self):
from tempfile import NamedTemporaryFile
from django.core.files import File
# does _get_units perform normally
with NamedTemporaryFile(prefix=u"upload_test", suffix=".txt") as f:
f.write("Test")
f.flush()
s = UploadScript(project=amcattest.create_test_project().id,
file=File(f))
self.assertEqual({u.name for u in s._get_units()}, {f.name})
# does a zip file work?
#handle, fn = mkstemp(suffix=".zip")
with NamedTemporaryFile(suffix=".zip") as f:
with zipfile.ZipFile(f, "w") as zf:
zf.writestr("test.txt", "TEST")
zf.writestr("x/test.txt", "TAST")
s = UploadScript(project=amcattest.create_test_project().id,
file=File(f))
self.assertEqual({f.name for f in s._get_units()}, {"test.txt", "x/test.txt"})
self.assertEqual({f.read() for f in s._get_units()}, {"TEST", "TAST"})
def upload_article(request, project):
plugin_type = UploadScript.get_plugin_type()
#scripts = (Datatable(PluginResource, rowlink="./upload-articles/{id}")
# .filter(plugin_type=plugin_type)
# .hide('id', 'class_name', 'plugin_type'))
scripts = (Datatable(
PluginResource,
rowlink="./upload-articles/{id}").filter(plugin_type=plugin_type).hide(
'id', 'plugin_type'))
can_create_plugin = False #Plugin.can_create(request.user)
return table_view(request,
project,
scripts,
selected='article sets',
template='navigator/project/upload.html',
can_create_plugin=can_create_plugin,
plugin_type=plugin_type)