def _populate_from_lines(self, lines):
mapper = self._mapper(next(lines))
context = self.Context(self.defaults)
for rows in iter_as_chunk(lines, self.chunksize):
entries = []
if mapper:
rows = [mapper(row) for row in rows]
try:
self.pre(rows, context)
for row in rows:
try:
entries.extend(self.create(row, context))
except Exception as e:
self.line_error(e, row, context)
context.line += 1
self.save(entries, context)
self.post(entries, context)
except Exception as e:
self.chunk_error(e, rows, context)
def save(self):
mlist = self.messaging_list
create = Recipient.objects.create
filter_recipient = Recipient.objects.filter
uploaded_file = self.cleaned_data['recipients']
# TODO: genexpr
def phones():
for line in uploaded_file:
# phone = PhoneField.filternumbers(smart_text(line.strip()))
phone = PhoneField.filternumbers(smart_str(line.strip()))
if phone:
yield phone
for phones in chunktools.iter_as_chunk(phones(), 256):
phones = frozenset(phones)
existing = frozenset(
filter_recipient(
messaging_list=mlist, phone__in=phones,
).values_list('phone', flat=True)
)
for phone in phones:
if phone not in existing:
create(messaging_list=mlist, phone=phone)
def _add_participants(activity, persons):
create_relation = partial(
Relation.objects.create,
subject_entity=activity,
user=activity.user,
# type_id=REL_OBJ_PART_2_ACTIVITY TODO: when orga can participate
)
users = []
# TODO: when orga can participate
for person in persons:
if isinstance(person, Contact):
create_relation(object_entity=person,
type_id=act_constants.REL_OBJ_PART_2_ACTIVITY)
if person.is_user:
users.append(person.is_user)
else:
create_relation(object_entity=person,
type_id=act_constants.REL_OBJ_ACTIVITY_SUBJECT)
for calendars_chunk in iter_as_chunk(
Calendar.objects.get_default_calendars(users).values(),
256,
):
activity.calendars.add(*calendars_chunk)
def test_iter_as_chunks02(self):
assert len(self.DATA_UNIX) % 5 == 0
chunks = [*chunktools.iter_as_chunk(self.DATA_UNIX, 5)]
self.assertEqual(16, len(chunks))
for i, chunk in enumerate(chunks):
self.assertEqual(5, len(chunk), f'Bad size for chunk {i}: {chunk}')
self.assertIsInstance(chunk, list)
self.assertEqual(self.DATA_UNIX, ''.join(''.join(chunk) for chunk in chunks))
def test_iter_as_chunks03(self):
data = self.DATA_UNIX + '9'
assert len(data) % 5 == 1
chunks = [*chunktools.iter_as_chunk(data, 5)]
self.assertEqual(17, len(chunks))
for i, chunk in enumerate(chunks[:-1]):
self.assertEqual(5, len(chunk), f'Bad size for chunk {i}: {chunk}')
self.assertEqual(['9'], chunks[-1])
self.assertEqual(data, ''.join(''.join(chunk) for chunk in chunks))
def test_iter_as_chunks03(self):
data = self.DATA_UNIX + '9'
assert len(data) % 5 == 1
chunks = list(chunktools.iter_as_chunk(data, 5))
self.assertEqual(17, len(chunks))
for i, chunk in enumerate(chunks[:-1]):
self.assertEqual(5, len(chunk),
'Bad size for chunk {}: {}'.format(i, chunk))
self.assertEqual(['9'], chunks[-1])
self.assertEqual(data, ''.join(''.join(chunk) for chunk in chunks))
def test_iter_as_chunks02(self):
assert len(self.DATA_UNIX) % 5 == 0
chunks = list(chunktools.iter_as_chunk(self.DATA_UNIX, 5))
self.assertEqual(16, len(chunks))
for i, chunk in enumerate(chunks):
self.assertEqual(5, len(chunk),
'Bad size for chunk {}: {}'.format(i, chunk))
self.assertIsInstance(chunk, list)
self.assertEqual(self.DATA_UNIX,
''.join(''.join(chunk) for chunk in chunks))
def save(self, *args, **kwargs):
instance = super().save(*args, **kwargs)
cdata = self.cleaned_data
get_key = self.subcell_key
calendars = [
*Calendar.objects.get_default_calendars(
part_user.is_user
for part_user in cdata.get(get_key(UsersSubCell), ())
).values()
]
i_participate, my_calendar = cdata.get(
get_key(MyParticipationSubCell), (False, None)
)
if i_participate:
calendars.append(my_calendar)
for calendars_chunk in iter_as_chunk(calendars, 256):
instance.calendars.add(*calendars_chunk)
return instance
def save(self): # TODO: factorise with MailingListAddRecipientsForm.save() ??
ml = self.ml
create = EmailRecipient.objects.create
filter_ = EmailRecipient.objects.filter
uploaded_file = self.cleaned_data['recipients']
# addresses = chunktools.iter_splitlinechunks(uploaded_file.chunks(),
# parser=self.filter_mail_chunk,
# )
# TODO: genexpr
def addresses():
for line in uploaded_file:
address = self.filter_mail_chunk(line)
if address:
yield address
# for recipients in chunktools.iter_as_chunk(addresses, 256):
for recipients in chunktools.iter_as_chunk(addresses(), 256):
recipients = frozenset(recipients)
existing = frozenset(filter_(ml=ml, address__in=recipients).values_list('address', flat=True))
for address in recipients:
if not address in existing:
create(ml=ml, address=address)
def test_iter_as_chunks01(self):
chunks = [*chunktools.iter_as_chunk(self.DATA_UNIX, 1000)]
self.assertEqual(1, len(chunks))
self.assertEqual(self.DATA_UNIX, ''.join(chunks[0]))
def chunks(self, chunk_size, source=None):
source = source if source is not None else self.DATA_UNIX
for chunk in chunktools.iter_as_chunk(source, chunk_size):
yield ''.join(chunk)