def export_invitations(self,output,allfields=False,encrypt=None,sign=False,format='csv'):
"""export invitations, sorted by primary (id, or uuid if joint), to output.
allfields is used for backup and writes all columns.
encrypt=to io_key, sign=with default key.
"""
from ekklesia.data import DataTable
session = self.session
Invitation = self.Invitation
membercls = self.member_class
if allfields:
if membercls:
columns = ['uuid']+list(self.invite_columns)
columns.remove('id')
else: columns = self.invite_columns
else: # restricted
columns = ['uuid','code']
if not membercls: columns += ['email']
if encrypt: encrypt = [self.io_key]
writer = DataTable(columns,coltypes=self.invite_types,gpg=self.gpg,
dataformat='invitation',fileformat=format,version=self.version)
writer.open(output,'w',encrypt=encrypt,sign=sign)
count = 0
for inv in session.query(Invitation).order_by(
Invitation.id if membercls else Invitation.uuid).yield_per(1000):
extra = {}
if membercls: extra['uuid'] = inv.member.uuid
writer.write(inv,extra)
count += 1
writer.close()
self.info('%i exported invitations', count)
def table_io(self, ids, fmt, encrypt=False, sign=False):
from ekklesia.data import objects_equal
columns = ('a', 'b', 'c')
coltypes = {'a': int, 'b': int, 'c': int}
t = DataTable(columns,
coltypes=coltypes,
gpg=ids['id1'],
fileformat=fmt,
required=False)
if fmt in ('json', 'jsondict'): f = {}
else: f = StringIO()
t.open(f, 'w', [receiver] if encrypt else False, sign)
for i in range(3):
t.write({'a': i, 'b': 2, 'c': 3})
if fmt in ('json', 'jsondict'):
f2 = t.close()
assert f is f2
else:
t.close()
f.seek(0)
t = DataTable(columns,
coltypes=coltypes,
gpg=ids['id2'],
fileformat=fmt)
t.open(f, 'r', encrypt, sender if sign else False)
i = 0
for row in t:
assert row == {'a': i, 'b': 2, 'c': 3}
i += 1
t.close()
def export_invitations(self,
output,
allfields=False,
encrypt=None,
sign=False,
format='csv'):
from ekklesia.data import DataTable
session = self.session
Invitation = self.Invitation
membercls = self.member_class
if allfields:
columns = self.invite_columns
if membercls: columns = ['uuid'] + columns
else: # restricted
columns = ['uuid', 'code']
if not membercls: columns.append('email')
if encrypt: encrypt = [encrypt]
writer = DataTable(columns,
coltypes=self.invite_types,
gpg=self.gpg,
dataformat='invitation',
fileformat=format,
version=self.version)
writer.open(output, 'w', encrypt=encrypt, sign=sign)
count = 0
for inv in session.query(Invitation).order_by(
Invitation.id if membercls else Invitation.uuid):
extra = {}
if membercls: extra['uuid'] = inv.member.uuid
writer.write(inv, extra)
count += 1
writer.close()
self.info('%i exported invitations', count)
def test_table_bad_write(self):
t, f = DataTable(('a',)), StringIO()
t.open(f,'w')
t.write({'a':0})
t.close()
f.seek(0)
t = DataTable(('a',))
t.open(f,'r')
with raises(AssertionError): t.write({'a':1})
def test_table_bad_write(self):
t, f = DataTable(('a', )), StringIO()
t.open(f, 'w')
t.write({'a': 0})
t.close()
f.seek(0)
t = DataTable(('a', ))
t.open(f, 'r')
with raises(AssertionError):
t.write({'a': 1})
def export_members(self,
output,
encrypt=None,
sign=False,
allfields=False,
format='csv'):
from ekklesia.data import DataTable
session = self.session
Department, Member = self.Department, self.Member
if allfields: columns = self.member_columns + ['department']
else: columns = ('uuid', 'email')
writer = DataTable(columns,
coltypes=self.member_types,
gpg=self.gpg,
dataformat='member',
fileformat=format,
version=self.version)
if encrypt: encrypt = [encrypt]
writer.open(output[0], 'w', encrypt=encrypt, sign=sign)
count = 0
if allfields:
query = session.query(Member).order_by(
Member.memberno if 'memberno' in columns else Member.uuid)
else:
query = session.query(Member.uuid, Member.email).filter(
Member.email != None).order_by(Member.uuid)
extra = {}
for member in query:
if allfields and member.department:
depid = member.department.id if self.department_spec == 'number' else member.department.name
extra = dict(department=depid)
writer.write(member, extra)
count += 1
writer.close()
self.info('%i exported members', count)
if not allfields: return
dwriter = DataTable(['id', 'name', 'parent', 'depth'],
gpg=self.gpg,
dataformat='department',
fileformat=format,
version=self.version)
dwriter.open(output[1], 'w', encrypt=encrypt, sign=sign)
count = 0
for dep in session.query(Department).order_by(Department.depth,
Department.id):
if dep.parent:
extra = dict(parent=dep.parent.id if self.department_spec ==
'number' else dep.parent.name)
else:
extra = {}
dwriter.write(dep, extra)
count += 1
dwriter.close()
self.info('%i exported departments', count)
def table_io(self,
fmt,
obj=False,
missing=False,
ignore=True,
required=False,
extra=False):
from ekklesia.data import objects_equal
columns = ('a', 'b', 'c')
coltypes = {'a': int, 'b': int, 'c': (int, )}
t = DataTable(columns,
coltypes=coltypes,
fileformat=fmt,
ignore=ignore,
required=required)
if fmt in ('json', 'jsondict'): f = {}
else: f = StringIO()
t.open(f, 'w')
if obj:
t.write(Obj(a=0))
t.write(Obj(a=1))
elif missing:
try:
t.write({'a': 0, 'b': 2})
assert ignore
except:
assert not ignore
return
elif extra:
try:
t.write({'a': 0, 'b': 2, 'c': [3, 4], 'd': 4})
assert ignore
except:
assert not ignore
return
else:
for i in range(3):
t.write({'a': i, 'b': 2, 'c': [3, 4]})
if fmt in ('json', 'jsondict'):
f2 = t.close()
assert f is f2
else:
t.close()
f.seek(0)
t = DataTable(columns, coltypes=coltypes, fileformat=fmt)
t.open(f, 'r')
i = 0
for row in t:
if obj:
assert objects_equal(Obj(**row), Obj(a=i))
else:
if missing: assert row == {'a': 0, 'b': 2, 'c': []}
else: assert row == {'a': i, 'b': 2, 'c': [3, 4]}
i += 1
t.close()
def export_members(self,output,encrypt=None,sign=False,allfields=False,format='csv'):
"""export data, sorted by primary (uuid, or id with member_no), to output.
allfields is used for backup and writes all columns.
without allfields, data for the invitation DB is generated.
output is [members,departments] if allfields, else just [members].
encrypt=to io_key, sign=with default key.
"""
from ekklesia.data import DataTable
session = self.session
Department, Member = self.Department, self.Member
if allfields:
columns = list(self.member_columns)+['departments']
if not self.member_no: columns.remove('id')
dataformat = 'member'
else:
columns = ('uuid','email')
dataformat = 'invitation'
writer = DataTable(columns,coltypes=self.member_types,gpg=self.gpg,
dataformat=dataformat,fileformat=format,version=self.version)
if encrypt: encrypt = [self.io_key]
writer.open(output[0],'w',encrypt=encrypt,sign=sign)
count = 0
if allfields:
query = session.query(Member).order_by(Member.id if self.member_no else Member.uuid)
else:
query = session.query(Member.uuid,Member.email,Member.registered).order_by(Member.uuid)
if 'registered' in self.member_import:
query = query.filter_by(registered=None) # don't export registered
extra = {}
for member in query.yield_per(1000):
if allfields and member.departments: # FIXME: use get_department?
if self.department_spec=='number':
depids = [department.id for department in member.departments]
else:
depids = [department.name for department in member.departments]
extra = dict(departments=depids)
writer.write(member,extra)
count += 1
writer.close()
self.info('%i exported members', count)
if not allfields: return
dwriter = DataTable(['id','name','parent','depth'],gpg=self.gpg,
dataformat='department',fileformat=format,version=self.version)
dwriter.open(output[1],'w',encrypt=encrypt,sign=sign)
count = 0
for dep in session.query(Department).order_by(Department.depth,Department.id).yield_per(1000):
if dep.parent:
extra = dict(parent=dep.parent.id if self.department_spec=='number' else dep.parent.name)
else: extra = {}
dwriter.write(dep,extra)
count += 1
dwriter.close()
self.info('%i exported departments', count)
def get_members(onlynew=False, crypto=True):
from ekklesia.data import DataTable
from accounts.models import Account, Invitation
if crypto:
gpg, verify, decrypt, sign, encrypt = api_crypto('members')
if crypto is True: crypto = gpg # not debug
else: verify = decrypt = sign = encrypt = False
columns = ['uuid']
twofactor = settings.TWO_FACTOR_SIGNUP
if twofactor: columns.append('activate')
writer = DataTable(columns,
gpg=crypto,
dataformat='member',
fileformat='json',
version=[1, 0])
writer.open(mode='w', encrypt=encrypt, sign=sign)
members = Account.objects.exclude(uuid=None)
stati = (Account.NEWMEMBER, )
if not onlynew: stati += (Account.MEMBER, Account.ELIGIBLE)
members = members.filter(
email_unconfirmed=None, # only confirmed emails
status__in=stati)
for member in members.values('uuid', 'status'):
if twofactor:
if member['status'] == Account.NEWMEMBER:
try:
inv = Invitation.objects.get(uuid=member['uuid'],
status=Invitation.REGISTERING)
except Invitation.DoesNotExist:
continue
member['activate'] = inv.secret
else:
member['activate'] = ''
writer.write(member)
return writer.close()
def get_invitations(onlychanged=False, crypto=True):
from ekklesia.data import DataTable
from idapi.mails import gnupg_init
from accounts.models import Invitation
if crypto:
gpg, verify, decrypt, sign, encrypt = api_crypto('invitations')
if crypto is True: crypto = gpg # not debug
else: verify = decrypt = sign = encrypt = False
writer = DataTable(['uuid', 'status'],
gpg=crypto,
dataformat='invitation',
fileformat='json',
version=[1, 0])
writer.open(mode='w', encrypt=encrypt, sign=sign)
count = 0
invs = Invitation.objects.exclude(status=Invitation.DELETED)
stati = {Invitation.REGISTERED: 'registered', Invitation.FAILED: 'failed'}
if not onlychanged:
stati.update({Invitation.REGISTERING: 'new', Invitation.NEW: 'new'})
for inv in invs.values('uuid', 'status'):
try:
inv['status'] = stati[inv['status']]
except KeyError:
continue # ignore status
writer.write(inv)
count += 1
return writer.close()
def table_io(self,fmt,obj=False,
missing=False,ignore=True,required=False,extra=False):
from ekklesia.data import objects_equal
columns = ('a','b','c')
coltypes = {'a':int,'b':int,'c':(int,)}
t = DataTable(columns,coltypes=coltypes,fileformat=fmt,ignore=ignore,required=required)
if fmt in ('json','jsondict'): f = {}
else: f = StringIO()
t.open(f,'w')
if obj:
t.write(Obj(a=0))
t.write(Obj(a=1))
elif missing:
try:
t.write({'a':0,'b':2})
assert ignore
except:
assert not ignore
return
elif extra:
try:
t.write({'a':0,'b':2,'c':[3,4],'d':4})
assert ignore
except:
assert not ignore
return
else:
for i in range(3): t.write({'a':i,'b':2,'c':[3,4]})
if fmt in ('json','jsondict'):
f2 = t.close()
assert f is f2
else:
t.close()
f.seek(0)
t = DataTable(columns,coltypes=coltypes,fileformat=fmt)
t.open(f,'r')
i = 0
for row in t:
if obj:
assert objects_equal(Obj(**row),Obj(a=i))
else:
if missing: assert row == {'a':0,'b':2,'c':[]}
else: assert row == {'a':i,'b':2,'c':[3,4]}
i+=1
t.close()
def table_io(self,ids,fmt,encrypt=False,sign=False):
from ekklesia.data import objects_equal
columns = ('a','b','c')
coltypes = {'a':int,'b':int,'c':int}
t = DataTable(columns,coltypes=coltypes,gpg=ids['id1'],fileformat=fmt,required=False)
if fmt in ('json','jsondict'): f = {}
else: f = StringIO()
t.open(f,'w',[receiver] if encrypt else False,sign)
for i in range(3): t.write({'a':i,'b':2,'c':3})
if fmt in ('json','jsondict'):
f2 = t.close()
assert f is f2
else:
t.close()
f.seek(0)
t = DataTable(columns,coltypes=coltypes,gpg=ids['id2'],fileformat=fmt)
t.open(f,'r',encrypt,sender if sign else False)
i = 0
for row in t:
assert row == {'a':i,'b':2,'c':3}
i+=1
t.close()
def sync_invitations(self,download=True,upload=True,dryrun=False,quick=False,input=None,output=None):
"sync invitations with ID server"
from ekklesia.backends import api_init
from ekklesia.data import DataTable
from six.moves import cStringIO as StringIO
import requests, json
session = self.session
Invitation = self.Invitation
membercls = self.member_class
check_email = self.invite_check_email
api = api_init(self.invite_api._asdict())
reply = False # whether server requested reply
if download: # download registered uuids(used codes), mark used
if input: input = json.load(input)
if not input: # pragma: no cover
url = self.invite_api.url
if quick: url+='?changed=1'
resp = api.get(url)
if resp.status_code != requests.codes.ok:
if self.debugging: open('invdown.html','w').write(resp.content)
assert False, 'cannot download used invite codes'
input = resp.json()
if not input:
self.warn("input is empty")
return
columns = ['uuid','status','echo']
if check_email: columns.append(check_email)
reader = DataTable(columns,coltypes=self.invite_types,required=('uuid','status'),gpg=self.gpg,
dataformat='invitation',fileformat=self.invite_api.format,version=self.version)
sign = self.invite_api.receiver if self.invite_api.sign else False
reader.open(input,'r',encrypt=self.invite_api.encrypt,sign=sign)
rcolumns, unknown = reader.get_columns()
if unknown: self.warn('ignoring unknown fields',unknown)
reply = 'echo' in rcolumns # reply?
if check_email: reply = reply or check_email in rcolumns
if upload:
# upload responses and non-uploaded,unused uuid&code
columns = ['uuid','code','status']
coltypes = self.invite_types.copy()
if check_email: coltypes[check_email] = bool
if download and reply:
if check_email and check_email in rcolumns: columns.append(check_email)
if 'echo' in rcolumns: columns.append('echo')
writer = DataTable(columns,coltypes=coltypes,gpg=self.gpg,
dataformat='invitation',fileformat=self.invite_api.format,version=self.version)
encrypt = [self.invite_api.receiver] if self.invite_api.encrypt else False
out = {}
writer.open(out,'w',encrypt=encrypt,sign=self.invite_api.sign)
if download: # process download and generate reply
if membercls: query = session.query(membercls)
else: query = session.query(Invitation)
count = 0
seen = set()
for data in reader: # only uploaded codes, reply optional
uuid = data['uuid']
if not uuid:
self.warn("uuid missing")
continue
if uuid in seen:
self.warn("member %s is duplicate" % uuid)
continue
seen.add(uuid)
status = data['status']
if not status in ('registered','failed','new') or (quick and status=='new'):
self.warn("invalid status %s for %s" % (status,uuid))
continue
inv = query.filter_by(uuid=uuid).first()
extra = {}
if membercls and inv:
inv = inv.invitation
if not inv:
self.error("member %s is unknown" % data['uuid'])
if check_email in columns and data[check_email]:
extra[check_email] = False
extra['uuid'] = uuid
writer.write(Invitation(status=IStatusType.deleted,code=''),extra)
continue
status = data['status'] # compare status
# new on new -> uploaded
# new on uploaded -> ignore
# registered/failed on uploaded -> registered/failed
# registered/failed on same -> ignore
# deleted on failed -> new
if status == IStatusType.new:
if inv.status == IStatusType.new:
inv.status = IStatusType.uploaded
inv.sent = ISentStatusType.unsent
elif inv.status != IStatusType.uploaded:
self.error("bad status %s for uuid %s, current %s",
status,data['uuid'],inv.status)
continue
elif inv.status == IStatusType.uploaded: # status in registered/failed
if inv.status != status: inv.change()
inv.status = status # upload confirmed or failed registration
inv.sent = ISentStatusType.unsent
elif status != inv.status:
self.error("bad status %s for uuid %s, current %s",
status, data['uuid'],inv.status)
continue
if upload and (status != IStatusType.new or reply): # write response for uploaded
if check_email and check_email in columns:
if member_class: extra[check_email] = inv.member.email == data[check_email]
else: extra[check_email] = inv.email == data[check_email]
if 'echo' in columns: extra['echo'] = data['echo']
if membercls: extra['uuid'] = data['uuid']
writer.write(inv,extra)
count += 1
self.info('%i codes used', count)
if not dryrun: session.commit()
if not upload: return
# process failed, which have already been deleted on the server and are ready for reset
count = 0
query = session.query(Invitation).filter_by(status=IStatusType.failed,
sent=ISentStatusType.sent)
for inv in query.yield_per(1000):
extra = {}
if membercls: uuid = inv.member.uuid
else: uuid = inv.uuid
if uuid in seen: continue # already replied
inv.reset()
count += 1
self.info('%i codes resetted', count)
if not dryrun: session.commit()
if not quick:
# append new invitations
count = 0
query = session.query(Invitation).filter_by(status=IStatusType.new)
for inv in query.yield_per(1000):
extra = {}
if membercls:
uuid = inv.member.uuid
extra['uuid'] = uuid
else: uuid = inv.uuid
writer.write(inv,extra)
count += 1
self.info('%i new codes uploaded', count)
writer.close()
if output:
json.dump(out,output)
elif not dryrun: # pragma: no cover
resp = api.post(self.invite_api.url,json=out)
if resp.status_code != requests.codes.ok:
if self.debugging: open('invup.html','w').write(resp.content)
assert False, 'cannot upload data'
def sync_members(self,download=True,upload=True,dryrun=False,quick=False,
input=None,output=None,invitations=None,format='csv'):
"sync members with ID server"
from ekklesia.data import DataTable
from ekklesia.backends import api_init
from six.moves import cStringIO as StringIO
import requests, json
session = self.session
Department, Member = self.Department, self.Member
check_email = self.check_email
check_member = self.check_member
coltypes = self.member_types.copy()
api = api_init(self.member_api._asdict())
if download: # download registered uuids
if input: input = json.load(input)
else: # pragma: no cover
url = self.member_api.url
if quick: url+='?new=1'
resp = api.get(url)
if resp.status_code != requests.codes.ok:
if self.debugging: open('memberdown.html','w').write(resp.content)
assert False, 'cannot download used uuids'
input = resp.json()
if not input:
self.warn("input is empty")
return
columns = ['uuid','echo']
if check_member: columns.append(check_member)
if check_email: columns.append(check_email)
reader = DataTable(columns,required=['uuid'],coltypes=coltypes,gpg=self.gpg,
dataformat='member',fileformat=self.member_api.format,version=self.version)
sign = self.member_api.receiver if self.member_api.sign else False
reader.open(input,'r',encrypt=self.member_api.encrypt,sign=sign)
columns = list(self.member_sync)
wcoltypes = dict(coltypes)
if check_member: wcoltypes[check_member] = bool
if check_email: wcoltypes[check_email] = bool
wcoltypes['departments'] = (int,) # replace by list of ids
if 'location' in self.member_sync:
wcoltypes['gpslat'] = wcoltypes['gpslng'] = float
columns += ['gpslat','gpslng']
if download:
rcolumns, unknown = reader.get_columns()
if unknown: self.warn(UnknownFieldsWarning('ignoring unknown fields %s',unknown))
if check_member and check_member in rcolumns: columns.append(check_member)
if check_email and check_email in rcolumns: columns.append(check_email)
if 'echo' in rcolumns: columns += ['echo']
encrypt = [self.member_api.receiver] if self.member_api.encrypt else False
encryptmail = [self.email_receiver] if self.member_api.encrypt else False
writer = DataTable(columns,coltypes=wcoltypes,gpg=self.gpg,
dataformat='member',fileformat=self.member_api.format,version=self.version)
out = {}
writer.open(out,'w',encrypt=encrypt,sign=self.member_api.sign)
if not quick and self.export_emails:
ewriter = DataTable(('uuid','email'),coltypes=coltypes,gpg=self.gpg,
dataformat='member',fileformat=format,version=self.version)
eout = output[2] if output else StringIO()
ewriter.open(eout,'w',encrypt=encryptmail,sign=self.member_api.sign)
def export(member,extra = {}):
if 'location' in self.member_import:
gps = self.gps_coord(self.get_location(member))
if gps: extra['gpslat'],extra['gpslng'] = gps
deps = self.get_departments(member)
if self.department_spec=='number':
deps = [dep.id for dep in deps]
else:
deps = [dep.name for dep in deps]
if deps:
extra['departments'] = deps
writer.write(member,extra)
if not quick and self.export_emails: ewriter.write(member)
if invitations: registered = {} # dict of registered uuids
mquery = session.query(Member)
count = 0
if download:
from datetime import datetime
seen = set()
for data in reader:
uuid = data['uuid']
if not uuid:
self.warn("uuid missing")
continue
if uuid in seen:
self.warn("member %s is duplicate" % uuid)
continue
seen.add(uuid)
member = mquery.filter_by(uuid=uuid).first()
extra = {}
if not member or (self.member_no and member.status == MStatusType.deleted):
self.warn("member %s is unknown" % uuid)
if check_member in columns and data[check_member]:
extra[check_member] = False
if check_email in columns and data[check_email]:
extra[check_email] = False
extra['departments'] = []
writer.write(Member(uuid=uuid,status=MStatusType.deleted),extra)
continue
if check_email in columns and data[check_email]:
extra[check_email] = member.email == data[check_email]
if check_member in columns:
if data[check_member]:
result = self.check_member_func(member,data[check_member])
if 'registered' in self.member_import:
if result and not member.registered:
member.registered = datetime.utcnow()
else:
if 'registered' in self.member_import and member.registered:
self.warn("member %s is already registered" % uuid)
result = None
extra[check_member] = result
elif 'registered' in self.member_import and not member.registered:
member.registered = datetime.utcnow()
if 'echo' in columns: extra['echo'] = data['echo']
if not dryrun: export(member,extra)
if invitations: registered[member.uuid] = True
count += 1
else:
for member in mquery.yield_per(1000):
if self.member_no and not member.memberno: continue # deleted
if not dryrun: export(member)
count += 1
self.info('%i members exported', count)
if not dryrun and 'registered' in self.member_import: session.commit()
writer.close()
if not quick and self.export_emails: ewriter.close()
if not quick and invitations:
iwriter = DataTable(('uuid','email'),coltypes=coltypes,gpg=self.gpg,
dataformat='member',fileformat=format,version=self.version)
# extra encrypt,sign
iwriter.open(invitations,'w',encrypt=encryptmail,sign=self.member_api.sign)
if self.member_no:
query = session.query(Member.uuid,Member.email,Member.memberno).\
filter(Member.email!=None,Member.memberno!=0)
else:
query = session.query(Member.uuid,Member.email).filter(Member.email!=None)
count = 0
for member in query.yield_per(1000):
if member.uuid in registered: continue # skip registered
iwriter.write(member)
count += 1
iwriter.close()
self.info('%i invitations exported', count)
if not upload: return
dwriter = DataTable(('id','name','parent','depth'),gpg=self.gpg,
dataformat='department',fileformat=self.member_api.format,version=self.version)
dout = {}
dwriter.open(dout,'w',encrypt=encrypt,sign=self.member_api.sign)
for dep in session.query(Department).order_by(Department.depth,Department.id).yield_per(1000):
if dep.parent:
extra = dict(parent=dep.parent.id)
# if self.department_spec=='number' else dep.parent.name
else: extra = {}
dwriter.write(dep,extra)
dwriter.close()
if output:
json.dump(out,output[0])
json.dump(dout,output[1])
elif not dryrun: # pragma: no cover
resp = api.post(self.member_api.url,json=dict(members=out,departments=dout))
if resp.status_code != requests.codes.ok:
if self.debugging: open('memberup.html','w').write(resp.content)
assert False, 'cannot upload data'
if not self.export_emails or output or quick: return
# pragma: no cover
from ekklesia.mail import create_mail, smtp_init
smtp = smtp_init(self.smtpconfig)
smtp.open()
self.info('sending email data')
msg = create_mail(self.gpgconfig['sender'],self.email_receiver,'Email data',eout.getvalue())
eout.close()
msg, results = self.gpg.encrypt(msg,default_key=True,verify=True, inline=True)
assert msg and results, 'error encrypting message'
if not dryrun: smtp.send(msg)
smtp.close()
def sync_invitations(self,
download=True,
upload=True,
dryrun=False,
quick=False,
input=None,
output=None):
"""sync invitations with ID server"""
from ekklesia.backends import api_init
from ekklesia.data import DataTable
from six.moves import cStringIO as StringIO
import requests, json
session = self.session
Invitation = self.Invitation
membercls = self.member_class
check_email = self.invite_check_email
api = api_init(self.invite_api._asdict())
reply = False # whether server requested reply
if download: # download registered/failed/verified uuids(used codes), mark used
if input: input = json.load(input)
if not input: # pragma: no cover
url = self.invite_api.url
if quick: url += '?changed=1'
resp = api.get(url)
if resp.status_code != requests.codes.ok:
if self.debugging:
open('invdown.html', 'w').write(resp.content)
assert False, 'cannot download used invite codes'
input = resp.json() # only json?
if not input:
self.warn("input is empty")
return
columns = ['uuid', 'status', 'code', 'echo']
if check_email: columns.append(check_email)
reader = DataTable(columns,
coltypes=self.invite_types,
required=('uuid', 'status', 'code'),
gpg=self.gpg,
dataformat='invitation',
fileformat=self.invite_api.format,
version=self.version)
sign = self.invite_api.receiver if self.invite_api.sign else False
reader.open(input, 'r', encrypt=self.invite_api.encrypt, sign=sign)
rcolumns, unknown = reader.get_columns()
if unknown: self.warn('ignoring unknown fields', unknown)
reply = 'echo' in rcolumns # reply?
if check_email: reply = reply or check_email in rcolumns
if upload:
# upload responses and non-uploaded,unused uuid&code
columns = ['uuid', 'code', 'status']
coltypes = self.invite_types.copy()
if check_email: coltypes[check_email] = bool
if download and reply:
if check_email and check_email in rcolumns:
columns.append(check_email)
if 'echo' in rcolumns: columns.append('echo')
writer = DataTable(columns,
coltypes=coltypes,
gpg=self.gpg,
dataformat='invitation',
fileformat=self.invite_api.format,
version=self.version)
encrypt = [self.invite_api.receiver
] if self.invite_api.encrypt else False
out = {}
writer.open(out, 'w', encrypt=encrypt, sign=self.invite_api.sign)
if download: # process download and generate reply
if membercls: query = session.query(membercls)
else: query = session.query(Invitation)
count = 0
seen = set()
for data in reader: # only uploaded codes, reply optional
uuid = data['uuid']
if not uuid:
self.warn("uuid missing")
continue
if uuid in seen:
self.warn("member %s is duplicate" % uuid)
continue
seen.add(uuid)
valid = ('registered', 'failed', 'verified', 'reset')
if not quick: valid += ('new', 'verify')
status = data['status']
if not status in valid:
self.warn("invalid status %s for %s" % (status, uuid))
continue
inv = query.filter_by(uuid=uuid).first()
extra = {}
if membercls and inv:
inv = inv.invitation
if not inv:
self.error("member %s is unknown" % data['uuid'])
if check_email in columns and data[check_email]:
extra[check_email] = False
extra['uuid'] = uuid # works also for membercls
writer.write(
Invitation(status=IStatusType.deleted, code=''), extra)
continue
"""compare status and inv.status
sync state transitions:
backend,idserver -> target
new,- -> idserver:new
new,new -> idserver:new, backend:uploaded
verify,registered/- -> idserver:verify
-,* -> backend:deleted,error
uploaded,new/registering -> idserver:no response, backend:uploaded/ignore
uploaded,registered/failed -> backend:registered/failed
uploaded_verify,verify -> idserver:no response, backend:uploaded_verify/ignore
uploaded_verify,verified -> backend:verified
registered,registered -> backend:registered, idserver:delete
failed,failed -> backend:failed, idserver:delete (or new,new)
new/uploaded,reset -> backend:new, idserver:new
verify/uploaded_verify,reset -> backend:verify, idserver:verify
failed,- -> backend:new (check if fail not downloaded)
"""
if status == IStatusType.new:
if inv.status == IStatusType.new:
if inv.code == data['code']:
# code upload confirmed, prepare for sending
inv.status = IStatusType.uploaded
inv.sent = ISentStatusType.unsent
else:
# mismatch, new code needs to be uploaded
self.info(
"updating old code %s for uuid %s, new %s",
data['code'], data['uuid'], inv.code)
# write
elif inv.status != IStatusType.uploaded: # ignore with uploaded
self.error("bad status %s for uuid %s, current %s",
status, data['uuid'], inv.status)
continue
elif status == IStatusType.verify:
if inv.status == IStatusType.verify:
if inv.code == data['code']:
# code upload confirmed, prepare for sending
inv.status = IStatusType.uploaded_verify
inv.sent = ISentStatusType.unsent
else:
# mismatch, new code needs to be uploaded
self.info(
"updating old verify code %s for uuid %s, new %s",
data['code'], data['uuid'], inv.code)
# write
elif inv.status != IStatusType.uploaded_verify: # ignore with uploaded_verify
self.error("bad status %s for uuid %s, current %s",
status, data['uuid'], inv.status)
continue
elif status == IStatusType.reset:
if inv.status in FinalStates:
self.warn("ignoring reset for uuid %s, status %s",
data['uuid'], inv.status)
continue
inv.reset()
elif status in FinalStates:
if inv.status == IStatusType.uploaded_verify and status==IStatusType.verified or \
inv.status == IStatusType.uploaded and status!=IStatusType.verified:
inv.status = status # upload confirmed or failed registration/verification
inv.sent = ISentStatusType.unsent
inv.change()
elif status != inv.status:
self.error("bad status %s for uuid %s, current %s",
status, data['uuid'], inv.status)
continue
else:
self.error("bad status %s for uuid %s, current %s", status,
data['uuid'], inv.status)
continue
if upload and not inv.status in (IStatusType.uploaded,
IStatusType.uploaded_verify):
# write response for uploaded
if check_email and check_email in columns:
if member_class:
extra[check_email] = inv.member.email == data[
check_email]
else:
extra[check_email] = inv.email == data[check_email]
if 'echo' in columns: extra['echo'] = data['echo']
if membercls: extra['uuid'] = data['uuid']
writer.write(inv, extra)
count += 1
self.info('%i codes used', count)
if not dryrun: session.commit()
if not upload: return
# process failed, which have already been deleted on the server and are ready for reset
count = 0
query = session.query(Invitation).filter_by(status=IStatusType.failed,
sent=ISentStatusType.sent)
for inv in query.yield_per(1000):
extra = {}
if membercls: uuid = inv.member.uuid
else: uuid = inv.uuid
if uuid in seen: continue # already replied
inv.reset()
count += 1
self.info('%i codes resetted', count)
if not dryrun: session.commit()
if not quick:
# append new invitations
count = 0
fresh = (IStatusType.new, IStatusType.verify)
query = session.query(Invitation).filter(
Invitation.status.in_(fresh))
for inv in query.yield_per(1000):
extra = {}
if membercls:
uuid = inv.member.uuid
extra['uuid'] = uuid
else:
uuid = inv.uuid
writer.write(inv, extra)
count += 1
self.info('%i new codes uploaded', count)
writer.close()
if output:
json.dump(out, output)
elif not dryrun: # pragma: no cover
resp = api.post(self.invite_api.url, json=out)
if resp.status_code != requests.codes.ok:
if self.debugging: open('invup.html', 'w').write(resp.content)
assert False, 'cannot upload data'
def table_io(self,
ids,
fmt,
encrypt=False,
sign=False,
obj=False,
missing=False,
ignore=True,
required=False,
extra=False):
from ekklesia.data import objects_equal
columns = ('a', 'b', 'c')
coltypes = {'a': int, 'b': int, 'c': int}
t = DataTable(columns,
coltypes=coltypes,
gpg=ids['id1'],
fileformat=fmt,
ignore=ignore,
required=required)
if fmt in ('json', 'jsondict'): f = {}
else: f = StringIO()
t.open(f, 'w', receiver if encrypt else False, sign)
if obj:
t.write(Obj(a=0))
t.write(Obj(a=1))
elif missing:
try:
t.write({'a': 0, 'b': 2})
assert ignore
except:
assert not ignore
return
elif extra:
try:
t.write({'a': 0, 'b': 2, 'c': 3, 'd': 4})
assert ignore
except:
assert not ignore
return
else:
for i in range(3):
t.write({'a': i, 'b': 2, 'c': 3})
if fmt in ('json', 'jsondict'):
f2 = t.close()
assert f is f2
else:
t.close()
f.seek(0)
t = DataTable(columns,
coltypes=coltypes,
gpg=ids['id2'],
fileformat=fmt)
t.open(f, 'r', encrypt, sender if sign else False)
i = 0
for row in t:
if obj:
assert objects_equal(Obj(**row), Obj(a=i))
else:
if missing: assert row == {'a': 0, 'b': 2, 'c': None}
else: assert row == {'a': i, 'b': 2, 'c': 3}
i += 1
t.close()
def sync_members(self,
download=True,
upload=True,
dryrun=False,
input=None,
output=None,
invitations=None,
format='csv'):
# format for emails and invitations export
from ekklesia.data import DataTable
from ekklesia.backends import api_init
from six.moves import cStringIO as StringIO
import requests, datetime, json
session = self.session
Department, Member = self.Department, self.Member
check_email = self.check_email
check_member = self.check_member
coltypes = self.member_types.copy()
api = api_init(self.member_api._asdict())
if download: # download registered uuids
if input: input = json.load(input)
else:
resp = api.get(self.member_api.url)
assert resp.status_code == requests.codes.ok, 'cannot download used uuids'
input = resp.json()
columns = ['uuid', 'echo']
if check_member: columns.append(check_member)
if check_email: columns.append(check_email)
reader = DataTable(columns,
required=['uuid'],
coltypes=coltypes,
gpg=self.gpg,
dataformat='member',
fileformat=self.member_api.format,
version=self.version)
reader.open(input,
'r',
encrypt=self.member_api.encrypt,
sign=self.member_api.receiver)
columns = list(self.member_sync)
wcoltypes = dict(coltypes)
if check_member: wcoltypes[check_member] = bool
if check_email: wcoltypes[check_email] = bool
wcoltypes['department'] = int # replace by ids
if 'location' in self.member_sync:
wcoltypes['gpslat'] = wcoltypes['gpslng'] = float
columns += ['gpslat', 'gpslng']
if download:
rcolumns, unknown = reader.get_columns()
if unknown:
self.warn(
UnknownFieldsWarning('ignoring unknown fields %s',
unknown))
if check_member and check_member in rcolumns:
columns.append(check_member)
if check_email and check_email in rcolumns:
columns.append(check_email)
if 'echo' in rcolumns: columns += ['echo']
encrypt = [self.member_api.receiver
] if self.member_api.encrypt else False
encryptmail = [self.email_receiver
] if self.member_api.encrypt else False
writer = DataTable(columns,
coltypes=wcoltypes,
gpg=self.gpg,
dataformat='member',
fileformat=self.member_api.format,
version=self.version)
out = {}
writer.open(out, 'w', encrypt=encrypt, sign=self.member_api.sign)
if self.export_emails:
ewriter = DataTable(('uuid', 'email'),
coltypes=coltypes,
gpg=self.gpg,
dataformat='member',
fileformat=format,
version=self.version)
eout = output[2] if output else StringIO()
ewriter.open(eout,
'w',
encrypt=encryptmail,
sign=self.member_api.sign)
def export(member, extra={}):
if 'location' in self.member_import:
gps = self.gps_coord(self.get_location(member))
if gps: extra['gpslat'], extra['gpslng'] = gps
dep = self.get_department(member)
if dep:
extra[
'department'] = dep.id if self.department_spec == 'number' else dep.name
writer.write(member, extra)
if self.export_emails: ewriter.write(member)
if invitations: registered = {} # dict of registered uuids
mquery = session.query(Member)
count = 0
check_memberno = 'memberno' in self.member_columns
if download:
seen = set()
for data in reader:
uuid = data['uuid']
if not uuid:
self.warn("uuid missing")
continue
if uuid in seen:
self.warn("member %s is duplicate" % uuid)
continue
seen.add(uuid)
member = mquery.filter_by(uuid=uuid).first()
extra = {}
if not member or (check_memberno
and not member.memberno): # deleted
self.warn("member %s is unknown" % uuid)
if check_member in columns and data[check_member]:
extra[check_member] = False
if check_email in columns and data[check_email]:
extra[check_email] = False
writer.write(Member(uuid=uuid, status=StatusType.deleted),
extra)
continue
if check_email in columns and data[check_email]:
extra[check_email] = member.email == data[check_email]
if check_member in columns:
if data[check_member]:
result = self.check_member_func(
member, data[check_member])
if 'registered' in self.member_import:
if result and not member.registered:
member.registered = datetime.datetime.utcnow()
else:
result = None
extra[check_member] = result
if 'echo' in columns: extra['echo'] = data['echo']
if not dryrun: export(member, extra)
if invitations: registered[member.uuid] = True
count += 1
else:
for member in mquery:
if check_memberno and not member.memberno: continue # deleted
if not dryrun: export(member)
count += 1
self.info('%i members exported', count)
if not dryrun and 'registered' in self.member_import: session.commit()
writer.close()
if self.export_emails: ewriter.close()
if invitations:
iwriter = DataTable(('uuid', 'email'),
coltypes=coltypes,
gpg=self.gpg,
dataformat='member',
fileformat=format,
version=self.version)
# extra encrypt,sign
iwriter.open(invitations,
'w',
encrypt=encryptmail,
sign=self.member_api.sign)
if check_memberno:
query = session.query(Member.uuid,Member.email,Member.memberno).\
filter(Member.email!=None,Member.memberno!=0)
else:
query = session.query(
Member.uuid, Member.email).filter(Member.email != None)
count = 0
for member in query:
if member.uuid in registered: continue # skip registered
iwriter.write(member)
count += 1
iwriter.close()
self.info('%i invitations exported', count)
if not upload: return
dwriter = DataTable(('id', 'name', 'parent', 'depth'),
gpg=self.gpg,
dataformat='department',
fileformat=self.member_api.format,
version=self.version)
dout = {}
dwriter.open(dout, 'w', encrypt=encrypt, sign=self.member_api.sign)
for dep in session.query(Department).order_by(Department.depth,
Department.id):
if dep.parent:
extra = dict(parent=dep.parent.id if self.department_spec ==
'number' else dep.parent.name)
else:
extra = {}
dwriter.write(dep, extra)
dwriter.close()
if output:
json.dump(out, output[0])
json.dump(dout, output[1])
elif not dryrun:
r = api.post(self.member_api.url,
json=dict(members=out, departments=dout))
assert r.status_code == requests.codes.ok, 'cannot upload data'
if not self.export_emails or output: return
from ekklesia.mail import create_mail, smtp_init
smtp = smtp_init(self.smtpconfig)
smtp.open()
self.info('sending email data')
msg = create_mail(self.gpgconfig['sender'], self.email_receiver,
'Email data', eout.getvalue())
eout.close()
msg, results = self.gpg.encrypt(msg,
default_key=True,
verify=True,
inline=True)
assert msg and results, 'error encrypting message'
if not dryrun: smtp.send(msg)
smtp.close()
def export_members(self,
output,
encrypt=None,
sign=False,
allfields=False,
format='csv'):
"""export data, sorted by primary (uuid, unless memberno exists), to output.
allfields is used for backup and writes all columns.
without allfields, data for the invitation DB is generated.
output is [members,departments] if allfields, else just [members].
encrypt=to io_key, sign=with default key.
"""
from ekklesia.data import DataTable
session = self.session
Department, Member = self.Department, self.Member
if allfields:
columns = list(self.member_columns) + ['department']
dataformat = 'member'
else:
columns = ('uuid', 'email')
dataformat = 'invitation'
writer = DataTable(columns,
coltypes=self.member_types,
gpg=self.gpg,
dataformat=dataformat,
fileformat=format,
version=self.version)
if encrypt: encrypt = [self.io_key]
writer.open(output[0], 'w', encrypt=encrypt, sign=sign)
count = 0
if allfields:
query = session.query(Member).order_by(
Member.memberno if 'memberno' in columns else Member.uuid)
else:
query = session.query(Member.uuid, Member.email,
Member.registered).order_by(Member.uuid)
if 'registered' in self.member_import:
query = query.filter_by(
registered=None) # don't export registered
extra = {}
for member in query.yield_per(1000):
if allfields and member.department: # FIXME: use get_department?
depid = member.department.id if self.department_spec == 'number' else member.department.name
extra = dict(department=depid)
writer.write(member, extra)
count += 1
writer.close()
self.info('%i exported members', count)
if not allfields: return
dwriter = DataTable(['id', 'name', 'parent', 'depth'],
gpg=self.gpg,
dataformat='department',
fileformat=format,
version=self.version)
dwriter.open(output[1], 'w', encrypt=encrypt, sign=sign)
count = 0
for dep in session.query(Department).order_by(
Department.depth, Department.id).yield_per(1000):
if dep.parent:
extra = dict(parent=dep.parent.id if self.department_spec ==
'number' else dep.parent.name)
else:
extra = {}
dwriter.write(dep, extra)
count += 1
dwriter.close()
self.info('%i exported departments', count)
