def export_invitations(self,output,allfields=False,encrypt=None,sign=False,format='csv'): """export invitations, sorted by primary (id, or uuid if joint), to output. allfields is used for backup and writes all columns. encrypt=to io_key, sign=with default key. """ from ekklesia.data import DataTable session = self.session Invitation = self.Invitation membercls = self.member_class if allfields: if membercls: columns = ['uuid']+list(self.invite_columns) columns.remove('id') else: columns = self.invite_columns else: # restricted columns = ['uuid','code'] if not membercls: columns += ['email'] if encrypt: encrypt = [self.io_key] writer = DataTable(columns,coltypes=self.invite_types,gpg=self.gpg, dataformat='invitation',fileformat=format,version=self.version) writer.open(output,'w',encrypt=encrypt,sign=sign) count = 0 for inv in session.query(Invitation).order_by( Invitation.id if membercls else Invitation.uuid).yield_per(1000): extra = {} if membercls: extra['uuid'] = inv.member.uuid writer.write(inv,extra) count += 1 writer.close() self.info('%i exported invitations', count)
def table_io(self, ids, fmt, encrypt=False, sign=False): from ekklesia.data import objects_equal columns = ('a', 'b', 'c') coltypes = {'a': int, 'b': int, 'c': int} t = DataTable(columns, coltypes=coltypes, gpg=ids['id1'], fileformat=fmt, required=False) if fmt in ('json', 'jsondict'): f = {} else: f = StringIO() t.open(f, 'w', [receiver] if encrypt else False, sign) for i in range(3): t.write({'a': i, 'b': 2, 'c': 3}) if fmt in ('json', 'jsondict'): f2 = t.close() assert f is f2 else: t.close() f.seek(0) t = DataTable(columns, coltypes=coltypes, gpg=ids['id2'], fileformat=fmt) t.open(f, 'r', encrypt, sender if sign else False) i = 0 for row in t: assert row == {'a': i, 'b': 2, 'c': 3} i += 1 t.close()
def get_members(onlynew=False, crypto=True): from ekklesia.data import DataTable from accounts.models import Account, Invitation if crypto: gpg, verify, decrypt, sign, encrypt = api_crypto('members') if crypto is True: crypto = gpg # not debug else: verify = decrypt = sign = encrypt = False columns = ['uuid'] twofactor = settings.TWO_FACTOR_SIGNUP if twofactor: columns.append('activate') writer = DataTable(columns, gpg=crypto, dataformat='member', fileformat='json', version=[1, 0]) writer.open(mode='w', encrypt=encrypt, sign=sign) members = Account.objects.exclude(uuid=None) stati = (Account.NEWMEMBER, ) if not onlynew: stati += (Account.MEMBER, Account.ELIGIBLE) members = members.filter( email_unconfirmed=None, # only confirmed emails status__in=stati) for member in members.values('uuid', 'status'): if twofactor: if member['status'] == Account.NEWMEMBER: try: inv = Invitation.objects.get(uuid=member['uuid'], status=Invitation.REGISTERING) except Invitation.DoesNotExist: continue member['activate'] = inv.secret else: member['activate'] = '' writer.write(member) return writer.close()
def export_invitations(self, output, allfields=False, encrypt=None, sign=False, format='csv'): from ekklesia.data import DataTable session = self.session Invitation = self.Invitation membercls = self.member_class if allfields: columns = self.invite_columns if membercls: columns = ['uuid'] + columns else: # restricted columns = ['uuid', 'code'] if not membercls: columns.append('email') if encrypt: encrypt = [encrypt] writer = DataTable(columns, coltypes=self.invite_types, gpg=self.gpg, dataformat='invitation', fileformat=format, version=self.version) writer.open(output, 'w', encrypt=encrypt, sign=sign) count = 0 for inv in session.query(Invitation).order_by( Invitation.id if membercls else Invitation.uuid): extra = {} if membercls: extra['uuid'] = inv.member.uuid writer.write(inv, extra) count += 1 writer.close() self.info('%i exported invitations', count)
def get_invitations(onlychanged=False, crypto=True): from ekklesia.data import DataTable from idapi.mails import gnupg_init from accounts.models import Invitation if crypto: gpg, verify, decrypt, sign, encrypt = api_crypto('invitations') if crypto is True: crypto = gpg # not debug else: verify = decrypt = sign = encrypt = False writer = DataTable(['uuid', 'status'], gpg=crypto, dataformat='invitation', fileformat='json', version=[1, 0]) writer.open(mode='w', encrypt=encrypt, sign=sign) count = 0 invs = Invitation.objects.exclude(status=Invitation.DELETED) stati = {Invitation.REGISTERED: 'registered', Invitation.FAILED: 'failed'} if not onlychanged: stati.update({Invitation.REGISTERING: 'new', Invitation.NEW: 'new'}) for inv in invs.values('uuid', 'status'): try: inv['status'] = stati[inv['status']] except KeyError: continue # ignore status writer.write(inv) count += 1 return writer.close()
def test_table_bad_write(self): t, f = DataTable(('a',)), StringIO() t.open(f,'w') t.write({'a':0}) t.close() f.seek(0) t = DataTable(('a',)) t.open(f,'r') with raises(AssertionError): t.write({'a':1})
def test_table_bad_write(self): t, f = DataTable(('a', )), StringIO() t.open(f, 'w') t.write({'a': 0}) t.close() f.seek(0) t = DataTable(('a', )) t.open(f, 'r') with raises(AssertionError): t.write({'a': 1})
def export_members(self, output, encrypt=None, sign=False, allfields=False, format='csv'): from ekklesia.data import DataTable session = self.session Department, Member = self.Department, self.Member if allfields: columns = self.member_columns + ['department'] else: columns = ('uuid', 'email') writer = DataTable(columns, coltypes=self.member_types, gpg=self.gpg, dataformat='member', fileformat=format, version=self.version) if encrypt: encrypt = [encrypt] writer.open(output[0], 'w', encrypt=encrypt, sign=sign) count = 0 if allfields: query = session.query(Member).order_by( Member.memberno if 'memberno' in columns else Member.uuid) else: query = session.query(Member.uuid, Member.email).filter( Member.email != None).order_by(Member.uuid) extra = {} for member in query: if allfields and member.department: depid = member.department.id if self.department_spec == 'number' else member.department.name extra = dict(department=depid) writer.write(member, extra) count += 1 writer.close() self.info('%i exported members', count) if not allfields: return dwriter = DataTable(['id', 'name', 'parent', 'depth'], gpg=self.gpg, dataformat='department', fileformat=format, version=self.version) dwriter.open(output[1], 'w', encrypt=encrypt, sign=sign) count = 0 for dep in session.query(Department).order_by(Department.depth, Department.id): if dep.parent: extra = dict(parent=dep.parent.id if self.department_spec == 'number' else dep.parent.name) else: extra = {} dwriter.write(dep, extra) count += 1 dwriter.close() self.info('%i exported departments', count)
def export_members(self,output,encrypt=None,sign=False,allfields=False,format='csv'): """export data, sorted by primary (uuid, or id with member_no), to output. allfields is used for backup and writes all columns. without allfields, data for the invitation DB is generated. output is [members,departments] if allfields, else just [members]. encrypt=to io_key, sign=with default key. """ from ekklesia.data import DataTable session = self.session Department, Member = self.Department, self.Member if allfields: columns = list(self.member_columns)+['departments'] if not self.member_no: columns.remove('id') dataformat = 'member' else: columns = ('uuid','email') dataformat = 'invitation' writer = DataTable(columns,coltypes=self.member_types,gpg=self.gpg, dataformat=dataformat,fileformat=format,version=self.version) if encrypt: encrypt = [self.io_key] writer.open(output[0],'w',encrypt=encrypt,sign=sign) count = 0 if allfields: query = session.query(Member).order_by(Member.id if self.member_no else Member.uuid) else: query = session.query(Member.uuid,Member.email,Member.registered).order_by(Member.uuid) if 'registered' in self.member_import: query = query.filter_by(registered=None) # don't export registered extra = {} for member in query.yield_per(1000): if allfields and member.departments: # FIXME: use get_department? if self.department_spec=='number': depids = [department.id for department in member.departments] else: depids = [department.name for department in member.departments] extra = dict(departments=depids) writer.write(member,extra) count += 1 writer.close() self.info('%i exported members', count) if not allfields: return dwriter = DataTable(['id','name','parent','depth'],gpg=self.gpg, dataformat='department',fileformat=format,version=self.version) dwriter.open(output[1],'w',encrypt=encrypt,sign=sign) count = 0 for dep in session.query(Department).order_by(Department.depth,Department.id).yield_per(1000): if dep.parent: extra = dict(parent=dep.parent.id if self.department_spec=='number' else dep.parent.name) else: extra = {} dwriter.write(dep,extra) count += 1 dwriter.close() self.info('%i exported departments', count)
def table_io(self, fmt, obj=False, missing=False, ignore=True, required=False, extra=False): from ekklesia.data import objects_equal columns = ('a', 'b', 'c') coltypes = {'a': int, 'b': int, 'c': (int, )} t = DataTable(columns, coltypes=coltypes, fileformat=fmt, ignore=ignore, required=required) if fmt in ('json', 'jsondict'): f = {} else: f = StringIO() t.open(f, 'w') if obj: t.write(Obj(a=0)) t.write(Obj(a=1)) elif missing: try: t.write({'a': 0, 'b': 2}) assert ignore except: assert not ignore return elif extra: try: t.write({'a': 0, 'b': 2, 'c': [3, 4], 'd': 4}) assert ignore except: assert not ignore return else: for i in range(3): t.write({'a': i, 'b': 2, 'c': [3, 4]}) if fmt in ('json', 'jsondict'): f2 = t.close() assert f is f2 else: t.close() f.seek(0) t = DataTable(columns, coltypes=coltypes, fileformat=fmt) t.open(f, 'r') i = 0 for row in t: if obj: assert objects_equal(Obj(**row), Obj(a=i)) else: if missing: assert row == {'a': 0, 'b': 2, 'c': []} else: assert row == {'a': i, 'b': 2, 'c': [3, 4]} i += 1 t.close()
def table_io(self,ids,fmt,encrypt=False,sign=False): from ekklesia.data import objects_equal columns = ('a','b','c') coltypes = {'a':int,'b':int,'c':int} t = DataTable(columns,coltypes=coltypes,gpg=ids['id1'],fileformat=fmt,required=False) if fmt in ('json','jsondict'): f = {} else: f = StringIO() t.open(f,'w',[receiver] if encrypt else False,sign) for i in range(3): t.write({'a':i,'b':2,'c':3}) if fmt in ('json','jsondict'): f2 = t.close() assert f is f2 else: t.close() f.seek(0) t = DataTable(columns,coltypes=coltypes,gpg=ids['id2'],fileformat=fmt) t.open(f,'r',encrypt,sender if sign else False) i = 0 for row in t: assert row == {'a':i,'b':2,'c':3} i+=1 t.close()
def table_io(self,fmt,obj=False, missing=False,ignore=True,required=False,extra=False): from ekklesia.data import objects_equal columns = ('a','b','c') coltypes = {'a':int,'b':int,'c':(int,)} t = DataTable(columns,coltypes=coltypes,fileformat=fmt,ignore=ignore,required=required) if fmt in ('json','jsondict'): f = {} else: f = StringIO() t.open(f,'w') if obj: t.write(Obj(a=0)) t.write(Obj(a=1)) elif missing: try: t.write({'a':0,'b':2}) assert ignore except: assert not ignore return elif extra: try: t.write({'a':0,'b':2,'c':[3,4],'d':4}) assert ignore except: assert not ignore return else: for i in range(3): t.write({'a':i,'b':2,'c':[3,4]}) if fmt in ('json','jsondict'): f2 = t.close() assert f is f2 else: t.close() f.seek(0) t = DataTable(columns,coltypes=coltypes,fileformat=fmt) t.open(f,'r') i = 0 for row in t: if obj: assert objects_equal(Obj(**row),Obj(a=i)) else: if missing: assert row == {'a':0,'b':2,'c':[]} else: assert row == {'a':i,'b':2,'c':[3,4]} i+=1 t.close()
def sync_invitations(self,download=True,upload=True,dryrun=False,quick=False,input=None,output=None): "sync invitations with ID server" from ekklesia.backends import api_init from ekklesia.data import DataTable from six.moves import cStringIO as StringIO import requests, json session = self.session Invitation = self.Invitation membercls = self.member_class check_email = self.invite_check_email api = api_init(self.invite_api._asdict()) reply = False # whether server requested reply if download: # download registered uuids(used codes), mark used if input: input = json.load(input) if not input: # pragma: no cover url = self.invite_api.url if quick: url+='?changed=1' resp = api.get(url) if resp.status_code != requests.codes.ok: if self.debugging: open('invdown.html','w').write(resp.content) assert False, 'cannot download used invite codes' input = resp.json() if not input: self.warn("input is empty") return columns = ['uuid','status','echo'] if check_email: columns.append(check_email) reader = DataTable(columns,coltypes=self.invite_types,required=('uuid','status'),gpg=self.gpg, dataformat='invitation',fileformat=self.invite_api.format,version=self.version) sign = self.invite_api.receiver if self.invite_api.sign else False reader.open(input,'r',encrypt=self.invite_api.encrypt,sign=sign) rcolumns, unknown = reader.get_columns() if unknown: self.warn('ignoring unknown fields',unknown) reply = 'echo' in rcolumns # reply? if check_email: reply = reply or check_email in rcolumns if upload: # upload responses and non-uploaded,unused uuid&code columns = ['uuid','code','status'] coltypes = self.invite_types.copy() if check_email: coltypes[check_email] = bool if download and reply: if check_email and check_email in rcolumns: columns.append(check_email) if 'echo' in rcolumns: columns.append('echo') writer = DataTable(columns,coltypes=coltypes,gpg=self.gpg, dataformat='invitation',fileformat=self.invite_api.format,version=self.version) encrypt = [self.invite_api.receiver] if self.invite_api.encrypt else False out = {} writer.open(out,'w',encrypt=encrypt,sign=self.invite_api.sign) if download: # process download and generate reply if membercls: query = session.query(membercls) else: query = session.query(Invitation) count = 0 seen = set() for data in reader: # only uploaded codes, reply optional uuid = data['uuid'] if not uuid: self.warn("uuid missing") continue if uuid in seen: self.warn("member %s is duplicate" % uuid) continue seen.add(uuid) status = data['status'] if not status in ('registered','failed','new') or (quick and status=='new'): self.warn("invalid status %s for %s" % (status,uuid)) continue inv = query.filter_by(uuid=uuid).first() extra = {} if membercls and inv: inv = inv.invitation if not inv: self.error("member %s is unknown" % data['uuid']) if check_email in columns and data[check_email]: extra[check_email] = False extra['uuid'] = uuid writer.write(Invitation(status=IStatusType.deleted,code=''),extra) continue status = data['status'] # compare status # new on new -> uploaded # new on uploaded -> ignore # registered/failed on uploaded -> registered/failed # registered/failed on same -> ignore # deleted on failed -> new if status == IStatusType.new: if inv.status == IStatusType.new: inv.status = IStatusType.uploaded inv.sent = ISentStatusType.unsent elif inv.status != IStatusType.uploaded: self.error("bad status %s for uuid %s, current %s", status,data['uuid'],inv.status) continue elif inv.status == IStatusType.uploaded: # status in registered/failed if inv.status != status: inv.change() inv.status = status # upload confirmed or failed registration inv.sent = ISentStatusType.unsent elif status != inv.status: self.error("bad status %s for uuid %s, current %s", status, data['uuid'],inv.status) continue if upload and (status != IStatusType.new or reply): # write response for uploaded if check_email and check_email in columns: if member_class: extra[check_email] = inv.member.email == data[check_email] else: extra[check_email] = inv.email == data[check_email] if 'echo' in columns: extra['echo'] = data['echo'] if membercls: extra['uuid'] = data['uuid'] writer.write(inv,extra) count += 1 self.info('%i codes used', count) if not dryrun: session.commit() if not upload: return # process failed, which have already been deleted on the server and are ready for reset count = 0 query = session.query(Invitation).filter_by(status=IStatusType.failed, sent=ISentStatusType.sent) for inv in query.yield_per(1000): extra = {} if membercls: uuid = inv.member.uuid else: uuid = inv.uuid if uuid in seen: continue # already replied inv.reset() count += 1 self.info('%i codes resetted', count) if not dryrun: session.commit() if not quick: # append new invitations count = 0 query = session.query(Invitation).filter_by(status=IStatusType.new) for inv in query.yield_per(1000): extra = {} if membercls: uuid = inv.member.uuid extra['uuid'] = uuid else: uuid = inv.uuid writer.write(inv,extra) count += 1 self.info('%i new codes uploaded', count) writer.close() if output: json.dump(out,output) elif not dryrun: # pragma: no cover resp = api.post(self.invite_api.url,json=out) if resp.status_code != requests.codes.ok: if self.debugging: open('invup.html','w').write(resp.content) assert False, 'cannot upload data'
def sync_members(self,download=True,upload=True,dryrun=False,quick=False, input=None,output=None,invitations=None,format='csv'): "sync members with ID server" from ekklesia.data import DataTable from ekklesia.backends import api_init from six.moves import cStringIO as StringIO import requests, json session = self.session Department, Member = self.Department, self.Member check_email = self.check_email check_member = self.check_member coltypes = self.member_types.copy() api = api_init(self.member_api._asdict()) if download: # download registered uuids if input: input = json.load(input) else: # pragma: no cover url = self.member_api.url if quick: url+='?new=1' resp = api.get(url) if resp.status_code != requests.codes.ok: if self.debugging: open('memberdown.html','w').write(resp.content) assert False, 'cannot download used uuids' input = resp.json() if not input: self.warn("input is empty") return columns = ['uuid','echo'] if check_member: columns.append(check_member) if check_email: columns.append(check_email) reader = DataTable(columns,required=['uuid'],coltypes=coltypes,gpg=self.gpg, dataformat='member',fileformat=self.member_api.format,version=self.version) sign = self.member_api.receiver if self.member_api.sign else False reader.open(input,'r',encrypt=self.member_api.encrypt,sign=sign) columns = list(self.member_sync) wcoltypes = dict(coltypes) if check_member: wcoltypes[check_member] = bool if check_email: wcoltypes[check_email] = bool wcoltypes['departments'] = (int,) # replace by list of ids if 'location' in self.member_sync: wcoltypes['gpslat'] = wcoltypes['gpslng'] = float columns += ['gpslat','gpslng'] if download: rcolumns, unknown = reader.get_columns() if unknown: self.warn(UnknownFieldsWarning('ignoring unknown fields %s',unknown)) if check_member and check_member in rcolumns: columns.append(check_member) if check_email and check_email in rcolumns: columns.append(check_email) if 'echo' in rcolumns: columns += ['echo'] encrypt = [self.member_api.receiver] if self.member_api.encrypt else False encryptmail = [self.email_receiver] if self.member_api.encrypt else False writer = DataTable(columns,coltypes=wcoltypes,gpg=self.gpg, dataformat='member',fileformat=self.member_api.format,version=self.version) out = {} writer.open(out,'w',encrypt=encrypt,sign=self.member_api.sign) if not quick and self.export_emails: ewriter = DataTable(('uuid','email'),coltypes=coltypes,gpg=self.gpg, dataformat='member',fileformat=format,version=self.version) eout = output[2] if output else StringIO() ewriter.open(eout,'w',encrypt=encryptmail,sign=self.member_api.sign) def export(member,extra = {}): if 'location' in self.member_import: gps = self.gps_coord(self.get_location(member)) if gps: extra['gpslat'],extra['gpslng'] = gps deps = self.get_departments(member) if self.department_spec=='number': deps = [dep.id for dep in deps] else: deps = [dep.name for dep in deps] if deps: extra['departments'] = deps writer.write(member,extra) if not quick and self.export_emails: ewriter.write(member) if invitations: registered = {} # dict of registered uuids mquery = session.query(Member) count = 0 if download: from datetime import datetime seen = set() for data in reader: uuid = data['uuid'] if not uuid: self.warn("uuid missing") continue if uuid in seen: self.warn("member %s is duplicate" % uuid) continue seen.add(uuid) member = mquery.filter_by(uuid=uuid).first() extra = {} if not member or (self.member_no and member.status == MStatusType.deleted): self.warn("member %s is unknown" % uuid) if check_member in columns and data[check_member]: extra[check_member] = False if check_email in columns and data[check_email]: extra[check_email] = False extra['departments'] = [] writer.write(Member(uuid=uuid,status=MStatusType.deleted),extra) continue if check_email in columns and data[check_email]: extra[check_email] = member.email == data[check_email] if check_member in columns: if data[check_member]: result = self.check_member_func(member,data[check_member]) if 'registered' in self.member_import: if result and not member.registered: member.registered = datetime.utcnow() else: if 'registered' in self.member_import and member.registered: self.warn("member %s is already registered" % uuid) result = None extra[check_member] = result elif 'registered' in self.member_import and not member.registered: member.registered = datetime.utcnow() if 'echo' in columns: extra['echo'] = data['echo'] if not dryrun: export(member,extra) if invitations: registered[member.uuid] = True count += 1 else: for member in mquery.yield_per(1000): if self.member_no and not member.memberno: continue # deleted if not dryrun: export(member) count += 1 self.info('%i members exported', count) if not dryrun and 'registered' in self.member_import: session.commit() writer.close() if not quick and self.export_emails: ewriter.close() if not quick and invitations: iwriter = DataTable(('uuid','email'),coltypes=coltypes,gpg=self.gpg, dataformat='member',fileformat=format,version=self.version) # extra encrypt,sign iwriter.open(invitations,'w',encrypt=encryptmail,sign=self.member_api.sign) if self.member_no: query = session.query(Member.uuid,Member.email,Member.memberno).\ filter(Member.email!=None,Member.memberno!=0) else: query = session.query(Member.uuid,Member.email).filter(Member.email!=None) count = 0 for member in query.yield_per(1000): if member.uuid in registered: continue # skip registered iwriter.write(member) count += 1 iwriter.close() self.info('%i invitations exported', count) if not upload: return dwriter = DataTable(('id','name','parent','depth'),gpg=self.gpg, dataformat='department',fileformat=self.member_api.format,version=self.version) dout = {} dwriter.open(dout,'w',encrypt=encrypt,sign=self.member_api.sign) for dep in session.query(Department).order_by(Department.depth,Department.id).yield_per(1000): if dep.parent: extra = dict(parent=dep.parent.id) # if self.department_spec=='number' else dep.parent.name else: extra = {} dwriter.write(dep,extra) dwriter.close() if output: json.dump(out,output[0]) json.dump(dout,output[1]) elif not dryrun: # pragma: no cover resp = api.post(self.member_api.url,json=dict(members=out,departments=dout)) if resp.status_code != requests.codes.ok: if self.debugging: open('memberup.html','w').write(resp.content) assert False, 'cannot upload data' if not self.export_emails or output or quick: return # pragma: no cover from ekklesia.mail import create_mail, smtp_init smtp = smtp_init(self.smtpconfig) smtp.open() self.info('sending email data') msg = create_mail(self.gpgconfig['sender'],self.email_receiver,'Email data',eout.getvalue()) eout.close() msg, results = self.gpg.encrypt(msg,default_key=True,verify=True, inline=True) assert msg and results, 'error encrypting message' if not dryrun: smtp.send(msg) smtp.close()
def sync_invitations(self, download=True, upload=True, dryrun=False, quick=False, input=None, output=None): """sync invitations with ID server""" from ekklesia.backends import api_init from ekklesia.data import DataTable from six.moves import cStringIO as StringIO import requests, json session = self.session Invitation = self.Invitation membercls = self.member_class check_email = self.invite_check_email api = api_init(self.invite_api._asdict()) reply = False # whether server requested reply if download: # download registered/failed/verified uuids(used codes), mark used if input: input = json.load(input) if not input: # pragma: no cover url = self.invite_api.url if quick: url += '?changed=1' resp = api.get(url) if resp.status_code != requests.codes.ok: if self.debugging: open('invdown.html', 'w').write(resp.content) assert False, 'cannot download used invite codes' input = resp.json() # only json? if not input: self.warn("input is empty") return columns = ['uuid', 'status', 'code', 'echo'] if check_email: columns.append(check_email) reader = DataTable(columns, coltypes=self.invite_types, required=('uuid', 'status', 'code'), gpg=self.gpg, dataformat='invitation', fileformat=self.invite_api.format, version=self.version) sign = self.invite_api.receiver if self.invite_api.sign else False reader.open(input, 'r', encrypt=self.invite_api.encrypt, sign=sign) rcolumns, unknown = reader.get_columns() if unknown: self.warn('ignoring unknown fields', unknown) reply = 'echo' in rcolumns # reply? if check_email: reply = reply or check_email in rcolumns if upload: # upload responses and non-uploaded,unused uuid&code columns = ['uuid', 'code', 'status'] coltypes = self.invite_types.copy() if check_email: coltypes[check_email] = bool if download and reply: if check_email and check_email in rcolumns: columns.append(check_email) if 'echo' in rcolumns: columns.append('echo') writer = DataTable(columns, coltypes=coltypes, gpg=self.gpg, dataformat='invitation', fileformat=self.invite_api.format, version=self.version) encrypt = [self.invite_api.receiver ] if self.invite_api.encrypt else False out = {} writer.open(out, 'w', encrypt=encrypt, sign=self.invite_api.sign) if download: # process download and generate reply if membercls: query = session.query(membercls) else: query = session.query(Invitation) count = 0 seen = set() for data in reader: # only uploaded codes, reply optional uuid = data['uuid'] if not uuid: self.warn("uuid missing") continue if uuid in seen: self.warn("member %s is duplicate" % uuid) continue seen.add(uuid) valid = ('registered', 'failed', 'verified', 'reset') if not quick: valid += ('new', 'verify') status = data['status'] if not status in valid: self.warn("invalid status %s for %s" % (status, uuid)) continue inv = query.filter_by(uuid=uuid).first() extra = {} if membercls and inv: inv = inv.invitation if not inv: self.error("member %s is unknown" % data['uuid']) if check_email in columns and data[check_email]: extra[check_email] = False extra['uuid'] = uuid # works also for membercls writer.write( Invitation(status=IStatusType.deleted, code=''), extra) continue """compare status and inv.status sync state transitions: backend,idserver -> target new,- -> idserver:new new,new -> idserver:new, backend:uploaded verify,registered/- -> idserver:verify -,* -> backend:deleted,error uploaded,new/registering -> idserver:no response, backend:uploaded/ignore uploaded,registered/failed -> backend:registered/failed uploaded_verify,verify -> idserver:no response, backend:uploaded_verify/ignore uploaded_verify,verified -> backend:verified registered,registered -> backend:registered, idserver:delete failed,failed -> backend:failed, idserver:delete (or new,new) new/uploaded,reset -> backend:new, idserver:new verify/uploaded_verify,reset -> backend:verify, idserver:verify failed,- -> backend:new (check if fail not downloaded) """ if status == IStatusType.new: if inv.status == IStatusType.new: if inv.code == data['code']: # code upload confirmed, prepare for sending inv.status = IStatusType.uploaded inv.sent = ISentStatusType.unsent else: # mismatch, new code needs to be uploaded self.info( "updating old code %s for uuid %s, new %s", data['code'], data['uuid'], inv.code) # write elif inv.status != IStatusType.uploaded: # ignore with uploaded self.error("bad status %s for uuid %s, current %s", status, data['uuid'], inv.status) continue elif status == IStatusType.verify: if inv.status == IStatusType.verify: if inv.code == data['code']: # code upload confirmed, prepare for sending inv.status = IStatusType.uploaded_verify inv.sent = ISentStatusType.unsent else: # mismatch, new code needs to be uploaded self.info( "updating old verify code %s for uuid %s, new %s", data['code'], data['uuid'], inv.code) # write elif inv.status != IStatusType.uploaded_verify: # ignore with uploaded_verify self.error("bad status %s for uuid %s, current %s", status, data['uuid'], inv.status) continue elif status == IStatusType.reset: if inv.status in FinalStates: self.warn("ignoring reset for uuid %s, status %s", data['uuid'], inv.status) continue inv.reset() elif status in FinalStates: if inv.status == IStatusType.uploaded_verify and status==IStatusType.verified or \ inv.status == IStatusType.uploaded and status!=IStatusType.verified: inv.status = status # upload confirmed or failed registration/verification inv.sent = ISentStatusType.unsent inv.change() elif status != inv.status: self.error("bad status %s for uuid %s, current %s", status, data['uuid'], inv.status) continue else: self.error("bad status %s for uuid %s, current %s", status, data['uuid'], inv.status) continue if upload and not inv.status in (IStatusType.uploaded, IStatusType.uploaded_verify): # write response for uploaded if check_email and check_email in columns: if member_class: extra[check_email] = inv.member.email == data[ check_email] else: extra[check_email] = inv.email == data[check_email] if 'echo' in columns: extra['echo'] = data['echo'] if membercls: extra['uuid'] = data['uuid'] writer.write(inv, extra) count += 1 self.info('%i codes used', count) if not dryrun: session.commit() if not upload: return # process failed, which have already been deleted on the server and are ready for reset count = 0 query = session.query(Invitation).filter_by(status=IStatusType.failed, sent=ISentStatusType.sent) for inv in query.yield_per(1000): extra = {} if membercls: uuid = inv.member.uuid else: uuid = inv.uuid if uuid in seen: continue # already replied inv.reset() count += 1 self.info('%i codes resetted', count) if not dryrun: session.commit() if not quick: # append new invitations count = 0 fresh = (IStatusType.new, IStatusType.verify) query = session.query(Invitation).filter( Invitation.status.in_(fresh)) for inv in query.yield_per(1000): extra = {} if membercls: uuid = inv.member.uuid extra['uuid'] = uuid else: uuid = inv.uuid writer.write(inv, extra) count += 1 self.info('%i new codes uploaded', count) writer.close() if output: json.dump(out, output) elif not dryrun: # pragma: no cover resp = api.post(self.invite_api.url, json=out) if resp.status_code != requests.codes.ok: if self.debugging: open('invup.html', 'w').write(resp.content) assert False, 'cannot upload data'
def table_io(self, ids, fmt, encrypt=False, sign=False, obj=False, missing=False, ignore=True, required=False, extra=False): from ekklesia.data import objects_equal columns = ('a', 'b', 'c') coltypes = {'a': int, 'b': int, 'c': int} t = DataTable(columns, coltypes=coltypes, gpg=ids['id1'], fileformat=fmt, ignore=ignore, required=required) if fmt in ('json', 'jsondict'): f = {} else: f = StringIO() t.open(f, 'w', receiver if encrypt else False, sign) if obj: t.write(Obj(a=0)) t.write(Obj(a=1)) elif missing: try: t.write({'a': 0, 'b': 2}) assert ignore except: assert not ignore return elif extra: try: t.write({'a': 0, 'b': 2, 'c': 3, 'd': 4}) assert ignore except: assert not ignore return else: for i in range(3): t.write({'a': i, 'b': 2, 'c': 3}) if fmt in ('json', 'jsondict'): f2 = t.close() assert f is f2 else: t.close() f.seek(0) t = DataTable(columns, coltypes=coltypes, gpg=ids['id2'], fileformat=fmt) t.open(f, 'r', encrypt, sender if sign else False) i = 0 for row in t: if obj: assert objects_equal(Obj(**row), Obj(a=i)) else: if missing: assert row == {'a': 0, 'b': 2, 'c': None} else: assert row == {'a': i, 'b': 2, 'c': 3} i += 1 t.close()
def sync_members(self, download=True, upload=True, dryrun=False, input=None, output=None, invitations=None, format='csv'): # format for emails and invitations export from ekklesia.data import DataTable from ekklesia.backends import api_init from six.moves import cStringIO as StringIO import requests, datetime, json session = self.session Department, Member = self.Department, self.Member check_email = self.check_email check_member = self.check_member coltypes = self.member_types.copy() api = api_init(self.member_api._asdict()) if download: # download registered uuids if input: input = json.load(input) else: resp = api.get(self.member_api.url) assert resp.status_code == requests.codes.ok, 'cannot download used uuids' input = resp.json() columns = ['uuid', 'echo'] if check_member: columns.append(check_member) if check_email: columns.append(check_email) reader = DataTable(columns, required=['uuid'], coltypes=coltypes, gpg=self.gpg, dataformat='member', fileformat=self.member_api.format, version=self.version) reader.open(input, 'r', encrypt=self.member_api.encrypt, sign=self.member_api.receiver) columns = list(self.member_sync) wcoltypes = dict(coltypes) if check_member: wcoltypes[check_member] = bool if check_email: wcoltypes[check_email] = bool wcoltypes['department'] = int # replace by ids if 'location' in self.member_sync: wcoltypes['gpslat'] = wcoltypes['gpslng'] = float columns += ['gpslat', 'gpslng'] if download: rcolumns, unknown = reader.get_columns() if unknown: self.warn( UnknownFieldsWarning('ignoring unknown fields %s', unknown)) if check_member and check_member in rcolumns: columns.append(check_member) if check_email and check_email in rcolumns: columns.append(check_email) if 'echo' in rcolumns: columns += ['echo'] encrypt = [self.member_api.receiver ] if self.member_api.encrypt else False encryptmail = [self.email_receiver ] if self.member_api.encrypt else False writer = DataTable(columns, coltypes=wcoltypes, gpg=self.gpg, dataformat='member', fileformat=self.member_api.format, version=self.version) out = {} writer.open(out, 'w', encrypt=encrypt, sign=self.member_api.sign) if self.export_emails: ewriter = DataTable(('uuid', 'email'), coltypes=coltypes, gpg=self.gpg, dataformat='member', fileformat=format, version=self.version) eout = output[2] if output else StringIO() ewriter.open(eout, 'w', encrypt=encryptmail, sign=self.member_api.sign) def export(member, extra={}): if 'location' in self.member_import: gps = self.gps_coord(self.get_location(member)) if gps: extra['gpslat'], extra['gpslng'] = gps dep = self.get_department(member) if dep: extra[ 'department'] = dep.id if self.department_spec == 'number' else dep.name writer.write(member, extra) if self.export_emails: ewriter.write(member) if invitations: registered = {} # dict of registered uuids mquery = session.query(Member) count = 0 check_memberno = 'memberno' in self.member_columns if download: seen = set() for data in reader: uuid = data['uuid'] if not uuid: self.warn("uuid missing") continue if uuid in seen: self.warn("member %s is duplicate" % uuid) continue seen.add(uuid) member = mquery.filter_by(uuid=uuid).first() extra = {} if not member or (check_memberno and not member.memberno): # deleted self.warn("member %s is unknown" % uuid) if check_member in columns and data[check_member]: extra[check_member] = False if check_email in columns and data[check_email]: extra[check_email] = False writer.write(Member(uuid=uuid, status=StatusType.deleted), extra) continue if check_email in columns and data[check_email]: extra[check_email] = member.email == data[check_email] if check_member in columns: if data[check_member]: result = self.check_member_func( member, data[check_member]) if 'registered' in self.member_import: if result and not member.registered: member.registered = datetime.datetime.utcnow() else: result = None extra[check_member] = result if 'echo' in columns: extra['echo'] = data['echo'] if not dryrun: export(member, extra) if invitations: registered[member.uuid] = True count += 1 else: for member in mquery: if check_memberno and not member.memberno: continue # deleted if not dryrun: export(member) count += 1 self.info('%i members exported', count) if not dryrun and 'registered' in self.member_import: session.commit() writer.close() if self.export_emails: ewriter.close() if invitations: iwriter = DataTable(('uuid', 'email'), coltypes=coltypes, gpg=self.gpg, dataformat='member', fileformat=format, version=self.version) # extra encrypt,sign iwriter.open(invitations, 'w', encrypt=encryptmail, sign=self.member_api.sign) if check_memberno: query = session.query(Member.uuid,Member.email,Member.memberno).\ filter(Member.email!=None,Member.memberno!=0) else: query = session.query( Member.uuid, Member.email).filter(Member.email != None) count = 0 for member in query: if member.uuid in registered: continue # skip registered iwriter.write(member) count += 1 iwriter.close() self.info('%i invitations exported', count) if not upload: return dwriter = DataTable(('id', 'name', 'parent', 'depth'), gpg=self.gpg, dataformat='department', fileformat=self.member_api.format, version=self.version) dout = {} dwriter.open(dout, 'w', encrypt=encrypt, sign=self.member_api.sign) for dep in session.query(Department).order_by(Department.depth, Department.id): if dep.parent: extra = dict(parent=dep.parent.id if self.department_spec == 'number' else dep.parent.name) else: extra = {} dwriter.write(dep, extra) dwriter.close() if output: json.dump(out, output[0]) json.dump(dout, output[1]) elif not dryrun: r = api.post(self.member_api.url, json=dict(members=out, departments=dout)) assert r.status_code == requests.codes.ok, 'cannot upload data' if not self.export_emails or output: return from ekklesia.mail import create_mail, smtp_init smtp = smtp_init(self.smtpconfig) smtp.open() self.info('sending email data') msg = create_mail(self.gpgconfig['sender'], self.email_receiver, 'Email data', eout.getvalue()) eout.close() msg, results = self.gpg.encrypt(msg, default_key=True, verify=True, inline=True) assert msg and results, 'error encrypting message' if not dryrun: smtp.send(msg) smtp.close()
def export_members(self, output, encrypt=None, sign=False, allfields=False, format='csv'): """export data, sorted by primary (uuid, unless memberno exists), to output. allfields is used for backup and writes all columns. without allfields, data for the invitation DB is generated. output is [members,departments] if allfields, else just [members]. encrypt=to io_key, sign=with default key. """ from ekklesia.data import DataTable session = self.session Department, Member = self.Department, self.Member if allfields: columns = list(self.member_columns) + ['department'] dataformat = 'member' else: columns = ('uuid', 'email') dataformat = 'invitation' writer = DataTable(columns, coltypes=self.member_types, gpg=self.gpg, dataformat=dataformat, fileformat=format, version=self.version) if encrypt: encrypt = [self.io_key] writer.open(output[0], 'w', encrypt=encrypt, sign=sign) count = 0 if allfields: query = session.query(Member).order_by( Member.memberno if 'memberno' in columns else Member.uuid) else: query = session.query(Member.uuid, Member.email, Member.registered).order_by(Member.uuid) if 'registered' in self.member_import: query = query.filter_by( registered=None) # don't export registered extra = {} for member in query.yield_per(1000): if allfields and member.department: # FIXME: use get_department? depid = member.department.id if self.department_spec == 'number' else member.department.name extra = dict(department=depid) writer.write(member, extra) count += 1 writer.close() self.info('%i exported members', count) if not allfields: return dwriter = DataTable(['id', 'name', 'parent', 'depth'], gpg=self.gpg, dataformat='department', fileformat=format, version=self.version) dwriter.open(output[1], 'w', encrypt=encrypt, sign=sign) count = 0 for dep in session.query(Department).order_by( Department.depth, Department.id).yield_per(1000): if dep.parent: extra = dict(parent=dep.parent.id if self.department_spec == 'number' else dep.parent.name) else: extra = {} dwriter.write(dep, extra) count += 1 dwriter.close() self.info('%i exported departments', count)