def sync_invitations(self,download=True,upload=True,dryrun=False,quick=False,input=None,output=None): "sync invitations with ID server" from ekklesia.backends import api_init from ekklesia.data import DataTable from six.moves import cStringIO as StringIO import requests, json session = self.session Invitation = self.Invitation membercls = self.member_class check_email = self.invite_check_email api = api_init(self.invite_api._asdict()) reply = False # whether server requested reply if download: # download registered uuids(used codes), mark used if input: input = json.load(input) if not input: # pragma: no cover url = self.invite_api.url if quick: url+='?changed=1' resp = api.get(url) if resp.status_code != requests.codes.ok: if self.debugging: open('invdown.html','w').write(resp.content) assert False, 'cannot download used invite codes' input = resp.json() if not input: self.warn("input is empty") return columns = ['uuid','status','echo'] if check_email: columns.append(check_email) reader = DataTable(columns,coltypes=self.invite_types,required=('uuid','status'),gpg=self.gpg, dataformat='invitation',fileformat=self.invite_api.format,version=self.version) sign = self.invite_api.receiver if self.invite_api.sign else False reader.open(input,'r',encrypt=self.invite_api.encrypt,sign=sign) rcolumns, unknown = reader.get_columns() if unknown: self.warn('ignoring unknown fields',unknown) reply = 'echo' in rcolumns # reply? if check_email: reply = reply or check_email in rcolumns if upload: # upload responses and non-uploaded,unused uuid&code columns = ['uuid','code','status'] coltypes = self.invite_types.copy() if check_email: coltypes[check_email] = bool if download and reply: if check_email and check_email in rcolumns: columns.append(check_email) if 'echo' in rcolumns: columns.append('echo') writer = DataTable(columns,coltypes=coltypes,gpg=self.gpg, dataformat='invitation',fileformat=self.invite_api.format,version=self.version) encrypt = [self.invite_api.receiver] if self.invite_api.encrypt else False out = {} writer.open(out,'w',encrypt=encrypt,sign=self.invite_api.sign) if download: # process download and generate reply if membercls: query = session.query(membercls) else: query = session.query(Invitation) count = 0 seen = set() for data in reader: # only uploaded codes, reply optional uuid = data['uuid'] if not uuid: self.warn("uuid missing") continue if uuid in seen: self.warn("member %s is duplicate" % uuid) continue seen.add(uuid) status = data['status'] if not status in ('registered','failed','new') or (quick and status=='new'): self.warn("invalid status %s for %s" % (status,uuid)) continue inv = query.filter_by(uuid=uuid).first() extra = {} if membercls and inv: inv = inv.invitation if not inv: self.error("member %s is unknown" % data['uuid']) if check_email in columns and data[check_email]: extra[check_email] = False extra['uuid'] = uuid writer.write(Invitation(status=IStatusType.deleted,code=''),extra) continue status = data['status'] # compare status # new on new -> uploaded # new on uploaded -> ignore # registered/failed on uploaded -> registered/failed # registered/failed on same -> ignore # deleted on failed -> new if status == IStatusType.new: if inv.status == IStatusType.new: inv.status = IStatusType.uploaded inv.sent = ISentStatusType.unsent elif inv.status != IStatusType.uploaded: self.error("bad status %s for uuid %s, current %s", status,data['uuid'],inv.status) continue elif inv.status == IStatusType.uploaded: # status in registered/failed if inv.status != status: inv.change() inv.status = status # upload confirmed or failed registration inv.sent = ISentStatusType.unsent elif status != inv.status: self.error("bad status %s for uuid %s, current %s", status, data['uuid'],inv.status) continue if upload and (status != IStatusType.new or reply): # write response for uploaded if check_email and check_email in columns: if member_class: extra[check_email] = inv.member.email == data[check_email] else: extra[check_email] = inv.email == data[check_email] if 'echo' in columns: extra['echo'] = data['echo'] if membercls: extra['uuid'] = data['uuid'] writer.write(inv,extra) count += 1 self.info('%i codes used', count) if not dryrun: session.commit() if not upload: return # process failed, which have already been deleted on the server and are ready for reset count = 0 query = session.query(Invitation).filter_by(status=IStatusType.failed, sent=ISentStatusType.sent) for inv in query.yield_per(1000): extra = {} if membercls: uuid = inv.member.uuid else: uuid = inv.uuid if uuid in seen: continue # already replied inv.reset() count += 1 self.info('%i codes resetted', count) if not dryrun: session.commit() if not quick: # append new invitations count = 0 query = session.query(Invitation).filter_by(status=IStatusType.new) for inv in query.yield_per(1000): extra = {} if membercls: uuid = inv.member.uuid extra['uuid'] = uuid else: uuid = inv.uuid writer.write(inv,extra) count += 1 self.info('%i new codes uploaded', count) writer.close() if output: json.dump(out,output) elif not dryrun: # pragma: no cover resp = api.post(self.invite_api.url,json=out) if resp.status_code != requests.codes.ok: if self.debugging: open('invup.html','w').write(resp.content) assert False, 'cannot upload data'
def sync_members(self, download=True, upload=True, dryrun=False, input=None, output=None, invitations=None, format='csv'): # format for emails and invitations export from ekklesia.data import DataTable from ekklesia.backends import api_init from six.moves import cStringIO as StringIO import requests, datetime, json session = self.session Department, Member = self.Department, self.Member check_email = self.check_email check_member = self.check_member coltypes = self.member_types.copy() api = api_init(self.member_api._asdict()) if download: # download registered uuids if input: input = json.load(input) else: resp = api.get(self.member_api.url) assert resp.status_code == requests.codes.ok, 'cannot download used uuids' input = resp.json() columns = ['uuid', 'echo'] if check_member: columns.append(check_member) if check_email: columns.append(check_email) reader = DataTable(columns, required=['uuid'], coltypes=coltypes, gpg=self.gpg, dataformat='member', fileformat=self.member_api.format, version=self.version) reader.open(input, 'r', encrypt=self.member_api.encrypt, sign=self.member_api.receiver) columns = list(self.member_sync) wcoltypes = dict(coltypes) if check_member: wcoltypes[check_member] = bool if check_email: wcoltypes[check_email] = bool wcoltypes['department'] = int # replace by ids if 'location' in self.member_sync: wcoltypes['gpslat'] = wcoltypes['gpslng'] = float columns += ['gpslat', 'gpslng'] if download: rcolumns, unknown = reader.get_columns() if unknown: self.warn( UnknownFieldsWarning('ignoring unknown fields %s', unknown)) if check_member and check_member in rcolumns: columns.append(check_member) if check_email and check_email in rcolumns: columns.append(check_email) if 'echo' in rcolumns: columns += ['echo'] encrypt = [self.member_api.receiver ] if self.member_api.encrypt else False encryptmail = [self.email_receiver ] if self.member_api.encrypt else False writer = DataTable(columns, coltypes=wcoltypes, gpg=self.gpg, dataformat='member', fileformat=self.member_api.format, version=self.version) out = {} writer.open(out, 'w', encrypt=encrypt, sign=self.member_api.sign) if self.export_emails: ewriter = DataTable(('uuid', 'email'), coltypes=coltypes, gpg=self.gpg, dataformat='member', fileformat=format, version=self.version) eout = output[2] if output else StringIO() ewriter.open(eout, 'w', encrypt=encryptmail, sign=self.member_api.sign) def export(member, extra={}): if 'location' in self.member_import: gps = self.gps_coord(self.get_location(member)) if gps: extra['gpslat'], extra['gpslng'] = gps dep = self.get_department(member) if dep: extra[ 'department'] = dep.id if self.department_spec == 'number' else dep.name writer.write(member, extra) if self.export_emails: ewriter.write(member) if invitations: registered = {} # dict of registered uuids mquery = session.query(Member) count = 0 check_memberno = 'memberno' in self.member_columns if download: seen = set() for data in reader: uuid = data['uuid'] if not uuid: self.warn("uuid missing") continue if uuid in seen: self.warn("member %s is duplicate" % uuid) continue seen.add(uuid) member = mquery.filter_by(uuid=uuid).first() extra = {} if not member or (check_memberno and not member.memberno): # deleted self.warn("member %s is unknown" % uuid) if check_member in columns and data[check_member]: extra[check_member] = False if check_email in columns and data[check_email]: extra[check_email] = False writer.write(Member(uuid=uuid, status=StatusType.deleted), extra) continue if check_email in columns and data[check_email]: extra[check_email] = member.email == data[check_email] if check_member in columns: if data[check_member]: result = self.check_member_func( member, data[check_member]) if 'registered' in self.member_import: if result and not member.registered: member.registered = datetime.datetime.utcnow() else: result = None extra[check_member] = result if 'echo' in columns: extra['echo'] = data['echo'] if not dryrun: export(member, extra) if invitations: registered[member.uuid] = True count += 1 else: for member in mquery: if check_memberno and not member.memberno: continue # deleted if not dryrun: export(member) count += 1 self.info('%i members exported', count) if not dryrun and 'registered' in self.member_import: session.commit() writer.close() if self.export_emails: ewriter.close() if invitations: iwriter = DataTable(('uuid', 'email'), coltypes=coltypes, gpg=self.gpg, dataformat='member', fileformat=format, version=self.version) # extra encrypt,sign iwriter.open(invitations, 'w', encrypt=encryptmail, sign=self.member_api.sign) if check_memberno: query = session.query(Member.uuid,Member.email,Member.memberno).\ filter(Member.email!=None,Member.memberno!=0) else: query = session.query( Member.uuid, Member.email).filter(Member.email != None) count = 0 for member in query: if member.uuid in registered: continue # skip registered iwriter.write(member) count += 1 iwriter.close() self.info('%i invitations exported', count) if not upload: return dwriter = DataTable(('id', 'name', 'parent', 'depth'), gpg=self.gpg, dataformat='department', fileformat=self.member_api.format, version=self.version) dout = {} dwriter.open(dout, 'w', encrypt=encrypt, sign=self.member_api.sign) for dep in session.query(Department).order_by(Department.depth, Department.id): if dep.parent: extra = dict(parent=dep.parent.id if self.department_spec == 'number' else dep.parent.name) else: extra = {} dwriter.write(dep, extra) dwriter.close() if output: json.dump(out, output[0]) json.dump(dout, output[1]) elif not dryrun: r = api.post(self.member_api.url, json=dict(members=out, departments=dout)) assert r.status_code == requests.codes.ok, 'cannot upload data' if not self.export_emails or output: return from ekklesia.mail import create_mail, smtp_init smtp = smtp_init(self.smtpconfig) smtp.open() self.info('sending email data') msg = create_mail(self.gpgconfig['sender'], self.email_receiver, 'Email data', eout.getvalue()) eout.close() msg, results = self.gpg.encrypt(msg, default_key=True, verify=True, inline=True) assert msg and results, 'error encrypting message' if not dryrun: smtp.send(msg) smtp.close()
def sync_invitations(self, download=True, upload=True, dryrun=False, quick=False, input=None, output=None): """sync invitations with ID server""" from ekklesia.backends import api_init from ekklesia.data import DataTable from six.moves import cStringIO as StringIO import requests, json session = self.session Invitation = self.Invitation membercls = self.member_class check_email = self.invite_check_email api = api_init(self.invite_api._asdict()) reply = False # whether server requested reply if download: # download registered/failed/verified uuids(used codes), mark used if input: input = json.load(input) if not input: # pragma: no cover url = self.invite_api.url if quick: url += '?changed=1' resp = api.get(url) if resp.status_code != requests.codes.ok: if self.debugging: open('invdown.html', 'w').write(resp.content) assert False, 'cannot download used invite codes' input = resp.json() # only json? if not input: self.warn("input is empty") return columns = ['uuid', 'status', 'code', 'echo'] if check_email: columns.append(check_email) reader = DataTable(columns, coltypes=self.invite_types, required=('uuid', 'status', 'code'), gpg=self.gpg, dataformat='invitation', fileformat=self.invite_api.format, version=self.version) sign = self.invite_api.receiver if self.invite_api.sign else False reader.open(input, 'r', encrypt=self.invite_api.encrypt, sign=sign) rcolumns, unknown = reader.get_columns() if unknown: self.warn('ignoring unknown fields', unknown) reply = 'echo' in rcolumns # reply? if check_email: reply = reply or check_email in rcolumns if upload: # upload responses and non-uploaded,unused uuid&code columns = ['uuid', 'code', 'status'] coltypes = self.invite_types.copy() if check_email: coltypes[check_email] = bool if download and reply: if check_email and check_email in rcolumns: columns.append(check_email) if 'echo' in rcolumns: columns.append('echo') writer = DataTable(columns, coltypes=coltypes, gpg=self.gpg, dataformat='invitation', fileformat=self.invite_api.format, version=self.version) encrypt = [self.invite_api.receiver ] if self.invite_api.encrypt else False out = {} writer.open(out, 'w', encrypt=encrypt, sign=self.invite_api.sign) if download: # process download and generate reply if membercls: query = session.query(membercls) else: query = session.query(Invitation) count = 0 seen = set() for data in reader: # only uploaded codes, reply optional uuid = data['uuid'] if not uuid: self.warn("uuid missing") continue if uuid in seen: self.warn("member %s is duplicate" % uuid) continue seen.add(uuid) valid = ('registered', 'failed', 'verified', 'reset') if not quick: valid += ('new', 'verify') status = data['status'] if not status in valid: self.warn("invalid status %s for %s" % (status, uuid)) continue inv = query.filter_by(uuid=uuid).first() extra = {} if membercls and inv: inv = inv.invitation if not inv: self.error("member %s is unknown" % data['uuid']) if check_email in columns and data[check_email]: extra[check_email] = False extra['uuid'] = uuid # works also for membercls writer.write( Invitation(status=IStatusType.deleted, code=''), extra) continue """compare status and inv.status sync state transitions: backend,idserver -> target new,- -> idserver:new new,new -> idserver:new, backend:uploaded verify,registered/- -> idserver:verify -,* -> backend:deleted,error uploaded,new/registering -> idserver:no response, backend:uploaded/ignore uploaded,registered/failed -> backend:registered/failed uploaded_verify,verify -> idserver:no response, backend:uploaded_verify/ignore uploaded_verify,verified -> backend:verified registered,registered -> backend:registered, idserver:delete failed,failed -> backend:failed, idserver:delete (or new,new) new/uploaded,reset -> backend:new, idserver:new verify/uploaded_verify,reset -> backend:verify, idserver:verify failed,- -> backend:new (check if fail not downloaded) """ if status == IStatusType.new: if inv.status == IStatusType.new: if inv.code == data['code']: # code upload confirmed, prepare for sending inv.status = IStatusType.uploaded inv.sent = ISentStatusType.unsent else: # mismatch, new code needs to be uploaded self.info( "updating old code %s for uuid %s, new %s", data['code'], data['uuid'], inv.code) # write elif inv.status != IStatusType.uploaded: # ignore with uploaded self.error("bad status %s for uuid %s, current %s", status, data['uuid'], inv.status) continue elif status == IStatusType.verify: if inv.status == IStatusType.verify: if inv.code == data['code']: # code upload confirmed, prepare for sending inv.status = IStatusType.uploaded_verify inv.sent = ISentStatusType.unsent else: # mismatch, new code needs to be uploaded self.info( "updating old verify code %s for uuid %s, new %s", data['code'], data['uuid'], inv.code) # write elif inv.status != IStatusType.uploaded_verify: # ignore with uploaded_verify self.error("bad status %s for uuid %s, current %s", status, data['uuid'], inv.status) continue elif status == IStatusType.reset: if inv.status in FinalStates: self.warn("ignoring reset for uuid %s, status %s", data['uuid'], inv.status) continue inv.reset() elif status in FinalStates: if inv.status == IStatusType.uploaded_verify and status==IStatusType.verified or \ inv.status == IStatusType.uploaded and status!=IStatusType.verified: inv.status = status # upload confirmed or failed registration/verification inv.sent = ISentStatusType.unsent inv.change() elif status != inv.status: self.error("bad status %s for uuid %s, current %s", status, data['uuid'], inv.status) continue else: self.error("bad status %s for uuid %s, current %s", status, data['uuid'], inv.status) continue if upload and not inv.status in (IStatusType.uploaded, IStatusType.uploaded_verify): # write response for uploaded if check_email and check_email in columns: if member_class: extra[check_email] = inv.member.email == data[ check_email] else: extra[check_email] = inv.email == data[check_email] if 'echo' in columns: extra['echo'] = data['echo'] if membercls: extra['uuid'] = data['uuid'] writer.write(inv, extra) count += 1 self.info('%i codes used', count) if not dryrun: session.commit() if not upload: return # process failed, which have already been deleted on the server and are ready for reset count = 0 query = session.query(Invitation).filter_by(status=IStatusType.failed, sent=ISentStatusType.sent) for inv in query.yield_per(1000): extra = {} if membercls: uuid = inv.member.uuid else: uuid = inv.uuid if uuid in seen: continue # already replied inv.reset() count += 1 self.info('%i codes resetted', count) if not dryrun: session.commit() if not quick: # append new invitations count = 0 fresh = (IStatusType.new, IStatusType.verify) query = session.query(Invitation).filter( Invitation.status.in_(fresh)) for inv in query.yield_per(1000): extra = {} if membercls: uuid = inv.member.uuid extra['uuid'] = uuid else: uuid = inv.uuid writer.write(inv, extra) count += 1 self.info('%i new codes uploaded', count) writer.close() if output: json.dump(out, output) elif not dryrun: # pragma: no cover resp = api.post(self.invite_api.url, json=out) if resp.status_code != requests.codes.ok: if self.debugging: open('invup.html', 'w').write(resp.content) assert False, 'cannot upload data'
def sync_members(self,download=True,upload=True,dryrun=False,quick=False, input=None,output=None,invitations=None,format='csv'): "sync members with ID server" from ekklesia.data import DataTable from ekklesia.backends import api_init from six.moves import cStringIO as StringIO import requests, json session = self.session Department, Member = self.Department, self.Member check_email = self.check_email check_member = self.check_member coltypes = self.member_types.copy() api = api_init(self.member_api._asdict()) if download: # download registered uuids if input: input = json.load(input) else: # pragma: no cover url = self.member_api.url if quick: url+='?new=1' resp = api.get(url) if resp.status_code != requests.codes.ok: if self.debugging: open('memberdown.html','w').write(resp.content) assert False, 'cannot download used uuids' input = resp.json() if not input: self.warn("input is empty") return columns = ['uuid','echo'] if check_member: columns.append(check_member) if check_email: columns.append(check_email) reader = DataTable(columns,required=['uuid'],coltypes=coltypes,gpg=self.gpg, dataformat='member',fileformat=self.member_api.format,version=self.version) sign = self.member_api.receiver if self.member_api.sign else False reader.open(input,'r',encrypt=self.member_api.encrypt,sign=sign) columns = list(self.member_sync) wcoltypes = dict(coltypes) if check_member: wcoltypes[check_member] = bool if check_email: wcoltypes[check_email] = bool wcoltypes['departments'] = (int,) # replace by list of ids if 'location' in self.member_sync: wcoltypes['gpslat'] = wcoltypes['gpslng'] = float columns += ['gpslat','gpslng'] if download: rcolumns, unknown = reader.get_columns() if unknown: self.warn(UnknownFieldsWarning('ignoring unknown fields %s',unknown)) if check_member and check_member in rcolumns: columns.append(check_member) if check_email and check_email in rcolumns: columns.append(check_email) if 'echo' in rcolumns: columns += ['echo'] encrypt = [self.member_api.receiver] if self.member_api.encrypt else False encryptmail = [self.email_receiver] if self.member_api.encrypt else False writer = DataTable(columns,coltypes=wcoltypes,gpg=self.gpg, dataformat='member',fileformat=self.member_api.format,version=self.version) out = {} writer.open(out,'w',encrypt=encrypt,sign=self.member_api.sign) if not quick and self.export_emails: ewriter = DataTable(('uuid','email'),coltypes=coltypes,gpg=self.gpg, dataformat='member',fileformat=format,version=self.version) eout = output[2] if output else StringIO() ewriter.open(eout,'w',encrypt=encryptmail,sign=self.member_api.sign) def export(member,extra = {}): if 'location' in self.member_import: gps = self.gps_coord(self.get_location(member)) if gps: extra['gpslat'],extra['gpslng'] = gps deps = self.get_departments(member) if self.department_spec=='number': deps = [dep.id for dep in deps] else: deps = [dep.name for dep in deps] if deps: extra['departments'] = deps writer.write(member,extra) if not quick and self.export_emails: ewriter.write(member) if invitations: registered = {} # dict of registered uuids mquery = session.query(Member) count = 0 if download: from datetime import datetime seen = set() for data in reader: uuid = data['uuid'] if not uuid: self.warn("uuid missing") continue if uuid in seen: self.warn("member %s is duplicate" % uuid) continue seen.add(uuid) member = mquery.filter_by(uuid=uuid).first() extra = {} if not member or (self.member_no and member.status == MStatusType.deleted): self.warn("member %s is unknown" % uuid) if check_member in columns and data[check_member]: extra[check_member] = False if check_email in columns and data[check_email]: extra[check_email] = False extra['departments'] = [] writer.write(Member(uuid=uuid,status=MStatusType.deleted),extra) continue if check_email in columns and data[check_email]: extra[check_email] = member.email == data[check_email] if check_member in columns: if data[check_member]: result = self.check_member_func(member,data[check_member]) if 'registered' in self.member_import: if result and not member.registered: member.registered = datetime.utcnow() else: if 'registered' in self.member_import and member.registered: self.warn("member %s is already registered" % uuid) result = None extra[check_member] = result elif 'registered' in self.member_import and not member.registered: member.registered = datetime.utcnow() if 'echo' in columns: extra['echo'] = data['echo'] if not dryrun: export(member,extra) if invitations: registered[member.uuid] = True count += 1 else: for member in mquery.yield_per(1000): if self.member_no and not member.memberno: continue # deleted if not dryrun: export(member) count += 1 self.info('%i members exported', count) if not dryrun and 'registered' in self.member_import: session.commit() writer.close() if not quick and self.export_emails: ewriter.close() if not quick and invitations: iwriter = DataTable(('uuid','email'),coltypes=coltypes,gpg=self.gpg, dataformat='member',fileformat=format,version=self.version) # extra encrypt,sign iwriter.open(invitations,'w',encrypt=encryptmail,sign=self.member_api.sign) if self.member_no: query = session.query(Member.uuid,Member.email,Member.memberno).\ filter(Member.email!=None,Member.memberno!=0) else: query = session.query(Member.uuid,Member.email).filter(Member.email!=None) count = 0 for member in query.yield_per(1000): if member.uuid in registered: continue # skip registered iwriter.write(member) count += 1 iwriter.close() self.info('%i invitations exported', count) if not upload: return dwriter = DataTable(('id','name','parent','depth'),gpg=self.gpg, dataformat='department',fileformat=self.member_api.format,version=self.version) dout = {} dwriter.open(dout,'w',encrypt=encrypt,sign=self.member_api.sign) for dep in session.query(Department).order_by(Department.depth,Department.id).yield_per(1000): if dep.parent: extra = dict(parent=dep.parent.id) # if self.department_spec=='number' else dep.parent.name else: extra = {} dwriter.write(dep,extra) dwriter.close() if output: json.dump(out,output[0]) json.dump(dout,output[1]) elif not dryrun: # pragma: no cover resp = api.post(self.member_api.url,json=dict(members=out,departments=dout)) if resp.status_code != requests.codes.ok: if self.debugging: open('memberup.html','w').write(resp.content) assert False, 'cannot upload data' if not self.export_emails or output or quick: return # pragma: no cover from ekklesia.mail import create_mail, smtp_init smtp = smtp_init(self.smtpconfig) smtp.open() self.info('sending email data') msg = create_mail(self.gpgconfig['sender'],self.email_receiver,'Email data',eout.getvalue()) eout.close() msg, results = self.gpg.encrypt(msg,default_key=True,verify=True, inline=True) assert msg and results, 'error encrypting message' if not dryrun: smtp.send(msg) smtp.close()