def import_invitations(self,input,decrypt=False,verify=False, allfields=False,sync=False,dryrun=False,format='csv'): """import data from input. allfields is used for restore and requires all columns. if sync, uuids not seen in input are set to status deleted. decrypt=with the default key, verify=check whether its signed with io_key. """ from ekklesia.data import DataTable session = self.session Invitation = self.Invitation membercls = self.member_class columns = self.invite_columns if membercls: columns = list(columns)+['uuid'] columns.remove('id') if allfields: reqcolumns = columns elif membercls: reqcolumns = ['uuid'] else: reqcolumns = ['uuid','email'] reader = DataTable(columns,coltypes=self.invite_types,required=reqcolumns, dataformat='invitation',fileformat=format,version=self.version,gpg=self.gpg) if not allfields and verify: verify = self.io_key reader.open(input,'r',encrypt=decrypt,sign=verify) columns = reader.get_columns()[0] iquery = session.query(Invitation) count = 0 seen = set() for data in reader: uuid = data['uuid'] if not uuid: self.warn("uuid missing") continue assert not uuid in seen, "member %s is duplicate" % uuid seen.add(uuid) if membercls: member = session.query(membercls).filter_by(uuid=uuid).first() if member is None: self.warn("uuid %s not found" % uuid) continue if not member.email: # email removed, disable invitation inv = member.invitation if inv and not inv.status in (IStatusType.deleted,IStatusType.registered): self.info("scheduling invitation for uuid '%s' for deletion", member.uuid) if not dryrun: inv.delete() continue count += 1 if dryrun: continue if member.invitation is None: # create a new invitation session.add(Invitation(member=member,**data)) #new else: if not 'sent' in columns and data['status'] in (IStatusType.new,IStatusType.uploaded): data['sent'] = ISentStatusType.unsent member.invitation.update(**data) #update inv else: inv = iquery.filter_by(uuid=uuid).first() if not data['email']: # email removed, disable invitation if inv is None: self.warn("uuid %s not found" % uuid) continue if inv.status in (IStatusType.deleted,IStatusType.registered): continue self.info("scheduling invitation for uuid '%s' for deletion", inv.uuid) if not dryrun: inv.delete() continue # check whether email already used if not inv or inv.email != data['email']: # fixme: what if emails swapped? email = iquery.filter_by(email=data['email']).first() if email and (not inv or inv.uuid != email.uuid): self.error("ignoring: duplicate email %s" % data['email']) continue count += 1 if dryrun: continue if inv: # if email changed and code has been sent, reset invcode and lastchange, unless allfields is set needreset = not allfields and inv.status==IStatusType.uploaded and \ inv.sent==ISentStatusType.sent and 'email' in data and data['email']!=inv.email and \ (not 'code' in data or data['code']==inv.code) if not needreset: data['code'] = inv.code # preserve inv.update(**data) if needreset: inv.reset() else: session.add(Invitation(**data)) #new self.info('%i imported invitations', count) if sync: # deleted unseen invitations count = 0 for inv in session.query(membercls if membercls else Invitation).yield_per(1000): uuid = inv.uuid if uuid in seen: continue if membercls: inv = inv.invitation if not inv: continue if inv.status==IStatusType.deleted: continue inv.status = IStatusType.deleted self.info("invitation %s deleted" % uuid) count += 1 self.info('%i deleted invitations', count) if not dryrun: session.commit()
def sync_invitations(self,download=True,upload=True,dryrun=False,quick=False,input=None,output=None): "sync invitations with ID server" from ekklesia.backends import api_init from ekklesia.data import DataTable from six.moves import cStringIO as StringIO import requests, json session = self.session Invitation = self.Invitation membercls = self.member_class check_email = self.invite_check_email api = api_init(self.invite_api._asdict()) reply = False # whether server requested reply if download: # download registered uuids(used codes), mark used if input: input = json.load(input) if not input: # pragma: no cover url = self.invite_api.url if quick: url+='?changed=1' resp = api.get(url) if resp.status_code != requests.codes.ok: if self.debugging: open('invdown.html','w').write(resp.content) assert False, 'cannot download used invite codes' input = resp.json() if not input: self.warn("input is empty") return columns = ['uuid','status','echo'] if check_email: columns.append(check_email) reader = DataTable(columns,coltypes=self.invite_types,required=('uuid','status'),gpg=self.gpg, dataformat='invitation',fileformat=self.invite_api.format,version=self.version) sign = self.invite_api.receiver if self.invite_api.sign else False reader.open(input,'r',encrypt=self.invite_api.encrypt,sign=sign) rcolumns, unknown = reader.get_columns() if unknown: self.warn('ignoring unknown fields',unknown) reply = 'echo' in rcolumns # reply? if check_email: reply = reply or check_email in rcolumns if upload: # upload responses and non-uploaded,unused uuid&code columns = ['uuid','code','status'] coltypes = self.invite_types.copy() if check_email: coltypes[check_email] = bool if download and reply: if check_email and check_email in rcolumns: columns.append(check_email) if 'echo' in rcolumns: columns.append('echo') writer = DataTable(columns,coltypes=coltypes,gpg=self.gpg, dataformat='invitation',fileformat=self.invite_api.format,version=self.version) encrypt = [self.invite_api.receiver] if self.invite_api.encrypt else False out = {} writer.open(out,'w',encrypt=encrypt,sign=self.invite_api.sign) if download: # process download and generate reply if membercls: query = session.query(membercls) else: query = session.query(Invitation) count = 0 seen = set() for data in reader: # only uploaded codes, reply optional uuid = data['uuid'] if not uuid: self.warn("uuid missing") continue if uuid in seen: self.warn("member %s is duplicate" % uuid) continue seen.add(uuid) status = data['status'] if not status in ('registered','failed','new') or (quick and status=='new'): self.warn("invalid status %s for %s" % (status,uuid)) continue inv = query.filter_by(uuid=uuid).first() extra = {} if membercls and inv: inv = inv.invitation if not inv: self.error("member %s is unknown" % data['uuid']) if check_email in columns and data[check_email]: extra[check_email] = False extra['uuid'] = uuid writer.write(Invitation(status=IStatusType.deleted,code=''),extra) continue status = data['status'] # compare status # new on new -> uploaded # new on uploaded -> ignore # registered/failed on uploaded -> registered/failed # registered/failed on same -> ignore # deleted on failed -> new if status == IStatusType.new: if inv.status == IStatusType.new: inv.status = IStatusType.uploaded inv.sent = ISentStatusType.unsent elif inv.status != IStatusType.uploaded: self.error("bad status %s for uuid %s, current %s", status,data['uuid'],inv.status) continue elif inv.status == IStatusType.uploaded: # status in registered/failed if inv.status != status: inv.change() inv.status = status # upload confirmed or failed registration inv.sent = ISentStatusType.unsent elif status != inv.status: self.error("bad status %s for uuid %s, current %s", status, data['uuid'],inv.status) continue if upload and (status != IStatusType.new or reply): # write response for uploaded if check_email and check_email in columns: if member_class: extra[check_email] = inv.member.email == data[check_email] else: extra[check_email] = inv.email == data[check_email] if 'echo' in columns: extra['echo'] = data['echo'] if membercls: extra['uuid'] = data['uuid'] writer.write(inv,extra) count += 1 self.info('%i codes used', count) if not dryrun: session.commit() if not upload: return # process failed, which have already been deleted on the server and are ready for reset count = 0 query = session.query(Invitation).filter_by(status=IStatusType.failed, sent=ISentStatusType.sent) for inv in query.yield_per(1000): extra = {} if membercls: uuid = inv.member.uuid else: uuid = inv.uuid if uuid in seen: continue # already replied inv.reset() count += 1 self.info('%i codes resetted', count) if not dryrun: session.commit() if not quick: # append new invitations count = 0 query = session.query(Invitation).filter_by(status=IStatusType.new) for inv in query.yield_per(1000): extra = {} if membercls: uuid = inv.member.uuid extra['uuid'] = uuid else: uuid = inv.uuid writer.write(inv,extra) count += 1 self.info('%i new codes uploaded', count) writer.close() if output: json.dump(out,output) elif not dryrun: # pragma: no cover resp = api.post(self.invite_api.url,json=out) if resp.status_code != requests.codes.ok: if self.debugging: open('invup.html','w').write(resp.content) assert False, 'cannot upload data'
def sync_members(self, download=True, upload=True, dryrun=False, input=None, output=None, invitations=None, format='csv'): # format for emails and invitations export from ekklesia.data import DataTable from ekklesia.backends import api_init from six.moves import cStringIO as StringIO import requests, datetime, json session = self.session Department, Member = self.Department, self.Member check_email = self.check_email check_member = self.check_member coltypes = self.member_types.copy() api = api_init(self.member_api._asdict()) if download: # download registered uuids if input: input = json.load(input) else: resp = api.get(self.member_api.url) assert resp.status_code == requests.codes.ok, 'cannot download used uuids' input = resp.json() columns = ['uuid', 'echo'] if check_member: columns.append(check_member) if check_email: columns.append(check_email) reader = DataTable(columns, required=['uuid'], coltypes=coltypes, gpg=self.gpg, dataformat='member', fileformat=self.member_api.format, version=self.version) reader.open(input, 'r', encrypt=self.member_api.encrypt, sign=self.member_api.receiver) columns = list(self.member_sync) wcoltypes = dict(coltypes) if check_member: wcoltypes[check_member] = bool if check_email: wcoltypes[check_email] = bool wcoltypes['department'] = int # replace by ids if 'location' in self.member_sync: wcoltypes['gpslat'] = wcoltypes['gpslng'] = float columns += ['gpslat', 'gpslng'] if download: rcolumns, unknown = reader.get_columns() if unknown: self.warn( UnknownFieldsWarning('ignoring unknown fields %s', unknown)) if check_member and check_member in rcolumns: columns.append(check_member) if check_email and check_email in rcolumns: columns.append(check_email) if 'echo' in rcolumns: columns += ['echo'] encrypt = [self.member_api.receiver ] if self.member_api.encrypt else False encryptmail = [self.email_receiver ] if self.member_api.encrypt else False writer = DataTable(columns, coltypes=wcoltypes, gpg=self.gpg, dataformat='member', fileformat=self.member_api.format, version=self.version) out = {} writer.open(out, 'w', encrypt=encrypt, sign=self.member_api.sign) if self.export_emails: ewriter = DataTable(('uuid', 'email'), coltypes=coltypes, gpg=self.gpg, dataformat='member', fileformat=format, version=self.version) eout = output[2] if output else StringIO() ewriter.open(eout, 'w', encrypt=encryptmail, sign=self.member_api.sign) def export(member, extra={}): if 'location' in self.member_import: gps = self.gps_coord(self.get_location(member)) if gps: extra['gpslat'], extra['gpslng'] = gps dep = self.get_department(member) if dep: extra[ 'department'] = dep.id if self.department_spec == 'number' else dep.name writer.write(member, extra) if self.export_emails: ewriter.write(member) if invitations: registered = {} # dict of registered uuids mquery = session.query(Member) count = 0 check_memberno = 'memberno' in self.member_columns if download: seen = set() for data in reader: uuid = data['uuid'] if not uuid: self.warn("uuid missing") continue if uuid in seen: self.warn("member %s is duplicate" % uuid) continue seen.add(uuid) member = mquery.filter_by(uuid=uuid).first() extra = {} if not member or (check_memberno and not member.memberno): # deleted self.warn("member %s is unknown" % uuid) if check_member in columns and data[check_member]: extra[check_member] = False if check_email in columns and data[check_email]: extra[check_email] = False writer.write(Member(uuid=uuid, status=StatusType.deleted), extra) continue if check_email in columns and data[check_email]: extra[check_email] = member.email == data[check_email] if check_member in columns: if data[check_member]: result = self.check_member_func( member, data[check_member]) if 'registered' in self.member_import: if result and not member.registered: member.registered = datetime.datetime.utcnow() else: result = None extra[check_member] = result if 'echo' in columns: extra['echo'] = data['echo'] if not dryrun: export(member, extra) if invitations: registered[member.uuid] = True count += 1 else: for member in mquery: if check_memberno and not member.memberno: continue # deleted if not dryrun: export(member) count += 1 self.info('%i members exported', count) if not dryrun and 'registered' in self.member_import: session.commit() writer.close() if self.export_emails: ewriter.close() if invitations: iwriter = DataTable(('uuid', 'email'), coltypes=coltypes, gpg=self.gpg, dataformat='member', fileformat=format, version=self.version) # extra encrypt,sign iwriter.open(invitations, 'w', encrypt=encryptmail, sign=self.member_api.sign) if check_memberno: query = session.query(Member.uuid,Member.email,Member.memberno).\ filter(Member.email!=None,Member.memberno!=0) else: query = session.query( Member.uuid, Member.email).filter(Member.email != None) count = 0 for member in query: if member.uuid in registered: continue # skip registered iwriter.write(member) count += 1 iwriter.close() self.info('%i invitations exported', count) if not upload: return dwriter = DataTable(('id', 'name', 'parent', 'depth'), gpg=self.gpg, dataformat='department', fileformat=self.member_api.format, version=self.version) dout = {} dwriter.open(dout, 'w', encrypt=encrypt, sign=self.member_api.sign) for dep in session.query(Department).order_by(Department.depth, Department.id): if dep.parent: extra = dict(parent=dep.parent.id if self.department_spec == 'number' else dep.parent.name) else: extra = {} dwriter.write(dep, extra) dwriter.close() if output: json.dump(out, output[0]) json.dump(dout, output[1]) elif not dryrun: r = api.post(self.member_api.url, json=dict(members=out, departments=dout)) assert r.status_code == requests.codes.ok, 'cannot upload data' if not self.export_emails or output: return from ekklesia.mail import create_mail, smtp_init smtp = smtp_init(self.smtpconfig) smtp.open() self.info('sending email data') msg = create_mail(self.gpgconfig['sender'], self.email_receiver, 'Email data', eout.getvalue()) eout.close() msg, results = self.gpg.encrypt(msg, default_key=True, verify=True, inline=True) assert msg and results, 'error encrypting message' if not dryrun: smtp.send(msg) smtp.close()
def sync_invitations(self, download=True, upload=True, dryrun=False, quick=False, input=None, output=None): """sync invitations with ID server""" from ekklesia.backends import api_init from ekklesia.data import DataTable from six.moves import cStringIO as StringIO import requests, json session = self.session Invitation = self.Invitation membercls = self.member_class check_email = self.invite_check_email api = api_init(self.invite_api._asdict()) reply = False # whether server requested reply if download: # download registered/failed/verified uuids(used codes), mark used if input: input = json.load(input) if not input: # pragma: no cover url = self.invite_api.url if quick: url += '?changed=1' resp = api.get(url) if resp.status_code != requests.codes.ok: if self.debugging: open('invdown.html', 'w').write(resp.content) assert False, 'cannot download used invite codes' input = resp.json() # only json? if not input: self.warn("input is empty") return columns = ['uuid', 'status', 'code', 'echo'] if check_email: columns.append(check_email) reader = DataTable(columns, coltypes=self.invite_types, required=('uuid', 'status', 'code'), gpg=self.gpg, dataformat='invitation', fileformat=self.invite_api.format, version=self.version) sign = self.invite_api.receiver if self.invite_api.sign else False reader.open(input, 'r', encrypt=self.invite_api.encrypt, sign=sign) rcolumns, unknown = reader.get_columns() if unknown: self.warn('ignoring unknown fields', unknown) reply = 'echo' in rcolumns # reply? if check_email: reply = reply or check_email in rcolumns if upload: # upload responses and non-uploaded,unused uuid&code columns = ['uuid', 'code', 'status'] coltypes = self.invite_types.copy() if check_email: coltypes[check_email] = bool if download and reply: if check_email and check_email in rcolumns: columns.append(check_email) if 'echo' in rcolumns: columns.append('echo') writer = DataTable(columns, coltypes=coltypes, gpg=self.gpg, dataformat='invitation', fileformat=self.invite_api.format, version=self.version) encrypt = [self.invite_api.receiver ] if self.invite_api.encrypt else False out = {} writer.open(out, 'w', encrypt=encrypt, sign=self.invite_api.sign) if download: # process download and generate reply if membercls: query = session.query(membercls) else: query = session.query(Invitation) count = 0 seen = set() for data in reader: # only uploaded codes, reply optional uuid = data['uuid'] if not uuid: self.warn("uuid missing") continue if uuid in seen: self.warn("member %s is duplicate" % uuid) continue seen.add(uuid) valid = ('registered', 'failed', 'verified', 'reset') if not quick: valid += ('new', 'verify') status = data['status'] if not status in valid: self.warn("invalid status %s for %s" % (status, uuid)) continue inv = query.filter_by(uuid=uuid).first() extra = {} if membercls and inv: inv = inv.invitation if not inv: self.error("member %s is unknown" % data['uuid']) if check_email in columns and data[check_email]: extra[check_email] = False extra['uuid'] = uuid # works also for membercls writer.write( Invitation(status=IStatusType.deleted, code=''), extra) continue """compare status and inv.status sync state transitions: backend,idserver -> target new,- -> idserver:new new,new -> idserver:new, backend:uploaded verify,registered/- -> idserver:verify -,* -> backend:deleted,error uploaded,new/registering -> idserver:no response, backend:uploaded/ignore uploaded,registered/failed -> backend:registered/failed uploaded_verify,verify -> idserver:no response, backend:uploaded_verify/ignore uploaded_verify,verified -> backend:verified registered,registered -> backend:registered, idserver:delete failed,failed -> backend:failed, idserver:delete (or new,new) new/uploaded,reset -> backend:new, idserver:new verify/uploaded_verify,reset -> backend:verify, idserver:verify failed,- -> backend:new (check if fail not downloaded) """ if status == IStatusType.new: if inv.status == IStatusType.new: if inv.code == data['code']: # code upload confirmed, prepare for sending inv.status = IStatusType.uploaded inv.sent = ISentStatusType.unsent else: # mismatch, new code needs to be uploaded self.info( "updating old code %s for uuid %s, new %s", data['code'], data['uuid'], inv.code) # write elif inv.status != IStatusType.uploaded: # ignore with uploaded self.error("bad status %s for uuid %s, current %s", status, data['uuid'], inv.status) continue elif status == IStatusType.verify: if inv.status == IStatusType.verify: if inv.code == data['code']: # code upload confirmed, prepare for sending inv.status = IStatusType.uploaded_verify inv.sent = ISentStatusType.unsent else: # mismatch, new code needs to be uploaded self.info( "updating old verify code %s for uuid %s, new %s", data['code'], data['uuid'], inv.code) # write elif inv.status != IStatusType.uploaded_verify: # ignore with uploaded_verify self.error("bad status %s for uuid %s, current %s", status, data['uuid'], inv.status) continue elif status == IStatusType.reset: if inv.status in FinalStates: self.warn("ignoring reset for uuid %s, status %s", data['uuid'], inv.status) continue inv.reset() elif status in FinalStates: if inv.status == IStatusType.uploaded_verify and status==IStatusType.verified or \ inv.status == IStatusType.uploaded and status!=IStatusType.verified: inv.status = status # upload confirmed or failed registration/verification inv.sent = ISentStatusType.unsent inv.change() elif status != inv.status: self.error("bad status %s for uuid %s, current %s", status, data['uuid'], inv.status) continue else: self.error("bad status %s for uuid %s, current %s", status, data['uuid'], inv.status) continue if upload and not inv.status in (IStatusType.uploaded, IStatusType.uploaded_verify): # write response for uploaded if check_email and check_email in columns: if member_class: extra[check_email] = inv.member.email == data[ check_email] else: extra[check_email] = inv.email == data[check_email] if 'echo' in columns: extra['echo'] = data['echo'] if membercls: extra['uuid'] = data['uuid'] writer.write(inv, extra) count += 1 self.info('%i codes used', count) if not dryrun: session.commit() if not upload: return # process failed, which have already been deleted on the server and are ready for reset count = 0 query = session.query(Invitation).filter_by(status=IStatusType.failed, sent=ISentStatusType.sent) for inv in query.yield_per(1000): extra = {} if membercls: uuid = inv.member.uuid else: uuid = inv.uuid if uuid in seen: continue # already replied inv.reset() count += 1 self.info('%i codes resetted', count) if not dryrun: session.commit() if not quick: # append new invitations count = 0 fresh = (IStatusType.new, IStatusType.verify) query = session.query(Invitation).filter( Invitation.status.in_(fresh)) for inv in query.yield_per(1000): extra = {} if membercls: uuid = inv.member.uuid extra['uuid'] = uuid else: uuid = inv.uuid writer.write(inv, extra) count += 1 self.info('%i new codes uploaded', count) writer.close() if output: json.dump(out, output) elif not dryrun: # pragma: no cover resp = api.post(self.invite_api.url, json=out) if resp.status_code != requests.codes.ok: if self.debugging: open('invup.html', 'w').write(resp.content) assert False, 'cannot upload data'
def import_members(self, memberfile=None, depfile=None, decrypt=False, verify=False, dryrun=False, allfields=False, format='csv'): from ekklesia.data import DataTable from sqlalchemy.orm import aliased session = self.session import_dep = self.department_spec != 'implicit' Department = self.Department depprimary = 'id' if self.department_spec == 'number' else 'name' dquery = session.query(Department) def get_department(id, create=False): # root dep: name,parent=None,depth=0 # fwd-ref: name,parent=None,depth=None dep = dquery.filter_by(**id).first() if not dep and create and not dryrun: dep = Department(parent=None, depth=None, **id) session.add(dep) return dep if depfile and import_dep and self.Department: columns = self.department_columns + ['parent'] if allfields: reqcolumns = columns else: reqcolumns = ('name', 'parent') reader = DataTable(columns, coltypes=self.department_types, required=reqcolumns, dataformat='department', fileformat=format, version=self.version, gpg=self.gpg) count = 0 reader.open(depfile, 'r', encrypt=decrypt, sign=verify) columns, tmp = reader.get_columns() seen = set() for data in reader: if not 'depth' in columns: data['depth'] = None # find existing parent by id, if not exist, create fwd-ref if not data['parent'] is None: data['parent'] = get_department( {depprimary: data['parent']}, create=True) # find existing dep by id id = data[depprimary] if not id: self.warn("identifier missing") continue assert not id in seen, "department %s is duplicate" % id seen.add(id) dep = get_department({depprimary: id}) if dep: parent = data['parent'] while parent: assert parent != dep, "cycle detected for department %s" % id parent = parent.parent # if not exist, create, set depth if parent.depth depth = data['parent'].depth if data['parent'] else None if not depth is None: depth += 1 if data['depth'] is None: data['depth'] = depth elif not depth is None: assert data['depth'] >= depth, "invalid depth" count += 1 if dryrun: continue if not dep: session.add(Department(**data)) else: dep.__init__(**data) self.info('%i imported departments', count) elif not import_dep: dquery.update(dict(depth=None)) # reset all depths Member = self.Member columns = self.member_columns + ['department'] if allfields: reqcolumns = columns else: reqcolumns = ['uuid', 'email'] if not import_dep and 'department' in self.member_import and not 'parent' in columns: columns.append('parent') reqcolumns.append('parent') reader = DataTable(columns, coltypes=self.member_types, required=reqcolumns, dataformat='member', fileformat=format, version=self.version, gpg=self.gpg) count = 0 reader.open(memberfile, 'r', encrypt=decrypt, sign=verify) columns, tmp = reader.get_columns() #primary either memberid or uuid #primary must exist for every member and be unique #primary+email must be unique if 'memberno' in columns: primary, primarykey = 'memberno', Member.memberno else: primary, primarykey = 'uuid', Member.uuid mquery = session.query(primarykey) seen, depseen = set(), set() for data in reader: id = data[primary] if not id: self.warn("uuid missing") continue assert not id in seen, "member %s is duplicate" % id seen.add(id) if 'department' in data: depid = data['department'] if import_dep: # all departments must exist dep = get_department({depprimary: depid}) assert dep, "unknown department %s" % depid else: # create departments from department and parent # find existing parent by id, if not exist, create fwd-ref parent = data['parent'] del data['parent'] if not parent is None: parent = get_department({'name': parent}, create=True) # find existing dep by id dep = get_department({'name': depid}) # if not exist, create, set depth if parent.depth if dep: sup = parent while sup: assert sup != dep, "cycle detected for department %s" % depid sup = sup.parent depseen.add(depid) if not dryrun: if not dep: dep = Department(name=depid, parent=parent, depth=None) session.add(dep) elif dep.parent != parent: # if parent changed and not seen, update assert not depid in depseen, "department %s is duplicate" % depid dep.__init__(parent=parent) data['department'] = dep if 'email' in columns and not data['email']: data['email'] = None member = mquery.filter_by(**{primary: id}).first() if data['email']: email = session.query(primarykey,Member.email).\ filter_by(email=data['email']).first() else: email = None if email and (not member or getattr(email, primary) != getattr(member, primary)): self.error("ignoring: duplicate email %s" % data['email']) continue count += 1 if dryrun: continue if member: member.__init__(**data) else: session.add(Member(**data)) #new self.info('%i imported members', count) # complete missing depths depalias = aliased(Department) fixdeps = session.query(Department.depth).filter_by(depth=None) if fixdeps.first(): fixdeps.filter_by(parent=None).update(dict(depth=0)) # set roots while fixdeps.first(): for sub in session.query(Department).join(depalias,Department.parent).\ filter(depalias.depth!=None): sub.depth = sub.parent.depth + 1 if not dryrun: session.commit()
def import_members(self, memberfile=None, depfile=None, decrypt=False, verify=False, dryrun=False, sync=False, allfields=False, format='csv'): """import data from memberfile, optionally depfile (if not implicit). allfields is used for restore and requires all columns. if sync, uuids not seen in input are set to status deleted. decrypt=with the default key, verify=check whether its signed with io_key. rows with missing uuids or duplicate uuids and emails are ignored. """ from ekklesia.data import DataTable, init_object from sqlalchemy.orm import aliased session = self.session import_dep = self.department_spec != 'implicit' # extra Department data? Department = self.Department depprimary = 'id' if self.department_spec == 'number' else 'name' dquery = session.query(Department) if verify: verify = self.io_key def get_department(id, create=False): # root dep: name,parent=None,depth=0 # fwd-ref: name,parent=None,depth=None dep = dquery.filter_by(**id).first() if not dep and create and not dryrun: # create forward reference w/o depth, calculate it later dep = Department(parent=None, depth=None, **id) session.add(dep) return dep if depfile and import_dep and self.Department: # import separate department data columns = list(self.department_columns) + ['parent'] if allfields: reqcolumns = columns else: reqcolumns = ('name', 'parent') reader = DataTable(columns, coltypes=self.department_types, required=reqcolumns, dataformat='department', fileformat=format, version=self.version, gpg=self.gpg) count = 0 reader.open(depfile, 'r', encrypt=decrypt, sign=verify) columns = reader.get_columns()[0] seen = set() # detect duplicates for data in reader: if not 'depth' in columns: data['depth'] = None # find existing parent by id, if not exist, create fwd-ref if not data['parent'] is None: data['parent'] = get_department( {depprimary: data['parent']}, create=True) # find existing dep by id id = data[depprimary] if not id: self.warn("identifier missing") continue assert not id in seen, "department %s is duplicate" % id seen.add(id) dep = get_department({depprimary: id}) if dep: parent = data['parent'] while parent: assert parent != dep, "cycle detected for department %s" % id parent = parent.parent # if not exist, create, set depth if parent.depth depth = data['parent'].depth if data['parent'] else None if not depth is None: depth += 1 if data['depth'] is None: data['depth'] = depth elif not depth is None: assert data['depth'] >= depth, "invalid depth" count += 1 if dryrun: continue if not dep: session.add(Department(**data)) else: dep.update(**data) self.info('%i imported departments', count) elif not import_dep: # implicit dquery.update(dict(depth=None)) # reset all depths Member = self.Member columns = list(self.member_columns) + ['department'] if allfields: reqcolumns = columns else: reqcolumns = ['uuid', 'email'] if not import_dep and 'department' in self.member_import and not 'parent' in columns: # implicit requires parent columns.append('parent') reqcolumns.append('parent') reader = DataTable(columns, coltypes=self.member_types, required=reqcolumns, dataformat='member', fileformat=format, version=self.version, gpg=self.gpg) count = 0 reader.open(memberfile, 'r', encrypt=decrypt, sign=verify) columns = reader.get_columns()[0] #primary either memberid or uuid #primary must exist for every member and be unique #primary+email must be unique if 'memberno' in columns: primary, primarykey = 'memberno', Member.memberno else: primary, primarykey = 'uuid', Member.uuid mquery = session.query(Member) seen, depseen = set(), set() # detect duplicates for data in reader: id = data[primary] if not id: self.warn("uuid missing") continue assert not id in seen, "member %s is duplicate" % id seen.add(id) if 'department' in data: depid = data['department'] if import_dep: # all departments must exist dep = get_department({depprimary: depid}) assert dep, "unknown department %s" % depid else: # implicit departments # create departments from department and parent # find existing parent by id, if not exist, create fwd-ref parent = data['parent'] del data['parent'] if not parent is None: parent = get_department({'name': parent}, create=True) # find existing dep by id dep = get_department({'name': depid}) # if not exist, create, set depth if parent.depth if dep: sup = parent while sup: assert sup != dep, "cycle detected for department %s" % depid sup = sup.parent depseen.add(depid) if not dryrun: if not dep: dep = Department(name=depid, parent=parent, depth=None) session.add(dep) elif dep.parent != parent: # if parent changed and not seen, update assert not depid in depseen, "department %s is duplicate" % depid dep.update(parent=parent) data['department'] = dep if 'email' in columns and not data['email']: data['email'] = None # make sure it's None # find existing member member = mquery.filter_by(**{primary: id}).first() if data['email'] and not (member and member.email == data['email']): # email already used by other member? if session.query( primarykey, Member.email).filter_by(email=data['email']).first(): self.error("ignoring: duplicate email %s" % data['email']) continue count += 1 if dryrun: continue if member: if member.email != data['email']: self.email_change(member, data) if data.get('status') == 'deleted': self.delete_member(member) member.update(**data) else: session.add(Member(**data)) #new self.info('%i imported members', count) if sync: # deleted unseen members count = 0 for member in session.query(Member).yield_per(1000): if member.status == MStatusType.deleted or member.uuid in seen: continue self.delete_member(member) member.status = MStatusType.deleted self.info("member %s deleted" % member.uuid) count += 1 self.info('%i deleted members', count) # complete missing depths depalias = aliased(Department) fixdeps = session.query(Department.depth).filter_by(depth=None) if fixdeps.first(): # any departments with missing depths? fixdeps.filter_by(parent=None).update(dict(depth=0)) # set roots while fixdeps.first(): # fill from roots to leaves for sub in session.query(Department).join(depalias,Department.parent).\ filter(depalias.depth!=None).yield_per(1000): sub.depth = sub.parent.depth + 1 if not dryrun: session.commit()
def sync_members(self,download=True,upload=True,dryrun=False,quick=False, input=None,output=None,invitations=None,format='csv'): "sync members with ID server" from ekklesia.data import DataTable from ekklesia.backends import api_init from six.moves import cStringIO as StringIO import requests, json session = self.session Department, Member = self.Department, self.Member check_email = self.check_email check_member = self.check_member coltypes = self.member_types.copy() api = api_init(self.member_api._asdict()) if download: # download registered uuids if input: input = json.load(input) else: # pragma: no cover url = self.member_api.url if quick: url+='?new=1' resp = api.get(url) if resp.status_code != requests.codes.ok: if self.debugging: open('memberdown.html','w').write(resp.content) assert False, 'cannot download used uuids' input = resp.json() if not input: self.warn("input is empty") return columns = ['uuid','echo'] if check_member: columns.append(check_member) if check_email: columns.append(check_email) reader = DataTable(columns,required=['uuid'],coltypes=coltypes,gpg=self.gpg, dataformat='member',fileformat=self.member_api.format,version=self.version) sign = self.member_api.receiver if self.member_api.sign else False reader.open(input,'r',encrypt=self.member_api.encrypt,sign=sign) columns = list(self.member_sync) wcoltypes = dict(coltypes) if check_member: wcoltypes[check_member] = bool if check_email: wcoltypes[check_email] = bool wcoltypes['departments'] = (int,) # replace by list of ids if 'location' in self.member_sync: wcoltypes['gpslat'] = wcoltypes['gpslng'] = float columns += ['gpslat','gpslng'] if download: rcolumns, unknown = reader.get_columns() if unknown: self.warn(UnknownFieldsWarning('ignoring unknown fields %s',unknown)) if check_member and check_member in rcolumns: columns.append(check_member) if check_email and check_email in rcolumns: columns.append(check_email) if 'echo' in rcolumns: columns += ['echo'] encrypt = [self.member_api.receiver] if self.member_api.encrypt else False encryptmail = [self.email_receiver] if self.member_api.encrypt else False writer = DataTable(columns,coltypes=wcoltypes,gpg=self.gpg, dataformat='member',fileformat=self.member_api.format,version=self.version) out = {} writer.open(out,'w',encrypt=encrypt,sign=self.member_api.sign) if not quick and self.export_emails: ewriter = DataTable(('uuid','email'),coltypes=coltypes,gpg=self.gpg, dataformat='member',fileformat=format,version=self.version) eout = output[2] if output else StringIO() ewriter.open(eout,'w',encrypt=encryptmail,sign=self.member_api.sign) def export(member,extra = {}): if 'location' in self.member_import: gps = self.gps_coord(self.get_location(member)) if gps: extra['gpslat'],extra['gpslng'] = gps deps = self.get_departments(member) if self.department_spec=='number': deps = [dep.id for dep in deps] else: deps = [dep.name for dep in deps] if deps: extra['departments'] = deps writer.write(member,extra) if not quick and self.export_emails: ewriter.write(member) if invitations: registered = {} # dict of registered uuids mquery = session.query(Member) count = 0 if download: from datetime import datetime seen = set() for data in reader: uuid = data['uuid'] if not uuid: self.warn("uuid missing") continue if uuid in seen: self.warn("member %s is duplicate" % uuid) continue seen.add(uuid) member = mquery.filter_by(uuid=uuid).first() extra = {} if not member or (self.member_no and member.status == MStatusType.deleted): self.warn("member %s is unknown" % uuid) if check_member in columns and data[check_member]: extra[check_member] = False if check_email in columns and data[check_email]: extra[check_email] = False extra['departments'] = [] writer.write(Member(uuid=uuid,status=MStatusType.deleted),extra) continue if check_email in columns and data[check_email]: extra[check_email] = member.email == data[check_email] if check_member in columns: if data[check_member]: result = self.check_member_func(member,data[check_member]) if 'registered' in self.member_import: if result and not member.registered: member.registered = datetime.utcnow() else: if 'registered' in self.member_import and member.registered: self.warn("member %s is already registered" % uuid) result = None extra[check_member] = result elif 'registered' in self.member_import and not member.registered: member.registered = datetime.utcnow() if 'echo' in columns: extra['echo'] = data['echo'] if not dryrun: export(member,extra) if invitations: registered[member.uuid] = True count += 1 else: for member in mquery.yield_per(1000): if self.member_no and not member.memberno: continue # deleted if not dryrun: export(member) count += 1 self.info('%i members exported', count) if not dryrun and 'registered' in self.member_import: session.commit() writer.close() if not quick and self.export_emails: ewriter.close() if not quick and invitations: iwriter = DataTable(('uuid','email'),coltypes=coltypes,gpg=self.gpg, dataformat='member',fileformat=format,version=self.version) # extra encrypt,sign iwriter.open(invitations,'w',encrypt=encryptmail,sign=self.member_api.sign) if self.member_no: query = session.query(Member.uuid,Member.email,Member.memberno).\ filter(Member.email!=None,Member.memberno!=0) else: query = session.query(Member.uuid,Member.email).filter(Member.email!=None) count = 0 for member in query.yield_per(1000): if member.uuid in registered: continue # skip registered iwriter.write(member) count += 1 iwriter.close() self.info('%i invitations exported', count) if not upload: return dwriter = DataTable(('id','name','parent','depth'),gpg=self.gpg, dataformat='department',fileformat=self.member_api.format,version=self.version) dout = {} dwriter.open(dout,'w',encrypt=encrypt,sign=self.member_api.sign) for dep in session.query(Department).order_by(Department.depth,Department.id).yield_per(1000): if dep.parent: extra = dict(parent=dep.parent.id) # if self.department_spec=='number' else dep.parent.name else: extra = {} dwriter.write(dep,extra) dwriter.close() if output: json.dump(out,output[0]) json.dump(dout,output[1]) elif not dryrun: # pragma: no cover resp = api.post(self.member_api.url,json=dict(members=out,departments=dout)) if resp.status_code != requests.codes.ok: if self.debugging: open('memberup.html','w').write(resp.content) assert False, 'cannot upload data' if not self.export_emails or output or quick: return # pragma: no cover from ekklesia.mail import create_mail, smtp_init smtp = smtp_init(self.smtpconfig) smtp.open() self.info('sending email data') msg = create_mail(self.gpgconfig['sender'],self.email_receiver,'Email data',eout.getvalue()) eout.close() msg, results = self.gpg.encrypt(msg,default_key=True,verify=True, inline=True) assert msg and results, 'error encrypting message' if not dryrun: smtp.send(msg) smtp.close()
def import_members(self,memberfile=None,depfile=None,decrypt=False,verify=False, dryrun=False,sync=False,allfields=False,format='csv'): """import data from memberfile, optionally depfile (if not implicit). allfields is used for restore and requires all columns. if sync, uuids not seen in input are set to status deleted. decrypt=with the default key, verify=check whether its signed with io_key. rows with missing uuids or duplicate uuids and emails are ignored. """ from ekklesia.data import DataTable, init_object from sqlalchemy.orm import aliased session = self.session import_dep = self.department_spec != 'implicit' # extra Department data? Department = self.Department depprimary = 'id' if self.department_spec=='number' else 'name' dquery = session.query(Department) if verify: verify = self.io_key def get_department(id,create=False): # root dep: name,parent=None,depth=0 # fwd-ref: name,parent=None,depth=None dep = dquery.filter_by(**id).first() if not dep and create and not dryrun: # create forward reference w/o depth, calculate it later dep = Department(parent=None,depth=None,**id) session.add(dep) return dep if depfile and import_dep and self.Department: # import separate department data columns = list(self.department_columns)+['parent'] if allfields: reqcolumns = columns else: reqcolumns = ('name','parent') reader = DataTable(columns,coltypes=self.department_types,required=reqcolumns, dataformat='department',fileformat=format,version=self.version,gpg=self.gpg) count = 0 reader.open(depfile,'r',encrypt=decrypt,sign=verify) columns = reader.get_columns()[0] seen = set() # detect duplicates for data in reader: if not 'depth' in columns: data['depth']=None # find existing parent by id, if not exist, create fwd-ref if not data['parent'] is None: data['parent'] = get_department({depprimary:data['parent']},create=True) # find existing dep by id id = data[depprimary] if not id: self.warn("identifier missing") continue assert not id in seen, "department %s is duplicate" % id seen.add(id) dep = get_department({depprimary:id}) if dep: parent = data['parent'] while parent: assert parent!=dep, "cycle detected for department %s" % id parent = parent.parent # if not exist, create, set depth if parent.depth depth = data['parent'].depth if data['parent'] else None if not depth is None: depth+=1 if data['depth'] is None: data['depth'] = depth elif not depth is None: assert data['depth']>=depth, "invalid depth" count += 1 if dryrun: continue if not dep: session.add(Department(**data)) else: dep.update(**data) self.info('%i imported departments', count) elif not import_dep: # implicit dquery.update(dict(depth=None)) # reset all depths Member = self.Member columns = list(self.member_columns)+['departments'] if allfields: reqcolumns = columns else: reqcolumns = ['uuid','email'] if self.member_no: reqcolumns.append('id') if not import_dep and 'departments' in self.member_import and not 'parent' in columns: # implicit requires parent columns.append('parent') reqcolumns.append('parent') reader = DataTable(columns,coltypes=self.member_types,required=reqcolumns, dataformat='member',fileformat=format,version=self.version,gpg=self.gpg) count = 0 reader.open(memberfile,'r',encrypt=decrypt,sign=verify) columns = reader.get_columns()[0] #primary either memberid or uuid #primary must exist for every member and be unique #primary+email must be unique if self.member_no: primary, primarykey = 'id', Member.id else: primary, primarykey = 'uuid', Member.uuid mquery = session.query(Member) seen, depseen = set(), set() # detect duplicates for data in reader: id = data[primary] if not id: self.warn("uuid missing") continue assert not id in seen, "member %s is duplicate" % id seen.add(id) if 'departments' in data: depids = data['departments'] if import_dep: # all departments must exist deps=set() for depid in depids: dep = get_department({depprimary:depid}) assert dep, "unknown department %s" % depid deps.add(dep) deps = list(deps) else: # implicit departments assert len(depids)==1, "only single implicit department supported" depid = depids[0] # create departments from department and parent # find existing parent by id, if not exist, create fwd-ref parent = data['parent'] del data['parent'] if not parent is None: parent = get_department({'name':parent},create=True) # find existing dep by id dep = get_department({'name':depid}) # if not exist, create, set depth if parent.depth if dep: sup = parent while sup: assert sup!=dep, "cycle detected for department %s" % depid sup = sup.parent depseen.add(depid) if not dryrun: if not dep: dep = Department(name=depid,parent=parent,depth=None) session.add(dep) elif dep.parent != parent: # if parent changed and not seen, update assert not depid in depseen, "department %s is duplicate" % depid dep.update(parent=parent) deps = [dep] data['departments'] = deps if 'email' in columns and not data['email']: data['email'] = None # make sure it's None # find existing member member = mquery.filter(primarykey==id).first() if data['email'] and not (member and member.email==data['email']): # email already used by other member? if session.query(primarykey,Member.email).filter_by(email=data['email']).first(): self.error("ignoring: duplicate email %s" % data['email']) continue count += 1 if dryrun: continue if member: if member.email != data['email']: self.email_change(member,data) if data.get('status') == 'deleted': self.delete_member(member) member.update(**data) else: session.add(Member(**data)) #new self.info('%i imported members', count) if sync: # deleted unseen members count = 0 for member in session.query(Member).yield_per(1000): if member.status==MStatusType.deleted or member.uuid in seen: continue self.delete_member(member) member.status = MStatusType.deleted self.info("member %s deleted" % member.uuid) count += 1 self.info('%i deleted members', count) # complete missing depths depalias = aliased(Department) fixdeps = session.query(Department.depth).filter_by(depth=None) if fixdeps.first(): # any departments with missing depths? fixdeps.filter_by(parent=None).update(dict(depth=0)) # set roots while fixdeps.first(): # fill from roots to leaves for sub in session.query(Department).join(depalias,Department.parent).\ filter(depalias.depth!=None).yield_per(1000): sub.depth = sub.parent.depth+1 if not dryrun: session.commit()
def import_invitations(self, input, decrypt=False, verify=False, allfields=False, dryrun=False, format='csv'): from ekklesia.data import DataTable from datetime import datetime session = self.session Invitation = self.Invitation membercls = self.member_class columns = self.invite_columns if membercls: columns.append('uuid') if allfields: reqcolumns = columns elif membercls: reqcolumns.append('uuid') else: reqcolumns = ['uuid', 'email'] reader = DataTable(columns, coltypes=self.invite_types, required=reqcolumns, dataformat='invitation', fileformat=format, version=self.version, gpg=self.gpg) reader.open(input, 'r', encrypt=decrypt, sign=verify) columns, tmp = reader.get_columns() recordchange = 'status' in columns and not 'lastchange' in columns iquery = session.query(Invitation) count = 0 seen = set() for data in reader: uuid = data['uuid'] if not uuid: self.warn("uuid missing") continue assert not uuid in seen, "member %s is duplicate" % uuid seen.add(uuid) if membercls: member = session.query(membercls).filter_by(uuid=uuid).first() if member is None: self.warn("uuid %s not found" % uuid) continue if not member.email: # email removed, disable invitation if member.invitation: self.info( "scheduling invitation for uuid '%s' for deletion", member.uuid) if not dryrun: member.invitation.status = StatusType.deleted member.invitation.lastchange = datetime.utcnow() continue count += 1 if dryrun: continue if member.invitation is None: # create a new invitation session.add(Invitation(member=member, **data)) #new else: if recordchange and data[ 'status'] != member.invitation.status: data['lastchange'] = datetime.utcnow() member.invitation.__init__(**data) #update inv else: inv = iquery.filter_by(uuid=uuid).first() if not data['email']: # email removed, disable invitation if inv is None: self.warn("uuid %s not found" % uuid) continue if inv.status == StatusType.deleted: continue self.info( "scheduling invitation for uuid '%s' for deletion", inv.uuid) if not dryrun: inv.status = StatusType.deleted inv.lastchange = datetime.utcnow() continue # check whether email already used if not inv or inv.email != data['email']: # fixme: what if emails swapped? email = iquery.filter_by(email=data['email']).first() if email and (not inv or inv.uuid != email.uuid): self.error("ignoring: duplicate email %s" % data['email']) continue count += 1 if dryrun: continue if inv: # if email changed and code has been sent, reset invcode and lastchange, unless allfields is set needreset = not allfields and inv.status==StatusType.uploaded and \ inv.sent==SentStatusType.sent and 'email' in data and data['email']!=inv.email and \ (not 'code' in data or data['code']==inv.code) if not needreset and recordchange and data[ 'status'] != inv.status: data['lastchange'] = datetime.utcnow() inv.__init__(**data) #update inv if needreset: inv.reset() else: session.add(Invitation(**data)) #new self.info('%i imported invitations', count) if not dryrun: session.commit()