def __init__(self, xml): """Initializes from a XML string Arguments: - xml -- file content of an XForm XML file """ try: self.document = parseString(xml) except ExpatError as e: raise XFormException('could not parse XML : ' + str(e)) instance = self.get_path(('h:html', 'h:head', 'model', 'instance')) forms = [child for child in instance.childNodes if child.nodeType == self.document.ELEMENT_NODE] if len(forms) != 1: raise XFormException('no unique form instance found') self.template = forms[0] self.name = self.template.nodeName formid = self.template.attributes.get('id') if not formid: raise XFormException('instance %s has no id attribute' % self.name) self.formid = formid.value #TODO implement 'required' and 'constraint' self.paths = [] self.add_paths(self.template, tuple()) self.clear() lo.debug('loaded XForm %s "%s" : %d paths' % ( self.name, self.formid, len(self.paths)))
def __setitem__(self, name, value): """Set value of a XForm field Raises XFormException if no field with given name can be found in this form. Arguments - name -- name of field, path parts separated by '/' - value -- value; int, float, date, time, and datetime will be converted to string; None will forestall the sending of this path """ if not name in self.paths: raise XFormException('path "%s" not found in form "%s"' % ( name, self.name)) if value is None: return if isinstance(value, int): value = str(value) elif isinstance(value, float): value = str(value) elif isinstance(value, datetime.time): value = value.strftime('%H:%M:%S.0') elif isinstance(value, datetime.date): value = value.strftime('%Y-%m-%d') elif isinstance(value, datetime.datetime): value = value.strftime('%Y-%m-%d %H:%M:%S.0') if not isinstance(value, str): raise XFormException('cannot convert value : ' + str(value)) self.items[name] = value lo.debug('setting %s[%s] = %s' % (self.formid, name, value))
def parse_datetime(self, value): try: value = datetime.datetime.strptime('2013/01/02 23:44:08.843', '%Y/%m/%d %H:%M:%S.%f') return value except ValueError: lo.debug('could not convert _date/_time value %s' % value) return value
def process(self): src = self.path1() dst = self.path2() lo.debug('converting "%s" to "%s"' % (src, dst)) subprocess.check_call([ self.config.convert_executable, src, '-quality', '100', '-resize', '%d>' % self.config.pixels, dst ])
def parse_datetime(self, value): try: value = datetime.datetime.strptime( '2013/01/02 23:44:08.843', '%Y/%m/%d %H:%M:%S.%f') return value except ValueError: lo.debug('could not convert _date/_time value %s' % value) return value
def run(self): n = 0 while not self.should_stop: lo.debug('uploader running n=%d' % n) if not self.dryrun and not self.try_connect(): lo.info('cannot connect; wait for 1 minute') for sec in range(60): time.sleep(1) if self.should_stop: break continue for name in sorted(self.model.tables): if self.should_stop: break table = self.model.tables[name] # be quiet when polling data apart from first time if n == 0: lo.info('sync data with MS-SQL table "%s"' % name) row = self.model.get_next_new(name) while row is not None: if self.should_stop: break rowname = table.rowname(row) if self.dryrun: lo.info('would upload form %s from table %s' % (rowname, name)) time.sleep(1) else: if self.try_send(table, row): lo.info('uploaded form %s from table %s' % (rowname, name)) self.model.mark_done(name, row) self.notify() row = self.model.get_next_new(name) seconds = self.interval while seconds > 0 and not self.should_stop: time.sleep(1) seconds -= 1 n += 1
def run(self): n = 0 while not self.should_stop: lo.debug('uploader running n=%d' % n) if not self.dryrun and not self.try_connect(): lo.info('cannot connect; wait for 1 minute') for sec in range(60): time.sleep(1) if self.should_stop: break continue for name in sorted(self.model.tables): if self.should_stop: break table = self.model.tables[name] # be quiet when polling data apart from first time if n == 0: lo.info('sync data with MS-SQL table "%s"' % name) row = self.model.get_next_new(name) while row is not None: if self.should_stop: break rowname = table.rowname(row) if self.dryrun: lo.info('would upload form %s from table %s' % ( rowname, name)) time.sleep(1) else: if self.try_send(table, row): lo.info('uploaded form %s from table %s' % ( rowname, name)) self.model.mark_done(name, row) self.notify() row = self.model.get_next_new(name) seconds = self.interval while seconds > 0 and not self.should_stop: time.sleep(1) seconds -= 1 n += 1
def discover_testsuites(paths=[]): ''' returns dictionary mapping name to python file for all testsuites discovered in the usual places: kvarq root path, user home directory, current working directory, KVARQ_TESTSUITES environment variable, and any more paths specified paths as arguments -- later occurrences of the same testsuite override previous ''' testsuite_paths = {} # 1) discover in root path root_base = os.path.abspath(os.path.join(get_root_path(), 'testsuites')) lo.debug('discovering testsuites in root path') add_testsuites_dir(testsuite_paths, root_base) # 2) discover from $HOME base = os.path.join(expanduser('~'), 'kvarq_testsuites') lo.debug('discovering testsuites in home directory') add_testsuites_dir(testsuite_paths, base) # 3) discover from CWD if not in root path cwd_base = os.path.abspath('testsuites') if cwd_base != root_base: lo.debug('discovering testsuites in current working directory') add_testsuites_dir(testsuite_paths, cwd_base) # 4) discover from KVARQ_TESTSUITES from_env = os.environ.get('KVARQ_TESTSUITES') if from_env: lo.debug('discovering testsuites in $KVARQ_TESTSUITES') for base in from_env.split(os.path.pathsep): add_testsuites_dir(testsuite_paths, base) # 5) explicitely specified paths for base in paths: if os.path.isdir(base): lo.debug('discovering testsuites in "%s"' % base) add_testsuites_dir(testsuite_paths, base) else: lo.warning('could not find directory "%s"' % base) return testsuite_paths
def fill_xform(self, row): xform = XForm(self.xform) ignored = [] lowerpaths = [path.lower() for path in xform.paths] for i, colname in enumerate(self.colnames): if colname in lowerpaths: idx = lowerpaths.index(colname) path = xform.paths[idx] # are stored as varchar(24)... if colname.endswith('_date') or colname.endswith('_time'): value = self.parse_datetime(row[i]) else: value = row[i] xform[path] = value else: ignored.append(colname) if ignored: lo.debug('created XForm : ignored %d values from db : %s' % (len(ignored), ignored)) return xform
def fill_xform(self, row): xform = XForm(self.xform) ignored = [] lowerpaths = [path.lower() for path in xform.paths] for i, colname in enumerate(self.colnames): if colname in lowerpaths: idx = lowerpaths.index(colname) path = xform.paths[idx] # are stored as varchar(24)... if colname.endswith('_date') or colname.endswith('_time'): value = self.parse_datetime(row[i]) else: value = row[i] xform[path] = value else: ignored.append(colname) if ignored: lo.debug('created XForm : ignored %d values from db : %s' % ( len(ignored), ignored)) return xform
def set_file(self, name, filename, mimetype=None): """Set value of a XForm "file type" field Raises XFormException if no field with given name can be found in this form. Arguments: - name -- name of field, path parts separated by '/' - filename -- name of file to be sent as value of field - mimetype (optional) -- will be used as the file's "Content-Type" """ if mimetype is None: mimetype = mimetypes.guess_type(filename)[0] if mimetypes is None: mimetype = 'application/binary' self[name] = os.path.basename(filename) self.filenames[name] = filename self.mimetypes[name] = mimetype lo.debug('(mimetype %s)' % mimetype)
def post_multipart(self, items): """Post items to server Sends the specified items to the server, raising an AggregateException in case of error. Arguments: - items -- a sequence of sequences ``(name, filename, value, file_content_type)`` such as returned by XForm.get_items() """ content_type, body = self.encode_multipart(items) self.request('POST', self.submission_uri, body, { 'Content-Type': content_type, 'Content-Length': len(body) }) r = self.conn.getresponse() r_body = r.read() lo.debug('POST %s -> status=%d reason=%s data="%s"' % ( self.submission_uri, r.status, r.reason, r_body)) if r.status == 404: lo.error('could not find form with specified id') lo.debug('response body : ' + r_body.decode('utf8')) raise AggregateFormNotFoundException('Form not found on server') if r.status != 201: lo.error('expected status=201 after posting, got ' + str(r.status)) lo.debug('response body : ' + r_body.decode('utf8')) raise AggregateException('Could not post multipart')
def get_authentication(self, method, uri): """Create new authentication token Arguments: - method -- HTTP method - uri -- URI for which authentication will be valid Return value: value that can be used for the 'Authorization' HTTP header """ realm = self.www_auth['realm'] snonce = self.www_auth['nonce'] qop = self.www_auth['qop'] nc = '%08d' % self.nc self.nc += 1 HA1 = hashlib.md5(':'.join([self.username, realm, self.password]).encode('utf8')).hexdigest() HA2 = hashlib.md5(':'.join([method, uri]).encode('utf8')).hexdigest() HAx = hashlib.md5(':'.join([HA1, snonce, nc, self.cnonce, qop, HA2]).encode('utf8')).hexdigest() lo.debug('DAA : username=%s realm=%s => HA1=%s' % (self.username, realm, HA1)) lo.debug('DAA : method=%s uri=%s => HA2=%s' % (method, uri, HA2)) lo.debug('DAA : snonce=%s nc=%s cnonce=%s qop=%s => response=%s' % ( snonce, nc, self.cnonce, qop, HAx)) auth = { 'username': self.username, 'realm': realm, 'nonce': snonce, 'uri': uri, 'response': HAx, 'cnonce': self.cnonce, 'algorithm': 'MD5', 'nc': nc, 'qop': qop, } auth = ', '.join([ '%s="%s"' % (key, value) for key, value in auth.items() ]) lo.debug('DAA response : ' + auth) return 'Digest ' + auth
def add_testsuites_dir(testsuite_paths, base): if not os.path.isdir(base): return for subdir in os.listdir(base): if not os.path.isdir(os.path.join(base, subdir)) or ( subdir[0] == '_' or subdir[0] == '.'): continue for fname in os.listdir(os.path.join(base, subdir)): if not fname.endswith('.py') or ( fname[0] == '_' or fname[0] == '.'): continue name = subdir + '/' + fname[:-3] path = os.path.join(base, subdir, fname) if name in testsuite_paths: lo.info('testsuite %s loaded from "%s"' % (name, path)) else: lo.debug('testsuite %s loaded from "%s"' % (name, path)) testsuite_paths[name] = path
def set_client(self, client): if self.client is None: self.button['state'] = 'normal' self.client = client lo.debug('using aggregate version ' + str(AGGREGATE_VERSION)) self.wm_title(client.url)
def connect(self, user=None, password=None): """Connect to ODK Aggregate server Connects to server, performing initial authentication and raising AggregateException in case of error. If the initial request was successful, the attribute .conn will be set to a value different fron None. Arguments: - user (optional) -- username to use for authentication - password (optional) -- password to use for authentication """ if self.scheme == 'https': self.conn = http.client.HTTPSConnection(self.address, self.port) #TODO check against provided certificate lo.info('SSL : server certificate NOT checked') else: self.conn = http.client.HTTPConnection(self.address, self.port) self.daa = None # raises ConnectionRefusedError self.request('HEAD', self.submission_uri, '') r = self.conn.getresponse() r_body = r.read() #cookie = r.getheader('Set-Cookie') #if cookie: # cookie = cookie[:cookie.index(';')] lo.debug("HEAD %s : status=%d reason=%s" % ( self.submission_url, r.status, r.reason)) # anonymous user has Data Collector rights -> status=204 if r.status == 204: self.daa = None lo.info('connected to %s (no authentication)' % self.url) # anonymous user has no Data Collector rights -> status=401 elif r.status == 401: lo.info('Aggregate replied status=401 -> digest access authentication (DAA)') if user is None or password is None: raise AggregateException('Must specify user/password for authentication') self.daa = DAA(r.getheader('www-authenticate'), user, password) #headers = create_headers(cookie) self.request('HEAD', self.submission_uri, '') r = self.conn.getresponse() r_body = r.read() lo.debug("server response DAA : status=%d reason=%s" % (r.status, r.reason)) if r.status == 401: lo.error('cannot authenticate : received second 401 response') raise AggregateException('Cannot authenticate') if r.status != 204: lo.error('expected status=204 (got %d) after authentication' % r.status) if r.status == 403: raise AggregateException( 'user "%s" is not allowed to post forms' % user) raise AggregateException('cannot authenticate') lo.info('connected to %s (authenticated as "%s")' % (self.url, user)) elif r.status == 404: raise AggregateException('Could not connect : path "%s" not found' % self.uri) else: raise AggregateException('Could not connect : unknown status')