def __init__(self, xml):
"""Initializes from a XML string
Arguments:
- xml -- file content of an XForm XML file
"""
try:
self.document = parseString(xml)
except ExpatError as e:
raise XFormException('could not parse XML : ' + str(e))
instance = self.get_path(('h:html', 'h:head', 'model', 'instance'))
forms = [child for child in instance.childNodes
if child.nodeType == self.document.ELEMENT_NODE]
if len(forms) != 1:
raise XFormException('no unique form instance found')
self.template = forms[0]
self.name = self.template.nodeName
formid = self.template.attributes.get('id')
if not formid:
raise XFormException('instance %s has no id attribute' % self.name)
self.formid = formid.value
#TODO implement 'required' and 'constraint'
self.paths = []
self.add_paths(self.template, tuple())
self.clear()
lo.debug('loaded XForm %s "%s" : %d paths' % (
self.name, self.formid, len(self.paths)))
def __setitem__(self, name, value):
"""Set value of a XForm field
Raises XFormException if no field with given name can be found
in this form.
Arguments
- name -- name of field, path parts separated by '/'
- value -- value; int, float, date, time, and datetime will be
converted to string; None will forestall the sending of
this path
"""
if not name in self.paths:
raise XFormException('path "%s" not found in form "%s"' % (
name, self.name))
if value is None:
return
if isinstance(value, int):
value = str(value)
elif isinstance(value, float):
value = str(value)
elif isinstance(value, datetime.time):
value = value.strftime('%H:%M:%S.0')
elif isinstance(value, datetime.date):
value = value.strftime('%Y-%m-%d')
elif isinstance(value, datetime.datetime):
value = value.strftime('%Y-%m-%d %H:%M:%S.0')
if not isinstance(value, str):
raise XFormException('cannot convert value : ' + str(value))
self.items[name] = value
lo.debug('setting %s[%s] = %s' % (self.formid, name, value))
def parse_datetime(self, value):
try:
value = datetime.datetime.strptime('2013/01/02 23:44:08.843',
'%Y/%m/%d %H:%M:%S.%f')
return value
except ValueError:
lo.debug('could not convert _date/_time value %s' % value)
return value
def process(self):
src = self.path1()
dst = self.path2()
lo.debug('converting "%s" to "%s"' % (src, dst))
subprocess.check_call([
self.config.convert_executable, src, '-quality', '100', '-resize',
'%d>' % self.config.pixels, dst
])
def parse_datetime(self, value):
try:
value = datetime.datetime.strptime(
'2013/01/02 23:44:08.843',
'%Y/%m/%d %H:%M:%S.%f')
return value
except ValueError:
lo.debug('could not convert _date/_time value %s' % value)
return value
def process(self):
src = self.path1()
dst = self.path2()
lo.debug('converting "%s" to "%s"' % (src, dst))
subprocess.check_call([
self.config.convert_executable,
src,
'-quality', '100',
'-resize', '%d>' % self.config.pixels,
dst
])
def run(self):
n = 0
while not self.should_stop:
lo.debug('uploader running n=%d' % n)
if not self.dryrun and not self.try_connect():
lo.info('cannot connect; wait for 1 minute')
for sec in range(60):
time.sleep(1)
if self.should_stop:
break
continue
for name in sorted(self.model.tables):
if self.should_stop:
break
table = self.model.tables[name]
# be quiet when polling data apart from first time
if n == 0:
lo.info('sync data with MS-SQL table "%s"' % name)
row = self.model.get_next_new(name)
while row is not None:
if self.should_stop:
break
rowname = table.rowname(row)
if self.dryrun:
lo.info('would upload form %s from table %s' %
(rowname, name))
time.sleep(1)
else:
if self.try_send(table, row):
lo.info('uploaded form %s from table %s' %
(rowname, name))
self.model.mark_done(name, row)
self.notify()
row = self.model.get_next_new(name)
seconds = self.interval
while seconds > 0 and not self.should_stop:
time.sleep(1)
seconds -= 1
n += 1
def run(self):
n = 0
while not self.should_stop:
lo.debug('uploader running n=%d' % n)
if not self.dryrun and not self.try_connect():
lo.info('cannot connect; wait for 1 minute')
for sec in range(60):
time.sleep(1)
if self.should_stop:
break
continue
for name in sorted(self.model.tables):
if self.should_stop:
break
table = self.model.tables[name]
# be quiet when polling data apart from first time
if n == 0:
lo.info('sync data with MS-SQL table "%s"' % name)
row = self.model.get_next_new(name)
while row is not None:
if self.should_stop:
break
rowname = table.rowname(row)
if self.dryrun:
lo.info('would upload form %s from table %s' % (
rowname, name))
time.sleep(1)
else:
if self.try_send(table, row):
lo.info('uploaded form %s from table %s' % (
rowname, name))
self.model.mark_done(name, row)
self.notify()
row = self.model.get_next_new(name)
seconds = self.interval
while seconds > 0 and not self.should_stop:
time.sleep(1)
seconds -= 1
n += 1
def discover_testsuites(paths=[]):
''' returns dictionary mapping name to python file for all
testsuites discovered in the usual places: kvarq root path,
user home directory, current working directory,
KVARQ_TESTSUITES environment variable, and any more paths
specified paths as arguments -- later occurrences of the same
testsuite override previous '''
testsuite_paths = {}
# 1) discover in root path
root_base = os.path.abspath(os.path.join(get_root_path(), 'testsuites'))
lo.debug('discovering testsuites in root path')
add_testsuites_dir(testsuite_paths, root_base)
# 2) discover from $HOME
base = os.path.join(expanduser('~'), 'kvarq_testsuites')
lo.debug('discovering testsuites in home directory')
add_testsuites_dir(testsuite_paths, base)
# 3) discover from CWD if not in root path
cwd_base = os.path.abspath('testsuites')
if cwd_base != root_base:
lo.debug('discovering testsuites in current working directory')
add_testsuites_dir(testsuite_paths, cwd_base)
# 4) discover from KVARQ_TESTSUITES
from_env = os.environ.get('KVARQ_TESTSUITES')
if from_env:
lo.debug('discovering testsuites in $KVARQ_TESTSUITES')
for base in from_env.split(os.path.pathsep):
add_testsuites_dir(testsuite_paths, base)
# 5) explicitely specified paths
for base in paths:
if os.path.isdir(base):
lo.debug('discovering testsuites in "%s"' % base)
add_testsuites_dir(testsuite_paths, base)
else:
lo.warning('could not find directory "%s"' % base)
return testsuite_paths
def fill_xform(self, row):
xform = XForm(self.xform)
ignored = []
lowerpaths = [path.lower() for path in xform.paths]
for i, colname in enumerate(self.colnames):
if colname in lowerpaths:
idx = lowerpaths.index(colname)
path = xform.paths[idx]
# are stored as varchar(24)...
if colname.endswith('_date') or colname.endswith('_time'):
value = self.parse_datetime(row[i])
else:
value = row[i]
xform[path] = value
else:
ignored.append(colname)
if ignored:
lo.debug('created XForm : ignored %d values from db : %s' %
(len(ignored), ignored))
return xform
def fill_xform(self, row):
xform = XForm(self.xform)
ignored = []
lowerpaths = [path.lower() for path in xform.paths]
for i, colname in enumerate(self.colnames):
if colname in lowerpaths:
idx = lowerpaths.index(colname)
path = xform.paths[idx]
# are stored as varchar(24)...
if colname.endswith('_date') or colname.endswith('_time'):
value = self.parse_datetime(row[i])
else:
value = row[i]
xform[path] = value
else:
ignored.append(colname)
if ignored:
lo.debug('created XForm : ignored %d values from db : %s' % (
len(ignored), ignored))
return xform
def set_file(self, name, filename, mimetype=None):
"""Set value of a XForm "file type" field
Raises XFormException if no field with given name can be found
in this form.
Arguments:
- name -- name of field, path parts separated by '/'
- filename -- name of file to be sent as value of field
- mimetype (optional) -- will be used as the file's
"Content-Type"
"""
if mimetype is None:
mimetype = mimetypes.guess_type(filename)[0]
if mimetypes is None:
mimetype = 'application/binary'
self[name] = os.path.basename(filename)
self.filenames[name] = filename
self.mimetypes[name] = mimetype
lo.debug('(mimetype %s)' % mimetype)
def post_multipart(self, items):
"""Post items to server
Sends the specified items to the server, raising an
AggregateException in case of error.
Arguments:
- items -- a sequence of sequences
``(name, filename, value, file_content_type)``
such as returned by XForm.get_items()
"""
content_type, body = self.encode_multipart(items)
self.request('POST', self.submission_uri, body, {
'Content-Type': content_type,
'Content-Length': len(body)
})
r = self.conn.getresponse()
r_body = r.read()
lo.debug('POST %s -> status=%d reason=%s data="%s"' % (
self.submission_uri, r.status, r.reason, r_body))
if r.status == 404:
lo.error('could not find form with specified id')
lo.debug('response body : ' + r_body.decode('utf8'))
raise AggregateFormNotFoundException('Form not found on server')
if r.status != 201:
lo.error('expected status=201 after posting, got ' + str(r.status))
lo.debug('response body : ' + r_body.decode('utf8'))
raise AggregateException('Could not post multipart')
def get_authentication(self, method, uri):
"""Create new authentication token
Arguments:
- method -- HTTP method
- uri -- URI for which authentication will be valid
Return value: value that can be used for the 'Authorization'
HTTP header
"""
realm = self.www_auth['realm']
snonce = self.www_auth['nonce']
qop = self.www_auth['qop']
nc = '%08d' % self.nc
self.nc += 1
HA1 = hashlib.md5(':'.join([self.username, realm, self.password]).encode('utf8')).hexdigest()
HA2 = hashlib.md5(':'.join([method, uri]).encode('utf8')).hexdigest()
HAx = hashlib.md5(':'.join([HA1, snonce, nc, self.cnonce, qop, HA2]).encode('utf8')).hexdigest()
lo.debug('DAA : username=%s realm=%s => HA1=%s' % (self.username, realm, HA1))
lo.debug('DAA : method=%s uri=%s => HA2=%s' % (method, uri, HA2))
lo.debug('DAA : snonce=%s nc=%s cnonce=%s qop=%s => response=%s' % (
snonce, nc, self.cnonce, qop, HAx))
auth = {
'username': self.username,
'realm': realm,
'nonce': snonce,
'uri': uri,
'response': HAx,
'cnonce': self.cnonce,
'algorithm': 'MD5',
'nc': nc,
'qop': qop,
}
auth = ', '.join([
'%s="%s"' % (key, value)
for key, value in auth.items()
])
lo.debug('DAA response : ' + auth)
return 'Digest ' + auth
def add_testsuites_dir(testsuite_paths, base):
if not os.path.isdir(base):
return
for subdir in os.listdir(base):
if not os.path.isdir(os.path.join(base, subdir)) or (
subdir[0] == '_' or subdir[0] == '.'):
continue
for fname in os.listdir(os.path.join(base, subdir)):
if not fname.endswith('.py') or (
fname[0] == '_' or fname[0] == '.'):
continue
name = subdir + '/' + fname[:-3]
path = os.path.join(base, subdir, fname)
if name in testsuite_paths:
lo.info('testsuite %s loaded from "%s"' % (name, path))
else:
lo.debug('testsuite %s loaded from "%s"' % (name, path))
testsuite_paths[name] = path
def set_client(self, client):
if self.client is None:
self.button['state'] = 'normal'
self.client = client
lo.debug('using aggregate version ' + str(AGGREGATE_VERSION))
self.wm_title(client.url)
def connect(self, user=None, password=None):
"""Connect to ODK Aggregate server
Connects to server, performing initial authentication and
raising AggregateException in case of error. If the initial
request was successful, the attribute .conn will be set to a
value different fron None.
Arguments:
- user (optional) -- username to use for authentication
- password (optional) -- password to use for authentication
"""
if self.scheme == 'https':
self.conn = http.client.HTTPSConnection(self.address, self.port)
#TODO check against provided certificate
lo.info('SSL : server certificate NOT checked')
else:
self.conn = http.client.HTTPConnection(self.address, self.port)
self.daa = None
# raises ConnectionRefusedError
self.request('HEAD', self.submission_uri, '')
r = self.conn.getresponse()
r_body = r.read()
#cookie = r.getheader('Set-Cookie')
#if cookie:
# cookie = cookie[:cookie.index(';')]
lo.debug("HEAD %s : status=%d reason=%s" % (
self.submission_url, r.status, r.reason))
# anonymous user has Data Collector rights -> status=204
if r.status == 204:
self.daa = None
lo.info('connected to %s (no authentication)' % self.url)
# anonymous user has no Data Collector rights -> status=401
elif r.status == 401:
lo.info('Aggregate replied status=401 -> digest access authentication (DAA)')
if user is None or password is None:
raise AggregateException('Must specify user/password for authentication')
self.daa = DAA(r.getheader('www-authenticate'), user, password)
#headers = create_headers(cookie)
self.request('HEAD', self.submission_uri, '')
r = self.conn.getresponse()
r_body = r.read()
lo.debug("server response DAA : status=%d reason=%s" % (r.status, r.reason))
if r.status == 401:
lo.error('cannot authenticate : received second 401 response')
raise AggregateException('Cannot authenticate')
if r.status != 204:
lo.error('expected status=204 (got %d) after authentication' %
r.status)
if r.status == 403:
raise AggregateException(
'user "%s" is not allowed to post forms' % user)
raise AggregateException('cannot authenticate')
lo.info('connected to %s (authenticated as "%s")' % (self.url, user))
elif r.status == 404:
raise AggregateException('Could not connect : path "%s" not found' %
self.uri)
else:
raise AggregateException('Could not connect : unknown status')
def set_client(self, client):
if self.client is None:
self.button['state'] = 'normal'
self.client = client
lo.debug('using aggregate version ' + str(AGGREGATE_VERSION))
self.wm_title(client.url)
