def __init__(self, xray_dir, id_re=re.compile('.*')):
self.xray_dir = xray_dir
self.id_re = id_re
if not os.path.isdir(xray_dir):
lo.error('could not find xray directory "%s"' % xray_dir)
return
intermediary_dir = os.path.join(xray_dir, self.intermediary)
if not os.path.isdir(intermediary_dir):
os.mkdir(intermediary_dir)
lo.info('created intermediary xray directory "%s"' % intermediary_dir)
self.done = set()
self.done_path = os.path.join(xray_dir, self.done_fname)
if os.path.exists(self.done_path):
self.done = set([
line.strip('\n\r')
for line in io.open(self.done_path).readlines()
])
self.ignored = set()
self.todo = set()
self.update(initial=True)
def post_multipart(self, items):
"""Post items to server
Sends the specified items to the server, raising an
AggregateException in case of error.
Arguments:
- items -- a sequence of sequences
``(name, filename, value, file_content_type)``
such as returned by XForm.get_items()
"""
content_type, body = self.encode_multipart(items)
self.request('POST', self.submission_uri, body, {
'Content-Type': content_type,
'Content-Length': len(body)
})
r = self.conn.getresponse()
r_body = r.read()
lo.debug('POST %s -> status=%d reason=%s data="%s"' % (
self.submission_uri, r.status, r.reason, r_body))
if r.status == 404:
lo.error('could not find form with specified id')
lo.debug('response body : ' + r_body.decode('utf8'))
raise AggregateFormNotFoundException('Form not found on server')
if r.status != 201:
lo.error('expected status=201 after posting, got ' + str(r.status))
lo.debug('response body : ' + r_body.decode('utf8'))
raise AggregateException('Could not post multipart')
def run_wrapped(self, callback):
try:
self.run(callback)
except Exception as e:
lo.error('unexpected error : ' + str(e))
log_e(lo)
callback()
def run_wrapped(self, callback):
try:
self.run(callback)
except Exception as e:
lo.error('unexpected error : ' + str(e))
log_e(lo)
callback()
def __init__(self, xray_dir, id_re=re.compile('.*')):
self.xray_dir = xray_dir
self.id_re = id_re
if not os.path.isdir(xray_dir):
lo.error('could not find xray directory "%s"' % xray_dir)
return
intermediary_dir = os.path.join(xray_dir, self.intermediary)
if not os.path.isdir(intermediary_dir):
os.mkdir(intermediary_dir)
lo.info('created intermediary xray directory "%s"' %
intermediary_dir)
self.done = set()
self.done_path = os.path.join(xray_dir, self.done_fname)
if os.path.exists(self.done_path):
self.done = set([
line.strip('\n\r')
for line in io.open(self.done_path).readlines()
])
self.ignored = set()
self.todo = set()
self.update(initial=True)
def try_send(self, table, row):
xform = table.fill_xform(row)
try:
self.client.post_multipart(xform.get_items())
return True
except AggregateException as e:
lo.error('could not connect : ' + str(e))
return False
except Exception as e:
lo.error('unexpected error while connecting : ' + str(e))
log_e(lo)
return False
def try_send(self, table, row):
xform = table.fill_xform(row)
try:
self.client.post_multipart(xform.get_items())
return True
except AggregateException as e:
lo.error('could not connect : ' + str(e))
return False
except Exception as e:
lo.error('unexpected error while connecting : ' + str(e))
log_e(lo)
return False
def load_config(candidates):
"""Returns first valid configuration (parsed using Config)"""
for filename in candidates:
try:
config = Config(filename)
lo.info('loaded config ' + os.path.abspath(filename))
return config
except ConfigException as e:
lo.warning('failed to load "%s" : %s' % (filename, str(e)))
lo.error('could not find any config file')
return None
def load_config(candidates):
"""Returns first valid configuration (parsed using Config)"""
for filename in candidates:
try:
config = Config(filename)
lo.info('loaded config ' + os.path.abspath(filename))
return config
except ConfigException as e:
lo.warning('failed to load "%s" : %s' % (filename, str(e)))
lo.error('could not find any config file')
return None
def try_connect(self):
if self.client.is_connected():
return True
try:
client.connect(self.username, self.password)
return True
except AggregateException as e:
lo.error('could not connect : ' + str(e))
return False
except Exception as e:
lo.error('unexpected error while connecting : ' + str(e))
log_e(lo)
return False
def try_connect(self):
if self.client.is_connected():
return True
try:
client.connect(self.username, self.password)
return True
except AggregateException as e:
lo.error('could not connect : ' + str(e))
return False
except Exception as e:
lo.error('unexpected error while connecting : ' + str(e))
log_e(lo)
return False
def load_testsuites(testsuite_paths, selection, raise_exception=False):
''' loads testsuites specified by their full name, group name,
or directly specify testsuite by path of python file '''
testsuites = {}
groups = {}
for name, path in testsuite_paths.items():
parts = name.split('/')
groups.setdefault(parts[0], {})[name] = path
for name_or_path in selection:
try:
if (os.path.isfile(name_or_path)
and not name_or_path in testsuite_paths
and not name_or_path in groups):
# load from path
parts = name_or_path.split(os.path.sep)
name = os.path.splitext(parts[-1])[0]
if len(parts) > 1:
name = parts[-2] + '/' + name
path = name_or_path
testsuites[name] = load_testsuite(path)
elif name_or_path in groups:
# load group
for name, path in groups[name_or_path].items():
testsuites[name] = load_testsuite(path)
else:
# load by full name
name = name_or_path
if name in testsuite_paths:
testsuites[name] = load_testsuite(testsuite_paths[name])
else:
lo.warning('could not find testsuite "%s"' % name)
except genes.TestsuiteLoadingException as e:
if raise_exception:
raise e
lo.error('could not load testsuite from "%s" : %s' % (name, e))
return testsuites
def run(self, callback):
try:
self.client.connect(self.config.username, self.config.password)
except AggregateException as e:
lo.error('could not connect : ' + str(e))
callback()
return
except Exception as e:
lo.error('unexpected error while connecting : ' + str(e))
log_e(lo)
callback()
return
while not self.cancel and self.xrays:
xray = self.xrays[0]
data = self.data[self.xrays[0]]
del self.xrays[0]
try:
form = XrayForm(self.config, self.store, xray, data)
form.process()
self.client.post_multipart(form.get_items())
self.store.mark_done(xray)
lo.info('uploaded image "%s" (%.2f kb)' % (
form.fname2, os.path.getsize(form.path2())/1024))
except Exception as e:
lo.error('could not upload image "%s" : %s' % (xray, str(e)))
log_e(lo)
if self.cancel:
lo.info('upload canceled')
callback()
def run(self, callback):
try:
self.client.connect(self.config.username, self.config.password)
except AggregateException as e:
lo.error('could not connect : ' + str(e))
callback()
return
except Exception as e:
lo.error('unexpected error while connecting : ' + str(e))
log_e(lo)
callback()
return
while not self.cancel and self.xrays:
xray = self.xrays[0]
data = self.data[self.xrays[0]]
del self.xrays[0]
try:
form = XrayForm(self.config, self.store, xray, data)
form.process()
self.client.post_multipart(form.get_items())
self.store.mark_done(xray)
lo.info('uploaded image "%s" (%.2f kb)' %
(form.fname2, os.path.getsize(form.path2()) / 1024))
except Exception as e:
lo.error('could not upload image "%s" : %s' % (xray, str(e)))
log_e(lo)
if self.cancel:
lo.info('upload canceled')
callback()
def connect(self, user=None, password=None):
"""Connect to ODK Aggregate server
Connects to server, performing initial authentication and
raising AggregateException in case of error. If the initial
request was successful, the attribute .conn will be set to a
value different fron None.
Arguments:
- user (optional) -- username to use for authentication
- password (optional) -- password to use for authentication
"""
if self.scheme == 'https':
self.conn = http.client.HTTPSConnection(self.address, self.port)
#TODO check against provided certificate
lo.info('SSL : server certificate NOT checked')
else:
self.conn = http.client.HTTPConnection(self.address, self.port)
self.daa = None
# raises ConnectionRefusedError
self.request('HEAD', self.submission_uri, '')
r = self.conn.getresponse()
r_body = r.read()
#cookie = r.getheader('Set-Cookie')
#if cookie:
# cookie = cookie[:cookie.index(';')]
lo.debug("HEAD %s : status=%d reason=%s" % (
self.submission_url, r.status, r.reason))
# anonymous user has Data Collector rights -> status=204
if r.status == 204:
self.daa = None
lo.info('connected to %s (no authentication)' % self.url)
# anonymous user has no Data Collector rights -> status=401
elif r.status == 401:
lo.info('Aggregate replied status=401 -> digest access authentication (DAA)')
if user is None or password is None:
raise AggregateException('Must specify user/password for authentication')
self.daa = DAA(r.getheader('www-authenticate'), user, password)
#headers = create_headers(cookie)
self.request('HEAD', self.submission_uri, '')
r = self.conn.getresponse()
r_body = r.read()
lo.debug("server response DAA : status=%d reason=%s" % (r.status, r.reason))
if r.status == 401:
lo.error('cannot authenticate : received second 401 response')
raise AggregateException('Cannot authenticate')
if r.status != 204:
lo.error('expected status=204 (got %d) after authentication' %
r.status)
if r.status == 403:
raise AggregateException(
'user "%s" is not allowed to post forms' % user)
raise AggregateException('cannot authenticate')
lo.info('connected to %s (authenticated as "%s")' % (self.url, user))
elif r.status == 404:
raise AggregateException('Could not connect : path "%s" not found' %
self.uri)
else:
raise AggregateException('Could not connect : unknown status')
if args.csv:
# post multiple forms
with io.open(args.csv) as csvfd:
reader = csv.reader(csvfd)
header = next(reader)
idxs = {}
for i, name in enumerate(header):
if name in form.paths:
idxs[name] = i
else:
lo.error('field "%s" not found in form "%s" -> IGNORING',
name, args.xform)
sys.exit(-1)
for row in reader:
form = XForm(form_xml)
for name in idxs:
form[name] = row[idxs[name]]
client.post_multipart(form.get_items())
lo.info('successfully posted form %s, "%s"',
args.xform, row[0])
else:
# post single form