def join_exit(self):
if self.uploader.is_alive():
lo.info('still waiting for uploader thread to shut down')
self.win.after(1000, self.join_exit)
else:
lo.info('uploader thread finished, exiting')
sys.exit(0)
def run(self, callback):
try:
self.client.connect(self.config.username, self.config.password)
except AggregateException as e:
lo.error('could not connect : ' + str(e))
callback()
return
except Exception as e:
lo.error('unexpected error while connecting : ' + str(e))
log_e(lo)
callback()
return
while not self.cancel and self.xrays:
xray = self.xrays[0]
data = self.data[self.xrays[0]]
del self.xrays[0]
try:
form = XrayForm(self.config, self.store, xray, data)
form.process()
self.client.post_multipart(form.get_items())
self.store.mark_done(xray)
lo.info('uploaded image "%s" (%.2f kb)' % (
form.fname2, os.path.getsize(form.path2())/1024))
except Exception as e:
lo.error('could not upload image "%s" : %s' % (xray, str(e)))
log_e(lo)
if self.cancel:
lo.info('upload canceled')
callback()
def __init__(self, xray_dir, id_re=re.compile('.*')):
self.xray_dir = xray_dir
self.id_re = id_re
if not os.path.isdir(xray_dir):
lo.error('could not find xray directory "%s"' % xray_dir)
return
intermediary_dir = os.path.join(xray_dir, self.intermediary)
if not os.path.isdir(intermediary_dir):
os.mkdir(intermediary_dir)
lo.info('created intermediary xray directory "%s"' % intermediary_dir)
self.done = set()
self.done_path = os.path.join(xray_dir, self.done_fname)
if os.path.exists(self.done_path):
self.done = set([
line.strip('\n\r')
for line in io.open(self.done_path).readlines()
])
self.ignored = set()
self.todo = set()
self.update(initial=True)
def initiate_join_exit(self):
if self.uploader.is_alive():
self.uploader.should_stop = True
lo.info('waiting for uploader thread to shut down')
self.win.after(1000, self.join_exit)
else:
sys.exit(0)
def run(self, callback):
try:
self.client.connect(self.config.username, self.config.password)
except AggregateException as e:
lo.error('could not connect : ' + str(e))
callback()
return
except Exception as e:
lo.error('unexpected error while connecting : ' + str(e))
log_e(lo)
callback()
return
while not self.cancel and self.xrays:
xray = self.xrays[0]
data = self.data[self.xrays[0]]
del self.xrays[0]
try:
form = XrayForm(self.config, self.store, xray, data)
form.process()
self.client.post_multipart(form.get_items())
self.store.mark_done(xray)
lo.info('uploaded image "%s" (%.2f kb)' %
(form.fname2, os.path.getsize(form.path2()) / 1024))
except Exception as e:
lo.error('could not upload image "%s" : %s' % (xray, str(e)))
log_e(lo)
if self.cancel:
lo.info('upload canceled')
callback()
def join_exit(self):
if self.uploader.is_alive():
lo.info('still waiting for uploader thread to shut down')
self.win.after(1000, self.join_exit)
else:
lo.info('uploader thread finished, exiting')
sys.exit(0)
def initiate_join_exit(self):
if self.uploader.is_alive():
self.uploader.should_stop = True
lo.info('waiting for uploader thread to shut down')
self.win.after(1000, self.join_exit)
else:
sys.exit(0)
def __init__(self, xray_dir, id_re=re.compile('.*')):
self.xray_dir = xray_dir
self.id_re = id_re
if not os.path.isdir(xray_dir):
lo.error('could not find xray directory "%s"' % xray_dir)
return
intermediary_dir = os.path.join(xray_dir, self.intermediary)
if not os.path.isdir(intermediary_dir):
os.mkdir(intermediary_dir)
lo.info('created intermediary xray directory "%s"' %
intermediary_dir)
self.done = set()
self.done_path = os.path.join(xray_dir, self.done_fname)
if os.path.exists(self.done_path):
self.done = set([
line.strip('\n\r')
for line in io.open(self.done_path).readlines()
])
self.ignored = set()
self.todo = set()
self.update(initial=True)
def interval_cb(self):
self.store.update()
if self.auto.get():
lo.info('starting automatic upload')
self.xray_list.listbox.selection_set(0, 'end')
self.button_cb()
else:
self.set_next_interval()
def interval_cb(self):
self.store.update()
if self.auto.get():
lo.info('starting automatic upload')
self.xray_list.listbox.selection_set(0, 'end')
self.button_cb()
else:
self.set_next_interval()
def load_config(candidates):
"""Returns first valid configuration (parsed using Config)"""
for filename in candidates:
try:
config = Config(filename)
lo.info('loaded config ' + os.path.abspath(filename))
return config
except ConfigException as e:
lo.warning('failed to load "%s" : %s' % (filename, str(e)))
lo.error('could not find any config file')
return None
def load_config(candidates):
"""Returns first valid configuration (parsed using Config)"""
for filename in candidates:
try:
config = Config(filename)
lo.info('loaded config ' + os.path.abspath(filename))
return config
except ConfigException as e:
lo.warning('failed to load "%s" : %s' % (filename, str(e)))
lo.error('could not find any config file')
return None
def load_get_testsuite(testsuites, name, testsuite_paths):
''' name can be full name or only filename part '''
if name in testsuites:
return testsuites[name]
if name in testsuite_paths:
return load_testsuite(testsuite_paths[name])
for fullname, path in testsuite_paths.items():
if fullname.split('/')[-1] == name:
lo.info('mapping testsuite "%s" to "%s"' % (name, fullname))
if fullname in testsuites:
return testsuites[fullname]
return load_testsuite(testsuite_paths[fullname])
return None
def run(self):
n = 0
while not self.should_stop:
lo.debug('uploader running n=%d' % n)
if not self.dryrun and not self.try_connect():
lo.info('cannot connect; wait for 1 minute')
for sec in range(60):
time.sleep(1)
if self.should_stop:
break
continue
for name in sorted(self.model.tables):
if self.should_stop:
break
table = self.model.tables[name]
# be quiet when polling data apart from first time
if n == 0:
lo.info('sync data with MS-SQL table "%s"' % name)
row = self.model.get_next_new(name)
while row is not None:
if self.should_stop:
break
rowname = table.rowname(row)
if self.dryrun:
lo.info('would upload form %s from table %s' % (
rowname, name))
time.sleep(1)
else:
if self.try_send(table, row):
lo.info('uploaded form %s from table %s' % (
rowname, name))
self.model.mark_done(name, row)
self.notify()
row = self.model.get_next_new(name)
seconds = self.interval
while seconds > 0 and not self.should_stop:
time.sleep(1)
seconds -= 1
n += 1
def run(self):
n = 0
while not self.should_stop:
lo.debug('uploader running n=%d' % n)
if not self.dryrun and not self.try_connect():
lo.info('cannot connect; wait for 1 minute')
for sec in range(60):
time.sleep(1)
if self.should_stop:
break
continue
for name in sorted(self.model.tables):
if self.should_stop:
break
table = self.model.tables[name]
# be quiet when polling data apart from first time
if n == 0:
lo.info('sync data with MS-SQL table "%s"' % name)
row = self.model.get_next_new(name)
while row is not None:
if self.should_stop:
break
rowname = table.rowname(row)
if self.dryrun:
lo.info('would upload form %s from table %s' %
(rowname, name))
time.sleep(1)
else:
if self.try_send(table, row):
lo.info('uploaded form %s from table %s' %
(rowname, name))
self.model.mark_done(name, row)
self.notify()
row = self.model.get_next_new(name)
seconds = self.interval
while seconds > 0 and not self.should_stop:
time.sleep(1)
seconds -= 1
n += 1
def add_testsuites_dir(testsuite_paths, base):
if not os.path.isdir(base):
return
for subdir in os.listdir(base):
if not os.path.isdir(os.path.join(base, subdir)) or (
subdir[0] == '_' or subdir[0] == '.'):
continue
for fname in os.listdir(os.path.join(base, subdir)):
if not fname.endswith('.py') or (
fname[0] == '_' or fname[0] == '.'):
continue
name = subdir + '/' + fname[:-3]
path = os.path.join(base, subdir, fname)
if name in testsuite_paths:
lo.info('testsuite %s loaded from "%s"' % (name, path))
else:
lo.debug('testsuite %s loaded from "%s"' % (name, path))
testsuite_paths[name] = path
def run_fake(self, callback):
import random, time
while not self.cancel and self.xrays:
lo.info('would start upload "%s" with data=%s' %
(self.xrays[0], str(self.data[self.xrays[0]])))
time.sleep(random.randint(3, 5))
lo.info('would have uploaded ' + self.xrays[0])
self.store.mark_done(self.xrays[0])
del self.xrays[0]
if self.cancel:
lo.info('upload canceled')
callback()
def run_fake(self, callback):
import random, time
while not self.cancel and self.xrays:
lo.info('would start upload "%s" with data=%s' % (
self.xrays[0], str(self.data[self.xrays[0]])))
time.sleep(random.randint(3,5))
lo.info('would have uploaded ' + self.xrays[0])
self.store.mark_done(self.xrays[0])
del self.xrays[0]
if self.cancel:
lo.info('upload canceled')
callback()
def update(self, initial=False):
todo = set()
# get list of files
done_n = 0
for fname in os.listdir(self.xray_dir):
if os.path.isdir(self.path(fname)):
continue
if self.ignore(fname):
continue
if fname in self.ignored:
continue
if not self.image(fname):
lo.warning('ignoring image "%s" : %s' %
(fname, self.invalid(fname)))
self.ignored.add(fname)
continue
if self.invalid(fname):
lo.info('ignoring file "%s" (unknown extension)' % fname)
self.ignored.add(fname)
continue
if fname in self.done:
done_n += 1
else:
todo.add(fname)
if len(todo) != len(self.todo):
if initial:
lo.info('scanned "%s" : %d files to upload (%d already done)' %
(os.path.abspath(
self.xray_dir), len(todo), len(self.done)))
else:
lo.info('rescanned "%s" : %d new images' % (os.path.abspath(
self.xray_dir), len(todo) - len(self.todo)))
self.todo = todo
def update(self, initial=False):
todo = set()
# get list of files
done_n = 0
for fname in os.listdir(self.xray_dir):
if os.path.isdir(self.path(fname)):
continue
if self.ignore(fname):
continue
if fname in self.ignored:
continue
if not self.image(fname):
lo.warning('ignoring image "%s" : %s' % (fname, self.invalid(fname)))
self.ignored.add(fname)
continue
if self.invalid(fname):
lo.info('ignoring file "%s" (unknown extension)' % fname)
self.ignored.add(fname)
continue
if fname in self.done:
done_n += 1
else:
todo.add(fname)
if len(todo) != len(self.todo):
if initial:
lo.info('scanned "%s" : %d files to upload (%d already done)' % (
os.path.abspath(self.xray_dir), len(todo), len(self.done)))
else:
lo.info('rescanned "%s" : %d new images' % (
os.path.abspath(self.xray_dir), len(todo) - len(self.todo)))
self.todo = todo
def connect(self, user=None, password=None):
"""Connect to ODK Aggregate server
Connects to server, performing initial authentication and
raising AggregateException in case of error. If the initial
request was successful, the attribute .conn will be set to a
value different fron None.
Arguments:
- user (optional) -- username to use for authentication
- password (optional) -- password to use for authentication
"""
if self.scheme == 'https':
self.conn = http.client.HTTPSConnection(self.address, self.port)
#TODO check against provided certificate
lo.info('SSL : server certificate NOT checked')
else:
self.conn = http.client.HTTPConnection(self.address, self.port)
self.daa = None
# raises ConnectionRefusedError
self.request('HEAD', self.submission_uri, '')
r = self.conn.getresponse()
r_body = r.read()
#cookie = r.getheader('Set-Cookie')
#if cookie:
# cookie = cookie[:cookie.index(';')]
lo.debug("HEAD %s : status=%d reason=%s" % (
self.submission_url, r.status, r.reason))
# anonymous user has Data Collector rights -> status=204
if r.status == 204:
self.daa = None
lo.info('connected to %s (no authentication)' % self.url)
# anonymous user has no Data Collector rights -> status=401
elif r.status == 401:
lo.info('Aggregate replied status=401 -> digest access authentication (DAA)')
if user is None or password is None:
raise AggregateException('Must specify user/password for authentication')
self.daa = DAA(r.getheader('www-authenticate'), user, password)
#headers = create_headers(cookie)
self.request('HEAD', self.submission_uri, '')
r = self.conn.getresponse()
r_body = r.read()
lo.debug("server response DAA : status=%d reason=%s" % (r.status, r.reason))
if r.status == 401:
lo.error('cannot authenticate : received second 401 response')
raise AggregateException('Cannot authenticate')
if r.status != 204:
lo.error('expected status=204 (got %d) after authentication' %
r.status)
if r.status == 403:
raise AggregateException(
'user "%s" is not allowed to post forms' % user)
raise AggregateException('cannot authenticate')
lo.info('connected to %s (authenticated as "%s")' % (self.url, user))
elif r.status == 404:
raise AggregateException('Could not connect : path "%s" not found' %
self.uri)
else:
raise AggregateException('Could not connect : unknown status')
idxs = {}
for i, name in enumerate(header):
if name in form.paths:
idxs[name] = i
else:
lo.error('field "%s" not found in form "%s" -> IGNORING',
name, args.xform)
sys.exit(-1)
for row in reader:
form = XForm(form_xml)
for name in idxs:
form[name] = row[idxs[name]]
client.post_multipart(form.get_items())
lo.info('successfully posted form %s, "%s"',
args.xform, row[0])
else:
# post single form
for fname in args.json:
with io.open(fname) as fd:
defaults = json.load(fd)
for name, value in defaults.items():
if os.path.isfile(value):
form.set_file(name, value)
else:
form[name] = value
for name, value in args.value:
def init(self):
lo.info('gathering initial data from MS-SQL database...')
def load_testsuite(path):
t0 = time.time()
testsuite = genes.load_testsuite(path)
lo.info('loaded testsuite from "%s" in %dms' % (
path, int(1e3*(time.time() - t0))))
return testsuite
def init(self):
lo.info('gathering initial data from MS-SQL database...')
