def join_exit(self): if self.uploader.is_alive(): lo.info('still waiting for uploader thread to shut down') self.win.after(1000, self.join_exit) else: lo.info('uploader thread finished, exiting') sys.exit(0)
def run(self, callback): try: self.client.connect(self.config.username, self.config.password) except AggregateException as e: lo.error('could not connect : ' + str(e)) callback() return except Exception as e: lo.error('unexpected error while connecting : ' + str(e)) log_e(lo) callback() return while not self.cancel and self.xrays: xray = self.xrays[0] data = self.data[self.xrays[0]] del self.xrays[0] try: form = XrayForm(self.config, self.store, xray, data) form.process() self.client.post_multipart(form.get_items()) self.store.mark_done(xray) lo.info('uploaded image "%s" (%.2f kb)' % ( form.fname2, os.path.getsize(form.path2())/1024)) except Exception as e: lo.error('could not upload image "%s" : %s' % (xray, str(e))) log_e(lo) if self.cancel: lo.info('upload canceled') callback()
def __init__(self, xray_dir, id_re=re.compile('.*')): self.xray_dir = xray_dir self.id_re = id_re if not os.path.isdir(xray_dir): lo.error('could not find xray directory "%s"' % xray_dir) return intermediary_dir = os.path.join(xray_dir, self.intermediary) if not os.path.isdir(intermediary_dir): os.mkdir(intermediary_dir) lo.info('created intermediary xray directory "%s"' % intermediary_dir) self.done = set() self.done_path = os.path.join(xray_dir, self.done_fname) if os.path.exists(self.done_path): self.done = set([ line.strip('\n\r') for line in io.open(self.done_path).readlines() ]) self.ignored = set() self.todo = set() self.update(initial=True)
def initiate_join_exit(self): if self.uploader.is_alive(): self.uploader.should_stop = True lo.info('waiting for uploader thread to shut down') self.win.after(1000, self.join_exit) else: sys.exit(0)
def run(self, callback): try: self.client.connect(self.config.username, self.config.password) except AggregateException as e: lo.error('could not connect : ' + str(e)) callback() return except Exception as e: lo.error('unexpected error while connecting : ' + str(e)) log_e(lo) callback() return while not self.cancel and self.xrays: xray = self.xrays[0] data = self.data[self.xrays[0]] del self.xrays[0] try: form = XrayForm(self.config, self.store, xray, data) form.process() self.client.post_multipart(form.get_items()) self.store.mark_done(xray) lo.info('uploaded image "%s" (%.2f kb)' % (form.fname2, os.path.getsize(form.path2()) / 1024)) except Exception as e: lo.error('could not upload image "%s" : %s' % (xray, str(e))) log_e(lo) if self.cancel: lo.info('upload canceled') callback()
def interval_cb(self): self.store.update() if self.auto.get(): lo.info('starting automatic upload') self.xray_list.listbox.selection_set(0, 'end') self.button_cb() else: self.set_next_interval()
def load_config(candidates): """Returns first valid configuration (parsed using Config)""" for filename in candidates: try: config = Config(filename) lo.info('loaded config ' + os.path.abspath(filename)) return config except ConfigException as e: lo.warning('failed to load "%s" : %s' % (filename, str(e))) lo.error('could not find any config file') return None
def load_get_testsuite(testsuites, name, testsuite_paths): ''' name can be full name or only filename part ''' if name in testsuites: return testsuites[name] if name in testsuite_paths: return load_testsuite(testsuite_paths[name]) for fullname, path in testsuite_paths.items(): if fullname.split('/')[-1] == name: lo.info('mapping testsuite "%s" to "%s"' % (name, fullname)) if fullname in testsuites: return testsuites[fullname] return load_testsuite(testsuite_paths[fullname]) return None
def run(self): n = 0 while not self.should_stop: lo.debug('uploader running n=%d' % n) if not self.dryrun and not self.try_connect(): lo.info('cannot connect; wait for 1 minute') for sec in range(60): time.sleep(1) if self.should_stop: break continue for name in sorted(self.model.tables): if self.should_stop: break table = self.model.tables[name] # be quiet when polling data apart from first time if n == 0: lo.info('sync data with MS-SQL table "%s"' % name) row = self.model.get_next_new(name) while row is not None: if self.should_stop: break rowname = table.rowname(row) if self.dryrun: lo.info('would upload form %s from table %s' % ( rowname, name)) time.sleep(1) else: if self.try_send(table, row): lo.info('uploaded form %s from table %s' % ( rowname, name)) self.model.mark_done(name, row) self.notify() row = self.model.get_next_new(name) seconds = self.interval while seconds > 0 and not self.should_stop: time.sleep(1) seconds -= 1 n += 1
def run(self): n = 0 while not self.should_stop: lo.debug('uploader running n=%d' % n) if not self.dryrun and not self.try_connect(): lo.info('cannot connect; wait for 1 minute') for sec in range(60): time.sleep(1) if self.should_stop: break continue for name in sorted(self.model.tables): if self.should_stop: break table = self.model.tables[name] # be quiet when polling data apart from first time if n == 0: lo.info('sync data with MS-SQL table "%s"' % name) row = self.model.get_next_new(name) while row is not None: if self.should_stop: break rowname = table.rowname(row) if self.dryrun: lo.info('would upload form %s from table %s' % (rowname, name)) time.sleep(1) else: if self.try_send(table, row): lo.info('uploaded form %s from table %s' % (rowname, name)) self.model.mark_done(name, row) self.notify() row = self.model.get_next_new(name) seconds = self.interval while seconds > 0 and not self.should_stop: time.sleep(1) seconds -= 1 n += 1
def add_testsuites_dir(testsuite_paths, base): if not os.path.isdir(base): return for subdir in os.listdir(base): if not os.path.isdir(os.path.join(base, subdir)) or ( subdir[0] == '_' or subdir[0] == '.'): continue for fname in os.listdir(os.path.join(base, subdir)): if not fname.endswith('.py') or ( fname[0] == '_' or fname[0] == '.'): continue name = subdir + '/' + fname[:-3] path = os.path.join(base, subdir, fname) if name in testsuite_paths: lo.info('testsuite %s loaded from "%s"' % (name, path)) else: lo.debug('testsuite %s loaded from "%s"' % (name, path)) testsuite_paths[name] = path
def run_fake(self, callback): import random, time while not self.cancel and self.xrays: lo.info('would start upload "%s" with data=%s' % (self.xrays[0], str(self.data[self.xrays[0]]))) time.sleep(random.randint(3, 5)) lo.info('would have uploaded ' + self.xrays[0]) self.store.mark_done(self.xrays[0]) del self.xrays[0] if self.cancel: lo.info('upload canceled') callback()
def run_fake(self, callback): import random, time while not self.cancel and self.xrays: lo.info('would start upload "%s" with data=%s' % ( self.xrays[0], str(self.data[self.xrays[0]]))) time.sleep(random.randint(3,5)) lo.info('would have uploaded ' + self.xrays[0]) self.store.mark_done(self.xrays[0]) del self.xrays[0] if self.cancel: lo.info('upload canceled') callback()
def update(self, initial=False): todo = set() # get list of files done_n = 0 for fname in os.listdir(self.xray_dir): if os.path.isdir(self.path(fname)): continue if self.ignore(fname): continue if fname in self.ignored: continue if not self.image(fname): lo.warning('ignoring image "%s" : %s' % (fname, self.invalid(fname))) self.ignored.add(fname) continue if self.invalid(fname): lo.info('ignoring file "%s" (unknown extension)' % fname) self.ignored.add(fname) continue if fname in self.done: done_n += 1 else: todo.add(fname) if len(todo) != len(self.todo): if initial: lo.info('scanned "%s" : %d files to upload (%d already done)' % (os.path.abspath( self.xray_dir), len(todo), len(self.done))) else: lo.info('rescanned "%s" : %d new images' % (os.path.abspath( self.xray_dir), len(todo) - len(self.todo))) self.todo = todo
def update(self, initial=False): todo = set() # get list of files done_n = 0 for fname in os.listdir(self.xray_dir): if os.path.isdir(self.path(fname)): continue if self.ignore(fname): continue if fname in self.ignored: continue if not self.image(fname): lo.warning('ignoring image "%s" : %s' % (fname, self.invalid(fname))) self.ignored.add(fname) continue if self.invalid(fname): lo.info('ignoring file "%s" (unknown extension)' % fname) self.ignored.add(fname) continue if fname in self.done: done_n += 1 else: todo.add(fname) if len(todo) != len(self.todo): if initial: lo.info('scanned "%s" : %d files to upload (%d already done)' % ( os.path.abspath(self.xray_dir), len(todo), len(self.done))) else: lo.info('rescanned "%s" : %d new images' % ( os.path.abspath(self.xray_dir), len(todo) - len(self.todo))) self.todo = todo
def connect(self, user=None, password=None): """Connect to ODK Aggregate server Connects to server, performing initial authentication and raising AggregateException in case of error. If the initial request was successful, the attribute .conn will be set to a value different fron None. Arguments: - user (optional) -- username to use for authentication - password (optional) -- password to use for authentication """ if self.scheme == 'https': self.conn = http.client.HTTPSConnection(self.address, self.port) #TODO check against provided certificate lo.info('SSL : server certificate NOT checked') else: self.conn = http.client.HTTPConnection(self.address, self.port) self.daa = None # raises ConnectionRefusedError self.request('HEAD', self.submission_uri, '') r = self.conn.getresponse() r_body = r.read() #cookie = r.getheader('Set-Cookie') #if cookie: # cookie = cookie[:cookie.index(';')] lo.debug("HEAD %s : status=%d reason=%s" % ( self.submission_url, r.status, r.reason)) # anonymous user has Data Collector rights -> status=204 if r.status == 204: self.daa = None lo.info('connected to %s (no authentication)' % self.url) # anonymous user has no Data Collector rights -> status=401 elif r.status == 401: lo.info('Aggregate replied status=401 -> digest access authentication (DAA)') if user is None or password is None: raise AggregateException('Must specify user/password for authentication') self.daa = DAA(r.getheader('www-authenticate'), user, password) #headers = create_headers(cookie) self.request('HEAD', self.submission_uri, '') r = self.conn.getresponse() r_body = r.read() lo.debug("server response DAA : status=%d reason=%s" % (r.status, r.reason)) if r.status == 401: lo.error('cannot authenticate : received second 401 response') raise AggregateException('Cannot authenticate') if r.status != 204: lo.error('expected status=204 (got %d) after authentication' % r.status) if r.status == 403: raise AggregateException( 'user "%s" is not allowed to post forms' % user) raise AggregateException('cannot authenticate') lo.info('connected to %s (authenticated as "%s")' % (self.url, user)) elif r.status == 404: raise AggregateException('Could not connect : path "%s" not found' % self.uri) else: raise AggregateException('Could not connect : unknown status')
idxs = {} for i, name in enumerate(header): if name in form.paths: idxs[name] = i else: lo.error('field "%s" not found in form "%s" -> IGNORING', name, args.xform) sys.exit(-1) for row in reader: form = XForm(form_xml) for name in idxs: form[name] = row[idxs[name]] client.post_multipart(form.get_items()) lo.info('successfully posted form %s, "%s"', args.xform, row[0]) else: # post single form for fname in args.json: with io.open(fname) as fd: defaults = json.load(fd) for name, value in defaults.items(): if os.path.isfile(value): form.set_file(name, value) else: form[name] = value for name, value in args.value:
def init(self): lo.info('gathering initial data from MS-SQL database...')
def load_testsuite(path): t0 = time.time() testsuite = genes.load_testsuite(path) lo.info('loaded testsuite from "%s" in %dms' % ( path, int(1e3*(time.time() - t0)))) return testsuite