def __init__(self, xray_dir, id_re=re.compile('.*')): self.xray_dir = xray_dir self.id_re = id_re if not os.path.isdir(xray_dir): lo.error('could not find xray directory "%s"' % xray_dir) return intermediary_dir = os.path.join(xray_dir, self.intermediary) if not os.path.isdir(intermediary_dir): os.mkdir(intermediary_dir) lo.info('created intermediary xray directory "%s"' % intermediary_dir) self.done = set() self.done_path = os.path.join(xray_dir, self.done_fname) if os.path.exists(self.done_path): self.done = set([ line.strip('\n\r') for line in io.open(self.done_path).readlines() ]) self.ignored = set() self.todo = set() self.update(initial=True)
def post_multipart(self, items): """Post items to server Sends the specified items to the server, raising an AggregateException in case of error. Arguments: - items -- a sequence of sequences ``(name, filename, value, file_content_type)`` such as returned by XForm.get_items() """ content_type, body = self.encode_multipart(items) self.request('POST', self.submission_uri, body, { 'Content-Type': content_type, 'Content-Length': len(body) }) r = self.conn.getresponse() r_body = r.read() lo.debug('POST %s -> status=%d reason=%s data="%s"' % ( self.submission_uri, r.status, r.reason, r_body)) if r.status == 404: lo.error('could not find form with specified id') lo.debug('response body : ' + r_body.decode('utf8')) raise AggregateFormNotFoundException('Form not found on server') if r.status != 201: lo.error('expected status=201 after posting, got ' + str(r.status)) lo.debug('response body : ' + r_body.decode('utf8')) raise AggregateException('Could not post multipart')
def run_wrapped(self, callback): try: self.run(callback) except Exception as e: lo.error('unexpected error : ' + str(e)) log_e(lo) callback()
def try_send(self, table, row): xform = table.fill_xform(row) try: self.client.post_multipart(xform.get_items()) return True except AggregateException as e: lo.error('could not connect : ' + str(e)) return False except Exception as e: lo.error('unexpected error while connecting : ' + str(e)) log_e(lo) return False
def load_config(candidates): """Returns first valid configuration (parsed using Config)""" for filename in candidates: try: config = Config(filename) lo.info('loaded config ' + os.path.abspath(filename)) return config except ConfigException as e: lo.warning('failed to load "%s" : %s' % (filename, str(e))) lo.error('could not find any config file') return None
def try_connect(self): if self.client.is_connected(): return True try: client.connect(self.username, self.password) return True except AggregateException as e: lo.error('could not connect : ' + str(e)) return False except Exception as e: lo.error('unexpected error while connecting : ' + str(e)) log_e(lo) return False
def load_testsuites(testsuite_paths, selection, raise_exception=False): ''' loads testsuites specified by their full name, group name, or directly specify testsuite by path of python file ''' testsuites = {} groups = {} for name, path in testsuite_paths.items(): parts = name.split('/') groups.setdefault(parts[0], {})[name] = path for name_or_path in selection: try: if (os.path.isfile(name_or_path) and not name_or_path in testsuite_paths and not name_or_path in groups): # load from path parts = name_or_path.split(os.path.sep) name = os.path.splitext(parts[-1])[0] if len(parts) > 1: name = parts[-2] + '/' + name path = name_or_path testsuites[name] = load_testsuite(path) elif name_or_path in groups: # load group for name, path in groups[name_or_path].items(): testsuites[name] = load_testsuite(path) else: # load by full name name = name_or_path if name in testsuite_paths: testsuites[name] = load_testsuite(testsuite_paths[name]) else: lo.warning('could not find testsuite "%s"' % name) except genes.TestsuiteLoadingException as e: if raise_exception: raise e lo.error('could not load testsuite from "%s" : %s' % (name, e)) return testsuites
def run(self, callback): try: self.client.connect(self.config.username, self.config.password) except AggregateException as e: lo.error('could not connect : ' + str(e)) callback() return except Exception as e: lo.error('unexpected error while connecting : ' + str(e)) log_e(lo) callback() return while not self.cancel and self.xrays: xray = self.xrays[0] data = self.data[self.xrays[0]] del self.xrays[0] try: form = XrayForm(self.config, self.store, xray, data) form.process() self.client.post_multipart(form.get_items()) self.store.mark_done(xray) lo.info('uploaded image "%s" (%.2f kb)' % ( form.fname2, os.path.getsize(form.path2())/1024)) except Exception as e: lo.error('could not upload image "%s" : %s' % (xray, str(e))) log_e(lo) if self.cancel: lo.info('upload canceled') callback()
def run(self, callback): try: self.client.connect(self.config.username, self.config.password) except AggregateException as e: lo.error('could not connect : ' + str(e)) callback() return except Exception as e: lo.error('unexpected error while connecting : ' + str(e)) log_e(lo) callback() return while not self.cancel and self.xrays: xray = self.xrays[0] data = self.data[self.xrays[0]] del self.xrays[0] try: form = XrayForm(self.config, self.store, xray, data) form.process() self.client.post_multipart(form.get_items()) self.store.mark_done(xray) lo.info('uploaded image "%s" (%.2f kb)' % (form.fname2, os.path.getsize(form.path2()) / 1024)) except Exception as e: lo.error('could not upload image "%s" : %s' % (xray, str(e))) log_e(lo) if self.cancel: lo.info('upload canceled') callback()
def connect(self, user=None, password=None): """Connect to ODK Aggregate server Connects to server, performing initial authentication and raising AggregateException in case of error. If the initial request was successful, the attribute .conn will be set to a value different fron None. Arguments: - user (optional) -- username to use for authentication - password (optional) -- password to use for authentication """ if self.scheme == 'https': self.conn = http.client.HTTPSConnection(self.address, self.port) #TODO check against provided certificate lo.info('SSL : server certificate NOT checked') else: self.conn = http.client.HTTPConnection(self.address, self.port) self.daa = None # raises ConnectionRefusedError self.request('HEAD', self.submission_uri, '') r = self.conn.getresponse() r_body = r.read() #cookie = r.getheader('Set-Cookie') #if cookie: # cookie = cookie[:cookie.index(';')] lo.debug("HEAD %s : status=%d reason=%s" % ( self.submission_url, r.status, r.reason)) # anonymous user has Data Collector rights -> status=204 if r.status == 204: self.daa = None lo.info('connected to %s (no authentication)' % self.url) # anonymous user has no Data Collector rights -> status=401 elif r.status == 401: lo.info('Aggregate replied status=401 -> digest access authentication (DAA)') if user is None or password is None: raise AggregateException('Must specify user/password for authentication') self.daa = DAA(r.getheader('www-authenticate'), user, password) #headers = create_headers(cookie) self.request('HEAD', self.submission_uri, '') r = self.conn.getresponse() r_body = r.read() lo.debug("server response DAA : status=%d reason=%s" % (r.status, r.reason)) if r.status == 401: lo.error('cannot authenticate : received second 401 response') raise AggregateException('Cannot authenticate') if r.status != 204: lo.error('expected status=204 (got %d) after authentication' % r.status) if r.status == 403: raise AggregateException( 'user "%s" is not allowed to post forms' % user) raise AggregateException('cannot authenticate') lo.info('connected to %s (authenticated as "%s")' % (self.url, user)) elif r.status == 404: raise AggregateException('Could not connect : path "%s" not found' % self.uri) else: raise AggregateException('Could not connect : unknown status')
if args.csv: # post multiple forms with io.open(args.csv) as csvfd: reader = csv.reader(csvfd) header = next(reader) idxs = {} for i, name in enumerate(header): if name in form.paths: idxs[name] = i else: lo.error('field "%s" not found in form "%s" -> IGNORING', name, args.xform) sys.exit(-1) for row in reader: form = XForm(form_xml) for name in idxs: form[name] = row[idxs[name]] client.post_multipart(form.get_items()) lo.info('successfully posted form %s, "%s"', args.xform, row[0]) else: # post single form