def __init__(self, version=None, owner=None, target=None, mode=None): self.owner = None if owner: self.owner = asstr(owner) self.target = target self.mode = None if mode: self.mode = asstr(mode) self.version = None if version: self.version = asstr(version) elif self.owner or self.mode: self.version = '1.0'
def __repr__(self): if self.version: if not (self.timestamp or self.checksum): raise ValueError( "neither 'timestamp' nor 'checksum' are defined") info = [self.header, '%', self.version] #if self.target: # info += ['\ntarget: ', self.target] if self.pusher: info += ['\npusher: ', self.pusher] if self.timestamp: info += ['\ntimestamp: ', str(self.timestamp)] if self.checksum: info += ['\nchecksum: ', asstr(self.checksum)] if self.parts: info += ['\nparts: ', str(self.parts)] for k in self.ignored: info += ['\n', k, ': ', self.ignored[k]] if self.pullers: info.append('\n---pullers---') for reader in self.pullers: info += ['\n', reader] return ''.join(info) elif self.timestamp: # former format, before :class:`Metadata` introduction if not self.timestamp_format or self.timestamp_format is True: self.timestamp_format = former_timestamp_format timestamp = time.strftime(self.timestamp_format, time.gmtime(self.timestamp)) if self.pullers: return '\n'.join([timestamp] + self.pullers) else: return timestamp else: return ''
def __init__(self, baseurl, username=None, password=None, certificate=None, verify_ssl=None, ssl_version=None): self.baseurl = asstr(baseurl) if not re.match('https?://[a-z]', baseurl): raise ValueError("wrong base url: '{}'", baseurl) parts = self.baseurl.split('/', 3) try: # basepath does not need to be quoted self.basepath = '/' + parts[3] except IndexError: self.basepath = None else: self.baseurl = '/'.join(parts[:3] + [quote(parts[3])]) self.session = requests.session() self.session.stream = True if username and password: self.session.auth = (username, password) if certificate: self.session.cert = certificate if verify_ssl is not None: self.session.verify = verify_ssl if ssl_version: self.session.mount( 'https://', make_https_adapter(parse_ssl_version(ssl_version))()) self.infinity_depth = None self.download_chunk_size = 1048576 self.retry_on_errno = [104, 110]
def __repr__(self): if self.version: if not (self.timestamp or self.checksum): raise ValueError( "neither 'timestamp' nor 'checksum' are defined") info = ['%'.join((self.header, self.version))] if self.pusher: info.append(': '.join(('pusher', self.pusher))) if self.timestamp: info.append(': '.join(('timestamp', str(self.timestamp)))) if self.checksum: info.append(': '.join(('checksum', asstr(self.checksum)))) if self.parts: info.append(': '.join(('parts', str(self.parts)))) for k in self.ignored: info.append(': '.join((k, self.ignored[k]))) info.append('---pullers---') if self.pullers: info += self.pullers return '\n'.join(info) elif self.timestamp: # former format, before :class:`Metadata` introduction if not self.timestamp_format or self.timestamp_format is True: self.timestamp_format = former_timestamp_format timestamp = time.strftime(self.timestamp_format, time.gmtime(self.timestamp)) if self.pullers: return '\n'.join([timestamp] + self.pullers) else: return timestamp else: return ''
def __getitem__(self, key): db = dbm.open(self.cache, 'c') try: value = asstr(db[key]) finally: db.close() timestamp, checksum = value.split(self.__separator__) return int(timestamp), checksum
def __init__(self, version=None, target=None, pusher=None, timestamp=None, timestamp_format=None, checksum=None, parts=None, pullers=[], **ignored): self.header = 'placeholder' if pusher: pusher = asstr(pusher) self.pusher = pusher self.target = target if timestamp: if isinstance(timestamp, basestring): if not timestamp_format or timestamp_format is True: timestamp_format = former_timestamp_format timestamp = time.strptime(timestamp, timestamp_format) if isinstance(timestamp, time.struct_time): timestamp = calendar.timegm(timestamp) timestamp = int(timestamp) self.timestamp = timestamp self.timestamp_format = timestamp_format self.checksum = checksum if version: version = asstr(version) elif pusher or checksum: version = '1.0' self.version = version self.pullers = pullers self.ignored = ignored if parts: parts = int(parts) self.parts = parts
def send(self, method, target, expected_codes, context=False, allow_redirects=False, retry_on_status_codes=[503, 504], retry_on_errno=None, subsequent_errors_on_retry=[], **kwargs): if retry_on_errno is None: retry_on_errno = self.retry_on_errno url = os.path.join(self.baseurl, quote(asstr(target))) counter = 0 while True: counter += 1 try: response = self.session.request( method, url, allow_redirects=allow_redirects, **kwargs) except requests.exceptions.ConnectionError as e: if hasattr(self, 'logger'): logger = self.logger else: logging.getLogger() while isinstance(e, Exception) and e.args: if (e.args[1:] and isinstance(e.args[1], EnvironmentError)) \ or e.args[0] == 'Connection aborted.': e1 = e.args[1] if e1.args and e1.args[0] == "(32, 'EPIPE')": e1 = OSError(32, e.args[0], *e1.args[1:]) try: if e1.args[0] in retry_on_errno: logger.debug( "on '%s %s', ignoring %s error: %s", method, target, e1.args[0], e1) continue except AttributeError: pass raise e1 else: e = e.args[0] logger.debug('%s', format_exc(e)) raise except OpenSSL.SSL.SysCallError as e: if e.args[0] in retry_on_errno: if hasattr(self, 'logger'): logger = self.logger else: logging.getLogger() logger.debug('on %s %s', method, target) logger.debug('ignoring %s error: %s', e.args[0], e) continue raise status_code = response.status_code if status_code in retry_on_status_codes: response.close() continue break if not isinstance(expected_codes, (list, tuple)): expected_codes = (expected_codes, ) if status_code not in expected_codes: response.close() if 1 < counter and status_code in subsequent_errors_on_retry and not context: # the request actually reached the host and may have been successful; # the target resource may be locked or missing, depending on the type of request; # if the response body is not expected (`not context`), ignore the errors # and try to silently return if hasattr(self, 'logger'): self.logger.debug( 'ignoring a %s error on retrying a %s request', status_code, method) pass else: raise UnexpectedResponse(method, url, status_code, expected_codes) if not context: response.close() return response