def download_index(self, target_file, force, verbose=True):
"""Download repository index
- verbose: If False, try not to print (LOG.info) anything to console
unless an actual download happens.
"""
index_url = url_join(self.url, [self.REMOTE_INDEX_FILENAME])
try:
idxgz_file = net.download_file(index_url, dirname(target_file), {
'use_cache': not force,
'save_properties': True,
'info': 'Get: [%s] :repository-index:' % (
urlparse(self.url).netloc)
})
except urllib2.HTTPError, e:
if e.code == 304: # Not Modified
# repository was not modified; no need to download index
if verbose:
LOG.info('Hit: [%s] :repository-index:',
urlparse(self.url).netloc)
return False
elif e.code == 404: # Not Found
raise ValueError, (
'{0.url} does not appear to be a valid repository '
'because {1} is missing.'.format(self, index_url))
raise
def get_repository(self, pyenv, osarch):
assert isinstance(pyenv, python.BasePythonEnvironment)
# Use full ActivePython version instead of pyver for ActiveState
# repositories, and let the AS server handle the redirection. We do this
# to control the repository URL for each and every ActivePython
# release. For eg., use /2.6.6.16/... instead of /2.6/... even though
# the actual repository path in our server uses 2.6.
if re.search(r'pypm.*\.activestate\.com', self.url):
ver = pyenv.apyver
else:
ver = pyenv.pyver
return RemoteRepository(url_join(self.url, [ver, osarch]))
def download_index(self,
target_file,
force,
verbose=True,
interactive=True):
"""Download repository index unless it was downloaded recently (Etag)
- force: Do not use cache; always download
- verbose: If False, try not to print (LOG.info) anything to console
unless an actual download happens.
Return True if download actually happened.
"""
def start_info(status):
if status == 'Hit' and not verbose:
return None
return '%s: [%s] :repository-index:' % (
status, six.moves.urlparse(self.url).netloc)
index_url = url_join(self.url, [self.REMOTE_INDEX_FILENAME])
try:
idxgz_file, downloaded = net.download_file(
index_url,
P.dirname(target_file), {
'use_cache': not force,
'save_properties': True,
'start_info': start_info,
},
interactive=interactive)
if not downloaded:
# index was not updated
return False
except six.moves.HTTPError as e:
if e.code == 404: # Not Found
raise ValueError(
'{0.url} does not appear to be a valid repository '
'because {1} is missing.'.format(self, index_url))
raise
# extract index.gz (REMOTE_INDEX_FILENAME) to index (target_file)
try:
with closing(gzip.open(idxgz_file, 'rb')) as f:
with open(target_file, 'wb') as f2:
f2.write(f.read())
except:
# If an error occurs during extraction, simply delete the index file
# (so that it will get properly synced during next sync)
sh.rm(target_file)
sh.rm(idxgz_file)
raise
return True
def get_repository(self, pyenv, osarch):
assert isinstance(pyenv, python.BasePythonEnvironment)
# Use full ActivePython version instead of pyver for ActiveState
# repositories, and let the AS server handle the redirection. We do this
# to control the repository URL for each and every ActivePython
# release. For eg., use /2.6.6.16/... instead of /2.6/... even though
# the actual repository path in our server uses 2.6.
if re.search(r'pypm.*\.activestate\.com', self.url):
ver = pyenv.apyver
else:
ver = pyenv.pyver
return RemoteRepository(url_join(self.url, [ver, osarch]))
def download_index(self, target_file, force, verbose=True, interactive=True):
"""Download repository index unless it was downloaded recently (Etag)
- force: Do not use cache; always download
- verbose: If False, try not to print (LOG.info) anything to console
unless an actual download happens.
Return True if download actually happened.
"""
def start_info(status):
if status == 'Hit' and not verbose:
return None
return '%s: [%s] :repository-index:' % (
status,
six.moves.urlparse(self.url).netloc)
index_url = url_join(self.url, [self.REMOTE_INDEX_FILENAME])
try:
idxgz_file, downloaded = net.download_file(index_url, P.dirname(target_file), {
'use_cache': not force,
'save_properties': True,
'start_info': start_info,
}, interactive=interactive)
if not downloaded:
# index was not updated
return False
except six.moves.HTTPError as e:
if e.code == 404: # Not Found
raise ValueError(
'{0.url} does not appear to be a valid repository '
'because {1} is missing.'.format(self, index_url))
raise
# extract index.gz (REMOTE_INDEX_FILENAME) to index (target_file)
try:
with closing(gzip.open(idxgz_file, 'rb')) as f:
with open(target_file, 'wb') as f2:
f2.write(f.read())
except:
# If an error occurs during extraction, simply delete the index file
# (so that it will get properly synced during next sync)
sh.rm(target_file)
sh.rm(idxgz_file)
raise
return True
def _query(self, *expr_list):
found = set()
# search every repository
for repo in self.repository_list:
db = self.rrmanager.get_index_db(repo)
with db.transaction() as session:
query = session.query(db.mapper_class)
for expr_fn in expr_list:
query = query.filter(expr_fn(db.mapper_class))
for pkg in query:
# return newly found items
if pkg.full_name not in found:
found.add(pkg.full_name)
# set download URL now
pkg.set_download_url(url_join(repo.url, [pkg.relpath]))
yield pkg
def _query(self, *expr_list):
found = set()
# search every repository
for repo in self.repository_list:
db = self.rrmanager.get_index_db(repo)
with db.transaction() as session:
query = session.query(db.mapper_class)
for expr_fn in expr_list:
query = query.filter(expr_fn(db.mapper_class))
for pkg in query:
# return newly found items
if pkg.full_name not in found:
found.add(pkg.full_name)
# set download URL now
pkg.set_download_url(
url_join(repo.url, [pkg.relpath]))
yield pkg
def get_repository(self, pyver, osarch):
return RemoteRepository(url_join(self.url, [pyver, osarch]))
