def _get_remote_info(pypi, distro_id, pypi_url):
server = xmlrpclib.Server(pypi_url)
res = server.search({'name': distro_id})
match = None
for x in res:
if utils.make_distro_id(x['name']) == utils.make_distro_id(distro_id):
match = x
break
if not match:
return None
data = {}
name = match['name']
urls = []
for rel in server.package_releases(name):
data = server.release_data(name, rel)
urls += [(rel, x) for x in server.release_urls(name, rel)]
kwargs = dict(data)
if kwargs.get('classifiers'):
kwargs['classifiers'] = \
u'\n'.join(kwargs['classifiers'])
return (name, kwargs, urls)
def _get_remote_info(pypi, distro_id, pypi_url):
server = xmlrpclib.Server(pypi_url)
res = server.search({'name': distro_id})
match = None
for x in res:
if utils.make_distro_id(x['name']) == utils.make_distro_id(distro_id):
match = x
break
if not match:
return None
data = {}
name = match['name']
urls = []
for rel in server.package_releases(name):
data = server.release_data(name, rel)
urls += [(rel, x) for x in server.release_urls(name, rel)]
kwargs = dict(data)
if kwargs.get('classifiers'):
kwargs['classifiers'] = \
u'\n'.join(kwargs['classifiers'])
return (name, kwargs, urls)
def find_req(self, reqstr, order_by='distro_id'):
ses = self.sessionmaker()
query = ses.query(model.SQLDistro)
pkgreqs = [x for x in pkg_resources.parse_requirements(reqstr)]
reqs = [model.SQLDistro.distro_id == \
utils.make_distro_id(x.project_name)
for x in pkgreqs]
query = query.filter(sa.or_(*reqs))
if order_by is not None:
vals = []
for x in order_by.split(','):
vals.append('distros_'+x.strip())
query = query.order_by(','.join(vals))
distros = [x for x in query
if self.has_role(x.distro_id,
READER_ROLE, MANAGER_ROLE,
model.OWNER_ROLE)]
res = []
for distro in distros:
entries = []
for f in self.get_files(distro.distro_id):
version = utils.parse_version(f)
if version:
for r in pkgreqs:
if version in r:
entries.append((f, version))
break
if entries:
res.append((distro, entries))
return res
def update_metadata(self, name, **kwargs):
distro_id = utils.make_distro_id(name)
ses = self.sessionmaker()
q = ses.query(model.SQLDistro)
distro = q.filter_by(distro_id=distro_id).all()
if len(distro) == 0:
if not self.has_role(None, ADD_DISTRO_ROLE, MANAGER_ROLE):
raise SecurityError('"%s" cannot add "%s" distro' %
(self.get_active_user(), distro_id))
distro = model.SQLDistro()
utils.update_obj(distro, **kwargs)
distro.distro_id = distro_id
distro.owner = self.get_active_user()
ses.add(distro)
self.logger.debug('Creating new distro "%s"' % distro_id)
else:
distro = distro[0]
if not self.has_role(distro_id, model.OWNER_ROLE, MANAGER_ROLE):
raise SecurityError('"%s" cannot manage "%s" distro' %
(self.get_active_user(), distro_id))
self.logger.debug('Updating distro "%s"' % distro_id)
utils.update_obj(distro, **kwargs)
distro.name = name
distro.last_updated = datetime.datetime.now()
ses.commit()
def update_metadata(self, name, **kwargs):
distro_id = utils.make_distro_id(name)
ses = self.sessionmaker()
q = ses.query(model.SQLDistro)
distro = q.filter_by(distro_id=distro_id).all()
if len(distro) == 0:
if not self.has_role(None, ADD_DISTRO_ROLE, MANAGER_ROLE):
raise SecurityError('"%s" cannot add "%s" distro' %
(self.get_active_user(), distro_id))
distro = model.SQLDistro()
utils.update_obj(distro, **kwargs)
distro.distro_id = distro_id
distro.owner = self.get_active_user()
ses.add(distro)
self.logger.debug('Creating new distro "%s"' % distro_id)
else:
distro = distro[0]
if not self.has_role(distro_id, model.OWNER_ROLE, MANAGER_ROLE):
raise SecurityError('"%s" cannot manage "%s" distro' %
(self.get_active_user(), distro_id))
self.logger.debug('Updating distro "%s"' % distro_id)
utils.update_obj(distro, **kwargs)
distro.name = name
distro.last_updated = datetime.datetime.now()
ses.commit()
def adddistro(self, pypi, filename, user_roles=[]):
name, ver = utils.parse_distro_from_filename(filename)
env = ve.VirtualEnv.create()
env.install_distro(filename)
md = env.get_distro(name).get_metadata_dict('PKG-INFO')
print
print 'Updated metadata for: ' + md['Name']
dictinfo = utils.pkg_info_as_distro(md)
distro_id = utils.make_distro_id(name)
pypi.update_metadata(**dictinfo)
print 'Adding file for: ' + md['Name']
self.addfile(pypi, distro_id, [filename])
if user_roles:
print 'Setting up default user roles for: ' + md['Name']
mapping = {}
for x in user_roles:
username, role = x.split(':')
if username not in mapping:
mapping[username] = []
roles = mapping[username]
roles.append(role)
for username, roles in mapping.items():
print ' %s: %s' % (username, ', '.join(roles))
pypi.security_manager.update_roles(distro_id=distro_id,
username=username,
roles=roles)
def get_distro(self, distro_id=None, distro_name=None):
if not distro_id:
distro_id = utils.make_distro_id(distro_name)
try:
if not self.has_role(distro_id,
READER_ROLE, MANAGER_ROLE, model.OWNER_ROLE):
raise SecurityError('Permission denied')
except ValueError, err:
return None
def get_distro(self, distro_id=None, distro_name=None):
if not distro_id:
distro_id = utils.make_distro_id(distro_name)
try:
if not self.has_role(distro_id, READER_ROLE, MANAGER_ROLE,
model.OWNER_ROLE):
raise SecurityError('Permission denied')
except ValueError, err:
return None
def respond_distro(self, req, distro_id):
yield u'<html><body><ul>\n'
distro_id = utils.make_distro_id(distro_id)
distro = self.pypi.get_distro(distro_id)
if distro is None and self.backup_pypis:
if not try_to_update(self.pypi, distro_id, self.backup_pypis):
raise NoSuchDistroError(distro_id)
for fname in self.pypi.get_files(distro_id):
base = os.path.basename(fname)
url = '../../d/' + distro_id + '/f/' + base
yield u'<li><a href="%s">%s</a></li>\n' % (url, base)
yield u'</ul></body></html>'
def respond_distro(self, req, distro_id):
yield u'<html><body><ul>\n'
distro_id = utils.make_distro_id(distro_id)
distro = self.pypi.get_distro(distro_id)
if distro is None and self.backup_pypis:
if not try_to_update(self.pypi, distro_id, self.backup_pypis):
raise HTTPNoSuchDistroError(distro_id)
for fname in self.pypi.get_files(distro_id):
base = os.path.basename(fname)
url = '../../d/'+distro_id+'/f/'+base
yield u'<li><a href="%s">%s</a></li>\n' % (url, base)
yield u'</ul></body></html>'
def save(self, req):
distro_id = self.distro.distro_id
indexname = self.indexname
updated = simplejson.loads(req.body or '{}')
entries = updated['entries']
if self.pypi.index_manager.has_index(distro_id, indexname):
self.pypi.index_manager.remove_index(distro_id, indexname)
for x in entries:
target_name, opt, target_version = utils.parse_req_parts(x)
if opt not in ('==', '='):
raise ValueError('Bad req option "%s" for "%s"' %
(str(opt), x))
target_distro_id = utils.make_distro_id(target_name)
self.pypi.index_manager.add_index_item(distro_id,
indexname,
target_distro_id,
target_version)
return http.ok([], simplejson.dumps({'entries': entries}))
def upload_files(self, name, content, **kwargs):
distro_id = utils.make_distro_id(name)
distro = self.get_distro(distro_id)
if not self.has_role(distro_id, model.OWNER_ROLE, MANAGER_ROLE):
raise SecurityError('"%s" is not the owner of "%s" distro' %
(self.get_active_user(), distro_id))
targetdir = os.path.join(self.basefiledir, distro_id[0], distro_id)
if not os.path.exists(targetdir):
os.makedirs(targetdir)
if not isinstance(content, (list, tuple)):
content = [content]
for content_item in content:
target = os.path.join(targetdir, content_item.filename)
content_item.save(target)
self.logger.debug('Added file "%s" to "%s"' %
(content_item.filename, distro_id))
self.update_updated(distro_id)
def upload_files(self, name, content, **kwargs):
distro_id = utils.make_distro_id(name)
distro = self.get_distro(distro_id)
if not self.has_role(distro_id, model.OWNER_ROLE, MANAGER_ROLE):
raise SecurityError('"%s" is not the owner of "%s" distro' %
(self.get_active_user(), distro_id))
targetdir = os.path.join(self.basefiledir, distro_id[0],
distro_id)
if not os.path.exists(targetdir):
os.makedirs(targetdir)
if not isinstance(content, (list, tuple)):
content = [content]
for content_item in content:
target = os.path.join(targetdir, content_item.filename)
content_item.save(target)
self.logger.debug('Added file "%s" to "%s"' %
(content_item.filename, distro_id))
self.update_updated(distro_id)
def put(self, req):
distro_id = self.distro.distro_id
updated = simplejson.loads(req.body or '{}')
altered = []
for indexdict in updated['indexes']:
indexname = indexdict['indexname']
altered.append(indexname)
entries = indexdict['entries']
self.pypi.index_manager.remove_index(distro_id, indexname)
for x in entries:
target_name, opt, target_version = utils.parse_req_parts(x)
if opt not in ('==', '='):
raise ValueError('Bad req option "%s" for "%s"' %
(str(opt), x))
target_distro_id = utils.make_distro_id(target_name)
self.pypi.index_manager.add_index_item(distro_id,
indexname,
target_distro_id,
target_version)
return http.ok([], simplejson.dumps({'indexes': altered}))
class PyPi(object):
"""Represents standard pypi functionality.
>>> pypi = PyPi('.', '', True)
>>> class Mock(object):
... def __init__(self, **kw):
... self.__dict__.update(kw)
>>> objs = {}
>>> class MockSessionMaker(object):
... def __init__(self, bind=None):
... self.bind = bind
... self.objs = objs
... def query(self, s):
... return self
... def filter_by(self, **kwargs):
... return self
... def all(self):
... return []
... def add(self, o):
... self.objs[o.username] = o
... def commit(self):
... pass
... def first(self):
... return None
... def execute(self, *args, **kwargs):
... return []
... def __iter__(self):
... return iter([])
>>> pypi._sessionmaker = MockSessionMaker
>>> pypi.register_user('foo', 'bar', 'bar', 'a')
>>> objs
{'foo': <clue.relmgr.model.SQLUser object ...>}
>>> pypi.perform_action('file_upload', name='foo', content='bar')
Traceback (most recent call last):
ValueError: no such distro, "foo"
"""
logger = utils.logger
_engine = None
_sessionmaker = None
_security_manager = None
_index_manager = None
def __init__(self, basefiledir, sqluri, self_register=False):
self.basefiledir = basefiledir
self.sqluri = sqluri
self.self_register = self_register
@property
def engine(self):
if self._engine is None:
self._engine = sa.create_engine(self.sqluri)
model.metadata.create_all(self.engine)
return self._engine
def setup_model(self):
# all that's required here is calling self.engine
self.engine
@property
def sessionmaker(self):
if self._sessionmaker is None:
self._sessionmaker = orm.sessionmaker(bind=self.engine)
return self._sessionmaker
@property
def security_manager(self):
if self._security_manager is None:
self._security_manager = model.SecurityManager(self.sessionmaker)
return self._security_manager
@property
def index_manager(self):
if self._index_manager is None:
self._index_manager = model.IndexManager(self.sessionmaker)
return self._index_manager
def register_user(self, name, password, confirm, email):
if not self.self_register:
raise SecurityError('Server does not permit self-registration')
if not name:
raise MissingParamError(name)
if not password:
raise MissingParamError(password)
if not confirm:
raise MissingParamError(email)
if password != confirm:
raise MismatchedPasswordsError()
if name == ANONYMOUS:
raise PyPiError('Cannot register reserved "anonymous" username')
self.security_manager.update_user(name, password, email)
def perform_action(self, action='', **kwargs):
if action not in self.actions:
self.logger.error('Action "%s" is not handled by this server' %
action)
raise PyPiError('Action "%s" is not handled by this server' %
action)
action = self.actions.get(action)
# the action method is unbound so we need to pass in self
action(self, **kwargs)
def get_active_user(self):
return 'eea'
return get_active_user()
def has_role(self, distro_id, *roles):
return True
# any user that has authenticated gets magical AUTHENTICATED_ROLE
if AUTHENTICATED_ROLE in roles and self.get_active_user() != ANONYMOUS:
return True
derived = self.security_manager.get_roles(self.get_active_user(),
distro_id, True)
for x in roles:
if x in derived:
return True
return False
def update_metadata(self, name, **kwargs):
distro_id = utils.make_distro_id(name)
ses = self.sessionmaker()
q = ses.query(model.SQLDistro)
distro = q.filter_by(distro_id=distro_id).all()
if len(distro) == 0:
if not self.has_role(None, ADD_DISTRO_ROLE, MANAGER_ROLE):
raise SecurityError('"%s" cannot add "%s" distro' %
(self.get_active_user(), distro_id))
distro = model.SQLDistro()
utils.update_obj(distro, **kwargs)
distro.distro_id = distro_id
distro.owner = self.get_active_user()
ses.add(distro)
self.logger.debug('Creating new distro "%s"' % distro_id)
else:
distro = distro[0]
if not self.has_role(distro_id, model.OWNER_ROLE, MANAGER_ROLE):
raise SecurityError('"%s" cannot manage "%s" distro' %
(self.get_active_user(), distro_id))
self.logger.debug('Updating distro "%s"' % distro_id)
utils.update_obj(distro, **kwargs)
distro.name = name
distro.last_updated = datetime.datetime.now()
ses.commit()
def update_updated(self, distro_id, last_updated=None):
ses = self.sessionmaker()
q = ses.query(model.SQLDistro)
distro = q.filter_by(distro_id=distro_id).first()
if last_updated is None:
last_updated = datetime.datetime.now()
distro.last_updated = last_updated
ses.commit()
def upload_files(self, name, content, **kwargs):
distro_id = utils.make_distro_id(name)
distro = self.get_distro(distro_id)
if not self.has_role(distro_id, model.OWNER_ROLE, MANAGER_ROLE):
raise SecurityError('"%s" is not the owner of "%s" distro' %
(self.get_active_user(), distro_id))
targetdir = os.path.join(self.basefiledir, distro_id[0], distro_id)
if not os.path.exists(targetdir):
os.makedirs(targetdir)
if not isinstance(content, (list, tuple)):
content = [content]
for content_item in content:
target = os.path.join(targetdir, content_item.filename)
content_item.save(target)
self.logger.debug('Added file "%s" to "%s"' %
(content_item.filename, distro_id))
self.update_updated(distro_id)
def get_indexes(self, distro_id):
return [x for x in self.index_manager.get_indexes(distro_id)]
def get_index(self, distro_id, indexname):
if not self.has_role(distro_id, READER_ROLE, MANAGER_ROLE,
model.OWNER_ROLE):
raise SecurityError('Permission denied')
indexes = self.index_manager.get_indexes(distro_id)
index = indexes[indexname]
res = []
for reqstr in index:
req = pkg_resources.Requirement.parse(reqstr)
targetdistro = self.get_distro(distro_name=req.project_name)
targetdir = os.path.join(self.basefiledir,
targetdistro.distro_id[0],
targetdistro.distro_id)
if not os.path.exists(targetdir):
continue
for fname in os.listdir(targetdir):
for d in package_index.distros_for_filename(
os.path.join(targetdir, fname)):
if d.version == req.specs[0][1]:
res.append((targetdistro, d, fname))
return res
def search(self, s, order_by='distro_id'):
ses = self.sessionmaker()
query = ses.query(model.SQLDistro)
expr = '%%%s%%' % s
query = query.filter(
sa.or_(model.SQLDistro.name.like(expr),
model.SQLDistro.description.like(expr),
model.SQLDistro.summary.like(expr)))
if order_by is not None:
vals = []
for x in order_by.split(','):
vals.append('distros_' + x.strip())
query = query.order_by(','.join(vals))
distros = [
x for x in query if self.has_role(x.distro_id, READER_ROLE,
MANAGER_ROLE, model.OWNER_ROLE)
]
return distros
def get_distros(self, order_by=None):
ses = self.sessionmaker()
query = ses.query(model.SQLDistro)
if order_by is not None:
vals = []
for x in order_by.split(','):
vals.append('distros_' + x.strip())
query = query.order_by(','.join(vals))
distros = [
x for x in query if self.has_role(x.distro_id, READER_ROLE,
MANAGER_ROLE, model.OWNER_ROLE)
]
return distros
def get_distro(self, distro_id=None, distro_name=None):
if not distro_id:
distro_id = utils.make_distro_id(distro_name)
try:
if not self.has_role(distro_id, READER_ROLE, MANAGER_ROLE,
model.OWNER_ROLE):
raise SecurityError('Permission denied')
except ValueError, err:
return None
ses = self.sessionmaker()
q = ses.query(model.SQLDistro)
if distro_name:
distro_id = utils.make_distro_id(distro_name)
return q.filter_by(distro_id=distro_id).first()
def get_distro(self, distro_id=None, distro_name=None):
if not distro_id:
distro_id = utils.make_distro_id(distro_name)
try:
if not self.has_role(distro_id,
READER_ROLE, MANAGER_ROLE, model.OWNER_ROLE):
raise SecurityError('Permission denied')
except ValueError, err:
return None
ses = self.sessionmaker()
q = ses.query(model.SQLDistro)
if distro_name:
distro_id = utils.make_distro_id(distro_name)
return q.filter_by(distro_id=distro_id).first()
def get_files(self, distro_id):
if not self.has_role(distro_id,
READER_ROLE, MANAGER_ROLE, model.OWNER_ROLE):
raise SecurityError('Permission denied')
if isinstance(distro_id, model.SQLDistro):
distro_id = distro_id.distro_id
distrodir = os.path.join(self.basefiledir, distro_id[0], distro_id)
res = []
if os.path.exists(distrodir):
for fname in os.listdir(distrodir):
res.append(os.path.join(distrodir, fname))
def get_distro_id(distro):
if isinstance(distro, basestring):
return utils.make_distro_id(distro)
return distro.distro_id
