def calculate():
projects = []
for name in Project.all():
for item in Project.list(name):
item = Project(family=item['family'], name=item['name'])
packages = Project.sort(item.packages)
packages = filter(
lambda o: o['tag'] == 'stable',
packages.values()
)
if not packages:
continue
pkg = Package(**packages[0])
item.dependents = pkg.get('dependents', [])
projects.append(item)
def _sort_by_time(item):
if 'update_at' in item:
return datetime.strptime(
item['updated_at'], '%Y-%m-%dT%H:%M:%SZ'
)
return None
popular = sorted(
projects, key=lambda o: len(o.dependents), reverse=True
)[:50]
latest = sorted(
projects, key=_sort_by_time, reverse=True
)[:50]
return {'popular': popular, 'latest': latest}
def calculate():
projects = []
for name in Project.all():
for item in Project.list(name):
item = Project(family=item['family'], name=item['name'])
packages = Project.sort(item.packages)
packages = filter(lambda o: o['tag'] == 'stable',
packages.values())
if not packages:
continue
pkg = Package(**packages[0])
item.dependents = pkg.get('dependents', [])
projects.append(item)
def _sort_by_time(item):
if 'update_at' in item:
return datetime.strptime(item['updated_at'], '%Y-%m-%dT%H:%M:%SZ')
return None
popular = sorted(projects, key=lambda o: len(o.dependents),
reverse=True)[:50]
latest = sorted(projects, key=_sort_by_time, reverse=True)[:50]
return {'popular': popular, 'latest': latest}
def _index(project, domain, config):
print(' sync: %(family)s/%(name)s' % project)
try:
index_search(project, 'update')
except:
print(' index: search error')
index_project(project, 'update')
url = '%s/%s/%s/' % (domain, project['family'], project['name'])
rv = requests.get(url)
if rv.status_code != 200:
raise Exception('%s: %s' % url, rv.status_code)
data = rv.json()
if 'packages' not in data:
data['packages'] = {}
me = Project(family=project['family'], name=project['name'])
if 'packages' in me:
packages = me['packages'].copy()
else:
packages = {}
for v in packages:
local = packages[v]
server = None
if v in data['packages']:
server = data['packages'][v]
if not server:
print(' delete: %s/%s@%s' % (me['family'], me['name'], v))
pkg = Package(family=me['family'], name=me['name'], version=v)
try:
extract_assets(pkg, 'delete')
except:
print(' delete: assets error')
pkg.delete()
# remove this version from project
Project(**me).remove(v)
elif 'md5' in server and \
('md5' not in local or local['md5'] != server['md5']):
print(' create: %s/%s@%s' % (me['family'], me['name'], v))
_fetch(server, domain, config)
# add this version to project
Project(**me).update(server)
for v in data['packages']:
if v not in packages:
pkg = data['packages'][v]
print(' create: %s/%s@%s' % (pkg['family'], pkg['name'], v))
_fetch(pkg, domain, config)
# add this version to project
Project(**me).update(pkg)
return True
def initdependents():
from yuan.tasks.dependent import calculate_dependents
for name in Project.all():
for item in Project.list(name):
item = Project(family=item['family'], name=item['name'])
for key in item.packages:
pkg = Package(**item.packages[key])
print(pkg)
calculate_dependents(pkg, 'update')
def initassets():
from yuan.tasks import extract_assets
for name in Project.all():
for item in Project.list(name):
item = Project(family=item['family'], name=item['name'])
for key in item.packages:
pkg = Package(**item.packages[key])
print(pkg)
extract_assets(pkg, 'upload')
def _fetch(pkg, domain, config):
url = '%s/%s/%s/%s/' % (
domain, pkg['family'], pkg['name'], pkg['version'])
rv = requests.get(url)
print(' fetch: %s' % url)
if rv.status_code != 200:
raise Exception('%s: %s' % url, rv.status_code)
pkg = Package(**rv.json()).save()
url = '%s%s' % (url, pkg['filename'])
fpath = os.path.join(
config['WWW_ROOT'], 'repository',
pkg.family, pkg.name, pkg.version,
pkg['filename']
)
print(' save: %s' % fpath)
urllib.urlretrieve(url, fpath)
try:
extract_assets(pkg, 'upload')
except:
print(' extract: assets error')
def _index(project, domain, config):
print(' sync: %(family)s/%(name)s' % project)
url = '%s/%s/%s/' % (domain, project['family'], project['name'])
rv = requests.get(url)
if rv.status_code != 200:
raise Exception('%s: %s' % url, rv.status_code)
data = rv.json()
if 'packages' not in data:
data['packages'] = {}
me = Project(family=project['family'], name=project['name'])
if 'packages' in me:
packages = me['packages'].copy()
else:
packages = {}
for v in packages:
local = packages[v]
server = None
if v in data['packages']:
server = data['packages'][v]
if not server:
print(' delete: %s/%s@%s' % (me['family'], me['name'], v))
pkg = Package(family=me['family'], name=me['name'], version=v)
try:
extract_assets(pkg, 'delete')
except:
print(' delete: assets error')
pkg.delete()
# remove this version from project
Project(**me).remove(v)
elif 'md5' not in server:
print(' error: md5 not in remote')
continue
elif 'md5' not in local or local['md5'] != server['md5']:
print(' create: %s/%s@%s' % (me['family'], me['name'], v))
_fetch(server, domain, config)
# add this version to project
Project(**me).update(server)
elif local['md5'] == server['md5']:
print(' warn: %s/%s@%s, same md5' % (
me['family'], me['name'], v
))
Project(**me).save()
continue
for v in data['packages']:
if v not in packages:
pkg = data['packages'][v]
print(' create: %s/%s@%s' % (pkg['family'], pkg['name'], v))
_fetch(pkg, domain, config)
# add this version to project
Project(**me).update(pkg)
project = Project(family=project['family'], name=project['name'])
try:
index_search(project, 'update')
except:
print(' index: search error')
index_project(project, 'update')
return True