def mirror(url, config): """sync a mirror site.""" print(' mirror: %s' % url) rv = requests.get(url) if rv.status_code != 200: raise Exception('%s: %s' % url, rv.status_code) data = rv.json() rv = urlparse(url) domain = '%s://%s/repository' % (rv.scheme, rv.netloc) def index_with_ctx(config, project): app = Flask('mirror') app.config = config with app.test_request_context(): _index(project, domain, config) jobs = [] for project in data: if not isinstance(project, dict): raise Exception('Only mirror a selected family') me = Project(family=project['family'], name=project['name']) if 'updated_at' not in me or \ _strptime(me['updated_at']) < _strptime(project['updated_at']): jobs.append(gevent.spawn(index_with_ctx, config, project)) gevent.joinall(jobs)
def calculate(): projects = [] for name in Project.all(): for item in Project.list(name): item = Project(family=item['family'], name=item['name']) packages = Project.sort(item.packages) packages = filter(lambda o: o['tag'] == 'stable', packages.values()) if not packages: continue pkg = Package(**packages[0]) item.dependents = pkg.get('dependents', []) projects.append(item) def _sort_by_time(item): if 'update_at' in item: return datetime.strptime(item['updated_at'], '%Y-%m-%dT%H:%M:%SZ') return None popular = sorted(projects, key=lambda o: len(o.dependents), reverse=True)[:50] latest = sorted(projects, key=_sort_by_time, reverse=True)[:50] return {'popular': popular, 'latest': latest}
def initsearch(): """init search engine.""" from yuan.search import index_project for name in Project.all(): for item in Project.list(name): print '%(family)s/%(name)s' % item item = Project(family=item['family'], name=item['name']) index_project(item, 'update')
def initdependents(): from yuan.tasks.dependent import calculate_dependents for name in Project.all(): for item in Project.list(name): item = Project(family=item['family'], name=item['name']) for key in item.packages: pkg = Package(**item.packages[key]) print(pkg) calculate_dependents(pkg, 'update')
def initassets(): from yuan.tasks import extract_assets for name in Project.all(): for item in Project.list(name): item = Project(family=item['family'], name=item['name']) for key in item.packages: pkg = Package(**item.packages[key]) print(pkg) extract_assets(pkg, 'upload')
def _index(project, domain, config): print(' sync: %(family)s/%(name)s' % project) url = '%s/%s/%s/' % (domain, project['family'], project['name']) rv = requests.get(url) if rv.status_code != 200: raise Exception('%s: %s' % url, rv.status_code) data = rv.json() if 'packages' not in data: data['packages'] = {} me = Project(family=project['family'], name=project['name']) if 'packages' in me: packages = me['packages'].copy() else: packages = {} for v in packages: local = packages[v] server = None if v in data['packages']: server = data['packages'][v] if not server: print(' delete: %s/%s@%s' % (me['family'], me['name'], v)) pkg = Package(family=me['family'], name=me['name'], version=v) try: extract_assets(pkg, 'delete') except: print(' delete: assets error') pkg.delete() # remove this version from project Project(**me).remove(v) elif 'md5' not in server: print(' error: md5 not in remote') continue elif 'md5' not in local or local['md5'] != server['md5']: print(' create: %s/%s@%s' % (me['family'], me['name'], v)) _fetch(server, domain, config) # add this version to project Project(**me).update(server) elif local['md5'] == server['md5']: print(' warn: %s/%s@%s, same md5' % ( me['family'], me['name'], v )) Project(**me).save() continue for v in data['packages']: if v not in packages: pkg = data['packages'][v] print(' create: %s/%s@%s' % (pkg['family'], pkg['name'], v)) _fetch(pkg, domain, config) # add this version to project Project(**me).update(pkg) project = Project(family=project['family'], name=project['name']) try: index_search(project, 'update') except: print(' index: search error') index_project(project, 'update') return True