Esempio n. 1
0
def index():
    """index projects."""
    from yuan.models import index_project
    for name in Project.all():
        for item in Project.list(name):
            print '%(family)s/%(name)s' % item
            index_project(item, 'sync')
Esempio n. 2
0
def calculate():
    projects = []

    for name in Project.all():
        for item in Project.list(name):
            item = Project(family=item['family'], name=item['name'])
            packages = Project.sort(item.packages)

            packages = filter(
                lambda o: o['tag'] == 'stable',
                packages.values()
            )

            if not packages:
                continue

            pkg = Package(**packages[0])
            item.dependents = pkg.get('dependents', [])
            projects.append(item)

    def _sort_by_time(item):
        if 'update_at' in item:
            return datetime.strptime(
                item['updated_at'], '%Y-%m-%dT%H:%M:%SZ'
            )
        return None

    popular = sorted(
        projects, key=lambda o: len(o.dependents), reverse=True
    )[:50]

    latest = sorted(
        projects, key=_sort_by_time, reverse=True
    )[:50]
    return {'popular': popular, 'latest': latest}
Esempio n. 3
0
def initsearch():
    """init search engine."""
    from yuan.search import index_project
    for name in Project.all():
        for item in Project.list(name):
            print '%(family)s/%(name)s' % item
            item = Project(family=item['family'], name=item['name'])
            index_project(item, 'update')
Esempio n. 4
0
def initdependents():
    from yuan.tasks.dependent import calculate_dependents
    for name in Project.all():
        for item in Project.list(name):
            item = Project(family=item['family'], name=item['name'])
            for key in item.packages:
                pkg = Package(**item.packages[key])
                print(pkg)
                calculate_dependents(pkg, 'update')
Esempio n. 5
0
def initassets():
    from yuan.tasks import extract_assets
    for name in Project.all():
        for item in Project.list(name):
            item = Project(family=item['family'], name=item['name'])
            for key in item.packages:
                pkg = Package(**item.packages[key])
                print(pkg)
                extract_assets(pkg, 'upload')
Esempio n. 6
0
File: mirror.py Progetto: souse/yuan
def mirror(url, config):
    """sync a mirror site."""

    print('  mirror: %s' % url)
    rv = requests.get(url)
    if rv.status_code != 200:
        raise Exception('%s: %s' % url, rv.status_code)

    data = rv.json()

    rv = urlparse(url)
    domain = '%s://%s/repository' % (rv.scheme, rv.netloc)

    def index_with_ctx(config, project):
        app = Flask('mirror')
        app.config = config
        with app.test_request_context():
            _index(project, domain, config)

    jobs = []
    for project in data:
        if not isinstance(project, dict):
            raise Exception('Only mirror a selected family')

        me = Project(family=project['family'], name=project['name'])
        if 'updated_at' not in me or \
           _strptime(me['updated_at']) < _strptime(project['updated_at']):
            jobs.append(gevent.spawn(index_with_ctx, config, project))

    gevent.joinall(jobs)
Esempio n. 7
0
File: status.py Progetto: souse/yuan
def calculate():
    projects = []

    for name in Project.all():
        for item in Project.list(name):
            item = Project(family=item['family'], name=item['name'])
            packages = Project.sort(item.packages)

            packages = filter(lambda o: o['tag'] == 'stable',
                              packages.values())

            if not packages:
                continue

            pkg = Package(**packages[0])
            item.dependents = pkg.get('dependents', [])
            projects.append(item)

    def _sort_by_time(item):
        if 'update_at' in item:
            return datetime.strptime(item['updated_at'], '%Y-%m-%dT%H:%M:%SZ')
        return None

    popular = sorted(projects, key=lambda o: len(o.dependents),
                     reverse=True)[:50]

    latest = sorted(projects, key=_sort_by_time, reverse=True)[:50]
    return {'popular': popular, 'latest': latest}
Esempio n. 8
0
File: mirror.py Progetto: souse/yuan
def _index(project, domain, config):
    print('    sync: %(family)s/%(name)s' % project)
    url = '%s/%s/%s/' % (domain, project['family'], project['name'])
    rv = requests.get(url)
    if rv.status_code != 200:
        raise Exception('%s: %s' % url, rv.status_code)
    data = rv.json()
    if 'packages' not in data:
        data['packages'] = {}

    me = Project(family=project['family'], name=project['name'])

    if 'packages' in me:
        packages = me['packages'].copy()
    else:
        packages = {}

    for v in packages:
        local = packages[v]
        server = None
        if v in data['packages']:
            server = data['packages'][v]

        if not server:
            print('  delete: %s/%s@%s' % (me['family'], me['name'], v))
            pkg = Package(family=me['family'], name=me['name'], version=v)
            try:
                extract_assets(pkg, 'delete')
            except:
                print('  delete: assets error')

            pkg.delete()
            # remove this version from project
            Project(**me).remove(v)
        elif 'md5' not in server:
            print('  error: md5 not in remote')
            continue
        elif 'md5' not in local or local['md5'] != server['md5']:
            print('  create: %s/%s@%s' % (me['family'], me['name'], v))
            _fetch(server, domain, config)
            # add this version to project
            Project(**me).update(server)
        elif local['md5'] == server['md5']:
            print('    warn: %s/%s@%s, same md5' % (
                me['family'], me['name'], v
            ))
            Project(**me).save()
            continue

    for v in data['packages']:
        if v not in packages:
            pkg = data['packages'][v]
            print('  create: %s/%s@%s' % (pkg['family'], pkg['name'], v))
            _fetch(pkg, domain, config)
            # add this version to project
            Project(**me).update(pkg)

    project = Project(family=project['family'], name=project['name'])
    try:
        index_search(project, 'update')
    except:
        print('  index: search error')
    index_project(project, 'update')
    return True