Exemplo n.º 1
0
def determine_versions():
    """
    Scrape the Swift website to find all available versions.

    >>> determine_versions()
    [Version(3.0.1), Version(3.0), ...]
    """

    request = requests.get(download_url)
    soup = BeautifulSoup(request.text, 'html.parser')
    releases = soup.find_all('a')

    versions = {}

    for a in releases:
        url = urljoin(download_url, a['href'])
        version, platform = parse_url(url)

        if version and platform:
            if version in versions:
                versions[version].binaries[platform] = url
            else:
                versions[version] = Version(version, {platform: url})

    return [version for (name, version) in versions.items()]
Exemplo n.º 2
0
def import_manifest(arch,version):
    v = Version.get_by_key_name(version)
    if v is None or not v.imported:
        m = fetcher.fetch(arch,version,'manifest.xml')
        if m is not None:
            m = Manifest(m)
            #xg_on = db.create_transaction_options(xg=True)
            v = version_ok(arch,version)
            prev = db.GqlQuery('select * from Version where imported = True and arch = {0}'.format(arch)).fetch(1)
            if prev is not None and len(prev) > 0:
                prev = prev[0]
                from htmldiff import Changeset
                pmanifest = Manifest(fetcher.fetch(arch,prev.value,'manifest.xml'))
                changes = Changeset(pmanifest,m)
                to_delete = [ pmanifest.files[x] for x in changes.dels | changes.changes 
                        if pmanifest.files[x]['path'].endswith('entity') and pmanifest.files[x]['path'].startswith('game/resources0.s2z')]
                to_import = [ m.files[x] for x in changes.adds | changes.changes if 
                        m.files[x]['path'].endswith('entity') and m.files[x]['path'].startswith('game/resources0.s2z')]
                total = len(to_delete)
                current = 1
                del(changes)
                del(m)
                del(pmanifest)
                for file in to_delete:
                    e = Node.get_by_key_name('|'.join([file['version'],file['path']]))
                    if e is not None:
                        logging.info('[{1}/{2}] Deleting {0} entity group'.format('|'.join([file['version'],file['path']]),current,total))
                        db.run_in_transaction(delete_group,e)
                    current += 1
                del(to_delete)
            else:
                prev = None
                to_import = [x for x in m.files.values() if x['path'].endswith('entity') and x['path'].startswith('game/resources0.s2z')]
            
            total = len(to_import)
            current = 1
            for file in to_import:
                if file['path'].endswith('.entity'):
                    e = Node.get_by_key_name('|'.join([file['version'],file['path']]))
                    if e is None:
                        data = fetcher.fetch(arch,file['version'],file['path'])
                        #if data is None:
                            #continue
                        logging.info('[%d/%d] importing %s %s into db' % (current,total,file['version'],file['path']))
                        db.run_in_transaction(parse_entity,data,file['version'],file['path'],[version])
                        #db.run_in_transaction_options(xg_on,parse_entity,file['version'],file['path'],[version])
                    #elif version not in e.versions:
                        #db.run_in_transaction(set_version,e,version)
                current += 1
            v.imported = True
            v.put()
            if prev is not None:
                prev.imported = False
                prev.put()
Exemplo n.º 3
0
 def _get_repo_versions(repo):
     repo_tags = sorted(list(repo.tags),
                        key=lambda t: t.commit.committed_date)
     tags = zip(list(repo_tags)[1:], list(repo_tags))
     versions = list(
         map(
             lambda tag: Version(
                 tag[0], DataExtractor._version_files(tag[0], tag[1])),
             tags))
     return sorted(versions,
                   key=lambda version: version._commit._commit_date)
Exemplo n.º 4
0
 def get_repo_versions(project, repo):
     commits_files = DataExtractor._get_commits_files(project, repo)
     commits_versions = DataExtractor.get_commits_between_versions(repo)
     tags_commits = dict(map(lambda t: (t.commit, t), repo.tags))
     versions = []
     for v, commits in commits_versions.items():
         files = reduce(
             list.__add__,
             list(map(lambda c: commits_files.get(c.hexsha, []), commits)),
             [])
         versions.append(Version(tags_commits[v], files))
     return sorted(versions,
                   key=lambda version: version._commit._commit_date)
Exemplo n.º 5
0
def save_version(version, commit=False):
    if os.path.exists(version.path):
        existing_version = Version.fromfile(version.path)

        if version != existing_version:
            print('Mismatched Data: {}'.format(version))
    else:
        print('Add {}'.format(version))
        version.save()

        if commit:
            subprocess.check_call(['git', 'add', version.path])
            subprocess.check_call(['git', 'commit', '-m', 'chore: Add {}'.format(version.version)])

        return True

    return False
Exemplo n.º 6
0
def save_version(version, commit=False):
    if os.path.exists(version.path):
        existing_version = Version.fromfile(version.path)

        if version != existing_version:
            print('Mismatched Data: {}'.format(version))
    else:
        print('Add {}'.format(version))
        version.save()

        if commit:
            subprocess.check_call(['git', 'add', version.path])
            subprocess.check_call([
                'git', 'commit', '-m', 'chore: Add {}'.format(version.version)
            ])

        return True

    return False
Exemplo n.º 7
0
def get_archive_contents_entry(tfile):

    basedir,tf = os.path.split(tfile)

    base, ext  = os.path.splitext(tf)

    splits = base.split('-')

    pkgname = '-'.join(splits[0:-1])
    ver = Version.parse(splits[-1])
    version = '({0} {1} {2})'.format(ver.major,
                                    ver.minor,
                                    ver.patch)

    with closing(tarfile.open(tfile)) as t:
        f = t.extractfile(base + '/docstring.txt')
        DOCSTRING = f.read().strip()

        f = t.extractfile(base + '/requirements.txt')
        REQUIREMENTS = f.read().strip()

        return '''({pkgname} . [{version} nil "{DOCSTRING}" tar])'''.format(**locals())
Exemplo n.º 8
0
    def get_page(self,arch,version):
        error = ''
        query = ''
        keywords = ''
        stored_key = self.request.get('stored_query')
        if stored_key:
            try:
                stored = StoredQuery.get(stored_key)
                if not stored:
                    error = 'Sorry, there was no such stored query'
                else:
                    query = stored.query
                    keywords = stored.keywords
            except:
                error = 'Sorry, there was no such stored query'
        else:
            query = self.request.get('query')
            query = urllib.unquote(query)
            keywords = self.request.get('keywords')

        template_values = {
            'query' : query,
            'version' : version,
            }

        query = query.strip()
        data = []

        v = Version.get_by_key_name(version)
        if arch != ARCHS.LINUX_RETAIL:
            return '<pre>Sorry, DB is disabled for RCT/SBT</pre>'
        if v is None or not v.imported:
            versions = get_versions(arch)
            versions.sort(key = lambda x: [int(y) for y in x.split('.')])
            if version == versions[-1]:
                error = "Sorry, this version is not imported into db yet, importing was put into queue"
                taskqueue.add(url='/import',params={'version' : version,'arch' : arch},queue_name='importer')
            else:
                self.redirect('/query/latest/?' + self.request.query_string)
        else:
            if len(query) > 0:
                for qline in query.splitlines():
                    operation = ''
                    if qline[0] == '&':
                        operation = '&'
                        qline = qline[1:]
                    elif qline[0] == '!':
                        operation = '!'
                        qline = qline[1:]
                    try:
                        qline = "Select * from Node where {0}".format(qline)
                        logging.info(qline)
                        q = db.GqlQuery(qline)
                    except:
                        error = 'Sorry this query was malformed'
                        q = None
                    if q is not None:
                        pb = q._proto_query
                        inequalities = {}
                        for k,v in pb.filters().iteritems():
                            #keywords.append(k[0])
                            if k[1] == '!=':
                                inequalities[k[0]] = set([v[1][0]._Literal__value for v in v])
                        #logging.info(inequalities)
                        #for k in pb.orderings():
                            #keywords.append(k[0])
                        if len(pb.orderings()) > 0:
                            error = 'Sorry order by is not allowed'
                        else:

                            try:
                                result = q.fetch(1000)
                                #logging.info('results: {0}'.format(len(result)))
                                _result = []
                                for r in result:
                                    ok = True
                                    for prop,keys in inequalities.iteritems():
                                        l = getattr(r,prop)
                                        if isinstance(l,list):
                                            l = set(l)
                                        else:
                                            l = set([l])
                                        if len(l & keys) > 0:
                                            ok = False
                                            #logging.info('filtered')
                                            #logging.info(l)
                                            break
                                        if not ok: break
                                    if ok:
                                        _result.append(r)
                                result = _result
                                _data = []
                                for node in result:
                                    root = node
                                    while root.parent() is not None:
                                        root = root.parent()
                                    _data.append((root.key().name(),root,node))
                                if operation == '&':
                                    roots = set([x[0] for x in _data])
                                    _data = [x for x in data if x[0] in roots]
                                    data = _data
                                elif operation == '!':
                                    roots = [x[0] for x in _data]
                                    _data = [x for x in data if x[0] not in roots]
                                    data = _data
                                else:
                                    data.extend(_data)

                            except datastore_errors.NeedIndexError, exc:
                                x = str(exc)
                                error = 'Sorry, this query is not possible without additional indices'
Exemplo n.º 9
0
 def test_version_not_snapshot(self):
     version = Version('2.2.0', {})
     self.assertFalse(version.is_snapshot)
Exemplo n.º 10
0
 def test_version_doesnt_support_unknown_platform(self):
     version = Version('2.2.0', {})
     self.assertFalse(version.supports_platform('test'))
Exemplo n.º 11
0
 def test_version_supports_known_platform(self):
     version = Version('2.2.0', {'test': 'http://example.com/test.pkg'})
     self.assertTrue(version.supports_platform('test'))
Exemplo n.º 12
0
 def test_version_is_snapshot(self):
     version = Version('2.2.1-SNAPSHOT-2016-04-23-a', {})
     self.assertTrue(version.is_snapshot)
Exemplo n.º 13
0
    def get_page(self,arch,version,hero):
        v = Version.get_by_key_name(version)
        if arch != fetcher.ARCHS.LINUX_RETAIL:
            return '<pre>Sorry, DB is disabled for RCT/SBT</pre>'
        elif v is None or not v.imported:
            versions = get_versions()
            versions.sort(key = lambda x: [int(y) for y in x.split('.')])
            if version == versions[-1]:
                self.response.out.write("Sorry, this version is not imported into db yet, importing was put into queue")
                taskqueue.add(url='/import',params={'version' : version,'arch' : arch},queue_name='importer')
            else:
                self.redirect('/heroes/latest/?' + self.request.query_string)
            return None
        else:
            if hero is None:
                manifest = fetcher.fetch(arch,version,'manifest.xml')
                manifest = Manifest(manifest)
                query = "Select * from Node where tag='hero'".format(version)
                q = db.GqlQuery(query)
                result = q.fetch(1000)
                result = [_ for _ in result if _.name not in ['wl_Warlock']]
                for hero in result:
                    if hasattr(hero,'attackprojectile') and hero.attackprojectile != '':
                        projectile = db.GqlQuery("Select * from Node where name='{0}'".format(hero.attackprojectile)).fetch(1)[0]
                        if hasattr(projectile,'speed'):
                            hero.projectilespeed = projectile.speed
                        else:
                            hero.projectilespeed = '""'
                    else:
                        hero.projectilespeed = '""'

                    #get url for icon
                    icon = hero.icon.replace('.tga','.dds')
                    path = '/'.join(hero.key().name().split('|')[1].split('/')[:-1])
                    path = '/'.join([path,icon])
                    path = path.replace('game/resources0.s2z','game/textures.s2z/00000000')
                    if path in manifest.files:
                        path = '/'.join([manifest.files[path]['version'],path])
                    else:
                        logging.info("Failed to create url for hero icon :( :")
                        logging.info(icon)
                        logging.info(path)
                    hero.iconurl = path
                template_values = {}
                template_values['data'] = result
                template_values['stringtables'] = get_stringtables_entities(arch,version)
                template_name = self.request.get('template')
                if template_name and template_name == 'csv':
                    template = templates.get_template('heroes.csv')        
                else:
                    template = templates.get_template('heroes.html')        
                #self.response.out.write(template.render(template_values))
                #return None
                return template.render(template_values)
            else:
                hero = db.GqlQuery("Select * from Node where tag='hero' and name = :1",hero).fetch(1)
                if len(hero) == 0:
                    return 'Sorry, such hero is not found'
                hero = hero[0]
                #get url for icon
                manifest = fetcher.fetch(arch,version,'manifest.xml')
                manifest = Manifest(manifest)
                icon = hero.icon.replace('.tga','.dds')
                path = '/'.join(hero.key().name().split('|')[1].split('/')[:-1])
                path = '/'.join([path,icon])
                path = path.replace('game/resources0.s2z','game/textures.s2z/00000000')
                path = '/'.join([manifest.files[path]['version'],path])
                hero.iconurl = path
                abilities = db.GqlQuery("Select * from Node where tag='ability' and name in :1",[hero.inventory0,hero.inventory1,hero.inventory2,hero.inventory3]).fetch(10)
                for a in abilities:
                    icon = a.icon.replace('.tga','.dds')
                    path = '/'.join(a.key().name().split('|')[1].split('/')[:-1])
                    path = '/'.join([path,icon])
                    path = path.replace('game/resources0.s2z','game/textures.s2z/00000000')
                    path = '/'.join([manifest.files[path]['version'],path])
                    a.iconurl = path

                #abilities = dict([(a.name,a) for a in abilities])
                template_values = {}
                template_values['entity'] = hero
                template_values['version'] = version
                template_values['abilities'] = abilities
                template_values['stringtables'] = get_stringtables_entities(arch,version)
                template = templates.get_template('hero.html')        
                return template.render(template_values)
Exemplo n.º 14
0
 def test_version_doesnt_support_unknown_platform(self):
     version = Version('2.2.0', {})
     self.assertFalse(version.supports_platform('test'))
Exemplo n.º 15
0
 def test_version_supports_known_platform(self):
     version = Version('2.2.0', {'test': 'http://example.com/test.pkg'})
     self.assertTrue(version.supports_platform('test'))
Exemplo n.º 16
0
        return

    # deprecieated
    @abc.abstractmethod
    def get_devo_old(self, delta=0):
        """Save the data object to the output."""
        return


AbstractDevoSource.register(UtmostDevoSource)

devo_source = UtmostDevoSource()
get_devo = devo_source.get_devo
get_devo_old = devo_source.get_devo_old

V = Version()

from shadow import BOT_TOKEN, CREATOR_ID, BOT_ID

TELEGRAM_URL = 'https://api.telegram.org/bot' + BOT_TOKEN
TELEGRAM_URL_SEND = TELEGRAM_URL + '/sendMessage'
TELEGRAM_URL_SEND_PHOTO = TELEGRAM_URL + '/sendPhoto'
TELEGRAM_URL_CHAT_ACTION = TELEGRAM_URL + '/sendChatAction'
JSON_HEADER = {'Content-Type': 'application/json;charset=utf-8'}

LOG_SENT = '{} {} sent to uid {} ({})'
LOG_ENQUEUED = 'Enqueued {} to uid {} ({})'
LOG_DID_NOT_SEND = 'Did not send {} to uid {} ({}): {}'
LOG_ERROR_SENDING = 'Error sending {} to uid {} ({}):\n{}'
LOG_ERROR_DAILY = 'Error enqueueing dailies:\n'
LOG_ERROR_QUERY = 'Error querying uid {} ({}): {}'
Exemplo n.º 17
0
 def test_version_snapshot_is_pre_release(self):
     version = Version('2.2.1-SNAPSHOT-2016-04-23-a', {})
     self.assertTrue(version.is_pre_release)
Exemplo n.º 18
0
        DOCSTRING = f.read().strip()

        f = t.extractfile(base + '/requirements.txt')
        REQUIREMENTS = f.read().strip()

        return '''({pkgname} . [{version} nil "{DOCSTRING}" tar])'''.format(**locals())

# build up a dictionary 
mostrecent = {}

for tf in tfs:
    basedir, tf = os.path.split(tf)
    base, ext = os.path.splitext(tf)
    splits = base.split('-')
    pkgname = '-'.join(splits[0:-1])
    version = Version.parse(splits[-1])

    if pkgname in mostrecent:
        if mostrecent[pkgname] < version:
            mostrecent[pkgname] = version
    else:
        mostrecent[pkgname] = version

ARCHIVE_CONTENTS = '(1\n'
    
for pkg in mostrecent:
    tfile = '{0}{1}-{2}.tar'.format(root, pkg, mostrecent[pkg])
    ARCHIVE_CONTENTS += '   ' + get_archive_contents_entry(tfile) + '\n'

ARCHIVE_CONTENTS += ')'