Beispiel #1
0
    def get(self, request):
        if request.endswith(".zip") and "/" not in request:
            fileobj = File.get_by_filename(request)
        elif request.endswith(".zip") and "/" in request:
            fileobj = File.get_by_fullpath(request)
        else:
            fileobj = File.get_by_base62(request)

        if fileobj is None:
            self.write("404 Not Found")
            return self.set_status(404)

        return self.write("%s  %s" % (fileobj.filename, fileobj.md5sum))
Beispiel #2
0
    def method_get_builds(self):
        channels = self.params.get('channels', None)
        device = self.params.get('device', None)
        after = int(self.params.get('after', 0))
        if not channels or not device:
            self.set_status(500)
            return self.fail("Invalid Parameters")

        cache_key = 'api_after_%s_%s_%s' % (device, "_".join(sorted(channels)), str(round(after, -4)))
        result = cache.get(cache_key)
        if result is not None:
            return self.success(result)

        result = []

        if 'snapshot' in channels and 'RC' not in channels:
            channels.append('RC')

        for channel in channels:
            file_obj = File.get_build(channel, device, after)
            if file_obj is not None:
                changesfile = re.sub(file_obj.filename, "CHANGES.txt", file_obj.full_path)
                result.append({
                    'channel': channel,
                    'filename': file_obj.filename,
                    'url': "http://get.cm/get/%s" % file_obj.full_path,
                    'changes': "http://get.cm/get/%s" % changesfile,
                    'md5sum': file_obj.md5sum,
                    'timestamp': file_obj.date_created.strftime('%s')
                })

        cache.set(cache_key, result)

        return self.success(result)
Beispiel #3
0
    def get(self):
        device = self.request.arguments.get('device', [None])[0]
        type = self.request.arguments.get('type', [None])[0]
        files = File.browse(device, type)

        for fileobj in files:
            fileobj.base62 = base62_encode(fileobj.id)

        devicelist = Device.get_all()

        # Set default values for device names dictionary
        devicedict = {}
        for codename in devicelist:
            if codename in devicemap:
                devicedict[codename] = devicemap[codename]
            else:
                devicedict[codename] = codename

        return self.render(
            "browse.mako", {
                'request_type': type,
                'request_device': device,
                'devices': devicelist,
                'devicenames': devicedict,
                'files': files
            })
Beispiel #4
0
    def method_get_all_builds(self):
        channels = self.params.get('channels', None)
        device = self.params.get('device', None)
        limit = int(self.params.get('limit', 3))
        if not channels or not device:
            self.set_status(500)
            return self.fail("Invalid Parameters")

        cache_key = 'api_all_%s_%s_%s' % (device, "_".join(sorted(channels)), str(limit))
        result = cache.get(cache_key)
        if result is not None:
            return self.success(result)

        result = []

        if 'snapshot' in channels and 'RC' not in channels:
            channels.append('RC')

        for channel in channels:
            files = File.browse(device, channel, limit)
            for file_obj in files:
                if file_obj is not None:
                    changesfile = re.sub(file_obj.filename, "CHANGES.txt", file_obj.full_path)
                    result.append({
                        'channel': channel,
                        'filename': file_obj.filename,
                        'url': "http://getcm.dianlujitao.com/get/%s" % file_obj.full_path,
                        'changes': "http://getcm.dianlujitao.com/get/%s" % changesfile,
                        'md5sum': file_obj.md5sum,
                        'timestamp': file_obj.date_created.strftime('%s')
                    })

        cache.set(cache_key, result)

        return self.success(result)
Beispiel #5
0
    def run(self):
        for build in self.get_builds():
            artifactlist = self.get_artifact(build)
            if artifactlist:
                if os.path.exists("/opt/www/mirror/jenkins/%s" %
                                  artifactlist[0][0].split("/")[5]):
                    print "Exists, skipping."
                    continue
                for artifactdata in artifactlist:
                    artifact, timestamp = artifactdata
                    full_path = "jenkins/%s/%s" % (artifact.split("/")[5],
                                                   artifact.split("/")[-1])
                    fileobj = File.get_by_fullpath(full_path)
                    if not fileobj:
                        base = "artifacts/%s" % artifact.replace(
                            "http://jenkins.cyanogenmod.com/job/android/", "")
                        build_number = base.split("/")[1]
                        fname = base.split("/")[-1]
                        build_type = "stable"
                        if "NIGHTLY" in artifact:
                            build_type = "nightly"
                        if "SNAPSHOT" in artifact:
                            #build_type = "snapshot"
                            build_type = "nightly"
                        if "EXPERIMENTAL" in artifact:
                            if "-M" in artifact:
                                build_type = "snapshot"
                            else:
                                build_type = "test"
                        if "-RC" in artifact:
                            build_type = "RC"
                        try:
                            os.mkdir("/opt/www/mirror/jenkins/%s" %
                                     build_number)
                        except:
                            pass
                        download_cmd = "wget -O /opt/www/mirror/jenkins/%s/%s %s" % (
                            build_number, fname, artifact)

                        # Only download CHANGES.txt and build.prop locally
                        if not fname.endswith(".zip"):
                            print "Running: %s" % download_cmd
                            os.system(download_cmd)

                        # Download all artifacts to the mirror
                        mirror_cmd = "ssh -p2200 [email protected] \"/root/add.sh /srv/mirror/jenkins/%s %s %s\"" % (
                            build_number, artifact, fname)
                        print "Running: %s" % mirror_cmd
                        os.system(mirror_cmd)

                        # Run the build.prop through getcm.addfile
                        if fname.endswith(".zip"):
                            addfile_cmd = "/usr/local/bin/getcm.addfile --timestamp %s --file /opt/www/mirror/jenkins/%s/%s --fullpath jenkins/%s/%s --type %s --config %s" % (
                                timestamp, build_number, fname, build_number,
                                fname, build_type, self.configPath)
                            print "Running: %s" % addfile_cmd
                            os.system(addfile_cmd)
Beispiel #6
0
    def get(self, request):
        request = request + ".zip"

        if "/" in request:
            fileobj = File.get_by_fullpath(request)
        elif "/" not in request:
            fileobj = File.get_by_filename(request)

        if fileobj is None and "/" not in request:
            self.write("404 Not Found")
            return self.set_status(404)
        elif fileobj is None:
            full_path = request
        else:
            full_path = fileobj.full_path

        url = self.mirrorpool.next() % full_path

        return self.redirect(url)
Beispiel #7
0
 def run(self):
     for build in self.get_builds():
         artifactdata = self.get_artifact(build)
         if artifactdata:
             artifact, timestamp = artifactdata
             full_path = artifact.replace("http://jenkins.cyanogenmod.com/job/android","artifacts")
             fileobj = File.get_by_fullpath(full_path)
             if not fileobj:
                 base = "artifacts/%s" % artifact.replace("http://jenkins.cyanogenmod.com/job/android/", "")
                 cmd = "getcm.addfile --timestamp %s --url %s --fullpath %s --type nightly --config %s" % (timestamp, artifact, base, self.configPath)
                 print "Running: %s" % cmd
                 os.system(cmd)
Beispiel #8
0
    def get(self, request):
        request = request + ".zip"

        if request.endswith("-latest.zip"):
            device = request.replace("-latest.zip", "")
            fileobj = File.get_latest_by_device(device)
        elif "/" in request:
            fileobj = File.get_by_fullpath(request)
        elif "/" not in request:
            fileobj = File.get_by_filename(request)

        if fileobj is None and "/" not in request:
            self.write("404 Not Found")
            return self.set_status(404)
        elif fileobj is None:
            full_path = request
        else:
            full_path = fileobj.full_path

        url = self.mirrorpool.next() % full_path

        return self.redirect(url)
Beispiel #9
0
    def run(self):
        for build in self.get_builds():
            artifactlist = self.get_artifact(build)
            if artifactlist:
                if os.path.exists("/opt/www/mirror/jenkins/%s" % artifactlist[0][0].split("/")[5]):
                    print "Exists, skipping."
                    continue
                for artifactdata in artifactlist:
                    artifact, timestamp = artifactdata
                    full_path = "jenkins/%s/%s" % (artifact.split("/")[5], artifact.split("/")[-1])
                    fileobj = File.get_by_fullpath(full_path)
                    if not fileobj:
                        base = "artifacts/%s" % artifact.replace("http://jenkins.cyanogenmod.com/job/android/", "")
                        build_number = base.split("/")[1]
                        fname = base.split("/")[-1]
                        build_type = "stable"
                        if "NIGHTLY" in artifact:
                            build_type = "nightly"
                        if "SNAPSHOT" in artifact:
                            #build_type = "snapshot"
                            build_type = "nightly"
                        if "EXPERIMENTAL" in artifact:
                            if "-M" in artifact:
                                build_type = "snapshot"
                            else:
                                build_type = "test"
                        if "-RC" in artifact:
                            build_type = "RC"
                        try:
                            os.mkdir("/opt/www/mirror/jenkins/%s" % build_number)
                        except:
                            pass
                        download_cmd = "wget -O /opt/www/mirror/jenkins/%s/%s %s" % (build_number, fname, artifact)

                        # Only download CHANGES.txt and build.prop locally
                        if not fname.endswith(".zip"):
                            print "Running: %s" % download_cmd
                            os.system(download_cmd)

                        # Download all artifacts to the mirror
                        mirror_cmd = "ssh -p22 [email protected] \"/root/add.sh /opt/mirror/jenkins/%s %s %s\"" % (build_number, artifact, fname)
                        print "Running: %s" % mirror_cmd
                        os.system(mirror_cmd)

                        # Run the build.prop through getcm.addfile
                        if fname.endswith(".zip"):
                            addfile_cmd = "/usr/local/bin/getcm.addfile --file /opt/www/mirror/jenkins/%s/%s --fullpath jenkins/%s/%s --type %s --config %s" % (build_number, fname, build_number, fname, build_type, self.configPath)
                            print "Running: %s" % addfile_cmd
                            os.system(addfile_cmd)
Beispiel #10
0
 def run(self):
     for build in self.get_builds():
         artifactlist = self.get_artifact(build)
         if artifactlist:
           for artifactdata in artifactlist:
             artifact, timestamp = artifactdata
             full_path = "jenkins/%s/%s" % (artifact.split("/")[5], artifact.split("/")[-1])
             if os.path.exists("/opt/www/mirror/%s" % full_path):
                 print "Exists, skipping."
                 continue
             fileobj = File.get_by_fullpath(full_path)
             if not fileobj:
                 base = "artifacts/%s" % artifact.replace("http://jenkins.cyanogenmod.com/job/android/", "")
                 build_number = base.split("/")[1]
                 fname = base.split("/")[-1]
                 build_type = "stable"
                 if "NIGHTLY" in artifact:
                     build_type = "nightly"
                 if "SNAPSHOT" in artifact:
                     #build_type = "snapshot"
                     build_type = "nightly"
                 if "EXPERIMENTAL" in artifact:
                     if "-M" in artifact:
                         build_type = "snapshot"
                     else:
                         #build_type = "experimental"
                         build_type = "nightly"
                 if "-RC" in artifact:
                     build_type = "RC"
                 #cmd = "/usr/local/bin/getcm.addfile --timestamp %s --url %s --fullpath %s --type %s --config %s" % (timestamp, artifact, base, build_type, self.configPath)
                 try:
                     os.mkdir("/opt/www/mirror/jenkins/%s" % build_number)
                 except:
                     pass
                 download_cmd = "wget -O /opt/www/mirror/jenkins/%s/%s %s" % (build_number, fname, artifact)
                 print "Running: %s" % download_cmd
                 os.system(download_cmd)
                 if (fname != "CHANGES.txt"):
                     mirror_cmd = "ssh -p2200 [email protected] \"/root/add.sh /srv/mirror/jenkins/%s %s %s\"" % (build_number, artifact, fname)
                     print "Running: %s" % mirror_cmd
                     os.system(mirror_cmd)
                     addfile_cmd = "/usr/local/bin/getcm.addfile --timestamp %s --file /opt/www/mirror/jenkins/%s/%s --fullpath jenkins/%s/%s --type %s --config %s" % (timestamp, build_number, fname, build_number, fname, build_type, self.configPath)
                     print "Running: %s" % addfile_cmd
                     os.system(addfile_cmd)
Beispiel #11
0
    def get(self):
        device = self.request.arguments.get('device', [None])[0]
        type = self.request.arguments.get('type', [None])[0]
        files = File.browse(device, type)

        for fileobj in files:
            fileobj.base62 = base62_encode(fileobj.id)

        devicelist = Device.get_all()

        # Set default values for device names dictionary
        devicedict = {}
        for codename in devicelist:
            if codename in devicemap:
                devicedict[codename] = devicemap[codename]
            else:
                devicedict[codename] = codename

        return self.render("browse.mako", {'request_type': type, 'request_device': device, 'devices': devicelist, 'devicenames': devicedict, 'files': files})
Beispiel #12
0
    def get(self, request):
        # Some torrent clients are retarded and urlencode the querystring.
        # if that happens, they don't deserve to download.
        if request.endswith("?webseed=1"):
            self.write("403 Forbidden")
            return self.set_status(403)

        fileobj = File.get_by_base62(request)
        if fileobj is None:
            self.write("404 Not Found")
            return self.set_status(404)

        # Determine mirror to use
        url = self.mirrorpool.next() % fileobj.full_path

        webseed = self.request.arguments.get('webseed', [None])[0]
        if webseed:
            url = url + "?" + urllib.urlencode({'webseed': webseed})
            logging.warn("Webseeding for '%s'" % fileobj.filename)

        return self.redirect(url)
Beispiel #13
0
    def method_get_all_builds(self):
        channels = self.params.get("channels", None)
        device = self.params.get("device", None)
        limit = int(self.params.get("limit", 3))
        if not channels or not device:
            self.set_status(500)
            return self.fail("Invalid Parameters")

        cache_key = "api_all_%s_%s_%s" % (device, "_".join(sorted(channels)), str(limit))
        result = cache.get(cache_key)
        if result is not None:
            return self.success(result)

        result = []

        if "snapshot" in channels and "RC" not in channels:
            channels.append("RC")

        for channel in channels:
            files = File.browse(device, channel, limit)
            for file_obj in files:
                if file_obj is not None:
                    changesfile = re.sub(file_obj.filename, "CHANGES.txt", file_obj.full_path)
                    result.append(
                        {
                            "channel": channel,
                            "filename": file_obj.filename,
                            "url": "http://getcm.thebronasium.com/get/%s" % file_obj.full_path,
                            "changes": "http://getcm.thebronasium.com/get/%s" % changesfile,
                            "md5sum": file_obj.md5sum,
                            "timestamp": file_obj.date_created.strftime("%s"),
                        }
                    )

        cache.set(cache_key, result)

        return self.success(result)
Beispiel #14
0
def process_file(args):
    init_database(create_engine(args.db_uri))
    session = DBSession()

    ota = OTAPackage(args.file)

    md5hash = md5sum(args.file)
    new = File.get_by_md5sum(md5hash)
    if new is None:
        new = File()
        new.md5sum = md5hash

    if args.url:
        new.filename = os.path.basename(args.url)
    else:
        new.filename = os.path.basename(args.file)

    if args.full_path:
        new.full_path = args.full_path
    else:
        new.full_path = args.file.replace(args.base_path, "")



    new.type = args.type
    new.size = os.path.getsize(args.file)
    new.device = ota.build_prop.get('ro.cm.device', 'unknown')
    if args.timestamp is not None:
        new.date_created = datetime.fromtimestamp(args.timestamp)
    else:
        new.date_created = datetime.fromtimestamp(os.path.getmtime(args.file))

    logging.debug("Filename = %s", new.filename)
    logging.debug("Type = %s", new.type)
    logging.debug("Device = %s", new.device)
    logging.debug("MD5 = %s", new.md5sum)

    try:
        session.add(new)
        session.commit()
        logging.info("File added successfully!")
    except IntegrityError:
        session.rollback()
        logging.error("File already exists: '%s'", new.filename)

    if args.url:
        logging.info("Removing temporary file '%s'", args.file)
        os.remove(args.file)
Beispiel #15
0
def process_file(args):
    init_database(create_engine(args.db_uri))
    session = DBSession()

    ota = OTAPackage(args.file)

    md5hash = md5sum(args.file)
    new = File.get_by_md5sum(md5hash)
    if new is None:
        new = File()
        new.md5sum = md5hash

    if args.url:
        new.filename = os.path.basename(args.url)
    else:
        new.filename = os.path.basename(args.file)

    if args.full_path:
        new.full_path = args.full_path
    else:
        new.full_path = args.file.replace(args.base_path, "")

    info_hash = create_torrent(args.file,
                               "/opt/www/torrents/%s.torrent" % new.filename,
                               new.full_path)

    new.type = args.type
    new.info_hash = info_hash
    new.size = os.path.getsize(args.file)
    new.device = ota.build_prop.get('ro.cm.device', 'unknown')
    if args.timestamp is not None:
        new.date_created = datetime.fromtimestamp(args.timestamp)
    else:
        new.date_created = datetime.fromtimestamp(os.path.getmtime(args.file))

    logging.debug("Filename = %s", new.filename)
    logging.debug("Type = %s", new.type)
    logging.debug("Device = %s", new.device)
    logging.debug("MD5 = %s", new.md5sum)

    try:
        session.add(new)
        session.commit()
        logging.info("File added successfully!")
    except IntegrityError:
        session.rollback()
        logging.error("File already exists: '%s'", new.filename)

    if args.url:
        logging.info("Removing temporary file '%s'", args.file)
        os.remove(args.file)
Beispiel #16
0
 def get(self):
     device = self.request.arguments.get('device', [None])[0]
     type = self.request.arguments.get('type', [None])[0]
     files = File.browse(device, type, 100)
     self.set_header('Content-Type', "application/xml; charset=utf-8")
     self.render("rss.mako", {'files': files})
Beispiel #17
0
def process_file(args):
    init_database(create_engine(args.db_uri))
    session = DBSession()
    build_prop = os.path.dirname(args.file) + "/build.prop"
    build_prop_raw = open(build_prop).read()

    device_name = None
    for line in build_prop_raw.split("\n"):
        if line.startswith("ro.cm.device"):
            k, device_name = line.split("=")

    if device_name is None:
        device_name = "unknown"

    # Determine md5sum
    build_number = args.full_path.split("/")[1]
    zip_name = args.full_path.split("/")[-1]
    md5_url = "http://jenkins.cyanogenmod.com/job/android/%s/artifact/archive/%s.md5sum" % (build_number, zip_name)
    md5hash = urllib2.urlopen(md5_url).read().split(" ")[0]

    # Determine filesize
    file_url = "/job/android/%s/artifact/archive/%s" % (build_number, zip_name)
    conn = httplib.HTTPConnection("jenkins.cyanogenmod.com")
    conn.request("HEAD", file_url)
    res = conn.getresponse()
    file_size = res.getheader('content-length')

    new = File.get_by_md5sum(md5hash)
    if new is None:
        new = File()
        new.md5sum = md5hash

    if args.url:
        new.filename = os.path.basename(args.url)
    else:
        new.filename = os.path.basename(args.file)

    if args.full_path:
        new.full_path = args.full_path
    else:
        new.full_path = args.file.replace(args.base_path, "")

    new.type = args.type
    new.info_hash = None
    new.size = file_size
    new.device = device_name
    if args.timestamp is not None:
        new.date_created = datetime.fromtimestamp(args.timestamp)
    else:
        new.date_created = datetime.fromtimestamp(os.path.getmtime(args.file))

    logging.debug("Filename = %s", new.filename)
    logging.debug("Type = %s", new.type)
    logging.debug("Device = %s", new.device)
    logging.debug("MD5 = %s", new.md5sum)
    logging.debug("Size = %s", new.size)

    try:
        session.add(new)
        session.commit()
        logging.info("File added successfully!")
    except IntegrityError:
        session.rollback()
        logging.error("File already exists: '%s'", new.filename)

    if args.url:
        logging.info("Removing temporary file '%s'", args.file)
        os.remove(args.file)
Beispiel #18
0
def process_file(args):
    init_database(create_engine(args.db_uri))
    session = DBSession()
    build_prop = os.path.dirname(args.file) + "/build.prop"
    build_prop_raw = open(build_prop).read()

    device_name = None
    build_date = None
    for line in build_prop_raw.split("\n"):
        if line.startswith("ro.cm.device"):
            k, device_name = line.split("=")
        if line.startswith("ro.build.date.utc"):
            k, build_date = line.split("=")

    if device_name is None:
        device_name = "unknown"

    # Determine md5sum
    build_number = args.full_path.split("/")[1]
    zip_name = args.full_path.split("/")[-1]
    md5_url = "http://jenkins.cyanogenmod.com/job/android/%s/artifact/archive/%s.md5sum" % (build_number, zip_name)
    md5hash = urllib2.urlopen(md5_url).read().split(" ")[0]

    # Determine filesize
    file_url = "/job/android/%s/artifact/archive/%s" % (build_number, zip_name)
    conn = httplib.HTTPConnection("jenkins.cyanogenmod.com")
    conn.request("HEAD", file_url)
    res = conn.getresponse()
    file_size = res.getheader('content-length')

    new = File.get_by_md5sum(md5hash)
    if new is None:
        new = File()
        new.md5sum = md5hash

    if args.url:
        new.filename = os.path.basename(args.url)
    else:
        new.filename = os.path.basename(args.file)

    if args.full_path:
        new.full_path = args.full_path
    else:
        new.full_path = args.file.replace(args.base_path, "")

    new.type = args.type
    new.info_hash = None
    new.size = file_size
    new.device = device_name
    if args.timestamp is not None:
        new.date_created = datetime.fromtimestamp(args.timestamp)
    elif build_date is not None:
        new.date_created = datetime.fromtimestamp(int(build_date))
    else:
        new.date_created = datetime.fromtimestamp(os.path.getmtime(args.file))

    logging.debug("Filename = %s", new.filename)
    logging.debug("Type = %s", new.type)
    logging.debug("Device = %s", new.device)
    logging.debug("MD5 = %s", new.md5sum)
    logging.debug("Size = %s", new.size)

    try:
        session.add(new)
        session.commit()
        logging.info("File added successfully!")
    except IntegrityError:
        session.rollback()
        logging.error("File already exists: '%s'", new.filename)

    if args.url:
        logging.info("Removing temporary file '%s'", args.file)
        os.remove(args.file)