def pre_validate(self, form):
if not self._deb_file_input or not isinstance(self._deb_file_input,
FileStorage):
return
# Save the deb file to a tmp file
_, debtmp = mkstemp()
with open(debtmp, "wb") as tmp:
tmp.write(self._deb_file_input.read())
# Open the deb file and parse control information
deb_obj = debfile.DebPackage(debtmp)
tags = TagSection(deb_obj.control_content("control"))
pkg_name = tags.get('Package', None)
if pkg_name is None:
# Remove the tmpfile
os.unlink(debtmp)
raise wtf.ValidationError("Invalid deb control file, "
"package name is empty.")
# Check 'unique' property on pkg_name
with db_scoped_session() as se:
obj_t = se.query(Item).filter_by(pkg_name=pkg_name).first()
if obj_t != None and hasattr(form,
'iid') and obj_t.iid != form.iid.data:
# obj exists, raise error
raise wtf.ValidationError("The same package name found in "
"%s" % obj_t)
# Validation successful, fill the tmpfile path
self._deb_tmpfile = debtmp
self.data = debtmp
def find_package_debs(package_name, work_directory):
"""Find package debs."""
deb_paths = []
cache = apt.Cache()
for filename in os.listdir(work_directory):
file_path = os.path.join(work_directory, filename)
if not file_path.endswith('.deb'):
continue
# Matching package name.
deb = debfile.DebPackage(file_path)
if deb.pkgname == package_name:
deb_paths.append(file_path)
continue
# Also include -dev packages that depend on the runtime package.
pkg = cache[deb.pkgname]
if pkg.section != 'libdevel' and pkg.section != 'universe/libdevel':
continue
# But ignore -dbg packages.
if deb.pkgname.endswith('-dbg'):
continue
for dependency in deb.depends:
if any(dep[0] == package_name for dep in dependency):
deb_paths.append(file_path)
break
return deb_paths
def populate_obj(self, obj, name, override=False):
if not isinstance(obj, Item):
raise TypeError("obj argument must be of type Item")
if not override:
return
if not self._deb_file_input or not isinstance(self._deb_file_input,
FileStorage):
return
# clean old deb file first
if obj.pkg_path:
old_local_cache_path = path.join(app.config["DEB_UPLOAD_PATH"],
obj.pkg_path)
if path.exists(old_local_cache_path):
os.unlink(old_local_cache_path)
sha1 = hashlib.sha1()
sha1.update(open(self._deb_tmpfile, 'rb').read())
deb_sha1_digest = sha1.hexdigest()
# Get package information from the deb file and populate the Item object
deb_obj = debfile.DebPackage(self._deb_tmpfile)
obj.control = deb_obj.control_content("control")
tags = TagSection(obj.control)
obj.pkg_dependencies = tags.get("Depends", "")
obj.pkg_predepends = tags.get("Pre-Depends", "")
obj.pkg_conflicts = tags.get("Conflicts", "")
obj.pkg_version = tags.get("Version", "")
obj.pkg_signature = deb_sha1_digest
obj.description = tags.get("Description", "")
pkg_name = tags.get("Package", None)
obj.pkg_name = pkg_name
pkg_fullname = obj.pkg_name + '-' + str(obj.pkg_version)
base_path = path.join("packages", obj.pkg_name, pkg_fullname, 'assets')
obj.pkg_assets_path = base_path
pkg_path, pkg_s3_key_path = self._gen_pkg_paths(
obj, pkg_fullname, self._deb_file_input.filename)
obj.pkg_path = pkg_s3_key_path
pkg_local_cache_path = path.join(app.config["DEB_UPLOAD_PATH"],
pkg_s3_key_path)
# Local package path
pkg_local_cache_dir = path.dirname(pkg_local_cache_path)
if not path.exists(pkg_local_cache_dir):
os.makedirs(pkg_local_cache_dir)
# Move tmp deb file to the cache folder
shutil.move(self._deb_tmpfile, pkg_local_cache_path)
self._deb_tmpfile = pkg_local_cache_path
def after_populate_obj(self, obj):
if not self._deb_file_input or not isinstance(self._deb_file_input,
FileStorage):
return
# Rebuild index and upload the deb file, can only be called in after_model_change()
# We need to postpone upload because the item id (primary key) won't be set until the transaction is done
# Sanity checks
if not obj or not obj.iid:
raise Exception("Deb uploading failed: item or item id is none.")
# We need some extra information of the package added to the index file
# dpkg_scan takes an additional file 'overrides' when generating the index file
changelog = obj.changelog
if changelog is None:
changelog = " "
summary = obj.summary
if summary is None:
summary = " "
# (obj.pkg_name, "filename", "null"),
pkg_overrides = [(obj.pkg_name, "itemid", obj.iid),
(obj.pkg_name, "itemname", obj.display_name),
(obj.pkg_name, "authorid", obj.author_id),
(obj.pkg_name, "price", str(obj.price)),
(obj.pkg_name, "pkgversion", obj.pkg_version),
(obj.pkg_name, "pkgchangelog", changelog),
(obj.pkg_name, "pkgsummary", summary)]
deb_obj = debfile.DebPackage(self._deb_tmpfile)
# Check if it's a tweak
tweak_file = detect_tweak(deb_obj.filelist)
if tweak_file is not None:
tweak_file = 'file:///' + tweak_file
pkg_overrides.append((
obj.pkg_name,
"Respring",
"YES",
))
pkg_overrides.append((
obj.pkg_name,
"TweakLib",
tweak_file,
))
index_s3_key_path = "Packages.gz"
pkg_index_file = path.join(app.config["DEB_UPLOAD_PATH"],
app.config["PKG_INDEX_FILE_NAME"])
dpkg_update_index.delay(app.config["DEB_UPLOAD_PATH"], self._s3_bucket,
index_s3_key_path, pkg_index_file,
pkg_overrides)
# Upload deb file to S3
self._upload_to_s3(obj.pkg_path)
def get_package_name(package_file):
pkg_cmd, install_options, uninstall_opts = get_package_command()
if "dpkg" == pkg_cmd:
deb_file = debfile.DebPackage(package_file)
package_name = deb_file.pkgname
elif "rpm" == pkg_cmd:
ts = rpm.ts()
ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES)
with open(package_file) as fd:
pkg_hdr = ts.hdrFromFdno(fd)
package_name = pkg_hdr[rpm.RPMTAG_NAME]
else:
raise exception.UnprocessableEntity(
_("Unhandled package type: %s)") % pkg_cmd)
return package_name
def config_puppetlabs_repo():
dist = platform.dist()[-1]
url = 'http://apt.puppetlabs.com/puppetlabs-release-{}.deb'.format(dist)
filename = '/tmp/puppet_apt.deb'
try:
urllib.urlretrieve(url, filename)
except IOError:
print "Could not install puppet"
raise
deb_pkg = debfile.DebPackage(filename)
if deb_pkg.check():
deb_pkg.install()
cache = apt.cache.Cache()
cache.update()
cache.open()
def get_deb_dist_arch(deb, basename_re=DEB_BASENAME_RE):
"""
Return the distribution and architecture for a `*.deb` file.
The distribution is taken from what ever is left when the
architecture, package name, and version are removed:
package-name-dist_version_arch.deb
"""
deb_pkg = debfile.DebPackage(deb)
arch = deb_pkg['Architecture']
package = deb_pkg['Package']
version = deb_pkg['Version']
deb_basename_match = re.match(
DEB_BASENAME_RE.format(arch=arch, package=package, version=version),
os.path.splitext(os.path.basename(deb))[0])
if deb_basename_match is None:
dist = None
else:
dist = deb_basename_match.group(2)
return dist, arch
def index(self):
form = PackageAssetsUploadForm(request.form)
# Populate choices for select field
with db_scoped_session() as s:
items = s.query(Item).all()
form.item_id.choices = [(item.iid, item.display_name)
for item in items]
if helpers.validate_form_on_submit(form):
# from boto.s3.key import Key
# Get file data from request.files
app_icon = request.files["app_icon"]
screenshot1 = request.files["screenshot1"]
screenshot2 = request.files["screenshot2"]
screenshot3 = request.files["screenshot3"]
screenshot4 = request.files["screenshot4"]
# Get package file
package_file = request.files["package_file"]
# Get banner image file
banner_img_file = request.files["banner_image"]
# Get pkg_assets_path
item = s.query(Item).get(form.item_id.data)
_, debTmpFile = mkstemp()
with open(debTmpFile, "wb") as local_deb_file:
local_deb_file.write(package_file.read())
sha1 = hashlib.sha1()
sha1.update(open(debTmpFile, 'rb').read())
deb_sha1_digest = sha1.hexdigest()
try:
# Verify and update item information based on package file
deb_obj = debfile.DebPackage(debTmpFile)
item.control = deb_obj.control_content("control")
tags = TagSection(item.control)
item.pkg_dependencies = tags.get("Depends", "")
item.pkg_predepends = tags.get("Pre-Depends", "")
item.pkg_conflicts = tags.get("Conflicts", "")
item.pkg_version = tags.get("Version", "")
item.pkg_signature = deb_sha1_digest
item.description = tags.get("Description", "")
# Create local package path
pkg_fullname = item.pkg_name + '-' + str(item.pkg_version)
base_path = path.join(
"packages",
item.pkg_name,
pkg_fullname,
'assets')
item.pkg_assets_path = base_path
pkg_name = tags.get("Package", None)
if (pkg_name is not None) and pkg_name != item.pkg_name:
# Check if the name already exists
t_item = s.query(Item).filter_by(pkg_name=pkg_name).first()
if t_item is None or t_item.iid == item.iid:
item.pkg_name = pkg_name
else:
flash("Package name '%s' is used by another "
"item(%d)." % (pkg_name, t_item.iid))
s.rollback()
os.unlink(debTmpFile)
return redirect(url_for(".index"))
# Build package path
pkg_path = path.join(
"packages",
item.pkg_name)
assets_bucket = app.config.get("S3_ASSETS_BUCKET")
pkg_bucket = app.config.get("S3_PKG_BUCKET")
pkg_s3_key_path = generate_bucket_key(
pkg_path,
pkg_fullname,
package_file.filename)
item.pkg_path = pkg_s3_key_path
pkg_local_cache_path = path.join(
app.config["DEB_UPLOAD_PATH"],
pkg_s3_key_path)
# Local package path
pkg_local_cache_dir = path.dirname(pkg_local_cache_path)
if not path.exists(pkg_local_cache_dir):
print("Creating path %s" % pkg_local_cache_dir)
os.makedirs(pkg_local_cache_dir)
# Move tmp deb file to the cache folder
shutil.move(debTmpFile, pkg_local_cache_path)
pkg_overrides = [(item.pkg_name, "itemid", item.iid),
(item.pkg_name, "itemname", item.display_name),
(item.pkg_name, "filename", "null")]
# Check if it's a tweak
tweak_file = detect_tweak(deb_obj.filelist)
if tweak_file is not None:
tweak_file = 'file:///' + tweak_file
pkg_overrides.append((item.pkg_name, "Respring", "YES",))
pkg_overrides.append((item.pkg_name, "TweakLib", tweak_file,))
index_s3_key_path = "Packages.gz"
# Upload deb file
upload_to_s3.delay(pkg_bucket,
pkg_s3_key_path,
pkg_local_cache_path)
pkg_index_file = path.join(app.config["DEB_UPLOAD_PATH"],
app.config["PKG_INDEX_FILE_NAME"]
)
# Update and upload package index
dpkg_update_index.delay(app.config["DEB_UPLOAD_PATH"],
pkg_bucket,
index_s3_key_path,
pkg_index_file,
pkg_overrides)
# Upload icon
icon_base_path = path.join(base_path, "icons")
icon_s3_path = generate_bucket_key(icon_base_path,
"app_icon",
app_icon.filename)
_, icon_tmpfile = mkstemp()
with open(icon_tmpfile, "wb") as tmp:
tmp.write(app_icon.read())
upload_to_s3.delay(assets_bucket, icon_s3_path, icon_tmpfile, True)
# Upload screenshot 1
ss_base_path = path.join(base_path, "screenshots")
sshot_s3_path = generate_bucket_key(ss_base_path,
"screenshot1",
screenshot1.filename)
_, sshot_tmpfile = mkstemp()
with open(sshot_tmpfile, "wb") as tmp:
tmp.write(screenshot1.read())
upload_to_s3.delay(assets_bucket, sshot_s3_path, sshot_tmpfile, True)
# Upload screenshot 2
ss_base_path = path.join(base_path, "screenshots")
sshot_s3_path = generate_bucket_key(ss_base_path,
"screenshot2",
screenshot2.filename)
_, sshot_tmpfile = mkstemp()
with open(sshot_tmpfile, "wb") as tmp:
tmp.write(screenshot2.read())
upload_to_s3.delay(assets_bucket, sshot_s3_path, sshot_tmpfile, True)
# Upload screenshot 3
ss_base_path = path.join(base_path, "screenshots")
sshot_s3_path = generate_bucket_key(ss_base_path,
"screenshot3",
screenshot3.filename)
_, sshot_tmpfile = mkstemp()
with open(sshot_tmpfile, "wb") as tmp:
tmp.write(screenshot3.read())
upload_to_s3.delay(assets_bucket, sshot_s3_path, sshot_tmpfile, True)
# Upload screenshot 4
ss_base_path = path.join(base_path, "screenshots")
sshot_s3_path = generate_bucket_key(ss_base_path,
"screenshot4",
screenshot4.filename)
_, sshot_tmpfile = mkstemp()
with open(sshot_tmpfile, "wb") as tmp:
tmp.write(screenshot4.read())
upload_to_s3.delay(assets_bucket, sshot_s3_path, sshot_tmpfile, True)
# Upload youtube video id
youtube_id_path = path.join(base_path, "videos")
youtube_s3_filename = "youtube-%s" % form.youtube_video_id.data
youtube_id_s3_path = path.join(youtube_id_path,
youtube_s3_filename)
_, youtube_id_tmpfile = mkstemp()
with open(youtube_id_tmpfile, "wb") as tmp:
tmp.write(form.youtube_video_id.data)
upload_to_s3.delay(assets_bucket, youtube_id_s3_path, youtube_id_tmpfile, True)
# Upload banner image
banner_img_path = path.join(base_path, "banners")
banner_img_s3_path = generate_bucket_key(banner_img_path,
"banner_image",
banner_img_file.filename)
_, banner_img_tmpfile = mkstemp()
with open(banner_img_tmpfile, "wb") as tmp:
tmp.write(banner_img_file.read())
# Add banner item to the database
with db_scoped_session() as ses:
banner = ses.query(Banner).filter_by(item_id = item.iid).first()
if banner is None:
banner = Banner(item_id=item.iid)
ses.add(banner)
ses.commit()
upload_to_s3.delay(assets_bucket, banner_img_s3_path, banner_img_tmpfile, True)
except Exception as e:
s.rollback()
raise e
# Commit changes
s.commit()
flash("Assets uploaded successfully")
return redirect(url_for('.index'))
context = {'form': form}
return self.render(self.template_name, **context)