def test_s3join(self): """ Verify that s3 pathing is assembled correctly """ self.assertEqual(s3join('a','b','c'),'a/b/c') # Double slashes should become single slashes: self.assertEqual(s3join('a/','/b/','/c'),'a/b/c') # Verify that trailing / is preserved: self.assertEqual(s3join('a','b','c/'),'a/b/c/') return
def test_s3join(self): """ Verify that s3 pathing is assembled correctly """ self.assertEqual(s3join('a', 'b', 'c'), 'a/b/c') # Double slashes should become single slashes: self.assertEqual(s3join('a/', '/b/', '/c'), 'a/b/c') # Verify that trailing / is preserved: self.assertEqual(s3join('a', 'b', 'c/'), 'a/b/c/') return
def upload_repodata(context): """ Upload repodata to the specified bucket. """ upload_directory(context, context.working_dir, context.opts.path, context.s3_rpm_items) # ALWAYS delete the existing s3 metadata items and upload the new ones. # We NEVER use check_items here: # Delete old metadata: for item in context.s3_repodata_items: verbose("Deleting old metadata file: %s", item.name) if not context.opts.dry_run: item.delete() # Delete any --remove'd RPM's: for item in context.s3_rpm_items: for remove_rpm in context.opts.remove: if fnmatch.fnmatch(item.name, remove_rpm): verbose("Deleting: %s", item.name) if not context.opts.dry_run: item.delete() # Upload new metadata: repo_dest = s3join(context.opts.path, REPODATA) upload_directory(context, context.working_dir_repodata, repo_dest) return
def upload_repodata(context): """ Upload repodata to the specified bucket. """ upload_directory( context, context.working_dir, context.opts.path, context.s3_rpm_items) # ALWAYS delete the existing s3 metadata items and upload the new ones. # We NEVER use check_items here: # Delete old metadata: for item in context.s3_repodata_items: verbose("Deleting old metadata file: %s", item.name) if not context.opts.dry_run: item.delete() # Delete any --remove'd RPM's: for item in context.s3_rpm_items: for remove_rpm in context.opts.remove: if fnmatch.fnmatch(item.name, remove_rpm): verbose("Deleting: %s", item.name) if not context.opts.dry_run: item.delete() # Upload new metadata: repo_dest = s3join(context.opts.path, REPODATA) upload_directory(context, context.working_dir_repodata, repo_dest) return
def upload_directory(context, dir_path, upload_prefix, check_items=[]): """ Upload all the files in the directory 'dir_path' into the s3 bucket. The variable 'upload_prefix' is the path relative to the s3 bucket. The list item 'check_items' is a list of existing s3 items at this path. If an item to be uploaded is found in check_items, it is skipped. """ items_by_name = dict(zip(map(lambda x: os.path.basename(x.name), check_items), check_items)) # Upload RPM's: for filename in os.listdir(dir_path): filepath = os.path.join(dir_path, filename) remote_item = items_by_name.get(filename, None) # Skip any non-file arguments: if not os.path.isfile(filepath): continue # Skip anything that doesn't need to be uploaded: if not should_upload(filepath, remote_item, context.opts.force_upload): verbose('File "%s" already exists in S3 location "%s" skipping upload', filename, upload_prefix) continue # Perform the upload: dest_path = s3join(upload_prefix, filename) item_key = boto.s3.key.Key(context.s3_bucket) item_key.key = dest_path if not context.opts.dry_run: item_key.set_contents_from_filename( filepath, cb=get_progress_fn(context.opts.verbose, "Uploading: %s" % dest_path) ) else: verbose("Uploading: %s" % dest_path) return
def list_metadata(context): """ List the current repo metadata items in s3, storing in s3_repodata_items. """ context.s3_repodata_path = s3join(context.opts.path, REPODATA) key_list = context.s3_bucket.list(prefix=context.s3_repodata_path) context.s3_repodata_items = [] for item in key_list: if item.name.find(FOLDER_SUFFIX) != -1: continue context.s3_repodata_items.append(item) return
def print_lists(context): """ Print repo info and bail. """ def list_item(item): print "\t%s - %ib - %s" % (item.name, item.size, item.last_modified) print "Repo info for %s:" % (s3join(context.opts.bucket, context.opts.path)) for metadata_item in context.s3_repodata_items: list_item(metadata_item) for rpm_item in context.s3_rpm_items: list_item(rpm_item) return
def print_lists(context): """ Print repo info and bail. """ def list_item(item): print "\t%s - %ib - %s" % ( item.name, item.size, item.last_modified) print "Repo info for %s:" % (s3join(context.opts.bucket, context.opts.path)) for metadata_item in context.s3_repodata_items: list_item(metadata_item) for rpm_item in context.s3_rpm_items: list_item(rpm_item) return
def upload_directory(context, dir_path, upload_prefix, check_items=[]): """ Upload all the files in the directory 'dir_path' into the s3 bucket. The variable 'upload_prefix' is the path relative to the s3 bucket. The list item 'check_items' is a list of existing s3 items at this path. If an item to be uploaded is found in check_items, it is skipped. """ items_by_name = dict( zip(map(lambda x: os.path.basename(x.name), check_items), check_items)) # Upload RPM's: for filename in os.listdir(dir_path): filepath = os.path.join(dir_path, filename) remote_item = items_by_name.get(filename, None) # Skip any non-file arguments: if not os.path.isfile(filepath): continue # Skip anything that doesn't need to be uploaded: if not should_upload(filepath, remote_item, context.opts.force_upload): verbose( 'File "%s" already exists in S3 location "%s" skipping upload', filename, upload_prefix) continue # Perform the upload: dest_path = s3join(upload_prefix, filename) item_key = boto.s3.key.Key(context.s3_bucket) item_key.key = dest_path if not context.opts.dry_run: item_key.set_contents_from_filename( filepath, cb=get_progress_fn(context.opts.verbose, "Uploading: %s" % dest_path)) else: verbose("Uploading: %s" % dest_path) return