def getall(self, dict_filter=None): """ :rtype: List of tuple :return: list of key/report :todo: add a filter capability """ nmapreportlist = [] for key in bucket_lister(self.bucket): if isinstance(key, Key): nmapreportjson = json.loads(key.get_contents_as_string()) nmapreport = NmapParser.parse_fromdict(nmapreportjson) nmapreportlist.append((key.key, nmapreport)) return nmapreportlist
def _get_current_pause_events(self, pipeline_system=None, event_id=None): """ Returns the current pause status of one or all pipeline systems and one or all events. Arguments: pipeline_system (str): Pipeline system name for which to return status, e.g. edxapp, ecommerce, etc., None for all systems event_id (str): Event ID for which to return status, None for all events Returns: dict(pipeline_system: list()): Dictionary with: keys: pipeline_system names values: lists of pipeline events as dicts """ pause_status = defaultdict(list) for key in bucket_lister(self.pipeline_bucket, prefix=self.CURRENT_DIRECTORY): # Only read YAML files - at least, files with a .yml suffix. if not key.name.endswith('.yml'): continue try: pause_data = yaml.safe_load(key.get_contents_as_string()) except yaml.YAMLError: log.warning('Unable to load file as YAML: %s - continuing...', key.name) continue if pause_data is None: log.warning('Unable to load file as YAML: %s - continuing...', key.name) continue # If pipeline system was specified and this pause file isn't for that system, ignore it. if pipeline_system and pipeline_system != pause_data[ 'pipeline_system']: continue # If event ID was specified and this pause file isn't for that event, ignore it. if event_id and event_id != pause_data['event_id']: continue # Add the base key name to the returned pause data. pause_data['key_name'] = os.path.basename(key.name) # Add the pause data from this file to the returned data. pause_status[pause_data['pipeline_system']].append(pause_data) return pause_status
def list_command(s3_bucket, args): logger.debug('listing bucket %s using prefix "%s"', s3_bucket.name, args.prefix) for key in bucket_lister(s3_bucket, prefix=args.prefix): last_modified = (iso8601.parse_date( key.last_modified).replace(tzinfo=None)) if args.after and last_modified < args.after: logger.debug('discarding key %s', key.name) continue if args.before and args.before < last_modified: logger.debug('discarding key %s', key.name) continue print args.format % { 'bucket': s3_bucket.name, 'b': s3_bucket.name, 'name': key.name, 'n': key.name, 'last_modified': key.last_modified, 'lm': key.last_modified, 'modified': key.last_modified, 'm': key.last_modified, }
def list_command(s3_bucket, args): logger.debug('listing bucket %s using prefix "%s"', s3_bucket.name, args.prefix) for key in bucket_lister(s3_bucket, prefix=args.prefix): last_modified = (iso8601 .parse_date(key.last_modified) .replace(tzinfo=None)) if args.after and last_modified < args.after: logger.debug('discarding key %s', key.name) continue if args.before and args.before < last_modified: logger.debug('discarding key %s', key.name) continue print args.format % { 'bucket': s3_bucket.name, 'b': s3_bucket.name, 'name': key.name, 'n': key.name, 'last_modified': key.last_modified, 'lm': key.last_modified, 'modified': key.last_modified, 'm': key.last_modified, }
def s3List(s3dir): b = s3PubBucket() return bucket_lister(b, s3dir)
def s3_list(s3_dir): from boto.s3.bucketlistresultset import bucket_lister b = s3_get_bucket() return bucket_lister(b, s3_dir)
def upload_s3(self, arg, dirname, names): """ This is the callback to os.path.walk and where much of the work happens """ bucket, key, bucket_name, root_dir = arg # Skip files and directories we don't want to sync for pattern in self.EXCLUDE_LIST: if fnmatch(os.path.basename(dirname), pattern): if self.verbosity > 1: print 'Skipping: %s (rule: %s)' % (names, pattern) del names[:] return # Later we assume the MEDIA_ROOT ends with a trailing slash if not root_dir.endswith(os.path.sep): root_dir = root_dir + os.path.sep list_prefix = dirname[len(root_dir):] if self.prefix: list_prefix = '%s/%s' % (self.prefix, list_prefix) s3_list = bucket_lister(bucket, prefix=list_prefix) for name in names: bad_name = False for pattern in self.EXCLUDE_LIST: if fnmatch(name, pattern): bad_name = True # Skip files we don't want to sync if bad_name: if self.verbosity > 1: print 'Skipping: %s (rule: %s)' % (names, pattern) continue filename = os.path.join(dirname, name) if os.path.isdir(filename): continue # Don't try to upload directories file_key = filename[len(root_dir):] if self.prefix: file_key = '%s/%s' % (self.prefix, file_key) # Check if file on S3 is older than local file, if so, upload # TODO: check if hash chunk corresponds if not self.do_force: s3_key = self.find_key_in_list(s3_list, file_key) if s3_key: s3_datetime = datetime.datetime(*time.strptime( s3_key.last_modified, '%Y-%m-%dT%H:%M:%S.000Z')[0:6]) local_datetime = datetime.datetime.utcfromtimestamp( os.stat(filename).st_mtime) if local_datetime < s3_datetime: self.skip_count += 1 if self.verbosity > 1: print "File %s hasn't been modified since last " \ "being uploaded" % (file_key) if file_key in self.s3_files: self.files_processed.add(file_key) del self.s3_files[file_key] continue if file_key in self.s3_files: self.files_processed.add(file_key) del self.s3_files[file_key] # File is newer, let's process and upload if self.verbosity > 0: print "Uploading %s..." % file_key if self.dry_run: self.upload_count += 1 continue try: upload_file_to_s3(file_key, filename, key, do_gzip=self.do_gzip, do_expires=self.do_expires, verbosity=self.verbosity) except boto.exception.S3CreateError, e: # TODO: retry to create a few times print "Failed to upload: %s" % e except Exception, e: print e raise