def getall(self, dict_filter=None):
"""
:rtype: List of tuple
:return: list of key/report
:todo: add a filter capability
"""
nmapreportlist = []
for key in bucket_lister(self.bucket):
if isinstance(key, Key):
nmapreportjson = json.loads(key.get_contents_as_string())
nmapreport = NmapParser.parse_fromdict(nmapreportjson)
nmapreportlist.append((key.key, nmapreport))
return nmapreportlist
def getall(self, dict_filter=None):
"""
:rtype: List of tuple
:return: list of key/report
:todo: add a filter capability
"""
nmapreportlist = []
for key in bucket_lister(self.bucket):
if isinstance(key, Key):
nmapreportjson = json.loads(key.get_contents_as_string())
nmapreport = NmapParser.parse_fromdict(nmapreportjson)
nmapreportlist.append((key.key, nmapreport))
return nmapreportlist
def _get_current_pause_events(self, pipeline_system=None, event_id=None):
"""
Returns the current pause status of one or all pipeline systems and one or all events.
Arguments:
pipeline_system (str):
Pipeline system name for which to return status, e.g. edxapp, ecommerce, etc., None for all systems
event_id (str):
Event ID for which to return status, None for all events
Returns:
dict(pipeline_system: list()):
Dictionary with:
keys: pipeline_system names
values: lists of pipeline events as dicts
"""
pause_status = defaultdict(list)
for key in bucket_lister(self.pipeline_bucket,
prefix=self.CURRENT_DIRECTORY):
# Only read YAML files - at least, files with a .yml suffix.
if not key.name.endswith('.yml'):
continue
try:
pause_data = yaml.safe_load(key.get_contents_as_string())
except yaml.YAMLError:
log.warning('Unable to load file as YAML: %s - continuing...',
key.name)
continue
if pause_data is None:
log.warning('Unable to load file as YAML: %s - continuing...',
key.name)
continue
# If pipeline system was specified and this pause file isn't for that system, ignore it.
if pipeline_system and pipeline_system != pause_data[
'pipeline_system']:
continue
# If event ID was specified and this pause file isn't for that event, ignore it.
if event_id and event_id != pause_data['event_id']:
continue
# Add the base key name to the returned pause data.
pause_data['key_name'] = os.path.basename(key.name)
# Add the pause data from this file to the returned data.
pause_status[pause_data['pipeline_system']].append(pause_data)
return pause_status
def list_command(s3_bucket, args):
logger.debug('listing bucket %s using prefix "%s"', s3_bucket.name,
args.prefix)
for key in bucket_lister(s3_bucket, prefix=args.prefix):
last_modified = (iso8601.parse_date(
key.last_modified).replace(tzinfo=None))
if args.after and last_modified < args.after:
logger.debug('discarding key %s', key.name)
continue
if args.before and args.before < last_modified:
logger.debug('discarding key %s', key.name)
continue
print args.format % {
'bucket': s3_bucket.name,
'b': s3_bucket.name,
'name': key.name,
'n': key.name,
'last_modified': key.last_modified,
'lm': key.last_modified,
'modified': key.last_modified,
'm': key.last_modified,
}
def list_command(s3_bucket, args):
logger.debug('listing bucket %s using prefix "%s"',
s3_bucket.name, args.prefix)
for key in bucket_lister(s3_bucket, prefix=args.prefix):
last_modified = (iso8601
.parse_date(key.last_modified)
.replace(tzinfo=None))
if args.after and last_modified < args.after:
logger.debug('discarding key %s', key.name)
continue
if args.before and args.before < last_modified:
logger.debug('discarding key %s', key.name)
continue
print args.format % {
'bucket': s3_bucket.name,
'b': s3_bucket.name,
'name': key.name,
'n': key.name,
'last_modified': key.last_modified,
'lm': key.last_modified,
'modified': key.last_modified,
'm': key.last_modified,
}
def s3List(s3dir): b = s3PubBucket() return bucket_lister(b, s3dir)
def s3_list(s3_dir):
from boto.s3.bucketlistresultset import bucket_lister
b = s3_get_bucket()
return bucket_lister(b, s3_dir)
def upload_s3(self, arg, dirname, names):
"""
This is the callback to os.path.walk and where much of the work happens
"""
bucket, key, bucket_name, root_dir = arg
# Skip files and directories we don't want to sync
for pattern in self.EXCLUDE_LIST:
if fnmatch(os.path.basename(dirname), pattern):
if self.verbosity > 1:
print 'Skipping: %s (rule: %s)' % (names, pattern)
del names[:]
return
# Later we assume the MEDIA_ROOT ends with a trailing slash
if not root_dir.endswith(os.path.sep):
root_dir = root_dir + os.path.sep
list_prefix = dirname[len(root_dir):]
if self.prefix:
list_prefix = '%s/%s' % (self.prefix, list_prefix)
s3_list = bucket_lister(bucket, prefix=list_prefix)
for name in names:
bad_name = False
for pattern in self.EXCLUDE_LIST:
if fnmatch(name, pattern):
bad_name = True # Skip files we don't want to sync
if bad_name:
if self.verbosity > 1:
print 'Skipping: %s (rule: %s)' % (names, pattern)
continue
filename = os.path.join(dirname, name)
if os.path.isdir(filename):
continue # Don't try to upload directories
file_key = filename[len(root_dir):]
if self.prefix:
file_key = '%s/%s' % (self.prefix, file_key)
# Check if file on S3 is older than local file, if so, upload
# TODO: check if hash chunk corresponds
if not self.do_force:
s3_key = self.find_key_in_list(s3_list, file_key)
if s3_key:
s3_datetime = datetime.datetime(*time.strptime(
s3_key.last_modified, '%Y-%m-%dT%H:%M:%S.000Z')[0:6])
local_datetime = datetime.datetime.utcfromtimestamp(
os.stat(filename).st_mtime)
if local_datetime < s3_datetime:
self.skip_count += 1
if self.verbosity > 1:
print "File %s hasn't been modified since last " \
"being uploaded" % (file_key)
if file_key in self.s3_files:
self.files_processed.add(file_key)
del self.s3_files[file_key]
continue
if file_key in self.s3_files:
self.files_processed.add(file_key)
del self.s3_files[file_key]
# File is newer, let's process and upload
if self.verbosity > 0:
print "Uploading %s..." % file_key
if self.dry_run:
self.upload_count += 1
continue
try:
upload_file_to_s3(file_key, filename, key,
do_gzip=self.do_gzip, do_expires=self.do_expires,
verbosity=self.verbosity)
except boto.exception.S3CreateError, e:
# TODO: retry to create a few times
print "Failed to upload: %s" % e
except Exception, e:
print e
raise
def s3_list(s3_dir): from boto.s3.bucketlistresultset import bucket_lister b = s3_get_bucket() return bucket_lister(b, s3_dir)
def s3List(s3dir):
b = s3PubBucket()
return bucket_lister(b, s3dir)
