def setup_bucket(s3, dirname, bucket_name): """Ensures the given bucket exists and prepares it for a duplicity run """ if not s3.lookup(bucket_name): s3.create_bucket(bucket_name) time.sleep(5) bucket = s3.get_bucket(bucket_name) # tag this bucket with the directory so we know what it # is when we retrieve it after the terrible fire or burglary tags = Tags() tagset = TagSet() tagset.add_tag('path', dirname) tags.add_tag_set(tagset) bucket.set_tags(tags) # turn off any lifecycle rotations while we are in the middle of a backup to_glacier = Transition(days=1, storage_class='GLACIER') rule = Rule('movetoglacier', 'duplicity', 'Disabled', transition=to_glacier) lifecycle = Lifecycle() lifecycle.append(rule) bucket.configure_lifecycle(lifecycle) # rename the manifest files from their glacier-safe versions keys = bucket.list(prefix='_duplicity') for key in keys: key.copy(bucket_name, key.name.replace("_duplicity", "duplicity")) key.delete() return bucket
def __init__(self, bucket_name, s3_to_glacier_after_days=None): # create s3 connection # create bucket if does not exist # create S3 connection if archive_S3_bucket name is specified self.__bucket_name = bucket_name self.__s3_conn = boto.connect_s3() self.__bucket = self.__s3_conn.lookup(self.__bucket_name) if not self.__bucket: try: self.__bucket = self.__s3_conn.create_bucket( self.__bucket_name) if s3_to_glacier_after_days is not None: to_glacier = Transition(days=s3_to_glacier_after_days, storage_class='GLACIER') rule = Rule(id='archive-rule1', status='Enabled', transition=to_glacier) lifecycle = Lifecycle() lifecycle.append(rule) self.__bucket.configure_lifecycle(lifecycle) except S3CreateError: logger.error('failed to create S3 bucket[' + self.__bucket_name + ']. please check your AWS policy.') raise
def glacier(name): bucket = conn.get_bucket(name) to_glacier = boto.s3.lifecycle.Transition(days=30, storage_class='GLACIER') rule = Rule('ruleid', 'logs/', 'Enabled', transition=to_glacier) lifecycle = Lifecycle() lifecycle.append(rule) bucket.configure_lifecycle(lifecycle)
def expire(self, days, transition): self.days = int(days) self.transition = transition #using nargs in ArgumentParser leads to passing lists, use the robustness principle if type(self.transition) == list: self.transition = str(self.transition[0]) if self.transition == 'delete': pass if self.transition == 'glacier': lifecycle = Lifecycle() lifecycle_action = Transition(days=self.days, storage_class='GLACIER') rule = Rule('ruleid', 'logs/', 'Enabled', transition=lifecycle_action) lifecycle.append(rule) for key in self.key_list: content_type, unused = mimetypes.guess_type(key.name) if not content_type: content_type = 'text/plain' expire_time = datetime.utcnow() + timedelta(days=(self.days)) expire_time = expire_time.strftime(("%a, %d %b %Y %H:%M:%S GMT")) metadata = {'Expires': expire_time, 'Content-Type': content_type} if self.debug: debug_message(key.name, metadata) self.key_list_content.append(key) key.copy(self.bucket_name, key, metadata=metadata, preserve_acl=True) if self.debug: print debug_message('debug', "bucket: {x.bucket_name}, policy: {x.command}, days: {x.days}".format(x=self))
def setup_bucket(s3, dirname, bucket_name): """Ensures the given bucket exists and prepares it for a duplicity run """ if not s3.lookup(bucket_name): s3.create_bucket(bucket_name) time.sleep(5) bucket = s3.get_bucket(bucket_name) # tag this bucket with the directory so we know what it # is when we retrieve it after the terrible fire or burglary tags = Tags() tagset = TagSet() tagset.add_tag('path', dirname) tags.add_tag_set(tagset) bucket.set_tags(tags) # turn off any lifecycle rotations while we are in the middle of a backup to_glacier = Transition(days=1, storage_class='GLACIER') rule = Rule('movetoglacier', 'duplicity', 'Disabled', transition=to_glacier) lifecycle = Lifecycle() lifecycle.append(rule) bucket.configure_lifecycle(lifecycle) # rename the manifest files from their glacier-safe versions keys = bucket.list(prefix = '_duplicity') for key in keys: key.copy(bucket_name, key.name.replace("_duplicity", "duplicity")) key.delete() return bucket
def destroy_lifecycle_rule(connection, module): name = module.params.get("name") prefix = module.params.get("prefix") rule_id = module.params.get("rule_id") changed = False if prefix is None: prefix = "" try: bucket = connection.get_bucket(name) except S3ResponseError as e: module.fail_json(msg=e.message) # Get the bucket's current lifecycle rules try: current_lifecycle_obj = bucket.get_lifecycle_config() except S3ResponseError as e: if e.error_code == "NoSuchLifecycleConfiguration": module.exit_json(changed=changed) else: module.fail_json(msg=e.message) # Create lifecycle lifecycle_obj = Lifecycle() # Check if rule exists # If an ID exists, use that otherwise compare based on prefix if rule_id is not None: for existing_rule in current_lifecycle_obj: if rule_id == existing_rule.id: # We're not keeping the rule (i.e. deleting) so mark as changed changed = True else: lifecycle_obj.append(existing_rule) else: for existing_rule in current_lifecycle_obj: if prefix == existing_rule.prefix: # We're not keeping the rule (i.e. deleting) so mark as changed changed = True else: lifecycle_obj.append(existing_rule) # Write lifecycle to bucket or, if there no rules left, delete lifecycle configuration try: if lifecycle_obj: bucket.configure_lifecycle(lifecycle_obj) else: bucket.delete_lifecycle_configuration() except BotoServerError as e: module.fail_json(msg=e.message) module.exit_json(changed=changed)
def backup_bucket(bucketname): connect() bucket = s3.get_bucket(bucketname) to_glacier = Transition(days=1, storage_class='GLACIER') rule = Rule('ruleid', '/', 'Enabled', transition=to_glacier) lifecycle = Lifecycle() lifecycle.append(rule) bucket.configure_lifecycle(lifecycle) return True
def test_lifecycle_with_glacier_transition(self): lifecycle = Lifecycle() transition = Transition(days=30, storage_class='GLACIER') rule = Rule('myid', prefix='', status='Enabled', expiration=None, transition=transition) lifecycle.append(rule) self.assertTrue(self.bucket.configure_lifecycle(lifecycle)) response = self.bucket.get_lifecycle_config() transition = response[0].transition self.assertEqual(transition.days, 30) self.assertEqual(transition.storage_class, 'GLACIER') self.assertEqual(transition.date, None)
def main(): argument_spec = ec2_argument_spec() argument_spec.update(dict( bucket = dict(required=True), s3_url = dict(aliases=['S3_URL']), lifecycle = dict(type='list', required=True), ), ) module = AnsibleModule(argument_spec=argument_spec) if not HAS_BOTO: module.fail_json(msg='boto required for this module') bucket = module.params.get('bucket') s3_url = module.params.get('s3_url') lifecycle = module.params.get('lifecycle') region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module) s3 = get_s3_connection(module, aws_connect_kwargs, region, s3_url) validate_lifecycle(module, bucket, lifecycle) bucket_obj = bucket_lookup(module, s3, bucket) if not bucket_obj: module.fail_json(msg='Bucket %s does not exist' %bucket) lifecycle_config = None new_lifecycle = False try: lifecycle_config = bucket_obj.get_lifecycle_config() except boto.exception.S3ResponseError: new_lifecycle = True results = calculate_net_rules(lifecycle_config, lifecycle) if results.changed: try: if results.rules.__len__() > 0: lifecycle_config = Lifecycle() for rule in results.rules: lifecycle_config.append(rule) bucket_obj.configure_lifecycle(lifecycle_config) else: bucket_obj.delete_lifecycle_configuration() module.exit_json(bucket=bucket, changed=True, msg='Lifecycle Configuration deleted') except boto.exception.S3ResponseError, e: module.fail_json(bucket=bucket, changed=results.changed, msg="Error %s: %s" % (e.error_code, e.message), lifecycle_rules=map(lambda x: x.to_xml(), results.rules))
def set_bucket_lifetime(bucket_name, days=14, aws_access={}, conn=None): ''' Set an expiration on a bucket in S3. ''' conn = return_s3_connection(aws_access) if conn is None else conn bucket = conn.get_bucket(bucket_name) expiration = Expiration(days=days) rule = Rule(id='ruleid', prefix='', status='Enabled', expiration=expiration) lifecycle = Lifecycle() lifecycle.append(rule) return bucket.configure_lifecycle(lifecycle)
def test_lifecycle_with_glacier_transition(): conn = boto.s3.connect_to_region("us-west-1") bucket = conn.create_bucket("foobar") lifecycle = Lifecycle() transition = Transition(days=30, storage_class='GLACIER') rule = Rule('myid', prefix='', status='Enabled', expiration=None, transition=transition) lifecycle.append(rule) bucket.configure_lifecycle(lifecycle) response = bucket.get_lifecycle_config() transition = response[0].transition transition.days.should.equal(30) transition.storage_class.should.equal('GLACIER') transition.date.should.equal(None)
def freeze(self, time=0): if time and isinstance(time, timedelta): time += timedelta(hours=12) transition = Transition(days=time.days, storage_class='GLACIER') elif time and isinstance(time, datetime): transition = Transition(days=(time-datetime.now()).days, storage_class='GLACIER') else: transition = Transition(days=time, storage_class='GLACIER') key = self.key.key rule = Rule(key, key, 'Enabled', transition=transition) lifecycle = Lifecycle() lifecycle.append(rule) self.bucket.configure_lifecycle(lifecycle)
def create_s3(): """ Create the S3 buckets All the buckets use the galaxy name as the 'folder' :return: """ # Create the bucket for the images s3 = boto.connect_s3() images_bucket = 'icrar.{0}.galaxy-images'.format(env.project_name) bucket = s3.create_bucket(images_bucket) bucket.set_acl('public-read') bucket.configure_website(suffix='index.html') bucket.set_policy('''{ "Statement":[ { "Sid":"PublicReadForGetBucketObjects", "Effect":"Allow", "Principal": { "AWS": "*" }, "Action":["s3:GetObject"], "Resource":["arn:aws:s3:::%s/*"] } ] } ''' % images_bucket) # Create the bucket for the output files file_bucket = 'icrar.{0}.files'.format(env.project_name) s3.create_bucket(file_bucket) # Create the bucket for the stats files file_bucket = 'icrar.{0}.archive'.format(env.project_name) bucket = s3.create_bucket(file_bucket) to_glacier = Transition(days=10, storage_class='GLACIER') rule1 = Rule('rule01', status='Enabled', prefix='stats/', transition=to_glacier) rule2 = Rule('rule02', status='Enabled', prefix='logs/', expiration=Expiration(days=20)) lifecycle = Lifecycle() lifecycle.append(rule1) lifecycle.append(rule2) bucket.configure_lifecycle(lifecycle)
def create_s3(): """ Create the S3 buckets All the buckets use the galaxy name as the 'folder' :return: """ # Create the bucket for the images s3 = boto.connect_s3() images_bucket = "icrar.{0}.galaxy-images".format(env.project_name) bucket = s3.create_bucket(images_bucket) bucket.set_acl("public-read") bucket.configure_website(suffix="index.html") bucket.set_policy( """{ "Statement":[ { "Sid":"PublicReadForGetBucketObjects", "Effect":"Allow", "Principal": { "AWS": "*" }, "Action":["s3:GetObject"], "Resource":["arn:aws:s3:::%s/*"] } ] } """ % images_bucket ) # Create the bucket for the output files file_bucket = "icrar.{0}.files".format(env.project_name) s3.create_bucket(file_bucket) # Create the bucket for the stats files file_bucket = "icrar.{0}.archive".format(env.project_name) bucket = s3.create_bucket(file_bucket) to_glacier = Transition(days=10, storage_class="GLACIER") rule1 = Rule("rule01", status="Enabled", prefix="stats/", transition=to_glacier) rule2 = Rule("rule02", status="Enabled", prefix="logs/", expiration=Expiration(days=20)) lifecycle = Lifecycle() lifecycle.append(rule1) lifecycle.append(rule2) bucket.configure_lifecycle(lifecycle)
def lifecycle(): #transitions = Transitions() exp = Expiration(date="2018-06-13 07:05:00") #exp = Expiration(days=1) rule = Rule(id='rule-1', prefix='', status='Enabled', expiration=exp) lifecycle = Lifecycle() lifecycle.append(rule) bucket = conn.get_bucket(bucket_name) ret = bucket.configure_lifecycle(lifecycle) print "Bucket Lifecycle Set:", ret print "=========================" current = bucket.get_lifecycle_config() print "Bucket Lifecycle Conf:", current print "Tran:", current[0].transition print "Expi:", current[0].expiration print "========================="
def test_set_lifecycle_policy(): """ PUTs arbitraty lifecycle_policy and checks whether GET lifecycle_policy API call returns 200 and other lifecycle_policy metadata is as set in PUT call """ bucket = helpers.get_bucket() transitions = Transitions() transitions.add_transition(days=30, storage_class='STANDARD_IA') transitions.add_transition(days=90, storage_class='GLACIER') expiration = Expiration(days=120) rule = Rule(id='ruleid', prefix='logs/', status='Enabled', expiration=expiration, transition=transitions) lifecycle = Lifecycle() lifecycle.append(rule) assert bucket.configure_lifecycle(lifecycle) == True
def __init__(self, bucket_name, s3_to_glacier_after_days=None): # create s3 connection # create bucket if does not exist # create S3 connection if archive_S3_bucket name is specified self.__bucket_name = bucket_name self.__s3_conn = boto.connect_s3() self.__bucket = self.__s3_conn.lookup(self.__bucket_name) if not self.__bucket: try: self.__bucket = self.__s3_conn.create_bucket(self.__bucket_name) if s3_to_glacier_after_days is not None: to_glacier = Transition(days=s3_to_glacier_after_days, storage_class='GLACIER') rule = Rule(id='archive-rule1', status='Enabled', transition=to_glacier) lifecycle = Lifecycle() lifecycle.append(rule) self.__bucket.configure_lifecycle(lifecycle) except S3CreateError: logger.error('failed to create S3 bucket[' + self.__bucket_name + ']. please check your AWS policy.') raise
def set_transition_to_glacier(self, days, prefix=''): """ Set rules when the files should be moved to Amazon Glacier for archiving This method must be called before write/upload methods Not used at the time, but could be for archiving s3 broker files :param prefix: str, prefix :param days: int, num of days :return: None """ try: to_glacier = Transition(days=days, storage_class='GLACIER') rule = Rule(id='ruleid', prefix=prefix, status='Enabled', transition=to_glacier) lifecycle = Lifecycle() lifecycle.append(rule) self.bucket.configure_lifecycle(lifecycle) except Exception as e: logging.exception("S3Client.set_transition_to_glacier failed for bucket {}, error {}" "".format(self.bucket_name, e))
def backup_transition(bucket_name, bucket_folder): # Engaging to backup transition S3_BUCKET = bucket_name bucket = conn.get_bucket(S3_BUCKET) ruleid = "rule_id_"+str(bucket_folder) to_glacier = Transition(days=0, storage_class='GLACIER') rule = Rule(ruleid, bucket_folder, 'Enabled', transition=to_glacier) lifecycle = Lifecycle() lifecycle.append(rule) # ready to backup # configuring the bucket with this lifecycle policy bucket.configure_lifecycle(lifecycle) # retrieve the current lifecycle policy of the bucket. current = bucket.get_lifecycle_config() print current[0].transition
def push_code_to_Aws(dest): s3_connection = boto.connect_s3(aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY) try: bucket = s3_connection.get_bucket('calljson') except: bucket = s3_connection.create_bucket('calljson') expiration = Expiration(days=1) rule = Rule(id='ruleid', status='Enabled', expiration=expiration) lifecycle = Lifecycle() lifecycle.append(rule) bucket.configure_lifecycle(lifecycle) # create new key in s3 key = bucket.new_key(dest) key.content_type = 'text/plain' f = open(dest, 'r') mystring = f.read() key.set_contents_from_string(mystring, policy='public-read') time.sleep(2) url = key.generate_url(160) o = urlparse(url) return o.scheme + "://" + o.netloc + o.path
def cleanup_bucket(s3, bucket): """Glacier-proofs the bucket by renaming the .manifest files to not get moved to glacier via our lifecycle rule """ # this isn't proof against eventual consistency, but it helps time.sleep(10) keys = bucket.list() # rename all the manifest and signature files so they don't get moved to glacier for key in keys: if not key.name.startswith("_") and \ key.name.endswith(".manifest"): # or key.name.endswith(".sigtar.gz")): key.copy(bucket.name, "_" + key.name) key.delete() # re-establish our lifecycle rules to_glacier = Transition(days=1, storage_class='GLACIER') rule = Rule('movetoglacier', 'duplicity', 'Enabled', transition=to_glacier) lifecycle = Lifecycle() lifecycle.append(rule) bucket.configure_lifecycle(lifecycle)
def create_lifecycle_rule(connection, module): name = module.params.get("name") expiration_date = module.params.get("expiration_date") expiration_days = module.params.get("expiration_days") prefix = module.params.get("prefix") rule_id = module.params.get("rule_id") status = module.params.get("status") storage_class = module.params.get("storage_class") transition_date = module.params.get("transition_date") transition_days = module.params.get("transition_days") changed = False try: bucket = connection.get_bucket(name) except S3ResponseError as e: module.fail_json(msg=e.message) # Get the bucket's current lifecycle rules try: current_lifecycle_obj = bucket.get_lifecycle_config() except S3ResponseError as e: if e.error_code == "NoSuchLifecycleConfiguration": current_lifecycle_obj = Lifecycle() else: module.fail_json(msg=e.message) # Create expiration if expiration_days is not None: expiration_obj = Expiration(days=expiration_days) elif expiration_date is not None: expiration_obj = Expiration(date=expiration_date) else: expiration_obj = None # Create transition if transition_days is not None: transition_obj = Transition(days=transition_days, storage_class=storage_class.upper()) elif transition_date is not None: transition_obj = Transition(date=transition_date, storage_class=storage_class.upper()) else: transition_obj = None # Create rule rule = Rule(rule_id, prefix, status.title(), expiration_obj, transition_obj) # Create lifecycle lifecycle_obj = Lifecycle() appended = False # If current_lifecycle_obj is not None then we have rules to compare, otherwise just add the rule if current_lifecycle_obj: # If rule ID exists, use that for comparison otherwise compare based on prefix for existing_rule in current_lifecycle_obj: if rule.id == existing_rule.id: if compare_rule(rule, existing_rule): lifecycle_obj.append(rule) appended = True else: lifecycle_obj.append(rule) changed = True appended = True elif rule.prefix == existing_rule.prefix: existing_rule.id = None if compare_rule(rule, existing_rule): lifecycle_obj.append(rule) appended = True else: lifecycle_obj.append(rule) changed = True appended = True else: lifecycle_obj.append(existing_rule) # If nothing appended then append now as the rule must not exist if not appended: lifecycle_obj.append(rule) changed = True else: lifecycle_obj.append(rule) changed = True # Write lifecycle to bucket try: bucket.configure_lifecycle(lifecycle_obj) except S3ResponseError as e: module.fail_json(msg=e.message) module.exit_json(changed=changed)
transition_obj = None # Create rule rule = Rule(rule_id, prefix, status.title(), expiration_obj, transition_obj) # Create lifecycle lifecycle_obj = Lifecycle() appended = False # If current_lifecycle_obj is not None then we have rules to compare, otherwise just add the rule if current_lifecycle_obj: # If rule ID exists, use that for comparison otherwise compare based on prefix for existing_rule in current_lifecycle_obj: if rule.id == existing_rule.id: if compare_rule(rule, existing_rule): lifecycle_obj.append(rule) appended = True else: lifecycle_obj.append(rule) changed = True appended = True elif rule.prefix == existing_rule.prefix: existing_rule.id = None if compare_rule(rule, existing_rule): lifecycle_obj.append(rule) appended = True else: lifecycle_obj.append(rule) changed = True appended = True else:
def main(): # Get command line aruments opts = getInput() #Set-up files that need to be appended to saveOldFiles(opts.ident) def exit_handler(signum, trace): """ Handle exit smoothly and save progress """ print "Thank you for using FoREST-Cat. Saving progress." log("Recieved exit command. Saving progress\n", opts.ident) if "states" not in globals(): states = [] errors = [] events = [] if "sensors" in dir() and "r" in dir(): saveProgress(states, errors, events, sensors, r, opts.ident) log("Progress saved.\n", opts.ident) print "Exiting..." log("Exiting...\n\n", opts.ident) exit(0) #Install exit handler - program will exit when it receives SIGINT signal.signal(signal.SIGINT, exit_handler) print "Welcome to the FoREST-cat program for detecting errors and" + \ " rare events in data from multiple sensory modalities." #Initialize data if not opts.test: call_rsync(opts) #Initialize ravq if opts.restart: print "Loading stored ravq from file..." r = loadFromFile("ravq", opts.ident) opts.startDate = loadFromFile("time", opts.ident) if r == None or opts.startDate == None: log("Failed to load RAVQ. Closing...", opts.ident) sendEmail("Failed to load RAVQ, FoREST-cat is closing.", "FoREST-cat load fail", opts.ident, "*****@*****.**") exit() log("Loaded RAVQ", opts.ident) #if we are loading a pre-existing RAVQ, we need to #load sensors after loading it, so we know when the start date is log("loading sensors", opts.ident) sensors = SensorArray(opts.config, opts.startDate) initData = sensors.getNext(opts.timeInt) #this doesn't get input #if we want to be really efficient, fix this one day log("sensors loaded", opts.ident) saveToFile("sensors", sensors, opts.ident) else: #if we aren't loading a pre-existing RAVQ, we need to #load sensors before generating the new one, so we know #how many sensors there are log("loading sensors", opts.ident) sensors = SensorArray(opts.config, opts.startDate) initData = sensors.getNext(opts.timeInt) #this doesn't get input #if we want to be really efficient, fix this one day log("sensors loaded", opts.ident) saveToFile("sensors", sensors, opts.ident) log("Generating new RAVQ...", opts.ident) r = ARAVQ(opts.bufferSize, opts.epsilon, opts.delta, len(initData), opts.historySize, opts.learningRate) log("RAVQ generated.", opts.ident) #Set up Amazon Web Services stuff if not opts.test: from boto.s3.lifecycle import Lifecycle, Rule, Transition import boto s3Conn = boto.connect_s3() bucket = s3Conn.get_bucket("forest-cat") lifecycle = Lifecycle() for item in ["log", "ravq", "events", "errors", "states"]: #set rules for transition to Glacier to_glacier = Transition(days=7, storage_class="GLACIER") rule = Rule(item + "Rule", item, "Enabled", transition=to_glacier) lifecycle.append(rule) bucket.configure_lifecycle(lifecycle) #This is a dictionary so it can modified in the handler today = {0: datetime.date(datetime.now())} def alarm_handler(signum, frame): """ This function will get called any time the alarm goes off. It is defined here so that it will have access to main() local variables. This supports an event-driven design. """ log("Starting processing...", opts.ident) #get data #check for updates with rsync - RSA keys need to be #appropriately configured for this to work if not opts.test: if callRsync(opts): sensors.getData() states = [] errors = [] events = [] while sensors.keepGoing: #Get data data = sensors.getNext(opts.timeInt) if opts.verbose: log("Data retrieved", opts.ident) #send data to RAVQ vec, errs = r.input(data, r.prevVec)[2:] if opts.verbose: log("Input processed: " + str(vec), opts.ident) states.append((r.newWinnerIndex, sensors.currTime)) #Handle events - Check for both event types if r.newWinnerIndex != r.previousWinnerIndex: ev = Event(r.previousWinnerIndex, r.newWinnerIndex, vec, data[0].time, "state transition") if not r.eventState: eventAlertTransition(ev, opts.ident) r.eventState = True log("Potential event: " + str(ev), opts.ident) events.append( str(ev.prevState) + ", " + str(ev.newState) + ", " + str(ev.vector) + ", " + str(ev.time) + ", " + ev.reason + "\n") elif len(errs) > len(sensors) * opts.eventThreshold: ev = Event(r.previousWinnerIndex, r.newWinnerIndex, vec, data[0].time, "anomalous number of errors") if not r.eventState: eventAlertAnomalous(ev, opts.ident) r.eventState = True log("Potential event: " + str(ev), opts.ident) events.append(ev) elif len(errs) > 0: #Handle errors for e in errs: log(str(e), opts.ident) if not e.sensor.errorState: errorAlert(e, opts.ident) e.sensor.errorState = True errors.append(e.sensor + ", " + str(e.time) + ", " + str(e.value) + ", " + e.flag + ", " + str(e.replace) + "\n") else: r.eventState = False if opts.verbose: log("Timestep complete.\n", opts.ident) if sensors.currTime.hour == 0 and sensors.currTime.minute == 0: log("Day is now " + str(sensors.currTime), opts.ident) if not opts.test: #If this is a test, we don't expect more data to appear signal.alarm(60 * opts.refreshRate) #set next alarm log("Buffer emptied.\n\n", opts.ident) #Save stuff saveProgress(states, errors, events, sensors, r, opts.ident) if opts.test: #tests don't need to run indefinitely log("Since this is test mode, exiting...", opts.ident) exit(0) #If it's a new day, archive files in S3 if today[0] != datetime.date(datetime.now()) and not opts.test: for item in ["log"+opts.ident, "ravq"+opts.ident, \ "events"+opts.ident, "errors"+opts.ident, \ "states"+opts.ident]: #store in s3 try: #exception handling in boto documentation is kind of vague key = boto.s3.key.Key(bucket) key.key = item + str(today) key.set_contents_from_filename(item) #clear old files infile = open(item, "w+") infile.write("") infile.close() except Exception as e: sendEmail("Received execption " + str(e), "Something went wrong in boto", "*****@*****.**") today[0] = datetime.date(datetime.now()) print today, datetime.date(datetime.now()) signal.signal(signal.SIGALRM, alarm_handler) signal.alarm(1) #go off once at start-up while True: signal.pause() #process sleeps until alarm goes off
def main(): # Get command line aruments opts = getInput() #Set-up files that need to be appended to saveOldFiles(opts.ident) def exit_handler(signum, trace): """ Handle exit smoothly and save progress """ print "Thank you for using FoREST-Cat. Saving progress." log("Recieved exit command. Saving progress\n", opts.ident) if "states" not in globals(): states = [] errors = [] events = [] if "sensors" in dir() and "r" in dir(): saveProgress(states, errors, events, sensors, r, opts.ident) log("Progress saved.\n", opts.ident) print "Exiting..." log("Exiting...\n\n", opts.ident) exit(0) #Install exit handler - program will exit when it receives SIGINT signal.signal(signal.SIGINT, exit_handler) print "Welcome to the FoREST-cat program for detecting errors and" + \ " rare events in data from multiple sensory modalities." #Initialize data if not opts.test: call_rsync(opts) #Initialize ravq if opts.restart: print "Loading stored ravq from file..." r = loadFromFile("ravq", opts.ident) opts.startDate = loadFromFile("time", opts.ident) if r == None or opts.startDate == None: log("Failed to load RAVQ. Closing...", opts.ident) sendEmail("Failed to load RAVQ, FoREST-cat is closing.", "FoREST-cat load fail", opts.ident, "*****@*****.**") exit() log("Loaded RAVQ", opts.ident) #if we are loading a pre-existing RAVQ, we need to #load sensors after loading it, so we know when the start date is log("loading sensors", opts.ident) sensors = SensorArray(opts.config, opts.startDate) initData = sensors.getNext(opts.timeInt) #this doesn't get input #if we want to be really efficient, fix this one day log("sensors loaded", opts.ident) saveToFile("sensors", sensors, opts.ident) else: #if we aren't loading a pre-existing RAVQ, we need to #load sensors before generating the new one, so we know #how many sensors there are log("loading sensors", opts.ident) sensors = SensorArray(opts.config, opts.startDate) initData = sensors.getNext(opts.timeInt) #this doesn't get input #if we want to be really efficient, fix this one day log("sensors loaded", opts.ident) saveToFile("sensors", sensors, opts.ident) log("Generating new RAVQ...", opts.ident) r = ARAVQ(opts.bufferSize, opts.epsilon, opts.delta, len(initData), opts.historySize, opts.learningRate) log("RAVQ generated.", opts.ident) #Set up Amazon Web Services stuff if not opts.test: from boto.s3.lifecycle import Lifecycle, Rule, Transition import boto s3Conn = boto.connect_s3() bucket = s3Conn.get_bucket("forest-cat") lifecycle = Lifecycle() for item in ["log", "ravq", "events", "errors", "states"]: #set rules for transition to Glacier to_glacier = Transition(days=7, storage_class = "GLACIER") rule = Rule(item+"Rule", item, "Enabled", transition = to_glacier) lifecycle.append(rule) bucket.configure_lifecycle(lifecycle) #This is a dictionary so it can modified in the handler today = {0: datetime.date(datetime.now())} def alarm_handler(signum, frame): """ This function will get called any time the alarm goes off. It is defined here so that it will have access to main() local variables. This supports an event-driven design. """ log("Starting processing...", opts.ident) #get data #check for updates with rsync - RSA keys need to be #appropriately configured for this to work if not opts.test: if callRsync(opts): sensors.getData() states = [] errors = [] events = [] while sensors.keepGoing: #Get data data = sensors.getNext(opts.timeInt) if opts.verbose: log("Data retrieved", opts.ident) #send data to RAVQ vec, errs = r.input(data, r.prevVec)[2:] if opts.verbose: log("Input processed: " + str(vec), opts.ident) states.append((r.newWinnerIndex, sensors.currTime)) #Handle events - Check for both event types if r.newWinnerIndex != r.previousWinnerIndex: ev = Event(r.previousWinnerIndex, r.newWinnerIndex, vec, data[0].time, "state transition") if not r.eventState: eventAlertTransition(ev, opts.ident) r.eventState = True log("Potential event: " + str(ev), opts.ident) events.append(str(ev.prevState) + ", " + str(ev.newState) + ", " + str(ev.vector) + ", " + str(ev.time) + ", " + ev.reason + "\n") elif len(errs) > len(sensors)*opts.eventThreshold: ev = Event(r.previousWinnerIndex, r.newWinnerIndex, vec, data[0].time, "anomalous number of errors") if not r.eventState: eventAlertAnomalous(ev, opts.ident) r.eventState = True log("Potential event: " + str(ev), opts.ident) events.append(ev) elif len(errs) > 0: #Handle errors for e in errs: log(str(e), opts.ident) if not e.sensor.errorState: errorAlert(e, opts.ident) e.sensor.errorState = True errors.append(e.sensor + ", " + str(e.time) + ", " + str(e.value) + ", " + e.flag + ", " + str(e.replace) + "\n") else: r.eventState = False if opts.verbose: log("Timestep complete.\n", opts.ident) if sensors.currTime.hour == 0 and sensors.currTime.minute == 0: log("Day is now " + str(sensors.currTime), opts.ident) if not opts.test: #If this is a test, we don't expect more data to appear signal.alarm(60*opts.refreshRate) #set next alarm log("Buffer emptied.\n\n", opts.ident) #Save stuff saveProgress(states, errors, events, sensors, r, opts.ident) if opts.test: #tests don't need to run indefinitely log("Since this is test mode, exiting...", opts.ident) exit(0) #If it's a new day, archive files in S3 if today[0] != datetime.date(datetime.now()) and not opts.test: for item in ["log"+opts.ident, "ravq"+opts.ident, \ "events"+opts.ident, "errors"+opts.ident, \ "states"+opts.ident]: #store in s3 try: #exception handling in boto documentation is kind of vague key = boto.s3.key.Key(bucket) key.key = item+str(today) key.set_contents_from_filename(item) #clear old files infile = open(item, "w+") infile.write("") infile.close() except Exception as e: sendEmail("Received execption " + str(e), "Something went wrong in boto", "*****@*****.**") today[0] = datetime.date(datetime.now()) print today, datetime.date(datetime.now()) signal.signal(signal.SIGALRM, alarm_handler) signal.alarm(1) #go off once at start-up while True: signal.pause() #process sleeps until alarm goes off
#!/usr/bin/env python import boto.s3 REGION="us-west-2" BUCKET="mybucket" c = boto.s3.connect_to_region(REGION) bucket = c.get_bucket(BUCKET) from boto.s3.lifecycle import Lifecycle, Transition, Rule to_glacier = Transition(days=30, storage_class='GLACIER') rule = Rule('ruleid', 'logs/', 'Enabled', transition=to_glacier) lifecycle = Lifecycle() lifecycle.append(rule) bucket.configure_lifecycle(lifecycle) current = bucket.get_lifecycle_config() print current[0].transition
# Create rule rule = Rule(rule_id, prefix, status.title(), expiration_obj, transition_obj) # Create lifecycle lifecycle_obj = Lifecycle() appended = False # If current_lifecycle_obj is not None then we have rules to compare, otherwise just add the rule if current_lifecycle_obj: # If rule ID exists, use that for comparison otherwise compare based on prefix for existing_rule in current_lifecycle_obj: if rule.id == existing_rule.id: if compare_rule(rule, existing_rule): lifecycle_obj.append(rule) appended = True else: lifecycle_obj.append(rule) changed = True appended = True elif rule.prefix == existing_rule.prefix: existing_rule.id = None if compare_rule(rule, existing_rule): lifecycle_obj.append(rule) appended = True else: lifecycle_obj.append(rule) changed = True appended = True else:
def freeze_item(): to_glacier = Transition(days=30, storage_class='GLACIER') rule = Rule('ruleid', '/', 'Enabled', transition=to_glacier) lifecycle = Lifecycle() lifecycle.append(rule)
#!/usr/bin/env python import boto.s3 REGION = "us-west-2" BUCKET = "mybucket" c = boto.s3.connect_to_region(REGION) bucket = c.get_bucket(BUCKET) from boto.s3.lifecycle import Lifecycle, Transition, Rule to_glacier = Transition(days=30, storage_class='GLACIER') rule = Rule('ruleid', 'logs/', 'Enabled', transition=to_glacier) lifecycle = Lifecycle() lifecycle.append(rule) bucket.configure_lifecycle(lifecycle) current = bucket.get_lifecycle_config() print current[0].transition