def __init__(self, provider="other"): self.logger = logging.getLogger("DNSWatch.InstanceInfo") self.provider = provider if provider == "gce": self.cloud = GCE() elif provider == "aws": self.cloud = AWS()
def create_standby_valors(self): empty_valors = self.get_available_valors() # Check if the number of empty valors is less than NUM_STANDBY_VALORS # If so then create additional valors if len(empty_valors) <= NUM_STANDBY_VALORS: NUM_VALORS_TO_CREATE = NUM_STANDBY_VALORS - len(empty_valors) aws = AWS() for i in range(NUM_VALORS_TO_CREATE): create_standby_valors_thread = threading.Thread( target=self.create_and_launch_valor, args=( aws.get_subnet_id(), aws.get_sec_group().id, )) create_standby_valors_thread.start() elif len(empty_valors) > NUM_STANDBY_VALORS: # Number of empty valors is more than the required number of # standby valors so do nothing pass
def main(): choice = int(input("Would you like local (1) or AWS (2) recognition?: ")) # create a tracker that uses the classifier tracker = Tracker() recognizer = Recognizer("./knownfaces/") aws = AWS() # get a threaded video stream and start it # wake up time for standard webcam is >= 8 seconds video = WebCamStream() video.start() frame_count = 0 # count frames detected_count = 0 # count detected faces # while the 'q' key has not been pressed while cv2.waitKey(1) != ord('q'): frame = video.read() # get current frame gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # convert to gray faces = tracker.get_coordinates() # coordinates of detected faces for (x, y, w, h) in faces: draw_rectangle(frame, x, y, w, h) # highlight with rectangle # if more faces were detected since last time if new_count > detected_count: # update face count and attempt to recognize all faces # NOTE: could have it predict on only the latest detected faces detected_count = new_count if choice == 1: # local recognition success, prediction = recognizer.predict( gray_frame[y:y + w, x:x + h]) else: # aws recognition convert = bytes(cv2.imencode('.jpg', gray_frame)[1]) success, prediction = aws.compare(convert) if success: # send the predicted name to the server send_name(prediction) # detect new faces every 15 frames if frame_count == 15: frame_count = 0 new_count = tracker.detect_and_track(gray_frame) # found less faces, adjust detected if new_count < detected_count: detected_count = new_count else: # update object trackers tracker.update(gray_frame) # display frame # this is only for testing and verification cv2.imshow('Image', frame) frame_count += 1 # Clean up video.stop() cv2.destroyAllWindows()
def get_empty_valor(self, migration_source_valor_id=None): """ Get and return an available valor node. If less than the standby number of valors exist then create more standby valors """ empty_valors = self.get_available_valors(migration_source_valor_id) # Check if there are no empty valors # If there are none then create a valor node if not empty_valors: aws = AWS() valor_id = self.create_and_launch_valor(aws.get_subnet_id(), aws.get_sec_group().id) # Check the valor state and verify that it is 'RUNNING' self.verify_valor_running(valor_id) empty_valor = self.rethinkdb_manager.get_valor(valor_id) else: # Select a random index for the array of empty_valors # This essentially selects a random valor from the list of valors random_index = random.randint(0, len(empty_valors) - 1) # Check the valor state and verify that it is 'RUNNING' self.verify_valor_running(empty_valors[random_index]['valor_id']) empty_valor = empty_valors[random_index] return empty_valor
def valor_create_pool(self, number_of_valors): aws = AWS() return self.valor_manager.create_valor_pool(number_of_valors, aws.get_subnet_id(), aws.get_sec_group().id)
def _detect_provider(self): self.logger.info("Detecting cloud provider.") provider = "other" gce = GCE() aws = AWS() if gce.is_inside(): provider = "gce" elif aws.is_inside(): provider = "aws" self.logger.info("My cloud provider is: {}.".format(provider)) return provider
def start(self): self.db = AWS() self.reset_ptr_rx() self.set_mode(MODE.RXCONT) while True: time.sleep(.5) rssi_value = self.get_rssi_value() snr_value = self.get_pkt_snr_value() status = self.get_modem_status() sys.stdout.flush() sys.stdout.write("\r%d %d %d %d" % (rssi_value, snr_value, status['rx_ongoing'], status['modem_clear']))
def __init__(self): self.instances = {} self.volumes = {} self.snapshots = {} self.filtered_tag_keys = [] self.require_tags_instance = [] self.metric_prefix = 'resource_tagger_' self.metrics = {} self.aws = AWS() self.tag_default_copy_key = '' self.tag_default_copy_split = ['-', 2] self.setup()
def main2(): a = AWS('techops-newdev') s3 = a.session.client('s3') x = s3.get_object( Bucket='cb-techops-nonprod-secure-store', Key='dev/apps/cb-techops-nonprod-ert/1.0/api_credentials.ini') pprint(x['Body'].read().decode('UTF-8'))
def _get_all_instances(): try: connection = AWS(access_key=env.aws_access_key, secret_key=env.aws_secret_key, region=env.ec2_region).connect() except Exception: Notification("Could not connect to AWS").error() abort("Exited!") instances = [] for reservation in connection.get_all_instances( filters={'key-name': env.aws_key_name, 'instance-state-name': 'running'}): for host in reservation.instances: # build the user@hostname string for ssh to be used later instances.append('ubuntu@' + str(host.public_dns_name)) Notification('Got {} instances:\n{}'.format(len(instances), "\n".join(instances))).info() return instances
def lambda_handler(event, context): secrets = Secrets() which_clouds = [] for c in ['aws', 'heroku', 'do', 'cloudflare']: isEnabled = os.environ.get('USE_{}'.format(c.upper()), None) if isEnabled: which_clouds.append(c) end = last_day_of_month = date.today().replace(day=1) - timedelta(days=1) start = last_day_of_month.replace(day=1) results = [] for cloud in which_clouds: res = {} if cloud == 'aws': # AWS aws_cloud = AWS() res = aws_cloud.get(start, end) if cloud == 'heroku': heroku_cloud = Heroku(secrets.get('heroku')) res = heroku_cloud.get(start, end) if cloud == 'do': do_cloud = DO(secrets.get('do')) res = do_cloud.get(start, end) if cloud == 'cloudflare': cf_cloud = Cloudflare(secrets.get('cloudflare')) res = cf_cloud.get(start, end) results.append(res) if os.environ.get('SLACK_REPORT', False): from report import REPORT r = REPORT(results, secrets.get('slack'), start, end, True) r.slack() return 200
def _create_instance(num_instances=1): """ Creates a new EC2 Instance using boto. """ try: connection = AWS(access_key=env.aws_access_key, secret_key=env.aws_secret_key, region=env.ec2_region).connect() except Exception: Notification("Could not connect to AWS").error() abort("Exited!") aws_ami = prompt("Hit enter to use the default Ubuntu AMI or enter one:", default=env.ec2_ami) aws_security_groups = prompt("Enter the security group (must already exist)?", default=env.ec2_security_group) aws_instance_type = prompt("What instance type do you want to create? ", default=env.ec2_instance) aws_instance_key_name = prompt("Enter your key pair name (don't include .pem extension)", default=env.aws_key_name) BUILD_SERVER = dict(image_id=aws_ami, instance_type=aws_instance_type, security_groups=[aws_security_groups], key_name=aws_instance_key_name) Notification('Spinning up the instances...').info() # Create new instance using boto. for _ in range(int(num_instances)): reservation = connection.run_instances(**BUILD_SERVER) instance = reservation.instances[0] time.sleep(5) while instance.state != 'running': time.sleep(5) instance.update() Notification('-Instance {} is {}'.format(instance.id, instance.state)).info() # A new instance take a little while to allow connections so sleep for x seconds. Notification('Sleeping for {} seconds before attempting to connect...'.format(TEMP_SLEEP)).info() time.sleep(TEMP_SLEEP) return instance.public_dns_name
def __init__(self, aws_access_key_id, aws_secret_access_key, req_ratio=None, req_width=None, req_height=None, consideration_rate=None, dest=None, ): """ Performer constructor method. :args: :required_ratio: The `desired` ratio of the output image :consideration_rate: The approximation value. :max_ratio: The maximum ratio to consider. :min_ratio: The minimum ration to consider. """ if not req_ratio: req_ratio = 1.9 if not req_width: req_width = 1330 if not req_height: req_height = 700 if not consideration_rate: consideration_rate = 0.9 self.req_ratio = req_ratio self.consideration_rate = consideration_rate self.dest = dest if (self.req_ratio or self.consideration_rate) < 0: raise ValueError("The required_ration and consideration_rate should be positive value") self.min_ratio = self.req_ratio - self.consideration_rate self.max_ratio = self.req_ratio + self.consideration_rate if self.min_ratio < 0: self.min_ratio = 0 self.downloader = Downloader(project="news", progress_bar=False, dest=self.dest) self.resizer = Resizer(req_ratio=self.req_ratio, min_ratio=self.min_ratio, max_ratio=self.max_ratio) self.aws = AWS(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
class AMI(): u""" Manage AWS AMI Resources """ def __init__(self): u""" AMI Class contructor """ self.aws = AWS() def minimal_linux_ami(self): u""" Get Minimal AWS Linux AMI ID """ client = self.aws.get_client('ec2') try: res = client.describe_images( Owners=['self', '099720109477'], Filters=[{ 'Name': 'virtualization-type', 'Values': ['hvm'] }, { 'Name': 'root-device-type', 'Values': ['ebs'] }, { 'Name': 'architecture', 'Values': ['x86_64'] }, { 'Name': 'description', 'Values': ['Canonical, Ubuntu, 16.04 LTS, amd64 xenial image*'] }]) except ClientError as ex: print(ex) sys.exit() timestep = None current_time = datetime.now(timezone.utc) ami_id = None for image in res['Images']: if timestep: create_time = parse(image['CreationDate']) current_timestep = current_time - create_time if current_timestep < timestep: timestep = current_timestep ami_id = image['ImageId'] else: create_time = parse(image['CreationDate']) timestep = current_time - create_time ami_id = image['ImageId'] return ami_id
def main(): options, args = parse_args() if get_usage_ratio(options.mount_point) < options.ratio: logger.info("{0} usage ratio is less than {1}. Quitting".format( options.mount_point, options.ratio)) return logger.info("Proceeding with extending media volume") aws = AWS() instance_info = utils.get_instance_identity()['document'] volume_id = create_volume(aws.ec2, options.increment, instance_info['availabilityZone']) new_device_name = '/dev/' + increment_partition_name( get_sorted_partitions()[-1]) aws.ec2.attach_volume(volume_id, instance_info['instanceId'], new_device_name) extend_lvm(new_device_name, options.logical_group, options.logical_device)
def get_client_materials_from_s3(s3_path, region='us-east-1', akid=None, skid=None): logger.debug("Getting client materials from S3") if s3_path.startswith('s3://'): s3_path = s3_path[5:] path_split = s3_path.split('/') bucket_name = path_split[0] aws = AWS(region, akid, skid) bucket = aws.s3.get_bucket(bucket_name) key = bucket.get_key('/'.join(path_split[1:])) with TempDir() as temp_dir: temp_file_name = path.join(temp_dir, 'client_materials.tgz') with open(temp_file_name, 'w+') as temp_file: key.get_contents_to_file(temp_file) #tf = tarfile.open(temp_file.name) #tf.extractall(temp_dir) ghetto_tar(temp_file_name) print os.listdir(temp_dir)
def __init__(self): u""" AMI Class contructor """ self.aws = AWS()
def valor_create(self): aws = AWS() return self.valor_manager.create_valor(aws.get_subnet_id(), aws.get_sec_group().id)
def main(): ''' Gets arguments from the user, sign a SAML response and opens a session with the generated AccessKey. Mandatory parameters are: ''' # Parse Arguments args = get_args() # Check if the user provided file to load from if args.load_file: print "[+] Loading SAMLResponse from file..." saved_response = open(args.load_file, "r").read() arn, role_name = AWS.load(saved_response) aws_session_token = AWS.assume_role( AWS.TEMPLATES['role_arn'].format(arn=arn, role=role_name), AWS.TEMPLATES['principal_arn'].format(arn=arn), saved_response) # Open shell with the session AWS.apply_cli_session(aws_session_token["Credentials"], args.region) return # Set time parameters for assertion time = AWS.gen_timestamp(base_time=args.time) saml_expiration = AWS.gen_timestamp(base_time=args.time, minutes=int(args.saml_validity)) session_expiration = AWS.gen_timestamp(base_time=args.time, minutes=int(args.session_validity)) # Create the assertion print "[+] Creating the assertion" root = AWS.create_assertion(time, args.idp, args.user, saml_expiration, args.sp, session_expiration, args.session_name, args.roles, args.arn) # Sign the assertion print "[+] Signing the assertion with the private key provided" signed_root = AWS.sign_assertion(root, args.key, args.cert) # Insert signed assertion to saml response saml_response = AWS.TEMPLATES["response"].format( id=AWS.gen_id(), issue_instant=time, issuer=args.idp, assertion=etree.tostring(signed_root)) # Encode the saml response with B64 encoded_response = b64encode(saml_response) # Check if the user provided file to export to if args.out_file: print "[+] Writing the SAMLResponse to file: %s" % args.out_file with open(args.out_file, "w") as out_file: out_file.write(encoded_response) # Exit return # Assume role and get session token print "[+] Calling AssumeRoleWithSAML API" aws_session_token = AWS.assume_role( AWS.TEMPLATES['role_arn'].format(arn=args.arn, role=args.roles[0]), AWS.TEMPLATES['principal_arn'].format(arn=args.arn), encoded_response) # Open shell with the session print "[+] Opening a shell" AWS.apply_cli_session(aws_session_token["Credentials"], args.region)
class WebHookHandler(tornado.web.RequestHandler): p = Properties() with open("token.properties", "rb") as f: p.load(f, "utf-8") verify_token = p["verify_token"] page_access_token = p["page_access_token"] api_url = 'https://graph.facebook.com/v2.9/me/messages' api_headers = {'content-type': 'application/json'} images_root = "/var/www/like-av.xyz/images/" aws = AWS() dao = DAO() def get(self): if self.get_argument("hub.verify_token", "") == self.verify_token: self.write(self.get_argument("hub.challenge", "")) else: self.write('Error, wrong validation token') def post(self): print "receive!" data = json.loads(self.request.body) print data messaging_events = data["entry"][0]["messaging"] text = "" for event in messaging_events: sender = event["sender"]["id"] if ("message" in event and "text" in event["message"]): text = event["message"]["text"] self.sendTextMessage(sender, "給我正妹圖片") if ("message" in event and "attachments" in event["message"]): attachments = event["message"]["attachments"] print attachments if attachments[0]["type"] == "image": img_url = attachments[0]["payload"]["url"] print img_url self.sendTypingMessage(sender, "typing_on") img_bytes = urllib2.urlopen(img_url).read() result = self.aws.search_faces(img_bytes) self.sendTypingMessage(sender, "typing_off") if result is None: self.sendTextMessage(sender, "不是正妹所以找不到") else: pattern = re.compile("https://(.*)/(.*)\?(.*)") match = pattern.search(img_url) img_name = match.group(2) print img_name today = str(datetime.date.today()) self.saveImage(today, img_name, img_bytes) face_count = len(result) for i in xrange(2): face = result[i] if face_count > i else None if face is not None: actress = self.dao.find_one_actress_by_id( face.get("id")) if bool(actress): self.sendImageMessage( sender, face, today + img_name, actress) if ("postback" in event and "payload" in event["postback"]): payload = event["postback"]["payload"] feedback = payload.split(",") if feedback[0] == "O": ox = "like" file = self.images_root + feedback[2] with open(file, "rb") as img_file: self.aws.insert_index_face(feedback[1], img_file.read()) else: ox = "unlike" self.dao.update_one_feedback_by_id(feedback[1], ox, feedback[2]) self.sendTextMessage(sender, "感謝回饋") def saveImage(self, today, img_name, img_bytes): directory = self.images_root + today + "/" if not os.path.exists(directory): os.makedirs(directory) f = open(directory + img_name, 'wb') f.write(img_bytes) f.close() def sendTextMessage(self, sender, text): if len(text) <= 0: return data = {"recipient": {"id": sender}, "message": {"text": text}} params = {"access_token": self.page_access_token} r = requests.post(self.api_url, params=params, data=json.dumps(data), headers=self.api_headers) def sendImageMessage(self, sender, face, img_name, actress): attachment = { "type": "template", "payload": { "template_type": "generic", "elements": [{ "title": actress.get("name"), "image_url": actress.get("img"), "subtitle": "相似度: " + str(round(face.get("similarity"), 2)) + "%", "default_action": { "type": "web_url", # "url": "http://www.dmm.co.jp/mono/dvd/-/list/=/article=actress/id=" + face.get("id") + "/sort=date/", "url": "http://sp.dmm.co.jp/mono/list/index/shop/dvd/article/actress/id/" + face.get("id") + "/sort/date", # "url": "http://www.r18.com/videos/vod/movies/list/id=" + face.get("id") + "/sort=new/type=actress/", "webview_height_ratio": "compact" }, "buttons": [ # { # "type": "web_url", # "url": "http://sukebei.nyaa.se/?page=search&term=" + actress.get("name"), # "title": "去找片" # }, { "type": "postback", "title": "O 覺得像", "payload": "O," + face.get("id") + "," + img_name }, { "type": "postback", "title": "X 差很多", "payload": "X," + face.get("id") + "," + img_name } ] }] } } data = { "recipient": { "id": sender }, "message": { "attachment": attachment } } params = {"access_token": self.page_access_token} r = requests.post(self.api_url, params=params, data=json.dumps(data), headers=self.api_headers) def sendTypingMessage(self, sender, action): data = {"recipient": {"id": sender}, "sender_action": action} params = {"access_token": self.page_access_token} r = requests.post(self.api_url, params=params, data=json.dumps(data), headers=self.api_headers)
def __init__(self): self.aws = AWS() self.rethinkdb_manager = RethinkDbManager()
class ValorManager: def __init__(self): self.aws = AWS() self.rethinkdb_manager = RethinkDbManager() def get_stack_name(self): resource = boto3.resource('ec2') meta_data = get_instance_metadata(timeout=0.5, num_retries=2) myinstance = resource.Instance(meta_data['instance-id']) # Find the Stack name in the instance tags for tag in myinstance.tags: if 'aws:cloudformation:stack-name' in tag['Key']: return tag['Value'] def get_efs_mount(self): stack_name = self.get_stack_name() cloudformation = boto3.resource('cloudformation') efs_stack = cloudformation.Stack(stack_name) file_system_id = '' for output in efs_stack.outputs: if output['OutputKey'] == 'FileSystemID': file_system_id = output['OutputValue'] return file_system_id def get_valor_for_virtue(self, virtue_id): # Go through the list of valors and return the valor that has the specified # virtue running on it. for valor in self.list_valors(): for virtue in valor['virtues']: if virtue == virtue_id: return valor def get_available_valors(self, migration_source_valor_id=None): if not migration_source_valor_id: # Return valors that have the required number of virtues return [ valor for valor in self.list_valors() if (len(valor.get('virtues', [])) < MAX_VIRTUES_PER_VALOR) and (valor['state'] == 'RUNNING') ] else: # Return valors that have the required number of virtues and that do not # match the migration source valor return [ valor for valor in self.list_valors() if (len(valor.get('virtues', [])) < MAX_VIRTUES_PER_VALOR) and (valor['state'] == 'RUNNING') and ( valor['valor_id'] != migration_source_valor_id) ] def get_empty_valors(self): # Return valors that have no virtues on them return [ valor for valor in self.list_valors() if len(valor['virtues']) == 0 ] def create_and_launch_valor(self, subnet_id, security_group_id): valor_id = self.create_valor(subnet_id, security_group_id) self.launch_valor(valor_id) return valor_id def create_standby_valors(self): empty_valors = self.get_available_valors() # Check if the number of empty valors is less than NUM_STANDBY_VALORS # If so then create additional valors if len(empty_valors) <= NUM_STANDBY_VALORS: NUM_VALORS_TO_CREATE = NUM_STANDBY_VALORS - len(empty_valors) aws = AWS() for i in range(NUM_VALORS_TO_CREATE): create_standby_valors_thread = threading.Thread( target=self.create_and_launch_valor, args=( aws.get_subnet_id(), aws.get_sec_group().id, )) create_standby_valors_thread.start() elif len(empty_valors) > NUM_STANDBY_VALORS: # Number of empty valors is more than the required number of # standby valors so do nothing pass def get_empty_valor(self, migration_source_valor_id=None): """ Get and return an available valor node. If less than the standby number of valors exist then create more standby valors """ empty_valors = self.get_available_valors(migration_source_valor_id) # Check if there are no empty valors # If there are none then create a valor node if not empty_valors: aws = AWS() valor_id = self.create_and_launch_valor(aws.get_subnet_id(), aws.get_sec_group().id) # Check the valor state and verify that it is 'RUNNING' self.verify_valor_running(valor_id) empty_valor = self.rethinkdb_manager.get_valor(valor_id) else: # Select a random index for the array of empty_valors # This essentially selects a random valor from the list of valors random_index = random.randint(0, len(empty_valors) - 1) # Check the valor state and verify that it is 'RUNNING' self.verify_valor_running(empty_valors[random_index]['valor_id']) empty_valor = empty_valors[random_index] return empty_valor def list_valors(self): valors = self.rethinkdb_manager.list_valors() virtues = self.rethinkdb_manager.list_virtues() # Update each valor field with a virtues field. [valor.update({'virtues': []}) for valor in valors] # Update valors list with associated virtues for each valor for valor in valors: for virtue in virtues: if (valor['valor_id'] == virtue['valor_id']): valor['virtues'].append(virtue['virtue_id']) return valors def create_valor(self, subnet, sec_group): # Base Setup with open('/mnt/efs/valor/deploy/compute/' + 'setup.sh', 'r') as f: base_setup_data = f.read() base_setup_data = base_setup_data.replace('${1}', self.get_efs_mount()) # Xenblanket Setup with open('/mnt/efs/valor/deploy/compute/' + 'setup_xenblanket.sh', 'r') as f: xenblanket_setup_data = f.read() xenblanket_setup_data = xenblanket_setup_data.replace( '${1}', self.get_efs_mount()) # Gaius Setup with open('/mnt/efs/valor/deploy/compute/' + 'setup_gaius.sh', 'r') as f: gaius_setup_data = f.read() # Syslog-ng Setup with open('/mnt/efs/valor/deploy/compute/' + 'setup_syslog_ng.sh', 'r') as f: syslog_ng_setup_data = f.read() user_data = base_setup_data + xenblanket_setup_data + \ gaius_setup_data + syslog_ng_setup_data valor_config = { 'image_id': 'ami-0f9cf087c1f27d9b1', 'inst_type': 't2.xlarge', 'subnet_id': subnet, 'key_name': 'starlab-virtue-te', 'tag_key': 'Project', 'tag_value': 'Virtue', 'sec_group': sec_group, 'inst_profile_name': '', 'inst_profile_arn': '', 'user_data': user_data, } instance = self.aws.instance_create(**valor_config) self.setup_valor(instance) return instance.id def setup_valor(self, instance): router_ip = self.rethinkdb_manager.get_router()['address'] valor = Valor(instance.id) valor.connect_with_ssh() self.rethinkdb_manager.add_valor(valor) # Add the valor node to the router RouterManager(router_ip).add_valor(valor) valor.setup(router_ip) # valor.verify_setup() instance.wait_until_stopped() instance.reload() self.rethinkdb_manager.set_valor(valor.aws_instance.id, 'state', 'STOPPED') return instance.id def launch_valor(self, valor_id): instance = self.aws.instance_launch(valor_id) Valor(valor_id).connect_with_ssh() self.rethinkdb_manager.set_valor(valor_id, 'state', 'RUNNING') valor_ip = self.rethinkdb_manager.get_valor(valor_id)['address'] # Add NFS export line for valor to access email preferences dir try: line = subprocess.check_output("grep mnt/ost /etc/exports", shell=True).strip("\n") line_num = subprocess.check_output( "sed -n '/mnt\/ost/=' /etc/exports", shell=True).strip("\n") # Remove current line and replace with updated export ret = subprocess.check_call( "sudo sed -i '{}d' /etc/exports".format(line_num), shell=True) assert ret == 0 line += " {}(rw,sync,no_subtree_check)".format(valor_ip) ret = subprocess.check_call( 'echo "{}" | sudo tee -a /etc/exports'.format(line), shell=True) assert ret == 0 ret = subprocess.check_call(['sudo', 'exportfs', '-ra']) assert ret == 0 except Exception as e: print("Failed to append to NFS exports with message: {}".format(e)) return instance.id def verify_valor_running(self, valor_id): # Check the valor state and verify that it is 'RUNNING' valor_state = self.rethinkdb_manager.get_valor(valor_id)['state'] valor_wait_timeout = 60 valor_wait_count = 0 while valor_state != 'RUNNING': if valor_state == 'STOPPED': self.launch_valor(valor_id) break elif valor_state == 'CREATING': time.sleep(10) valor_state = self.rethinkdb_manager.get_valor( valor_id)['state'] valor_wait_count = valor_wait_count + 1 elif valor_wait_count >= valor_wait_timeout: Exception('ERROR: Timed out waiting for valor to reach ' '[RUNNING] state - current state is [{}]'.format( valor_state)) else: Exception( 'ERROR: Unexpected Error condition encountered while getting a ' 'valor') def create_valor_pool(self, number_of_valors, subnet, sec_group): valor_ids = [] for index in range(int(number_of_valors)): valor_id = self.create_valor(subnet, sec_group) valor_ids.append(valor_id) for valor_id in valor_ids: self.launch_valor(valor_id) return valor_ids def stop_valor(self, valor_id): valor_found = False for valor in self.get_empty_valors(): if valor_id == valor['valor_id']: valor_found = True if valor_found: instance = self.aws.instance_stop(valor_id) self.rethinkdb_manager.set_valor(valor_id, 'state', 'STOPPED') valor_ip = self.rethinkdb_manager.get_valor(valor_id)['address'] # Remove NFS export entry try: line = subprocess.check_output("grep mnt/ost /etc/exports", shell=True).strip("\n") line_num = subprocess.check_output( "sed -n '/mnt\/ost/=' /etc/exports", shell=True).strip("\n") # Remove current line and replace with updated export ret = subprocess.check_call( "sudo sed -i '{}d' /etc/exports".format(line_num), shell=True) assert ret == 0 line = line.replace( " {}(rw,sync,no_subtree_check)".format(valor_ip), "") ret = subprocess.check_call( 'echo "{}" | sudo tee -a /etc/exports'.format(line), shell=True) assert ret == 0 ret = subprocess.check_call(['sudo', 'exportfs', '-ra']) assert ret == 0 except Exception as e: print("Failed to remove NFS export with message: {}".format(e)) return instance.id else: virtues_on_valor = [ valor['virtues'] for valor in self.list_valors() if valor['valor_id'] == valor_id ] # If no valors are found then the valor_id does not exist if len(virtues_on_valor) == 0: raise Exception( 'ERROR: No Valor exists with the specified valor_id {}'. format(valor_id)) else: raise Exception( 'ERROR: Valor currently has the following Virtue/s running on it: ' '{}'.format(virtues_on_valor)) def destroy_valor(self, valor_id): valor_found = False for valor in self.get_empty_valors(): if valor_id == valor['valor_id']: valor_found = True if valor_found: self.create_standby_valors() self.aws.instance_destroy(valor_id, block=False) self.rethinkdb_manager.remove_valor(valor_id) return valor_id else: virtues_on_valor = [ valor['virtues'] for valor in self.list_valors() if valor['valor_id'] == valor_id ] # If no valors are found then the valor_id does not exist if len(virtues_on_valor) == 0: raise Exception( 'ERROR: No Valor exists with the specified valor_id {}'. format(valor_id)) else: raise Exception( 'ERROR: Valor currently has the following Virtue/s running on it: ' '{}'.format(virtues_on_valor)) def migrate_virtue(self, virtue_id, destination_valor_id): virtue = self.rethinkdb_manager.get_virtue(virtue_id) current_valor = rethinkdb.db('transducers').table('galahad').filter({ 'function': 'valor', 'address': virtue['address'] }).run(self.rethinkdb_manager.connection).next() destination_valor = self.rethinkdb_manager.get_valor( destination_valor_id) if current_valor['valor_id'] == destination_valor_id: raise Exception( ('ERROR: Source valor [{0}] and Destination Valor [{1}] ' 'are the same'.format(current_valor['valor_id'], destination_valor_id))) virtues_on_dst_valor = rethinkdb.db('transducers').table( 'galahad').filter({ 'function': 'virtue', 'address': destination_valor['address'] }).run(self.rethinkdb_manager.connection) dst_virtue_count = len(list(virtues_on_dst_valor)) if (dst_virtue_count >= MAX_VIRTUES_PER_VALOR): raise Exception(('ERROR: Destination Valor has too many ({0})' ' Virtues running on it' ' to migrate.'.format(dst_virtue_count))) rethinkdb.db("transducers").table("commands") \ .filter({'valor_ip': current_valor['address'], 'virtue_id': virtue_id}) \ .update({'valor_dest': destination_valor['address'], 'enabled': True}).run(self.rethinkdb_manager.connection) rethinkdb.db('transducers').table('commands') \ .filter({'virtue_id': virtue_id, 'transducer_id': 'introspection'}) \ .update({"valor_id": destination_valor_id}).run(self.rethinkdb_manager.connection) def add_virtue(self, valor_address, valor_id, virtue_id, efs_path, role_create=False): self.create_standby_valors() return self.rethinkdb_manager.add_virtue(valor_address, valor_id, virtue_id, efs_path, role_create) def list_virtues(self): return self.rethinkdb_manager.list_virtues() def auto_migration_start(self, migration_interval): self.rethinkdb_manager.auto_migration_start(migration_interval) def auto_migration_stop(self): self.rethinkdb_manager.auto_migration_stop() def auto_migration_status(self): return self.rethinkdb_manager.auto_migration_status() def is_auto_migration_on(self): return self.rethinkdb_manager.is_auto_migration_on()
def main(): """ Main data execution. """ # Variable to set exit code error_count = 0 # Gather Region data and connect regions = boto.ec2.regions() for region in regions: if region.name in BAD_REGIONS: continue conn = AWS(region.name) connect = conn.connect_to_ec2() # Create table for bad data table = PrettyTable( ['Instance ID', 'Server Name', 'Environment', 'Product']) table.hrules = True table.format = True # Retrieve instance data reservations = connect.get_all_instances() if reservations: LOG.info("Instances found in region {}!".format(region.name)) with open(HTML_FILE, 'a') as html: # Iterate through AWS instances, tag data for res in reservations: for inst in res.instances: for tag in inst.tags: # Checks if Tag key exists if "Name" in inst.tags and not None: amazon_name = inst.tags['Name'] else: amazon_name = "missing name" if "Product" in inst.tags and not None: amazon_product = inst.tags['Product'] else: amazon_product = "missing product" if "Environment" in inst.tags and not None: amazon_environment = inst.tags['Environment'] else: amazon_environment = "missing environment" continue # Check for bad tag values, and print those in table bad_environment = check_data(amazon_environment, CANONICAL_ENVIRONMENTS) bad_product = check_data(amazon_product, CANONICAL_PRODUCTS) if bad_product is not None: table.add_row( [inst.id, amazon_name, "+", amazon_product]) error_count += 1 elif bad_environment is not None: table.add_row([ inst.id, amazon_name, amazon_environment, "+" ]) error_count += 1 else: break # Print region name and coresponding data html.write("<h2> Region {}</h2>".format(region.name)) html.write(table.get_html_string()) else: LOG.info("There are no instances in region {}".format(region.name)) # Prints closing html tags with open(HTML_FILE, 'a') as html_close: html_close.write(''' </body> </html> ''') html_close.close() sys.exit(error_count)
from weather_master import PackageReader from aws import AWS print("Start Master Module Simulator") reader = PackageReader() print("\nRxDone") #payload = [0x0A, 0x05, 0x2E, 0x95, 0xF3, 0x71, 0x84, 0x00, 0x62, 0x11] payload = [ 0x10, 0x05, 0x2E, 0x8F, 0xED, 0x8C, 0x8E, 0x00, 0x64, 0x17, 0x3F, 0x2C, 0x23, 0x01, 0x89, 0x44 ] #10052E8FED8C8E0064173F2C23018944 print(bytes(payload).hex()) reader.read_package(payload, True) db = AWS() db.connect() db.publish_sensor_data(reader.data_pack, True) db.disconnect()
import os import shutil import sys import time import unittest # local modules from aws import AWS self = os.path.basename(sys.argv[0]) myName = os.path.splitext(self)[0] log = logging.getLogger(myName) logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') log.setLevel(logging.DEBUG) aws = AWS(logger=log) test_bucket = 'af-example1' test_instance_id = 'i-00d4bb1c2ee88275c' @unittest.skip("Skipping Test01") class Test01(unittest.TestCase): def test_get_buckets(self): buckets = aws.get_buckets() log.debug(buckets) self.assertGreater(len(buckets), 0) def test_get_instances(self): response = aws.get_instances() log.debug(response)
class InfrastructureTemplate(): u""" AWS Cloudformation Infrastructure Template """ def __init__(self): u""" Infrastructure Class Contructor """ self.aws = AWS() self.ami = AMI() self.ref_stack_id = Ref('AWS::StackId') self.ami_id = self.ami.minimal_linux_ami() # NOTE: Troposphere doesn't have a template feature to make KeyPairs # So handle this ad-hoc for now. self.keypair_name = 'test-deploy-keypair' if self.keypair_doesnt_exist(): self.create_keypair(self.keypair_name) self.deployment_bucket_prefix = 'test-deploy-bucket-' self.deployment_bucket_name = '{}{}'.format( self.deployment_bucket_prefix, uuid.uuid4().hex[:12].lower()) self.deployment_bucket_location = None if self.deploy_bucket_doesnt_exist(): self.deployment_bucket_location = self.create_deploy_bucket( self.deployment_bucket_name) else: self.deployment_bucket_location = self.get_bucket_url( self.deployment_bucket_name) self.server_certificate_name = 'test-deploy-certificate' self.server_certificate_arn = None if self.server_certificate_doesnt_exist(): self.server_certificate_arn = self.upload_server_certificate() self.template = Template() self.template.add_version('2010-09-09') self.template.add_description( 'AWS Cloudformation Template for autoscaled, load balance controlled EC2 service' ) self.template.add_parameter( Parameter('KeyName', Description='Name of an existing EC2 KeyPair', Default=self.keypair_name, Type='String')) self.template.add_parameter( Parameter('AmiId', Description='Lastest Minimal Linux AMI', Default=self.ami_id, Type='String')) self.template.add_parameter( Parameter('DeployBucketName', Description='Name of the deployment_bucket', Default=self.deployment_bucket_name, Type='String')) self.template.add_parameter( Parameter('DeployBucketLocation', Description='Location of the deployment_bucket', Default=self.deployment_bucket_location, Type='String')) self.template.add_parameter( Parameter('ServerCertificateArn', Description='Certificate ARN for the Load Balancer', Default=self.server_certificate_arn, Type='String')) self.sshlocation = self.template.add_parameter( Parameter( 'SSHLocation', Description= 'The IP address range that can be used to SSH to the EC2 instances', Type='String', MinLength='9', MaxLength='18', Default='0.0.0.0/0', AllowedPattern= r"(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})", ConstraintDescription=( "must be a valid IP CIDR range of the form x.x.x.x/x."))) self.vpc = self.template.add_resource( VPC('TestDeployVpc', CidrBlock='10.0.0.0/16', Tags=Tags(Application=self.ref_stack_id))) self.subnet = self.template.add_resource( Subnet('TestDeploySubnet', VpcId=Ref(self.vpc), CidrBlock='10.0.0.0/24', Tags=Tags(Application=self.ref_stack_id))) self.gateway = self.template.add_resource( InternetGateway('TestDeployGateway', Tags=Tags(Application=self.ref_stack_id))) self.gatewayattach = self.template.add_resource( VPCGatewayAttachment('AttachGateway', VpcId=Ref(self.vpc), InternetGatewayId=Ref(self.gateway))) self.route_table = self.template.add_resource( RouteTable('RouteTable', VpcId=Ref(self.vpc), Tags=Tags(Application=self.ref_stack_id))) self.route = self.template.add_resource( Route('Route', DependsOn='AttachGateway', GatewayId=Ref('TestDeployGateway'), DestinationCidrBlock='0.0.0.0/0', RouteTableId=Ref(self.route_table))) self.subnet_route_association = self.template.add_resource( SubnetRouteTableAssociation( 'SubnetRouteTableAssociation', SubnetId=Ref(self.subnet), RouteTableId=Ref(self.route_table), DependsOn=['TestDeploySubnet', 'RouteTable'])) self.network_acl = self.template.add_resource( NetworkAcl('NetworkAcl', VpcId=Ref(self.vpc), Tags=Tags(Application=self.ref_stack_id))) self.inbound_private_http = self.template.add_resource( NetworkAclEntry('InboundHTTP', NetworkAclId=Ref(self.network_acl), RuleNumber='100', Protocol='6', PortRange=PortRange(To='80', From='80'), Egress='false', RuleAction='allow', CidrBlock='0.0.0.0/0')) self.inbound_private_http_alt = self.template.add_resource( NetworkAclEntry('InboundHTTPAlt', NetworkAclId=Ref(self.network_acl), RuleNumber='101', Protocol='6', PortRange=PortRange(To='8000', From='8000'), Egress='false', RuleAction='allow', CidrBlock='0.0.0.0/0')) self.inbound_private_https = self.template.add_resource( NetworkAclEntry('InboundHTTPS', NetworkAclId=Ref(self.network_acl), RuleNumber='102', Protocol='6', PortRange=PortRange(To='443', From='443'), Egress='false', RuleAction='allow', CidrBlock='0.0.0.0/0')) self.inbound_ssh = self.template.add_resource( NetworkAclEntry('InboundSSH', NetworkAclId=Ref(self.network_acl), RuleNumber='103', Protocol='6', PortRange=PortRange(To='22', From='22'), Egress='false', RuleAction='allow', CidrBlock='0.0.0.0/0')) self.inbound_response = self.template.add_resource( NetworkAclEntry('InboundResponsePorts', NetworkAclId=Ref(self.network_acl), RuleNumber='104', Protocol='6', PortRange=PortRange(To='65535', From='1024'), Egress='false', RuleAction='allow', CidrBlock='0.0.0.0/0')) self.outbound_http = self.template.add_resource( NetworkAclEntry('OutboundHTTP', NetworkAclId=Ref(self.network_acl), RuleNumber='100', Protocol='6', PortRange=PortRange(To='80', From='80'), Egress='true', RuleAction='allow', CidrBlock='0.0.0.0/0')) self.outbound_https = self.template.add_resource( NetworkAclEntry('OutboundHTTPS', NetworkAclId=Ref(self.network_acl), RuleNumber='101', Protocol='6', PortRange=PortRange(To='443', From='443'), Egress='true', RuleAction='allow', CidrBlock='0.0.0.0/0')) self.outbound_response = self.template.add_resource( NetworkAclEntry('OutboundResponsePorts', NetworkAclId=Ref(self.network_acl), RuleNumber='102', Protocol='6', PortRange=PortRange(To='65535', From='1024'), Egress='true', RuleAction='allow', CidrBlock='0.0.0.0/0')) self.subnet_network_association = self.template.add_resource( SubnetNetworkAclAssociation( 'SubnetNetworkACLAssociation', SubnetId=Ref(self.subnet), NetworkAclId=Ref(self.network_acl), DependsOn=['TestDeploySubnet', 'NetworkAcl'])) self.instance_security_group = self.template.add_resource( SecurityGroup('InstanceSecurityGroup', GroupDescription='Open all ports', SecurityGroupIngress=[ SecurityGroupRule(IpProtocol='tcp', FromPort='22', ToPort='22', CidrIp='0.0.0.0/0'), SecurityGroupRule(IpProtocol='tcp', FromPort='1024', ToPort='65535', CidrIp='0.0.0.0/0') ], SecurityGroupEgress=[ SecurityGroupRule(IpProtocol='tcp', FromPort='1', ToPort='65535', CidrIp='0.0.0.0/0') ], VpcId=Ref(self.vpc))) self.instance = self.template.add_resource( Instance( 'TestDeployInstance', ImageId=Ref('AmiId'), InstanceType='t2.micro', KeyName=Ref('KeyName'), NetworkInterfaces=[ NetworkInterfaceProperty( GroupSet=[Ref('InstanceSecurityGroup')], AssociatePublicIpAddress='true', DeviceIndex='0', DeleteOnTermination='true', SubnetId=Ref('TestDeploySubnet')) ], UserData=Base64( Join('', [ "#!/bin/bash\n", "apt-get update\n", "apt-get -y install python python-pip python-setuptools\n", "mkdir aws-cfn-bootstrap-latest\n", "curl https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz | tar xz -C aws-cfn-bootstrap-latest --strip-components 1\n", "easy_install aws-cfn-bootstrap-latest\n", "/usr/local/bin/cfn-init --stack ", { "Ref": "AWS::StackName" }, " --resource TestDeployInstance", " --region ", { "Ref": "AWS::Region" }, "\n", "/usr/local/bin/cfn-signal --exit-code $? '", { "Ref": "WaitHandle" }, "'\n" "\n", "python -m SimpleHTTPServer 8000 2>&1 >/dev/null &\n", ])), DependsOn=['InstanceSecurityGroup', 'TestDeploySubnet'], Tags=Tags(Application=self.ref_stack_id))) # self.load_balancer_security_group = self.template.add_resource( # SecurityGroup('LoadBalancerSecurityGroup', # GroupDescription='Open all ports', # SecurityGroupIngress=[ # SecurityGroupRule( # IpProtocol='tcp', # FromPort='1', # ToPort='65535', # CidrIp='0.0.0.0/0')], # SecurityGroupEgress=[ # SecurityGroupRule( # IpProtocol='tcp', # FromPort='1', # ToPort='65535', # CidrIp='0.0.0.0/0')], # VpcId=Ref(self.vpc))) # self.launch_configuration = self.template.add_resource( # LaunchConfiguration('LaunchConfiguration', # AssociatePublicIpAddress=True, # UserData=Base64(Join('', [ # "#!/bin/bash\n", # "sudo pip install ansible\n", # "sudo pip install SimpleHTTPServer\n", # "python -m SimpleHTTPServer 8000 2>&1 >/dev/null &\n", # "cfn-signal -e 0", # " --resource AutoScalingGroup", # " --stack ", Ref("AWS::StackName"), # " --region ", Ref("AWS::Region"), "\n" # ])), # ImageId=Ref('AmiId'), # KeyName=Ref('KeyName'), # BlockDeviceMappings=[ # ec2.BlockDeviceMapping( # DeviceName="/dev/sda1", # Ebs=ec2.EBSBlockDevice( # VolumeSize="8" # ) # ) # ], # SecurityGroups=[Ref('InstanceSecurityGroup')], # InstanceType="t2.micro", # DependsOn='InstanceSecurityGroup')) # self.load_balancer = self.template.add_resource( # LoadBalancer( # "LoadBalancer", # ConnectionDrainingPolicy=elb.ConnectionDrainingPolicy( # Enabled=True, # Timeout=120, # ), # Subnets=[Ref('TestDeploySubnet')], # HealthCheck=elb.HealthCheck( # Target='HTTP:8000/', # HealthyThreshold='10', # UnhealthyThreshold='10', # Interval='300', # Timeout='60' # ), # Listeners=[ # elb.Listener( # LoadBalancerPort='443', # InstancePort='8000', # Protocol='HTTPS', # InstanceProtocol='HTTP', # SSLCertificateId=Ref('ServerCertificateArn') # ), # elb.Listener( # LoadBalancerPort='22', # InstancePort='22', # Protocol='TCP', # InstanceProtocol='TCP' # ) # ], # CrossZone=True, # SecurityGroups=[Ref('LoadBalancerSecurityGroup')], # LoadBalancerName='api-lb', # Scheme='internet-facing', # DependsOn=['LoadBalancerSecurityGroup', # 'TestDeploySubnet'])) # self.auto_scaling_group = self.template.add_resource( # AutoScalingGroup( # "AutoscalingGroup", # DesiredCapacity=1, # Tags=[ # Tag("Environment", 'EnvType', True) # ], # LaunchConfigurationName=Ref(self.launch_configuration), # MinSize=1, # MaxSize=2, # VPCZoneIdentifier=[Ref(self.subnet)], # LoadBalancerNames=[Ref(self.load_balancer)], # HealthCheckType='EC2', # UpdatePolicy=UpdatePolicy( # AutoScalingReplacingUpdate=AutoScalingReplacingUpdate( # WillReplace=True, # ), # AutoScalingRollingUpdate=AutoScalingRollingUpdate( # PauseTime='PT5M', # MinInstancesInService="1", # MaxBatchSize='1', # WaitOnResourceSignals=True # ) # ) # )) @staticmethod def get_bucket_url(bucket_name): u""" Generates the bucket location """ return 'https://{}.s3.amazonaws.com/'.format(bucket_name) def keypair_doesnt_exist(self): u""" Check to see if EC2 keypair exists """ client = self.aws.get_client('ec2') try: client.describe_key_pairs(KeyNames=[self.keypair_name]) except ClientError: return True return False def create_keypair(self, keypair_name): u""" Create a keypair resource """ client = self.aws.get_client('ec2') try: res = client.create_key_pair(KeyName=keypair_name) print(res['KeyMaterial']) open('ansible/test-deploy.pem', 'w').write(res['KeyMaterial']) except ClientError as ex: sys.exit(ex) def deploy_bucket_doesnt_exist(self): u""" Check to see if the deployment bucket exists """ client = self.aws.get_client('s3') try: res = client.list_buckets() for bucket in res['Buckets']: if self.deployment_bucket_prefix in bucket['Name']: self.delpoyment_bucket_name = bucket['Name'] return False return True except ClientError as ex: sys.exit(ex) def create_deploy_bucket(self, bucket_name): u""" Create static deployment bucket """ client = self.aws.get_client('s3') try: res = client.create_bucket(ACL='private', Bucket=bucket_name, CreateBucketConfiguration=\ {'LocationConstraint': self.aws.session.region_name}) print('Create Bucket: {}'.format(res)) return res['Location'] except ClientError as ex: sys.exit(ex) def server_certificate_doesnt_exist(self): u""" Check for the presence of the server certificate """ client = self.aws.get_client('iam') res = client.list_server_certificates() for cert in res['ServerCertificateMetadataList']: if self.server_certificate_name in cert['ServerCertificateName']: self.server_certificate_arn = cert['Arn'] return False return True def upload_server_certificate(self): u""" Upload server certificate to AWS ACM """ client = self.aws.get_client('iam') certificate = open('cert.pem', 'r').read() private_key = open('key.pem', 'r').read() try: res = client.upload_server_certificate( ServerCertificateName=self.server_certificate_name, CertificateBody=certificate, PrivateKey=private_key) from pprint import pprint as pp pp('Uploaded: {}'.format(res)) return res['ServerCertificateMetadata']['Arn'] except ClientError as ex: print(ex) sys.exit() def cleanup(self): u""" Cleanup static resources """ self.destroy_deploy_bucket() self.destroy_ec2_keypair(self.keypair_name) self.delete_server_certificate(self.server_certificate_name) def destroy_deploy_bucket(self): u""" Destroy static deployment bucket(s) """ client = self.aws.get_client('s3') buckets = client.list_buckets() for bucket in buckets['Buckets']: objects = client.list_objects_v2(Bucket=bucket['Name']) while True: if 'Contents' not in objects: break for s3_object in objects['Contents']: if 'test-deploy-bucket-' in s3_object['Key']: try: client.delete_object(Bucket=bucket['Name'], Key=s3_object['Key']) except ClientError as ex: print(ex) continue if objects['IsTruncated']: token = objects['ContinuationToken'] objects = client.list_objects_v2(bucket['Name'], ContinuationToken=token) else: break result = client.delete_bucket(Bucket=bucket['Name']) print(result) def destroy_ec2_keypair(self, keypair): u""" Destroy static ec2 keypair resources """ client = self.aws.get_client('ec2') try: client.delete_key_pair(KeyName=keypair) except ClientError as ex: print(ex) def delete_server_certificate(self, certificate_name): u""" Remove certificate from AWS Certificate Manager """ client = self.aws.get_client('iam') try: client.delete_server_certificate( ServerCertificateName=certificate_name) except ClientError as ex: print(ex) def print_template(self, output='yaml'): u""" Dump Cloudformation Template """ if 'yaml' not in output: print(self.template.to_json()) else: print(self.template.to_yaml()) def generate_template(self, output='yaml'): u""" Return the Cloudformation Template Body """ if 'yaml' not in output: return self.template.to_json() return self.template.to_yaml() def prepare_payload(self): u""" Collect all payload files in a zip archive """ if os.path.isfile('test-deploy-files.zip'): os.remove('test-deploy-files.zip') zipf = zipfile.ZipFile('test-deploy-files.zip', 'w', zipfile.ZIP_DEFLATED) self.zipdir('ansible/', zipf) zipf.close() @staticmethod def zipdir(path, zipf): u""" Put all the contents of a directory into a zipfile """ for root, _, files in os.walk(path): for file in files: zipf.write(os.path.join(root, file)) def deliver_payload(self, bucket_name): u""" Put the necessary files into S3 """ client = self.aws.get_client('s3') res = client.upload_file('test-deploy-files.zip', bucket_name, Key='test-delpoy-files.zip') from pprint import pprint as pp pp(res)
class LoRaMaster(LoRa): def __init(self): super(LoRaRcvCont, self).__init__(verbose) self.set_mode(MODE.STDBY) #self.set_dio_mapping([0] * 6) self.db = AWS() def on_rx_done(self): BOARD.led_on() self.clear_irq_flags(RxDone=1) reader.read_package(self.read_payload(nocheck=True), True) self.set_mode(MODE.SLEEP) self.reset_ptr_rx() BOARD.led_off() self.set_mode(MODE.RXCONT) if (reader.is_accepted): self.db.connect() self.db.publish_sensor_data(reader.data_pack, True) self.db.disconnect() def on_txdone(self): print("\nTxDone") print(self.get_irq_flags()) def on_cad_done(self): print("\non_CadDone") print(self.get_irq_flags()) def on_rx_timeout(self): print("\non_RxTimeout") print(self.get_irq_flags()) time.sleep(.5) self.set_mode(MODE.SLEEP) self.reset_ptr_rx() self.set_mode(MODE.RXCONT) def on_valid_header(self): print("\non_ValidHeader") print(self.get_irq_flags()) def on_payload_crc_error(self): print("\non_PayloadCrcError") print(self.get_irq_flags()) def on_fhss_change_channel(self): print("\non_Fhss_changeChannel") print(self.get_irq_flags()) def print_payload(self, payload): chex_sum = payload[0] cmd = payload[1] print("chex_sum: \t{}".format(chex_sum)) print("payload len: \t{}".format(len(payload))) if (chex_sum == len(payload)): print("payload of correct length") else: print("payload of incorrect length") print("command: \t{}".format(cmd)) def start(self): self.db = AWS() self.reset_ptr_rx() self.set_mode(MODE.RXCONT) while True: time.sleep(.5) rssi_value = self.get_rssi_value() snr_value = self.get_pkt_snr_value() status = self.get_modem_status() sys.stdout.flush() sys.stdout.write("\r%d %d %d %d" % (rssi_value, snr_value, status['rx_ongoing'], status['modem_clear']))
print("Unlocking Door") lock.unlock_door() print("Door Locked") # if audio is set to new in Firebase, download the latest audio file and play it via the speaker if audio == "new": fb.get_storage() os.system("omxplayer audioVisitor.mp3") fb.update_data({'doorbell/audio/state': 'waiting'}) if __name__ == '__main__': # Firebase class instance fb = Firebase() # AWS class instance aws = AWS(fb) # NewFace class instance nf = NewFace(fb) # Solenoid Lock class instance lock = Solenoid() # Face Check class instance fc = FaceCheck() # Doorbell button class instance bb = BellButton(aws, nf, switch_face_check_on, switch_face_check_off) # Start the BellButton Thread class bb.start() # Start the Face Check Thread class fc.start() # Turn on face check switch_face_check_on() # Run the listen function
class ETL: def __init__(self): self.dao = DAO() self.aws = AWS() def check_monthly_ranking(self): intervals = ["1_20", "21_40", "41_60", "61_80", "81_100"] for interval in intervals: url = "http://www.dmm.co.jp/mono/dvd/-/ranking/=/term=monthly/mode=actress/rank=" + interval r = requests.get(url) soup = bs(r.text) actresses = soup.find_all("td", {"class": "bd-b"}) for actress in actresses: actress_a = actress.find("a") pattern = re.compile( "/mono/dvd/-/list/=/article=actress/id=(.*)/") match = pattern.search(actress_a.get("href")) actress_id = match.group(1) detail = self.dao.find_one_actress_by_id(actress_id) if detail is None or detail.get("name") is None: img = actress_a.find("img") actress_img = img.get("src").replace('medium/', '') actress_name = img.get("alt").encode('utf-8') print actress_name self.dao.update_one_info_by_actress({ "id": actress_id, "name": actress_name, "img": actress_img }) self.aws.insert_index_face( actress_id, urllib2.urlopen(actress_img).read()) def check_new_actress(self): self.get_new_actress("http://actress.dmm.co.jp/-/top/", "act-box-125 group", 1, "/-/detail/=/actress_id=(.*)/") self.get_new_actress("http://www.dmm.co.jp/mono/dvd/-/actress/", "act-box-100 group mg-b20", 0, "/mono/dvd/-/list/=/article=actress/id=(.*)/") def get_new_actress(self, url, cssClass, index, hrefPattern): r = requests.get(url) soup = bs(r.text) act_box = soup.find_all("ul", {"class": cssClass}) actresses = act_box[index].find_all("a") for actress in actresses: pattern = re.compile(hrefPattern) match = pattern.search(actress.get("href")) actress_id = match.group(1) detail = self.dao.find_one_actress_by_id(actress_id) if detail is None or detail.get("name") is None: actress_img = actress.find("img").get("src").replace( 'medium/', '') actress_name = actress.text.encode('utf-8') print actress_name self.dao.update_one_info_by_actress({ "id": actress_id, "name": actress_name, "img": actress_img }) self.aws.insert_index_face(actress_id, urllib2.urlopen(actress_img).read()) def check_new_works(self): now = datetime.datetime.now() year = str(now.year) month = str(now.month) day = str(now.day) url = "http://www.dmm.co.jp/mono/dvd/-/calendar/=/month=" + month + "/year=" + year + "/day=" + day + "-" + day + "/" r = requests.get(url) soup = bs(r.text) print datetime.date.today() cal = soup.find("table", {"id": "monocal"}) works_list = cal.find_all("tr") if len(works_list) == 0: return for works in works_list: actress_tag = works.find("td", {"class": "info-01"}) if actress_tag is None or actress_tag.text == "----": continue pattern = re.compile("/mono/dvd/-/list/=/article=actress/id=(.*)/") match = pattern.search(actress_tag.find("a").get("href")) actress_id = match.group(1) if self.dao.find_one_actress_by_id(actress_id) is None: continue title_tag = works.find("td", {"class": "title-monocal"}) title = title_tag.find("a") title_name = title.text pattern = re.compile( ur"(^(【数量限定】|【DMM限定】|【DMM限定販売】|【アウトレット】|【特選アウトレット】)|(ブルーレイディスク)$)", re.UNICODE) match = re.search(pattern, title_name) if match: continue title_url = "http://www.dmm.co.jp" + title.get("href") detail = self.get_works_detail(title_url, actress_id) if detail is None: continue else: print detail.get("img") self.aws.insert_index_face( detail.get("id"), urllib2.urlopen(detail.get("img")).read()) def get_works_detail(self, url, actress_id): r = requests.get(url) soup = bs(r.text) sample = soup.find("div", {"class": "tx10 pd-3 lh4"}) if sample is None: return # No Image a_tag = sample.find("a") if a_tag is None: return print a_tag.get('href') performer = soup.find("span", {"id": "performer"}) performer_a_tag = performer.find_all("a") if len(performer_a_tag) == 1: pattern = re.compile("/mono/dvd/-/detail/=/cid=(.*)/") match = pattern.search(url) cid = match.group(1) print cid project_works = self.dao.find_one_works_by_id(actress_id) print project_works if bool(project_works) and any( cid in s for s in project_works.get("works")): return else: self.dao.update_one_works_by_id(actress_id, cid) self.get_sample(soup, actress_id) return {"id": actress_id, "img": a_tag.get('href')} else: return def get_sample(self, soup, actress_id): sample_image = soup.find("div", {"id": "sample-image-block"}) if sample_image is None: return sample_head = soup.find("div", {"class": "headline mg-b10 lh3"}) sample_small = sample_head.find("span", {"class": "nw"}) if sample_small is not None: return images = map(lambda tag: re.sub(r'-(\d+)', r'jp-\1', tag.get("src")), sample_image.find_all("img")) for image in images: print image self.aws.insert_index_face(actress_id, urllib2.urlopen(image).read())
def __init__(self): u""" Infrastructure Class Contructor """ self.aws = AWS() self.ami = AMI() self.ref_stack_id = Ref('AWS::StackId') self.ami_id = self.ami.minimal_linux_ami() # NOTE: Troposphere doesn't have a template feature to make KeyPairs # So handle this ad-hoc for now. self.keypair_name = 'test-deploy-keypair' if self.keypair_doesnt_exist(): self.create_keypair(self.keypair_name) self.deployment_bucket_prefix = 'test-deploy-bucket-' self.deployment_bucket_name = '{}{}'.format( self.deployment_bucket_prefix, uuid.uuid4().hex[:12].lower()) self.deployment_bucket_location = None if self.deploy_bucket_doesnt_exist(): self.deployment_bucket_location = self.create_deploy_bucket( self.deployment_bucket_name) else: self.deployment_bucket_location = self.get_bucket_url( self.deployment_bucket_name) self.server_certificate_name = 'test-deploy-certificate' self.server_certificate_arn = None if self.server_certificate_doesnt_exist(): self.server_certificate_arn = self.upload_server_certificate() self.template = Template() self.template.add_version('2010-09-09') self.template.add_description( 'AWS Cloudformation Template for autoscaled, load balance controlled EC2 service' ) self.template.add_parameter( Parameter('KeyName', Description='Name of an existing EC2 KeyPair', Default=self.keypair_name, Type='String')) self.template.add_parameter( Parameter('AmiId', Description='Lastest Minimal Linux AMI', Default=self.ami_id, Type='String')) self.template.add_parameter( Parameter('DeployBucketName', Description='Name of the deployment_bucket', Default=self.deployment_bucket_name, Type='String')) self.template.add_parameter( Parameter('DeployBucketLocation', Description='Location of the deployment_bucket', Default=self.deployment_bucket_location, Type='String')) self.template.add_parameter( Parameter('ServerCertificateArn', Description='Certificate ARN for the Load Balancer', Default=self.server_certificate_arn, Type='String')) self.sshlocation = self.template.add_parameter( Parameter( 'SSHLocation', Description= 'The IP address range that can be used to SSH to the EC2 instances', Type='String', MinLength='9', MaxLength='18', Default='0.0.0.0/0', AllowedPattern= r"(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})", ConstraintDescription=( "must be a valid IP CIDR range of the form x.x.x.x/x."))) self.vpc = self.template.add_resource( VPC('TestDeployVpc', CidrBlock='10.0.0.0/16', Tags=Tags(Application=self.ref_stack_id))) self.subnet = self.template.add_resource( Subnet('TestDeploySubnet', VpcId=Ref(self.vpc), CidrBlock='10.0.0.0/24', Tags=Tags(Application=self.ref_stack_id))) self.gateway = self.template.add_resource( InternetGateway('TestDeployGateway', Tags=Tags(Application=self.ref_stack_id))) self.gatewayattach = self.template.add_resource( VPCGatewayAttachment('AttachGateway', VpcId=Ref(self.vpc), InternetGatewayId=Ref(self.gateway))) self.route_table = self.template.add_resource( RouteTable('RouteTable', VpcId=Ref(self.vpc), Tags=Tags(Application=self.ref_stack_id))) self.route = self.template.add_resource( Route('Route', DependsOn='AttachGateway', GatewayId=Ref('TestDeployGateway'), DestinationCidrBlock='0.0.0.0/0', RouteTableId=Ref(self.route_table))) self.subnet_route_association = self.template.add_resource( SubnetRouteTableAssociation( 'SubnetRouteTableAssociation', SubnetId=Ref(self.subnet), RouteTableId=Ref(self.route_table), DependsOn=['TestDeploySubnet', 'RouteTable'])) self.network_acl = self.template.add_resource( NetworkAcl('NetworkAcl', VpcId=Ref(self.vpc), Tags=Tags(Application=self.ref_stack_id))) self.inbound_private_http = self.template.add_resource( NetworkAclEntry('InboundHTTP', NetworkAclId=Ref(self.network_acl), RuleNumber='100', Protocol='6', PortRange=PortRange(To='80', From='80'), Egress='false', RuleAction='allow', CidrBlock='0.0.0.0/0')) self.inbound_private_http_alt = self.template.add_resource( NetworkAclEntry('InboundHTTPAlt', NetworkAclId=Ref(self.network_acl), RuleNumber='101', Protocol='6', PortRange=PortRange(To='8000', From='8000'), Egress='false', RuleAction='allow', CidrBlock='0.0.0.0/0')) self.inbound_private_https = self.template.add_resource( NetworkAclEntry('InboundHTTPS', NetworkAclId=Ref(self.network_acl), RuleNumber='102', Protocol='6', PortRange=PortRange(To='443', From='443'), Egress='false', RuleAction='allow', CidrBlock='0.0.0.0/0')) self.inbound_ssh = self.template.add_resource( NetworkAclEntry('InboundSSH', NetworkAclId=Ref(self.network_acl), RuleNumber='103', Protocol='6', PortRange=PortRange(To='22', From='22'), Egress='false', RuleAction='allow', CidrBlock='0.0.0.0/0')) self.inbound_response = self.template.add_resource( NetworkAclEntry('InboundResponsePorts', NetworkAclId=Ref(self.network_acl), RuleNumber='104', Protocol='6', PortRange=PortRange(To='65535', From='1024'), Egress='false', RuleAction='allow', CidrBlock='0.0.0.0/0')) self.outbound_http = self.template.add_resource( NetworkAclEntry('OutboundHTTP', NetworkAclId=Ref(self.network_acl), RuleNumber='100', Protocol='6', PortRange=PortRange(To='80', From='80'), Egress='true', RuleAction='allow', CidrBlock='0.0.0.0/0')) self.outbound_https = self.template.add_resource( NetworkAclEntry('OutboundHTTPS', NetworkAclId=Ref(self.network_acl), RuleNumber='101', Protocol='6', PortRange=PortRange(To='443', From='443'), Egress='true', RuleAction='allow', CidrBlock='0.0.0.0/0')) self.outbound_response = self.template.add_resource( NetworkAclEntry('OutboundResponsePorts', NetworkAclId=Ref(self.network_acl), RuleNumber='102', Protocol='6', PortRange=PortRange(To='65535', From='1024'), Egress='true', RuleAction='allow', CidrBlock='0.0.0.0/0')) self.subnet_network_association = self.template.add_resource( SubnetNetworkAclAssociation( 'SubnetNetworkACLAssociation', SubnetId=Ref(self.subnet), NetworkAclId=Ref(self.network_acl), DependsOn=['TestDeploySubnet', 'NetworkAcl'])) self.instance_security_group = self.template.add_resource( SecurityGroup('InstanceSecurityGroup', GroupDescription='Open all ports', SecurityGroupIngress=[ SecurityGroupRule(IpProtocol='tcp', FromPort='22', ToPort='22', CidrIp='0.0.0.0/0'), SecurityGroupRule(IpProtocol='tcp', FromPort='1024', ToPort='65535', CidrIp='0.0.0.0/0') ], SecurityGroupEgress=[ SecurityGroupRule(IpProtocol='tcp', FromPort='1', ToPort='65535', CidrIp='0.0.0.0/0') ], VpcId=Ref(self.vpc))) self.instance = self.template.add_resource( Instance( 'TestDeployInstance', ImageId=Ref('AmiId'), InstanceType='t2.micro', KeyName=Ref('KeyName'), NetworkInterfaces=[ NetworkInterfaceProperty( GroupSet=[Ref('InstanceSecurityGroup')], AssociatePublicIpAddress='true', DeviceIndex='0', DeleteOnTermination='true', SubnetId=Ref('TestDeploySubnet')) ], UserData=Base64( Join('', [ "#!/bin/bash\n", "apt-get update\n", "apt-get -y install python python-pip python-setuptools\n", "mkdir aws-cfn-bootstrap-latest\n", "curl https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz | tar xz -C aws-cfn-bootstrap-latest --strip-components 1\n", "easy_install aws-cfn-bootstrap-latest\n", "/usr/local/bin/cfn-init --stack ", { "Ref": "AWS::StackName" }, " --resource TestDeployInstance", " --region ", { "Ref": "AWS::Region" }, "\n", "/usr/local/bin/cfn-signal --exit-code $? '", { "Ref": "WaitHandle" }, "'\n" "\n", "python -m SimpleHTTPServer 8000 2>&1 >/dev/null &\n", ])), DependsOn=['InstanceSecurityGroup', 'TestDeploySubnet'], Tags=Tags(Application=self.ref_stack_id)))
def search_amazon(request, isbn): from aws import AWS import xml.etree.ElementTree as ET import urllib2 import logging from datetime import datetime, date XMLNS = "{http://webservices.amazon.com/AWSECommerceService/2010-06-01}" import myconfig aws = AWS() request_url = aws.doItemLookUp(isbn) logging.info(request_url) result = urllib2.urlopen(request_url) tree = ET.parse(result) root = tree.getroot() asin="" detail_page_shop_url="" title="" author="" publisher="" publication_date="" price="" image_url="" err_msg="" for node in root.getchildren(): if node.tag == XMLNS + "Items": for subnode in node.getchildren(): if subnode.tag == XMLNS + "Request": for subnode2 in subnode.getchildren(): if subnode2.tag == XMLNS + "Errors": for errnode in subnode2.getchildren(): if errnode.tag == XMLNS + "Error": for msg in errnode.getchildren(): if msg.tag == XMLNS + "Message": err_msg = msg.text if subnode.tag == XMLNS + "Item": for item in subnode.getchildren(): if item.tag == XMLNS + "ASIN": asin = item.text if item.tag == XMLNS + "DetailPageURL": detail_page_shop_url = item.text if item.tag == XMLNS + "MediumImage": for imageurl in item.getchildren(): if imageurl.tag == XMLNS + "URL": image_url = imageurl.text if item.tag == XMLNS + "ItemAttributes": for item_attr in item.getchildren(): if item_attr.tag == XMLNS + "Title": title = item_attr.text if item_attr.tag == XMLNS + "Author": author = item_attr.text if item_attr.tag == XMLNS + "ListPrice": for listprice in item_attr.getchildren(): if listprice.tag == XMLNS + "Amount": price = listprice.text if item_attr.tag == XMLNS + "Publisher": publisher = item_attr.text if item_attr.tag == XMLNS + "PublicationDate": publication_date = item_attr.text try: price = int(price) except ValueError: price = 0 try: st = datetime.strptime(publication_date,'%Y-%m-%d') pub_date = date(st.year, st.month, st.day) except ValueError: try: st = datetime.strptime(publication_date,'%Y-%m') pub_date = date(st.year, st.month, st.day) except ValueError: pub_date = None book = Book(asin=asin, isbn=isbn,detail_page_shop_url=detail_page_shop_url,title=title,author=author, publisher=publisher,publication_date=pub_date,price=price,image_url=image_url, tags=[], deletion_reserve=False, lending=False, version=1) form = BookForm(instance=book) return render_to_response('register/search_result.html', {'form': form.as_widget()})
def __init(self): super(LoRaRcvCont, self).__init__(verbose) self.set_mode(MODE.STDBY) #self.set_dio_mapping([0] * 6) self.db = AWS()
def __init__(self): self.dao = DAO() self.aws = AWS()
class Resources(object): def __init__(self): self.instances = {} self.volumes = {} self.snapshots = {} self.filtered_tag_keys = [] self.require_tags_instance = [] self.metric_prefix = 'resource_tagger_' self.metrics = {} self.aws = AWS() self.tag_default_copy_key = '' self.tag_default_copy_split = ['-', 2] self.setup() def setup(self): self.instances.clear() self.volumes.clear() self.snapshots.clear() # Filters fk = os.getenv("TAG_FILTER_KEYS_INSTANCE" or []) if isinstance(fk, list): self.filtered_tag_keys = fk elif isinstance(fk, str): self.filtered_tag_keys = fk.split(',') else: self.filtered_tag_keys = [] # Require tags for Instance rt = os.getenv("TAG_REQUIRED_KEYS_INSTANCE" or []) if isinstance(rt, list): self.require_tags_instance = rt elif isinstance(rt, str): self.require_tags_instance = rt.split(',') else: self.require_tags_instance = [] # Copy tag value when it's defined self.tag_default_copy_key = os.getenv("TAG_DEFAULT_COPY_KEY" or '') ts = os.getenv("TAG_DEFAULT_COPY_SPLIT" or '') if isinstance(ts, list): self.tag_default_copy_split = ts elif isinstance(ts, str): self.tag_default_copy_split = ts.split(',') else: self.tag_default_copy_split = [] # Metrics self.metrics = { "total_instances": 0, "total_volumes": 0, "total_snapshots": 0, } def load_info_instances(self): for instance in self.aws.get_instances(): try: i = self.instances[instance["resourceId"]] except KeyError: self.instances[instance["resourceId"]] = {} i = self.instances[instance["resourceId"]] i["volumes"] = {} i["volumes_ebs"] = {} i["tags"] = copy.deepcopy(utils.tag_list_to_dict(instance["tags"])) try: i["imageId"] = instance["configuration"]["imageId"] except: pass for b in instance["configuration"]["blockDeviceMappings"]: try: i["volumes_ebs"][b["ebs"]["volumeId"]] = b["deviceName"] except KeyError: i["volumes"][b["deviceName"]] = b def init_info_instances(self): """Init will load instances to dict only if the instances is empty """ if len(self.instances) <= 0: self.load_info_instances() return def load_info_volumes(self): """Load all valumes that has not tags """ self.init_info_instances() volumes = self.aws.get_volumes() for vol in volumes: if 'Tags' in volumes[vol]: # Ignor volumes that alread have tags. # TODO: enforce default tags when it's not present. continue v = copy.deepcopy(volumes[vol]) if vol not in self.volumes: self.volumes[vol] = {} if v['SnapshotId'] != "": self.volumes[vol]["snapshoot"] = v['SnapshotId'] try: s = self.snapshots[v['SnapshotId']] except KeyError: self.snapshots[v['SnapshotId']] = {} s = self.snapshots[v['SnapshotId']] s["volume_id"] = vol if len(v["Attachments"]) > 0: for a in v["Attachments"]: try: cur_tags = self.tag_filter( self.instances[a['InstanceId']]["tags"]) # self.volumes[vol]["instance_tags"] = new_tags except KeyError: cur_tags = {"error": "Instance tag's is NotFound."} self.volumes[vol]["instance_attached"] = a['InstanceId'] self.volumes[vol]["attached"] = "yes" new_tags = copy.deepcopy(cur_tags) try: device = self.instances[ a['InstanceId']]["volumes_ebs"][vol] except KeyError: device = '' if 'Name' in new_tags: new_tags["Name"] += " " + device self.volumes[vol]["tags"] = new_tags else: self.volumes[vol]["attached"] = "no" return def show_report(self): if LOG_LEVEL == "DEBUG": print(">> EC2 Instances: ") pprint(self.instances) print(">> Snapshoots: ") pprint(self.snapshots) print(">> Untagged volumes: ") pprint(self.volumes) msg = ("Total untagged volumes: {}".format(len(self.volumes))) #logger.info(msg) print(msg) msg = ("Total snapshoots to tag: {}".format(len(self.snapshots))) #logger.info(msg) print(msg) def add_metrics(self): self.aws.add_metrics( data={ "name": "total_untagged_resources", "value": len(self.volumes), "dimensions": { "Name": "resource", "Value": "volumes" } }) def tag_filter(self, tags): tags_filtered = {} tags_filtered.clear() for t in tags: if t in self.filtered_tag_keys: tags_filtered.update({t: tags[t]}) return tags_filtered def apply_tags_volumes(self): self.load_info_volumes() messages = [] if len(self.volumes) <= 0: return for vol in self.volumes.keys(): volume = self.volumes[vol] try: if volume["attached"] != "yes": messages.append( "ignoring volume {}=unattached to instance".format( vol)) continue except KeyError: pass try: if len(volume["tags"]) <= 0: messages.append( "ignoring volume {}=empty tags".format(vol)) continue if 'error' in volume["tags"]: messages.append("ignoring volume {}={}".format( vol, str(volume["tags"]))) continue except KeyError as e: messages.append("ignoring volume {}=KeyError {}".format( vol, e)) continue msg = ("{}={}".format(vol, str(volume["tags"]))) logging.info(msg) messages.append(msg) self.aws.clients["ec2"].create_tags(Resources=[vol], Tags=utils.tag_dict_to_list( volume["tags"])) print("Tags applied to volumes: {}".format(json.dumps(messages))) return def apply_tags_snapshots(self): "TODO: use same tags of Volumes" return def apply_tags_instances(self): """ TODO: - use global/default tags. - Overwrite existing? - Useful to enforce to whole account; but in shared Account? - how to distinguish between two resources? """ return def apply_tags_ec2_resource(self, resource_id, tags): return self.aws.clients["ec2"].create_tags(Resources=[resource_id], Tags=tags) def apply_tags_images(self): "TODO: what tags to use? Globals?" return def apply_tags_from_event(self, event): if 'instance-id' in event["detail"]: return self.process_event_instance(event) elif 'event' in event["detail"]: if event["detail"]["event"] == "createVolume": return self.process_event_volume(event) return {"error": "event not found", "event": event} def process_event_instance(self, event): instance_id = event["detail"]["instance-id"] print("Processing InstanceID: {}".format(instance_id)) instance = self.aws.get_instance_tags_api(instance_id) if not instance: print("ERR - No Tags or resource found for ID: {}".format( instance_id)) return instance_tags = utils.tag_list_to_dict(instance["Tags"]) missing_keys = self.check_required_tags(instance_tags) if len(missing_keys) <= 0: print( "That's OK, all required Tags was defined to resource_id: {}". format(instance_id)) try: self.check_tags_instance_dm(instance["BlockDeviceMappings"], instance_tags) except Exception as e: print( "Error - Tagging process was not complete in instance devices: {}" .format(e)) pass return try: vpc_id = instance["VpcId"] except: vpc_id = '' vpc = self.aws.get_vpc_tags(vpc_id) if len(vpc) <= 0: print("ERR - No VPC [{}] found, skipping tagger".format(vpc_id)) return try: vpc_tags = utils.tag_list_to_dict(vpc[0]["tags"]) except KeyError: print( "ERR - No VPC [{}] tags found, skipping tagger".format(vpc_id)) return except: raise # discovery and fill tags to apply based on required (missing) tags_to_apply = self.mount_required_tags_instance( missing_keys=missing_keys, vpc_tags=vpc_tags, instance_tags=instance_tags) msg = { "msg": "Applying tags to Instance", "tags_to_apply": tags_to_apply, "resource_id": instance_id, "current_resource_tags": instance_tags, "current_vpc_tags": vpc_tags, "found_missing_keys": missing_keys } print(msg) self.apply_tags_ec2_resource( resource_id=instance["InstanceId"], tags=utils.tag_dict_to_list(tags_to_apply)) if 'BlockDeviceMappings' not in instance: return return self.check_tags_instance_dm(instance["BlockDeviceMappings"], { **instance_tags, **tags_to_apply }) def check_required_tags(self, instance_tags): missing_keys = [] missing_keys.clear() for k in self.require_tags_instance: try: v = instance_tags[k] except KeyError: missing_keys.append(k) return missing_keys def mount_required_tags_instance(self, missing_keys, vpc_tags, instance_tags): """ Check the missing keys and mount the tags to apply based on filters, VPC and Instance tags. """ tags_to_apply = {} cnt = 0 for k in missing_keys: try: tags_to_apply[k] = vpc_tags[k] except KeyError: try: if self.tag_default_copy_key not in instance_tags: continue copy_tag = instance_tags[self.tag_default_copy_key] tag_value = '' cnt = 0 sep = self.tag_default_copy_split[0] offset = int(self.tag_default_copy_split[1]) for v in copy_tag.split(sep): tag_value += v cnt += 1 if cnt >= offset: break tag_value += '-' tags_to_apply[k] = tag_value except Exception as e: print("Unexpected error: ", e) raise return tags_to_apply def mount_required_tags_volume(self, missing_keys, instance_tags, volume_tags): """ Check the missing keys and mount the tags to apply based on Required tags (Env var), Instance Tags and Current Volume tags. """ tags_to_apply = {} cnt = 0 for k in missing_keys: try: tags_to_apply[k] = instance_tags[k] except KeyError: try: tags_to_apply[k] = 'missing-value' except Exception as e: print("Unexpected error: ", e) raise return tags_to_apply def check_tags_instance_dm(self, devices, itags): """Apply tags to Instance Devices """ for device in devices: try: ebs_map = device["Ebs"] except KeyError: print( "Error - 'Ebs' info is not found on device map: {}".format( device)) continue except Exception as e: print("Error - Ebs_map on check_tags_instance_dm(): {}".format( e)) continue try: self.apply_tags_volume(ebs_map["VolumeId"], itags, device["DeviceName"]) except Exception as e: print( "Unkwnown error on check_tags_instance_dm(): {}".format(e)) pass def apply_tags_volume(self, volume_id, itags, device_name): if not itags: print("ignoring volume {}=empty tags".format(volume_id)) return volume = self.aws.get_volume_tags_api(volume_id) if not volume: print("ERR - No Tags or resource found for ID: {}".format( instance_id)) return volume_tags = utils.tag_list_to_dict(volume["Tags"]) missing_keys = self.check_required_tags(volume_tags) if len(missing_keys) <= 0: print( "That's OK, all required Tags was defined to resource_id: {}". format(volume_id)) return # TODO double check: instance_id (owner of itags) is the same of Attachments.InstanceId tags_to_apply = self.mount_required_tags_volume( missing_keys=missing_keys, instance_tags=itags, volume_tags=volume_tags) if 'Name' in tags_to_apply: tags_to_apply["Name"] += " " + device_name else: tags_to_apply["Name"] = "{} {}".format(itags["Name"], device_name) msg = { "msg": "Applying tags to Volume", "tags_to_apply": tags_to_apply, "resource_id": volume_id, "current_resource_tags": volume_tags, "current_instance_tags": itags, "found_missing_keys": missing_keys } print(msg) return self.apply_tags_ec2_resource( resource_id=volume_id, tags=utils.tag_dict_to_list(tags_to_apply))
def __init__(self, auth=None, region=None): AWS.__init__(self, auth, region)
class InstanceInfo: def __init__(self, provider="other"): self.logger = logging.getLogger("DNSWatch.InstanceInfo") self.provider = provider if provider == "gce": self.cloud = GCE() elif provider == "aws": self.cloud = AWS() def get_fqdn(self): return socket.getfqdn() def get_hostname(self): """Return only name before first dot.""" hostname = socket.gethostname() if hostname: try: hostname = hostname.split(".")[0] except ValueError: pass return hostname else: return None def get_private_ip(self): """ Return one IP address belongs to network interfaces used for external connections. """ self.logger.debug("Detecting private IP.") ip = None if self.provider in ["aws", "gce"]: ip = self._get_private_ip_cloud() else: ip = self._get_private_ip_other() self.logger.debug("My private IP: {}.".format(ip)) return ip def get_public_ip(self): self.logger.debug("Detecting public IP.") ip = None if self.provider in ["aws", "gce"]: ip = self._get_public_ip_cloud() else: ip = self._get_public_ip_other() self.logger.debug("My public IP: {}.".format(ip)) return ip def _get_private_ip_other(self): s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) ip = None interfaces = self._get_interfaces() if len(interfaces) > 1: self.logger.debug( "More than one interface found, using external "\ "connect to find proper IP.") # First method s.connect(("8.8.8.8", 53)) ip = s.getsockname()[0] else: interface = interfaces[0] # Second method ip = socket.inet_ntoa(fcntl.ioctl( s.fileno(), 0x8915, # SIOCGIFADDR struct.pack('256s', interface))[20:24]) s.close() return ip def _get_private_ip_cloud(self): return self.cloud.get_private_ip() def _get_public_ip_other(self): ip = None try: name = socket.gethostbyaddr(self.private_ip)[0] ip = self._query(name, "A", ["8.8.8.8", "8.8.4.4"])[0] #ip = self._query(name, "A", ["127.0.1.1"])[0] except: self.logger.error("Failed to find public IP.") return ip def _get_public_ip_cloud(self): return self.cloud.get_public_ip() def _get_interfaces(self): self.logger.debug("Getting network interfaces.") interfaces = list() with open("/proc/net/dev", "r") as dev_file: devices = dev_file.readlines() for dev in devices[2:]: dev_name = dev.split(":")[0].strip() if dev_name != "lo": interfaces.append(dev_name) self.logger.debug("Interfaces: {}.".format(interfaces)) return interfaces