def main(): # this should come from Django later ec2_access_key = 'svallero' ec2_secret_key = '5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8' #ec2_access_key = 'oneadmin' #ec2_secret_key = 'd2154097e7420fb39d8b101dd521cc29717772eb' ssh_key = 'sara' master_image = 'ami-00000984' master_flavour = 'm1.small' master_userdata = 'echo "pippo" > /root/pippo.txt' worker_flavour = 'm1.large' worker_userdata = 'echo "pippo" > /root/pippo.txt' shared_secret = 'pippo' check_queue_every = 15 min_job_waiting_time = 100 jobs_per_vm = 6 check_vms_every = 45 kill_idle_after = 3600 min_num_workers = 2 max_num_workers = 10 vm_deploy_time = 350 ########################################################################### try: import logging logging.getLogger('boto').setLevel(logging.DEBUG) logging.getLogger('urllib3').setLevel(logging.DEBUG) boto.set_file_logger('boto', 'boto.log') conn = boto.connect_ec2_endpoint( "https://one-master.to.infn.it/ec2api", aws_access_key_id=str(ec2_access_key), aws_secret_access_key=str(ec2_secret_key), validate_certs=True, #https_connection_factory=factory, #is_secure=True, debug=10) # conn.run_instances(master_image,instance_type=master_flavour,key_name=ssh_key) print(conn.get_params()) #conn.run_instances(master_image,instance_type=master_flavour) conn.get_all_reservations() #print ("Error: ",conn.ResponseError) #reservations = conn.get_all_reservations(dry_run = True) #reservations = conn.get_only_instances() # inst=reservations[0].instances #return inst[-1] #print inst #print (boto.exception.EC2ResponseError) #return 1 except: print("Ciccia!") errore = sys.exc_info()[0] print("Error: %s" % errore) raise
def __init__(self, download_path, bucket_name, num_processes=2, log_file=None, log_level=logging.INFO): self.download_path = download_path self.bucket_name = bucket_name self.num_processes = num_processes if log_file: boto.set_file_logger('boto-downloader', log_file, log_level) else: boto.set_stream_logger('boto-downloader', log_level) self.task_queue = multiprocessing.JoinableQueue() self.s3 = boto.connect_s3() self.bucket = self.s3.lookup(self.bucket_name) self.n_tasks = 0
def __init__(self, index_path, bucket_name, num_processes=2, log_file=None, log_level=logging.INFO): self.index_path = index_path self.bucket_name = bucket_name self.num_processes = num_processes if log_file: boto.set_file_logger('pynas-uploader', log_file, log_level) else: boto.set_stream_logger('pynas-uploader', log_level) self.task_queue = multiprocessing.JoinableQueue() self.status_queue = multiprocessing.Queue() self.s3 = boto.connect_s3() self.bucket = self.s3.lookup(self.bucket_name) self.index = pynas.index.Index(index_path) self.n_tasks = 0
def __init__(self): ScriptBase.__init__(self) self.wdir = boto.config.get('Pyami', 'working_dir') self.log_file = '%s.log' % self.instance_id self.log_path = os.path.join(self.wdir, self.log_file) boto.set_file_logger(self.name, self.log_path) self.src_name = boto.config.get(self.name, 'src_bucket') self.dst_name = boto.config.get(self.name, 'dst_bucket') self.replace = boto.config.getbool(self.name, 'replace_dst', True) s3 = boto.connect_s3() self.src = s3.lookup(self.src_name) if not self.src: boto.log.error('Source bucket does not exist: %s' % self.src_name) dest_access_key = boto.config.get(self.name, 'dest_aws_access_key_id', None) if dest_access_key: dest_secret_key = boto.config.get(self.name, 'dest_aws_secret_access_key', None) s3 = boto.connect(dest_access_key, dest_secret_key) self.dst = s3.lookup(self.dst_name) if not self.dst: self.dst = s3.create_bucket(self.dst_name)
def __init__(self): super(CopyBot, self).__init__() self.wdir = boto.config.get('Pyami', 'working_dir') self.log_file = '%s.log' % self.instance_id self.log_path = os.path.join(self.wdir, self.log_file) boto.set_file_logger(self.name, self.log_path) self.src_name = boto.config.get(self.name, 'src_bucket') self.dst_name = boto.config.get(self.name, 'dst_bucket') self.replace = boto.config.getbool(self.name, 'replace_dst', True) s3 = boto.connect_s3() self.src = s3.lookup(self.src_name) if not self.src: boto.log.error('Source bucket does not exist: %s' % self.src_name) dest_access_key = boto.config.get(self.name, 'dest_aws_access_key_id', None) if dest_access_key: dest_secret_key = boto.config.get(self.name, 'dest_aws_secret_access_key', None) s3 = boto.connect(dest_access_key, dest_secret_key) self.dst = s3.lookup(self.dst_name) if not self.dst: self.dst = s3.create_bucket(self.dst_name)
def __init__(self, config_file=None): super(SonOfMMM, self).__init__(config_file) self.log_file = '%s.log' % self.instance_id self.log_path = os.path.join(self.working_dir, self.log_file) boto.set_file_logger(self.name, self.log_path) if self.sd.has_option('ffmpeg_args'): self.command = '/usr/local/bin/ffmpeg ' + self.sd.get('ffmpeg_args') else: self.command = '/usr/local/bin/ffmpeg -y -i %s %s' self.output_mimetype = self.sd.get('output_mimetype') if self.sd.has_option('output_ext'): self.output_ext = self.sd.get('output_ext') else: self.output_ext = mimetypes.guess_extension(self.output_mimetype) self.output_bucket = self.sd.get_obj('output_bucket') self.input_bucket = self.sd.get_obj('input_bucket') # check to see if there are any messages queue # if not, create messages for all files in input_bucket m = self.input_queue.read(1) if not m: self.queue_files()
def __init__(self, config_file=None): Service.__init__(self, config_file) self.log_file = "%s.log" % self.instance_id self.log_path = os.path.join(self.working_dir, self.log_file) boto.set_file_logger(self.name, self.log_path) if self.sd.has_option("ffmpeg_args"): self.command = "/usr/local/bin/ffmpeg " + self.sd.get("ffmpeg_args") else: self.command = "/usr/local/bin/ffmpeg -y -i %s %s" self.output_mimetype = self.sd.get("output_mimetype") if self.sd.has_option("output_ext"): self.output_ext = self.sd.get("output_ext") else: self.output_ext = mimetypes.guess_extension(self.output_mimetype) self.output_bucket = self.sd.get_obj("output_bucket") self.input_bucket = self.sd.get_obj("input_bucket") # check to see if there are any messages queue # if not, create messages for all files in input_bucket m = self.input_queue.read(1) if not m: self.queue_files()
def __init__(self, config_file=None): super(SonOfMMM, self).__init__(config_file) self.log_file = '%s.log' % self.instance_id self.log_path = os.path.join(self.working_dir, self.log_file) boto.set_file_logger(self.name, self.log_path) if self.sd.has_option('ffmpeg_args'): self.command = '/usr/local/bin/ffmpeg ' + self.sd.get( 'ffmpeg_args') else: self.command = '/usr/local/bin/ffmpeg -y -i %s %s' self.output_mimetype = self.sd.get('output_mimetype') if self.sd.has_option('output_ext'): self.output_ext = self.sd.get('output_ext') else: self.output_ext = mimetypes.guess_extension(self.output_mimetype) self.output_bucket = self.sd.get_obj('output_bucket') self.input_bucket = self.sd.get_obj('input_bucket') # check to see if there are any messages queue # if not, create messages for all files in input_bucket m = self.input_queue.read(1) if not m: self.queue_files()
def run_scripts(self): scripts = config.get('Pyami', 'scripts') if scripts: for script in scripts.split(','): script = script.strip(" ") try: pos = script.rfind('.') if pos > 0: mod_name = script[0:pos] cls_name = script[pos+1:] cls = find_class(mod_name, cls_name) boto.log.info('Running Script: %s' % script) s = cls() s.main() else: boto.log.warning('Trouble parsing script: %s' % script) except Exception, e: boto.log.exception('Problem Running Script: %s. Startup process halting.' % script) raise e def main(self): self.run_scripts() self.notify('Startup Completed for %s' % config.get('Instance', 'instance-id')) if __name__ == "__main__": if not config.has_section('loggers'): boto.set_file_logger('startup', '/var/log/boto.log') sys.path.append(config.get('Pyami', 'working_dir')) su = Startup() su.main()
def load_packages(self): package_str = boto.config.get('Pyami', 'packages') if package_str: packages = package_str.split(',') for package in packages: package = package.strip() if package.startswith('s3:'): package = self.fetch_s3_file(package) if package: # if the "package" is really a .py file, it doesn't have to # be installed, just being in the working dir is enough if not package.endswith('.py'): self.run('easy_install -Z %s' % package, exit_on_error=False) def main(self): self.create_working_dir() self.load_boto() self.load_packages() self.notify('Bootstrap Completed for %s' % boto.config.get_instance('instance-id')) if __name__ == "__main__": # because bootstrap starts before any logging configuration can be loaded from # the boto config files, we will manually enable logging to /var/log/boto.log boto.set_file_logger('bootstrap', '/var/log/boto.log') bs = Bootstrap() bs.main()
script = script.strip(" ") try: pos = script.rfind('.') if pos > 0: mod_name = script[0:pos] cls_name = script[pos + 1:] cls = find_class(mod_name, cls_name) boto.log.info('Running Script: %s' % script) s = cls() s.main() else: boto.log.warning('Trouble parsing script: %s' % script) except Exception as e: boto.log.exception( 'Problem Running Script: %s. Startup process halting.' % script) raise e def main(self): self.run_scripts() self.notify('Startup Completed for %s' % config.get('Instance', 'instance-id')) if __name__ == "__main__": if not config.has_section('loggers'): boto.set_file_logger('startup', '/var/log/boto.log') sys.path.append(config.get('Pyami', 'working_dir')) su = Startup() su.main()
boto.log.exception('Problem Retrieving file: %s' % s3_file) path = None return path def load_packages(self): package_str = boto.config.get('Pyami', 'packages') if package_str: packages = package_str.split(',') for package in packages: package = package.strip() if package.startswith('s3:'): package = self.fetch_s3_file(package) if package: # if the "package" is really a .py file, it doesn't have to # be installed, just being in the working dir is enough if not package.endswith('.py'): self.run('easy_install -Z %s' % package, exit_on_error=False) def main(self): self.create_working_dir() self.load_boto() self.load_packages() self.notify('Bootstrap Completed for %s' % boto.config.get_instance('instance-id')) if __name__ == "__main__": # because bootstrap starts before any logging configuration can be loaded from # the boto config files, we will manually enable logging to /var/log/boto.log boto.set_file_logger('bootstrap', '/var/log/boto.log') bs = Bootstrap() bs.main()
# Set environment variables to secrets: # AWS_ACCESS_KEY_ID # AWS_SECRET_ACCESS_KEY from os import environ import logging import boto BUCKET_NAME = 'wpmedia.vivelohoy.com' if 'AWS_ACCESS_KEY_ID' not in environ or 'AWS_SECRET_ACCESS_KEY' not in environ: print 'Error: Environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY not set.' exit(1) boto.set_file_logger('boto-copier', 's3_copy.log', logging.INFO) s3 = boto.connect_s3() bucket = s3.get_bucket(BUCKET_NAME) for src_key in bucket.list(): if 'wp-content/' not in src_key.name: new_key = 'wp-content/' + src_key.name src_key.copy(BUCKET_NAME, new_key) print '{src} => {dest}'.format(src=src_key.name, dest=new_key)
else: print value sys.exit(1) return excepthook # It would be nice to get rid of all of the print statements in favor # of log messages; for now I'm just doing this to log object creation objLog = '/tmp/activity.log' log = logging.getLogger('objLog') activity_handler = logging.FileHandler(objLog) activity_handler.setFormatter(logging.Formatter('%(message)s')) log.addHandler(activity_handler) log.setLevel(logging.INFO) boto.set_file_logger('boto', '/tmp/boto.log', level=logging.DEBUG) arange = lambda x, y: [ chr(z) for z in range(ord(x), ord(y)) ] rand_from_list = lambda x: x[random.randint(0, len(x)-1)] random.seed() euca_version = open('/etc/eucalyptus/eucalyptus-version', 'r').read().replace('eee-', '').strip() ec2timeout = "60" mode = "system" host = "localhost" objTypes = ['keypair', 'object', 'bucket', 'volume', 'snapshot', 'rule', 'group'] userlist = {} accounts = {} debug = 1 local_ips = set() activity_log_version = 0