#--------------------------------------------------------------------------------------
"""
Contains the package dedicated preferences
"""


#--------------------------------------------------------------------------------------
def usage_description() :
    return " --instance-type='m1.large' --image-id='ami-1cdb2c75' --number-nodes=8"


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'amazon.cluster' )

a_container.add_option( PersistentOption( "--instance-type",
                                          metavar = "< EC2 instance type : 'c1.xlarge' or 'm1.large', for example >",
                                          choices = [ 'm1.large', 'c1.xlarge', 'm1.xlarge', ' m2.xlarge', 'm2.2xlarge', 'm2.4xlarge', 'c1.medium' ],
                                          action = "store",
                                          dest = "instance_type",
                                          help = "('%default', by default)",
                                          default = "c1.xlarge" ) )

a_container.add_option( PersistentOption( "--image-id",
                                          metavar = "< Amazon EC2 AMI ID >",
                                          action = "store",
                                          dest = "image_id",
                                          help = "('%default', by default)",
                                          default = None ) )

#--------------------------------------------------------------------------------------
def usage_description() :
    from cloudflu import common
    
    an_usage_description = ""
    an_usage_description += " --study-name=<my uploaded study>"
    an_usage_description += "[ --located-files='<file path 1>%s<file path 2>..']" % common.arg_list_separator()
    
    return an_usage_description


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'amazon.data_transfer.upload_resume' )

a_container.add_option( TransientOption( "--located-files",
                                         metavar = "< the list of file paths inside the study >",
                                         action = "store",
                                         dest = "located_files",
                                         default = None ) )


#--------------------------------------------------------------------------------------
def extract( the_option_parser ) :
    from cloudflu.common import print_d
    an_options, an_args = the_option_parser.parse_args()

    a_located_files = an_options.located_files
    if a_located_files != None :

#--------------------------------------------------------------------------------------
"""
Contains the package dedicated preferences
"""


#--------------------------------------------------------------------------------------
def usage_description() :
    return " --study-name='my booked study'"


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'amazon.study_seal' )

a_container.add_option( TransientOption( "--study-name",
                                         metavar = "< name of the existing study >",
                                         action = "store",
                                         dest = "study_name",
                                         default = None ) )


#--------------------------------------------------------------------------------------
def extract( the_option_parser ) :
    from cloudflu.common import print_d, print_i, print_e
    
    an_options, an_args = the_option_parser.parse_args()

    a_study_name = an_options.study_name
# --------------------------------------------------------------------------------------
"""
Contains the package dedicated preferences
"""


# --------------------------------------------------------------------------------------
def usage_description():
    return " --study-name=< an unique study name >"


# --------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption

a_container = OptionGroup("amazon.data_transfer.upload")

a_container.add_option(
    TransientOption(
        "--study-name",
        metavar="< an unique study name >",
        action="store",
        dest="study_name",
        help="(generated, by default)",
        default=None,
    )
)


# --------------------------------------------------------------------------------------
def extract(the_option_parser):
#--------------------------------------------------------------------------------------
def usage_description() :
    from cloudflu import common
    
    an_usage_description = ""
    an_usage_description += " --study-name=<my uploaded study>"
    an_usage_description += "[ --located-files='<file path 1>%s<file path 2>..']" % common.arg_list_separator()
    an_usage_description += " --output-dir='./tmp'"
    
    return an_usage_description


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'amazon.data_transfer.download' )

a_container.add_option( TransientOption( "--located-files",
                                         metavar = "< the list of file paths inside the study >",
                                         action = "store",
                                         dest = "located_files",
                                         default = None ) )

a_container.add_option( TransientOption( "--output-dir",
                                         metavar = "< location of the task defintion >",
                                         action = "store",
                                         dest = "output_dir",
                                         help = "(the same a 'study' name, by default)",
                                         default = None ) )

a_container.add_option( PersistentOption( "--fresh",

#--------------------------------------------------------------------------------------
"""
Contains the package dedicated preferences
"""


#--------------------------------------------------------------------------------------
def usage_description() :
    return " --number-threads=3"


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'common.algorithm' )

a_container.add_option( PersistentOption( "--number-threads",
                                          metavar = "< number of threads to use >",
                                          type = "int",
                                          action = "store",
                                          dest = "number_threads",
                                          help = "(%default, by default)",
                                          default = 3 ) )


#--------------------------------------------------------------------------------------
def extract( the_option_parser ) :
    an_options, an_args = the_option_parser.parse_args()

    a_number_threads = an_options.number_threads

#--------------------------------------------------------------------------------------
"""
Contains the package dedicated preferences
"""


#--------------------------------------------------------------------------------------
def usage_description() :
    return " --aws-access-key-id=${AWS_ACCESS_KEY_ID} --aws-secret-access-key=${AWS_SECRET_ACCESS_KEY}"


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'amazon.security' )

a_container.add_option( PersistentOption( "--aws-access-key-id",
                                          metavar = "< Amazon key id >",
                                          action = "store",
                                          dest = "aws_access_key_id",
                                          help = "(${AWS_ACCESS_KEY_ID}, by default)",
                                          default = None ) )

a_container.add_option( PersistentOption( "--aws-secret-access-key",
                                          metavar = "< Amazon secret key >",
                                          action = "store",
                                          dest = "aws_secret_access_key",
                                          help = "(${AWS_SECRET_ACCESS_KEY}, by default)",
                                          default = None ) )

#--------------------------------------------------------------------------------------
"""
Contains the package dedicated preferences
"""


#--------------------------------------------------------------------------------------
def usage_description() :
    return " --upload-seed-size=5160"


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'amazon.data_transfer.upload' )

a_container.add_option( PersistentOption( "--upload-seed-size",
                                          metavar = "< size of file pieces to be uploaded, in bytes >",
                                          type = "int",
                                          action = "store",
                                          dest = "upload_seed_size",
                                          help = "(%default, by default)",
                                          default = 65536 ) )


#--------------------------------------------------------------------------------------
def extract( the_option_parser ) :
    from cloudflu.common import print_d, print_i, print_e
    
    an_options, an_args = the_option_parser.parse_args()
Beispiel #9
0
# --------------------------------------------------------------------------------------
"""
Contains the package dedicated preferences
"""


# --------------------------------------------------------------------------------------
def usage_description():
    return " --debug"


# --------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption

a_container = OptionGroup("common")

a_container.add_option(
    PersistentOption(
        "--debug",
        metavar="< print debug information >",
        action="store_true",
        dest="enable_debug",
        help="(%default, by default)",
        default=False,
    )
)

a_container.add_option(
    PersistentOption(
        "--log-file",
# --------------------------------------------------------------------------------------
"""
This script is responsible for cluster environment setup for the given Amazon EC2 reservation
"""


# --------------------------------------------------------------------------------------
def usage_description():
    return " --output-dir='~/damBreak 2010-11-23 16:23' --before-hook='cloudflu-foam2vtk-before' --time-hook='cloudflu-foam2vtk' --after-hook='cloudflu-foam2vtk-after'"


# --------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption

a_container = OptionGroup("amazon.solver")

a_container.add_option(
    TransientOption(
        "--output-dir",
        metavar="< folder suffix for the output results >",
        action="store",
        dest="output_dir",
        help="(%default, by default)",
        default=None,
    )
)

a_container.add_option(
    PersistentOption(
        "--before-hook",
"""


#--------------------------------------------------------------------------------------
def usage_description() :
    an_usage_description = ' "mpirun --hostfile ~/.openmpi_hostfile -x LD_LIBRARY_PATH -x PATH -np ${a_number_processors} ${application} -case . -parallel"'
    an_usage_description += " --case-dir='.'"
    an_usage_description += " --solver-log='log.solver-run'"
    an_usage_description += " --watched-keyword='ExecutionTime = '"
    
    return an_usage_description


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'amazon.timestamps_upload' )

a_container.add_option( TransientOption( "--case-dir",
                                         metavar = "< location of files inside of the study >",
                                         action = "store",
                                         dest = "case_dir",
                                         help = "('%default', by default)",
                                         default = '.' ) )

a_container.add_option( TransientOption( "--solver-log",
                                         metavar = "< location of files inside of the study >",
                                         action = "store",
                                         dest = "solver_log",
                                         help = "('%default', by default)",
                                         default = 'log.solver-run' ) )
Beispiel #12
0
"""

#--------------------------------------------------------------------------------------
def usage_description() :
    from cloudflu import common
    
    an_usage_description = ""
    an_usage_description += " --study-name=<my uploaded study>"
    an_usage_description += " <file path 1>[ <file path 2>]"

    return an_usage_description


#------------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup
a_container = OptionGroup( 'amazon.rm', 'application specific' )

a_container.add_option( "--study-name",
                        metavar = "< name of the uploaded study >",
                        action = "store",
                        dest = "study_name",
                        default = None )


#------------------------------------------------------------------------------------------
def extract( the_option_parser ) :
    from cloudflu.common import print_d, print_i, print_e

    an_options, an_args = the_option_parser.parse_args()

    a_study_name = an_options.study_name
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.
##
## See http://sourceforge.net/apps/mediawiki/cloudflu
##
## Author : Alexey Petrov
##


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'amazon.cluster' )

a_container.add_option( PersistentOption( "--login-name",
                                          metavar = "< specifies the user to log in as on the remote machine >",
                                          action = "store",
                                          dest = "login_name",
                                          help = "(\"%default\", by default)",
                                          default = 'ubuntu' ) )

a_container.add_option( PersistentOption( "--host-port",
                                          metavar = "< port to ssh >",
                                          type = "int",
                                          action = "store",
                                          dest = "host_port",
                                          help = "(%default, by default)",
                                          default = 22 ) )

#--------------------------------------------------------------------------------------
"""
Contains the package dedicated preferences
"""


#--------------------------------------------------------------------------------------
def usage_description() :
    return " [--production]"


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'common.install' )

a_container.add_option( PersistentOption( "--production",
                                          metavar = "< deploy official production version >",
                                          action = "store_true",
                                          dest = "production",
                                          help = "(%default, by default)",
                                          default = True ) )

a_container.add_option( PersistentOption( "--url",
                                          metavar = "< package name or precise location to download from >",
                                          action = "store",
                                          dest = "url",
                                          help = "('%default', by default)",
                                          default = 'cloudflu' ) )
"""


#--------------------------------------------------------------------------------------
def usage_description() :
    from cloudflu import common

    an_usage_description = " --file-locations='< file1 location >%s< file2 location >..'" % common.arg_list_separator()
    an_usage_description += " < file 1 > < folder 2 > .."
    
    return an_usage_description


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'amazon.data_transfer.upload' )

a_container.add_option( TransientOption( "--file-locations",
                                         metavar = "< location of files inside of the study >",
                                         action = "store",
                                         dest = "file_locations",
                                         help = "('%default', by default)",
                                         default = '' ) )


#--------------------------------------------------------------------------------------
def extract( the_option_parser ) :
    from cloudflu.common import print_d, print_i, print_e
    
    an_options, an_args = the_option_parser.parse_args()
# ------------------------------------------------------------------------------------------
"""
This script is responsible for efficient uploading of multi file data
"""


# --------------------------------------------------------------------------------------
def usage_description():
    return " --study-name=< user study name >"


# --------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption

a_container = OptionGroup("amazon.data_transfer")

a_container.add_option(
    TransientOption(
        "--study-name",
        metavar="< existing study name >",
        action="store",
        dest="study_name",
        help="(read from standard input, if not given)",
        default=None,
    )
)

# --------------------------------------------------------------------------------------
def extract(the_option_parser):
    from cloudflu.common import print_d
# --------------------------------------------------------------------------------------
"""
This script is responsible for cluster environment setup for the given Amazon EC2 reservation
"""


# --------------------------------------------------------------------------------------
def usage_description():
    return " --case-dir='~/damBreak' --run-hook='Allrun'"


# --------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption

a_container = OptionGroup("amazon.solver")

a_container.add_option(
    TransientOption(
        "--case-dir",
        metavar="< location of the source OpenFOAM case dir >",
        action="store",
        dest="case_dir",
        default=None,
    )
)

a_container.add_option(
    PersistentOption(
        "--run-hook",
        metavar="< case dir executable in to be run in the cluster >",
Beispiel #18
0
##
## See http://sourceforge.net/apps/mediawiki/cloudflu
##
## Author : Alexey Petrov
##


#--------------------------------------------------------------------------------------
"""
Contains the package dedicated preferences
"""


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'amazon.cluster' )

a_container.add_option( PersistentOption( "--cluster-location",
                                          metavar = "< location of the cluster nodes : 'eu-west-1', 'us-east-1', 'us-west-1' or 'ap-southeast-1' >",
                                          choices = [ 'eu-west-1', 'us-east-1', 'us-west-1', 'ap-southeast-1' ],
                                          action = "store",
                                          dest = "location",
                                          help = "('%default', by default)",
                                          default = 'us-east-1' ) )


#------------------------------------------------------------------------------------------
def dump( the_identation_level, the_output ) :
    from cloudflu.preferences import dump_begin, dump_resume, dump_end
    dump_begin( the_identation_level, a_container, the_output )

#--------------------------------------------------------------------------------------
def usage_description() :
    import cloudflu.common as common

    an_usage_description = ""
    an_usage_description += " [ --script-file='./remote_adjust_profile.sh%s./remote_update_sources-list.sh' ]" % common.arg_list_separator()
    an_usage_description += " [ --sequence-file='./remote_sshd-config.sh' ]"

    return an_usage_description


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'common.ssh.run' )

import cloudflu.common as common
a_container.add_option( TransientOption( "--script-file",
                                         metavar = "< script (or list of scripts separated by '%s')"
                                         "or be executed on the remote host >" % common.arg_list_separator(),
                                         action = "store",
                                         dest = "script_file",
                                         default = None ) )

a_container.add_option( TransientOption( "--script-args",
                                         metavar = "< arguments (or list of arguments separated by '%s)"
                                         "or the remote script execution >" % common.arg_list_separator(),
                                         action = "store",
                                         dest = "script_args",
                                         default = None ) )

#--------------------------------------------------------------------------------------
"""
This script is responsible for the task packaging and sending it for execution in a cloud
"""


#--------------------------------------------------------------------------------------
def usage_description() :
    return " --instance-ord=0"


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'amazon.instance_extract' )

a_container.add_option( TransientOption( "--instance-ord",
                                         metavar = "< order number of the instance in the given reservation >",
                                         type = "int",
                                         action = "store",
                                         dest = "instance_ord",
                                         help = "(%default, by default)",
                                         default = 0 ) )


#--------------------------------------------------------------------------------------
def extract( the_option_parser ) :
    an_options, an_args = the_option_parser.parse_args()

    an_instance_ord = int( an_options.instance_ord )

#--------------------------------------------------------------------------------------
"""
This script upload a file from web and publish it as 'study file' in S3
"""


#--------------------------------------------------------------------------------------
def usage_description() :
    return " --remote-location='/mnt'"


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'amazon.credentials_deploy' )

a_container.add_option( TransientOption( "--aws-user-id",
                                         metavar = "< AWS User ID >",
                                         action = "store",
                                         dest = "aws_user_id",
                                         help = "(${AWS_USER_ID}, by default)",
                                         default = None ) )

a_container.add_option( TransientOption( "--ec2-private-key",
                                         metavar = "< EC2 Private Key >",
                                         action = "store",
                                         dest = "ec2_private_key",
                                         help = "(${EC2_PRIVATE_KEY}, by default)",
                                         default = None ) )
Beispiel #22
0
# --------------------------------------------------------------------------------------
"""
Contains the package dedicated preferences
"""


# --------------------------------------------------------------------------------------
def usage_description():
    return " [ --password='******' | --identity-file='~/.ssh/tmpraDD2j.pem' ] --host-port=22 --login-name='ubuntu' --host-name='ec2-184-73-11-20.compute-1.amazonaws.com'"


# --------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption

a_container = OptionGroup("common.ssh")

a_container.add_option(
    TransientOption(
        "--password",
        metavar="< password for the given host and login name >",
        action="store",
        dest="password",
        default=None,
    )
)

a_container.add_option(
    TransientOption(
        "--identity-file",
        metavar="< private key for RSA or DSA authentication >",
##


#------------------------------------------------------------------------------------------
"""
Removes whole appointed cloud study or just given files from this study
"""

#--------------------------------------------------------------------------------------
def usage_description() :
    return " --hostfile='/tmp/.openmpi_hostfile'"


#------------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'amazon.openmpi_config' )

a_container.add_option( PersistentOption( "--hostfile",
                                          metavar = "< location of the MPI 'hostfile' >",
                                          action = "store",
                                          dest = "hostfile",
                                          help = "(\"%default\", by default)",
                                          default = '/tmp/.openmpi_hostfile' ) )


#------------------------------------------------------------------------------------------
def extract( the_option_parser ) :
    an_options, an_args = the_option_parser.parse_args()

    a_hostfile = an_options.hostfile
    from cloudflu.common import print_d

#--------------------------------------------------------------------------------------
"""
Contains the package dedicated preferences
"""


#--------------------------------------------------------------------------------------
def usage_description() :
    return " --socket-timeout=2"


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'common.algorithm' )

a_container.add_option( PersistentOption( "--socket-timeout",
                                          metavar = "< socket timeout >",
                                          type = "int",
                                          action = "store",
                                          dest = "socket_timeout",
                                          help = "(%default, by default)",
                                          default = 7 ) )


#--------------------------------------------------------------------------------------
def extract( the_option_parser ) :
    an_options, an_args = the_option_parser.parse_args()

    a_socket_timeout = an_options.socket_timeout

#------------------------------------------------------------------------------------------
"""
This script is responsible for efficient downloading of multi file data
"""


#--------------------------------------------------------------------------------------
def usage_description() :
    return " [--booked]"


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'amazon.data_transfer.upload' )

a_container.add_option( TransientOption( "--booked",
                                         metavar = "< use already existing study >",
                                         action = "store_true",
                                         dest = "booked",
                                         help = "(%default, by default)",
                                         default = False ) )

#--------------------------------------------------------------------------------------
def extract( the_option_parser ) :
    from cloudflu.common import print_d
    an_options, an_args = the_option_parser.parse_args()

    a_booked = an_options.booked
    print_d( "a_booked = %s\n" % a_booked )
##
## See http://sourceforge.net/apps/mediawiki/cloudflu
##
## Author : Alexey Petrov
##


#--------------------------------------------------------------------------------------
"""
Contains the package dedicated preferences
"""


#--------------------------------------------------------------------------------------
from cloudflu.preferences import OptionGroup, PersistentOption, TransientOption
a_container = OptionGroup( 'common.proxy' )

a_container.add_option( PersistentOption( "--proxy-address",
                                          metavar = "< The proxy server address >",
                                          type = "str",
                                          action = "store",
                                          dest = "proxy_address",
                                          help = "(%default, by default)",
                                          default = None ) )

a_container.add_option( PersistentOption( "--proxy-port",
                                          metavar = "< The proxy server port >",
                                          type = "int",
                                          action = "store",
                                          dest = "proxy_port",
                                          help = "(%default, by default)",