Example #1
0
def setup(config_file, deployment_id, ssh_key_file):
    mesos_marathon_setup_cmd = "python mesos_marathon_setup.py --config_file " + config_file + \
                               " --deployment_id " + deployment_id + " --ssh_key_file " + ssh_key_file
    shell_call(mesos_marathon_setup_cmd)

    hydra_setup_cmd = "python hydra_setup_script.py --deployment_id " + deployment_id
    shell_call(hydra_setup_cmd)
Example #2
0
 def create(self):
     if self.image == "ubuntu-12-04" or self.image == "ubuntu-14-04":
         cmd = "gcloud compute disks create " + self.name + " --image " + self.image + " --type " + self.type + \
               " --size=" + self.size + " -q"
     elif self.image is None:
         cmd = "gcloud compute disks create " + self.name + " --type " + self.type + " --size=" + self.size + " -q"
     else:
         cmd = "gcloud compute disks create " + self.name + " --source-snapshot " + self.image + \
               " --type " + self.type + " --size=" + self.size + " -q"
     print("disk_cmd=%s" % cmd)
     shell_call(cmd)
Example #3
0
 def create(self):
     if self.image == "ubuntu-12-04" or self.image == "ubuntu-14-04":
         cmd = "gcloud compute disks create " + self.name + " --image " + self.image + " --type " + self.type + \
               " --size=" + self.size + " -q"
     elif self.image is None:
         cmd = "gcloud compute disks create " + self.name + " --type " + self.type + " --size=" + self.size + " -q"
     else:
         cmd = "gcloud compute disks create " + self.name + " --source-snapshot " + self.image + \
               " --type " + self.type + " --size=" + self.size + " -q"
     print ("disk_cmd=%s" % cmd)
     shell_call(cmd)
Example #4
0
def add(lang):
    langs = getlangs(lang)
    puts(u"Adding %s" % ', '.join(langs))
    for loc in langs:
        with indent(2):
            puts(u"Generating PO for %s" % loc)
        shell_call(u"django-admin.py makemessages -l %(lang)s "
                   u"-e py,html,email,txt" % {'lang': loc})
        for app in I18N_APPS:
            with indent(4):
                puts(u"Generating PO for app %s" % app)
            with chdir(os.path.join(REPO_ROOT, app)):
                shell_call(u"django-admin.py makemessages "
                           u"-d djangojs -l %(lang)s" % {'lang': loc})
        puts(colored.green("sucesssfuly generated %s" % loc))
Example #5
0
def add(lang):
    langs = getlangs(lang)
    puts("Adding %s" % ', '.join(langs))
    for loc in langs:
        with indent(2):
            puts("Generating PO for %s" % loc)
        shell_call("django-admin.py makemessages -l %(lang)s "
                   "-e py,html,email,txt" % {'lang': loc})
        for app in I18N_APPS:
            with indent(4):
                puts("Generating PO for app %s" % app)
            with chdir(os.path.join(REPO_ROOT, app)):
                shell_call("django-admin.py makemessages "
                           "-d djangojs -l %(lang)s" % {'lang': loc})
        puts(colored.green("sucesssfuly generated %s" % loc))
Example #6
0
def compile_mo(lang=None):
    langs = getlangs(lang)
    puts(u"Compiling %s" % ', '.join(langs))
    for loc in langs:
        with indent(2):
            puts(u"Compiling %s" % loc)
        shell_call(u"django-admin.py compilemessages -l %(lang)s "
                   % {'lang': loc})
        for app in I18N_APPS:
            with indent(4):
                puts(u"Compiling app %s" % app)
            with chdir(os.path.join(REPO_ROOT, app)):
                shell_call(u"django-admin.py compilemessages -l %(lang)s"
                           % {'lang': loc})
        puts(colored.green("sucesssfuly compiled %s" % loc))
Example #7
0
def compile_mo(lang=None):
    langs = getlangs(lang)
    puts("Compiling %s" % ', '.join(langs))
    for loc in langs:
        with indent(2):
            puts("Compiling %s" % loc)
        shell_call("django-admin.py compilemessages -l %(lang)s "
                   % {'lang': loc})
        for app in I18N_APPS:
            with indent(4):
                puts("Compiling app %s" % app)
            with chdir(os.path.join(REPO_ROOT, app)):
                shell_call("django-admin.py compilemessages -l %(lang)s"
                           % {'lang': loc})
        puts(colored.green("sucesssfuly compiled %s" % loc))
Example #8
0
    def create(self, common_section):
        pathname = "/tmp/gce_key.txt"

        tfile = open(pathname, 'w')
        with open(common_section.sshkey) as f:
            lines = f.readlines()
            tfile.writelines(self.user_name + ":" + lines[0])
        tfile.close()

        cmd = "gcloud compute instances create " + self.name + " --machine-type " + self.machine_type + \
              " --network " + common_section.network + \
              " --maintenance-policy MIGRATE --scopes https://www.googleapis.com/auth/cloud-platform " \
              "--disk name=" + self.disk_list[0].name + ",mode=rw,boot=yes,auto-delete=yes --disk name=" + \
              self.disk_list[1].name + \
              ",mode=rw,boot=no,auto-delete=yes --no-address --tags no-ip --metadata-from-file sshKeys=" + pathname
        print("create_instance_cmd = %s" % cmd)
        shell_call(cmd)
        self.ip = self.get_ip()
        return self.ip
Example #9
0
    def create(self, common_section):
        pathname = "/tmp/gce_key.txt"

        tfile = open(pathname, 'w')
        with open(common_section.sshkey) as f:
            lines = f.readlines()
            tfile.writelines(self.user_name + ":" + lines[0])
        tfile.close()

        cmd = "gcloud compute instances create " + self.name + " --machine-type " + self.machine_type + \
              " --network " + common_section.network + \
              " --maintenance-policy MIGRATE --scopes https://www.googleapis.com/auth/cloud-platform " \
              "--disk name=" + self.disk_list[0].name + ",mode=rw,boot=yes,auto-delete=yes --disk name=" + \
              self.disk_list[1].name + \
              ",mode=rw,boot=no,auto-delete=yes --no-address --tags no-ip --metadata-from-file sshKeys=" + pathname
        print("create_instance_cmd = %s" % cmd)
        shell_call(cmd)
        self.ip = self.get_ip()
        return self.ip
Example #10
0
 def join(self):
     """Uses pandaseq 2.7 to join the foward and reverse reads together.
     See https://github.com/neufeld/pandaseq"""
     # Special case for new primers that don't join #
     rev_primer_name = self.info['primers']['reverse']['name']
     not_joining_primers = ("1132R", "1000R")
     if rev_primer_name in not_joining_primers:
         print "No overlap special case"
         self.trim_and_concat.run()
         return
     # Special case for primers that highly overlap #
     high_overlap_primers = ("806R",)
     if rev_primer_name in high_overlap_primers:
         print "High overlap special case, using mothur"
         result = sh.mothur("#make.contigs(ffastq=%s, rfastq=%s);" % (self.uncomrpessed_pair.fwd, self.uncomrpessed_pair.rev))
         if "ERROR" in result.stdout: raise Exception("Mothur didn't run correctly")
         # Move things #
         #shutil.move(self.tax.centers.prefix_path + '.align', self.mothur_aligned)
         #shutil.move(self.tax.centers.prefix_path + '.align.report', self.p.mothur_report)
         return
     # Default case #
     command = 'pandaseq27 -T 1 -f %s -r %s -u %s -F 1> %s 2> %s'
     command = command % (self.fwd, self.rev, self.unassembled.path, self.assembled.path, self.assembled.p.out)
     shell_call(command) # Because it exits with status 1 https://github.com/neufeld/pandaseq/issues/40
Example #11
0
def run_experiment():
    shell_call(oracle_cmd)
    shell_call(sleep_cmd)
    shell_call(exp_cmd)

    output = os.popen(tail_cmd).read()
    sp = output.strip().split('\n')
    if 'CALL:' in sp[0] and 'ORACLE:' in sp[1]:
        t1 = int(sp[0].strip().split(':')[1])
        t2 = int(sp[1].strip().split(':')[1])
        return t2 - t1
    else:
        return -1
Example #12
0
from framework.utils.data.text_indexer import TextIndexer
from word_classifier.data import ClassifierData
from framework.evaluator import Evaluator
from shell_command import shell_call
import framework.utils.common as utils
from time import time
import numpy as np
import os
import sys

params = utils.load_param_file(sys.argv[1])

params['num_classes'] = len(params['keywords']) + 1
release_cmd = 'python3 ../tools/release_model.py %s' % sys.argv[1]
shell_call(release_cmd)
vocab_file = os.path.join(utils.get_dict_value(params, 'output_location'),
                          'vocab.pkl')
release_dir = os.path.join(utils.get_dict_value(params, 'output_location'),
                           params['model_name'])
graphdef_file = os.path.join(release_dir, params['model_name'] + '.graphdef')
ckpt = os.path.join(utils.get_dict_value(params, 'output_location'),
                    utils.get_dict_value(params, 'model_name') + '.ckpt')

e = Evaluator.load_graphdef(graphdef_file)
e.dump_variable_sizes()
i = TextIndexer.from_file(vocab_file)

test_data = ClassifierData.get_data_from_dirs(
    ['/mnt/work/training_data/statmt.tokenized/valid'], params=params)
#test_data = ClassifierData.get_data(params=params)
model_results = []
Example #13
0
def callcmd(cmd):
    print("executing: %s" % cmd)
    shell_call(cmd)
Example #14
0
else:
    keywords_cmd = ''
params_cmd = 'python3 %s/params2json.py %s' % (script_path, paramsfile)

params = utils.load_param_file(paramsfile)
model_name = params['model_name']

release_dir_name = model_name
release_cmds = []
release_cmds.append('mkdir %s' % os.path.join(model_dirname, release_dir_name))

release_files = [
    model_dirname + '/*.graphdef',
    model_dirname + '/*.json',
    model_dirname + '/release.timestamp.txt',
]
for src_file in release_files:
    release_cmds.append(
        'cp -rvf %s %s' %
        (src_file, os.path.join(model_dirname, release_dir_name)))

copy2repo = 'python3 %s/copy2repo.py --paramsfile %s --release_dir_name %s' % (
    script_path, os.path.join(model_dirname, 'params.py'), release_dir_name)

cmds = [freeze_cmd, params_cmd] + release_cmds
#cmds = [freeze_cmd, vocab_cmd, params_cmd] + release_cmds + [keywords_cmd, copy2repo]
for c in cmds:
    if len(c) > 0:
        print("EXECUTING: %s" % c)
        shell_call(c)
Example #15
0
                        help='SSH public key absolute path. It would be used to get passwordless login to cloud '
                             'instances. Default is ~/.ssh/id_rsa.pub')
    # parser.add_argument('--cont', '-t', action='store_true',
    #                    help='If your script fails because of any reason in middle of somethhing, use this flag. '
    #                         'This flag will resume the script from failed step. ')

    parser.add_argument('--clean', '-c', action='store_true', help='cleanup instances')
    args = parser.parse_args()

    ssh_key_file = args.ssh_key_file
    config_file = args.config_file
    deployment_id = args.deployment_id
    local_work_dir = os.environ['HOME']

    config = ConfigParser.ConfigParser()
    config.read(config_file)
    sections = config.sections()

    if args.clean:
        # TODO: Needs to be updated. This should be a function and should clean the instances according to supplied tag.
        print("==> Removing deployment nodes")
        f = open(local_work_dir + '/.' + deployment_id + '_mesos_all_ips', 'r')
        for ip in f:
            ip = ip.rstrip()
            # setup_helpers.delete_instance(config, ip)
        shell_call("rm " + local_work_dir + "/." + deployment_id + "_mesos_all_ips")
        shell_call("rm " + local_work_dir + "/." + deployment_id + "_mesos_masters_ips")
        shell_call("rm " + local_work_dir + "/." + deployment_id + "_mesos_slaves_ips")
    else:
        setup(config_file, deployment_id, ssh_key_file)
Example #16
0
 def join(self):
     """Uses pandaseq 2.8"""
     self.assembled.remove()
     command = 'pandaseq28 -T 1 -f %s -r %s -u %s -F 1> %s 2> %s'
     command = command % (self.fwd_path, self.rev_path, self.unassembled.path, self.assembled.path, self.assembled.p.out)
     shell_call(command) # Because it exits with status 1 https://github.com/neufeld/pandaseq/issues/40
Example #17
0
#! /usr/bin/python

#import module

from shell_command import shell_call

shell_call("ls *.py")

#Below link is contains all the shell command which you may want to execute through this program
#https://ss64.com/bash/