Пример #1
0
def get_json(url):
    logger.info("GET: %s" % url)
    options = webdriver.ChromeOptions()
    options.add_argument('--headless')
    options.add_argument('--no-sandbox')
    options.add_argument('--disable-gpu')
    options.add_argument('--window-size=1280x1696')
    options.add_argument('--user-data-dir=/tmp/user-data')
    options.add_argument('--hide-scrollbars')
    options.add_argument('--enable-logging')
    options.add_argument('--log-level=0')
    options.add_argument('--v=99')
    options.add_argument('--single-process')
    options.add_argument('--data-path=/tmp/data-path')
    options.add_argument('--ignore-certificate-errors')
    options.add_argument('--homedir=/tmp')
    options.add_argument('--disk-cache-dir=/tmp/cache-dir')
    options.add_argument(
        'user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'
    )
    options.binary_location = os.getcwd() + "/bin/headless-chromium"

    driver = webdriver.Chrome(chrome_options=options)

    driver.get(url)
    pre = driver.find_element_by_tag_name("pre").text

    try:
        return json.loads(pre)
    except Exception as e:
        logger.err(e)
        return None
    finally:
        driver.close()
Пример #2
0
def lambda_handler(event, context):
    file = scrape_d.get_content()

    # upload the csv to s3
    session = boto3.Session(aws_access_key_id=AWS_KEY,
                            aws_secret_access_key=AWS_SECRET,
                            region_name=REGION_NAME)
    s3 = session.resource('s3')
    bucket = s3.Bucket(S3_BUCKET)

    try:
        s3.meta.client.head_bucket(Bucket=S3_BUCKET)

        logger.info('Uploading result of scrape to S3')
        bucket.upload_file('/tmp/%s.csv' % file,
                           '%s.csv' % file,
                           ExtraArgs={'ACL': 'public-read'})

        logger.success('Uploaded %s.csv to S3' % file)
    except ClientError as e:
        # 404 error, bucket does not exist.
        error_code = e.response['Error']['Code']
        if error_code == '404':
            logger.err('Bucket(%s) does not exist' % S3_BUCKET)
        else:
            logger.err(e)
Пример #3
0
 def __init__(self, username, password=None, key_filename=None, host=DEFAULT_SSH_HOST, port=DEFAULT_SSH_PORT):
     pkey = os.path.expanduser(key_filename) if key_filename is not None else None
     self.send_lock = threading.Lock()
     self.ssh_client = paramiko.SSHClient()
     self.ssh_client.load_system_host_keys()
     self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
     connect_kw = {"username": username,
                   "compress": True,
                   "look_for_keys": True}
     if password is not None:
         connect_kw["password"] = password
         connect_kw["look_for_keys"] = False
     if key_filename is not None:
         connect_kw["key_filename"] = key_filename
         connect_kw["look_for_keys"] = False
     try:
         self.ssh_client.connect(host, port, **connect_kw)
     except paramiko.ssh_exception.PasswordRequiredException as e:
         if e.message == "Private key file is encrypted":
             raise SshRpcKeyEncrypted()
         raise SshRpcError()
     except paramiko.ssh_exception.SSHException as e:
         if e.message == "No authentication methods available":
             raise SshRpcKeyNoAuthMethod()
         if e.message == "Authentication failed.":
             raise SshRpcKeyAuthFailed()
         raise SshRpcError(e.message)
     except socket.error as e:
         log.err("Unable to establish connection, please try again later ({})".format(e))
         os._exit(1)
     self.ssh_transport = self.ssh_client.get_transport()
     self.ssh_transport.set_keepalive(30)
     self.ssh_channel = None
     self._server_update()
Пример #4
0
 def image(file):
     try:
         img = pygame.image.load(file)
         log.info(f"Loading texture: {file}")
         return img
     except:
         log.err(f"Failed to load texture: {file}")
         return unknown
Пример #5
0
 def do_check_grub(self):
     self.update_progress(_("Checking bootloader"),True)
     log(" --> Checking Grub configuration")
     if os.path.exists("/target/boot/grub/grub.cfg"):
         return True
     else:
         err("!No /target/boot/grub/grub.cfg file found!")
         return False
Пример #6
0
 def __remove__(file):
     try:
         os.remove(file)
     except PermissionError:
         time.sleep(3)
         try:
             os.remove(file)
         except Exception as e:
             log.err(f"Failed to remove swap file \"{file}\"")
Пример #7
0
def create_nfs_maps(cluster_fileserver, cluster_name):
    logger = logging.getLogger("cluster-creation")
    cmd = 'ssh batfs0101 /hub/sysadmin/bin/getauto.devel2'
    ret, out = do_cmd(cmd)
    if ret != 0:
        logger.error('Checkout failed')
    #logger.error(out)
    else:
        logger.info(out)
        logger.info('Checking passed')

    sfs_ah = '/usr/bin/echo -e "%s\tbatfs-%s-nfs:/vmgr/&" >> /var/tmp/auto.sfs.template.devel.bat' % (
        cluster_name, cluster_name)
    cmd = 'ssh batfs0101\t' "'%s'" % (sfs_ah)
    ret, out = do_cmd(cmd)
    logger.info(out)

    sfs_blr = '/usr/bin/echo -e "%s\tbatfs-%s-nfs:/vmgr/&" >> /var/tmp/sfs.worldwide.US-Billerica.devel' % (
        cluster_name, cluster_name)
    cmd = 'ssh batfs0101\t' "'%s'" % (sfs_blr)
    ret, out = do_cmd(cmd)
    logger.info(out)

    proc = subprocess.run(
        ['ssh', 'batfs0101', '/hub/sysadmin/bin/putauto.devel2'],
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        input='Y\n',
        encoding='ascii')
    if proc != 0:
        logger.info(proc.stdout)
    else:
        logger.err(proc.stderr)

    dfs_maps_batdfs = "ssh dc-00 'dfsutil link add \\\mathworks\\batdfsroot\\{} \\\\batfs-{}-cifs\\vmgr\\{}'".format(
        cluster_name, cluster_name, cluster_name)
    print(dfs_maps_batdfs)
    ret, out = do_cmd(dfs_maps_batdfs)
    logger.info(out)

    dfs_maps_blrbat = "ssh dc-00 'dfsutil link add \\\mathworks\\blrbatroot\\{} \\\\batfs-{}-cifs\\vmgr\{}'".format(
        cluster_name, cluster_name, cluster_name)
    print(dfs_maps_blrbat)
    ret, out = do_cmd(dfs_maps_blrbat)
    logger.info(out)

    dfs_pro_ttl = "ssh dc-00 'dfsutil property ttl set \\\mathworks\\batdfsroot\\{} 86400'".format(
        cluster_name)
    ret, out = do_cmd(dfs_pro_ttl)
    logger.info(out)

    dfs_pro_bat_ttl = "ssh dc-00 'dfsutil property ttl set \\\mathworks\\blrbatroot\{} 86400'".format(
        cluster_name)
    ret, out = do_cmd(dfs_pro_bat_ttl)
    logger.info(out)
Пример #8
0
 def reset(self):
     "Resets/restarts server uncleanly. No params."
     if self.reactor:
         args = sys.argv #take the original arguments
         args.insert(0, sys.executable) # add python
         os.execv(sys.executable, args) # replace self with new python.
         logger.err("Reset failed:", sys.executable, str(args))
         return # This will not return.  os.execv should overwrite us.
     else:
         logger.err("<Not resetting: no reactor>")
     pass
Пример #9
0
def get_op(x: str):
    """
    Get the opcode of a given number
    :param x: number
    :return: none, print opcode
    """
    to_plug = int(x) if x.find("0x") == -1 else int(x, 16)
    value = get_opcode(to_plug)
    if value is not None:
        excite(f'opcode for {x}: {value}')
    else:
        err(f'could not get opcode for {x}!')
Пример #10
0
    def __init__(self, setup):
        self.setup = setup

        # find the squashfs..
        self.media = config.get("loop_directory", "/dev/loop0")

        if (not os.path.exists(self.media)):
            err("Critical Error: Live medium (%s) not found!" % self.media)
            # sys.exit(1)
        inf("Using live medium: " + self.media)
        self.our_total = 0
        self.our_current = 0
Пример #11
0
def get_instr(args):
    """
    Get instructions for file with given arguments
    :param args: arguments
    :return: none, prints respectively
    """
    # Weird list parsing
    _len = calculate_real_len(args)
    if _len == 3:
        if args[1] is not None and args[2] is not None:
            get_instr_s(*args)
    elif _len == 1:
        get_instr_a(args[0])
    else:
        err(f'Invalid arguments! Given: {args}')
Пример #12
0
def parse(hexdump, from_loc: str, to_loc: str):
    """
    Parse a hex dump from a location to another location
    :param hexdump: dump contents
    :param from_loc: starting location
    :param to_loc: end location
    :return: lines of hex between
    """
    dump_as_arr = str(hexdump).split("\n")
    from_index = find(hexdump, from_loc)
    to_index = find(hexdump, to_loc)
    if int(from_loc, 16) > int(to_loc, 16):
        err(f'Cannot parse locations backwards! Asked to find {from_loc} to {to_loc}')
        return

    return dump_as_arr[from_index:to_index]
Пример #13
0
def load_config(config_path):
    """read config file"""
    if os.path.isfile(config_path):
        file = open(config_path, "r")
        content = file.read()
        inf("#Reading yaml file:" + config_path)
        log(content)
    else:
        err("{} doesn't exists. Please create config file!".format(
            config_path))
        return {}

    try:
        return yaml.load(content, Loader=yaml.FullLoader) or {}
    except:
        return yaml.load(content) or {}
Пример #14
0
def run(cmd, vital=True):
    inf("Running: "+cmd)
    if "||" in cmd:
        mode = cmd.split("||")[0].strip()
        cmd = cmd.split("||")[1].strip()
        if "{distro_codename}" in cmd:
            cmd = cmd.replace("{distro_codename}",
                              config.get("distro_codename", "linux"))
        if mode == "chroot":
            i = do_run_in_chroot(cmd)
        else:
            i = os.system(cmd)
    else:
        i = os.system(cmd)
    if vital and i != 0:
        err("Failed to run command (Exited with {}): {}".format(str(int(i/512)), cmd))
    return i
Пример #15
0
 def populateEmptyChunkInfo(self, dbName):
     """Populate self.emptyChunkInfo[dbName] with a set() containing
     chunkIds for the empty chunks of the particular db. If no empty
     chunk information can be found and loaded for the db, a default
     empty chunks file is used."""
     sanitizedDbName = filter(lambda c: (c in string.letters) or
                              (c in string.digits) or (c in ["_"]),
                              dbName)
     if sanitizedDbName != dbName:
         logger.wrn("WARNING, dbName=", dbName,
                    "contains questionable characters. sanitized=",
                    sanitizedDbName)
     name = "empty_%s.txt" % sanitizedDbName
     info = self.loadIntsFromFile(name)
     if not info and type(info) != type(list):
         logger.err("Couldn't find %s, using %s." % (name,
                    self.defaultEmptyChunks))
         self.emptyChunkInfo[dbName] = self.emptyChunkInfo[""]
     else:
         self.emptyChunkInfo[dbName] = info
     pass
Пример #16
0
    def answer(self, text, attachment=None, domain=None, title=None):
        receiver = 'chat_id' if self.dialog_id < 0 else 'user_id'
        receiver_id = abs(self.dialog_id)
        pars = {
            'message': text,
            receiver: unicode(receiver_id),
            #'access_token': tokens.get_token(),
            'guid': random.randint(0, 1000000),
        }

        if attachment is not None:
            pars['attachment'] = attachment
        if title is not None:
            pars['title'] = title
        if domain is not None:
            del pars[receiver]
            pars['domain'] = domain

        response = api.query(u'messages.send', args=pars, bot=self.bot)
        info = vk_utils.get_bot_info()
        if info is None:
            logger.err(u'Не удалось получить информацию о боте, ответ не отправлен. ')
            return None
        if response is None:
            logger.log(u'Ничего не пришло в ответ от Vk в classes.py@answer')
            return response
        if response.get('error') is None:
            if info['status'] != 'ok':
                #api.query(u'account.setOnline')
                vk_utils.set_bot_status(info, 'ok')
        else:
            logger.log(u'Пришла ошибка от VKAPI во время отправки сообщения. ' \
                + unicode(response['error']['error_msg'])) 
            if response['error']['error_code'] == 14: 
                vk_utils.set_bot_status(info, 'error', response['error']['captcha_sid'])

        return response
Пример #17
0
def do_api(path, account_id):
    if os.path.isfile(API_KEY_FILE):
        with open(API_KEY_FILE) as f:
            key = f.read().strip()
        r = requests.get(API_ENDPOINT + path, auth=(account_id, key))
        if r.status_code == 200:
            log.ok("Success!")
            return 0
        else:
            if r.status_code == 401:
                log.err("Invalid account_id and/or api_key given")
            elif r.status_code == 403:
                log.err("You have no permission to use the that resource")
            elif r.status_code == 404:
                log.err("The endpoint could not be found")
            return 1
    else:
        log.err("Could not find API key file [{key_file}]".format(key_file=API_KEY_FILE))
        return 1
Пример #18
0
def run(rpc, recipe, is_dev):
    lc = 0
    state = {"software_list": {}, "name": None, "is_dev": is_dev}
    try:
        for statement in rparser.parse(recipe):
            # Everything after # is comment.
            lc += 1
            command = statement[0]
            args = statement[1:]
            state = rpc.call("rc_" + command, {"args": args, "state": state})
            # recipe_execute_cmd(rpc, state, clean_line)
        return state
    except DocoptExit as e:
        log.err('Recipe contains an invalid command at Line {line}.'.format(line=lc))
        log.err(e)
    except RecipeRuntimeError as e:
        log.err('Recipe command on line {line} failed with error: {e}'.format(line=lc, e=e))
Пример #19
0
    def start_installation(self):

        # mount the media location.
        log(" --> Installation started")
        if(not os.path.exists("/target")):
            os.mkdir("/target")
        if(not os.path.exists("/source")):
            os.mkdir("/source")

        # Custom commands
        self.do_hook_commands("pre_install_hook")

        run("umount -lf /target/dev/shm")
        run("umount -lf /target/dev/pts")
        run("umount -lf /target/dev/")
        run("umount -lf /target/sys/")
        run("umount -lf /target/proc/")
        run("umount -lf /target/run/")

        self.mount_source()

        if self.setup.automated:
            self.create_partitions()
        else:
            self.format_partitions()
            self.mount_partitions()

        # Custom commands
        self.do_hook_commands("pre_rsync_hook")

        # Transfer the files
        SOURCE = "/source/"
        DEST = "/target/"

        self.our_current = 0
        # (Valid) assumption: num-of-files-to-copy ~= num-of-used-inodes-on-/
        self.our_total = int(subprocess.getoutput(
            "df --inodes /{src} | awk 'END{{ print $3 }}'".format(src=SOURCE.strip('/'))))
        log(" --> Copying {} files".format(self.our_total))

        if config.get("netinstall",False):
            self.run_and_update(config.package_manager("create_rootfs"))
            pkgs = open("branding/netinstall_packages.txt").read().split("\n")

        else:
            if config.get("use_rsync",True) and 0 == os.system("which rsync &>/dev/null"):
                EXCLUDE_DIRS = "dev/* proc/* sys/* tmp/* run/* mnt/* media/* lost+found source target".split()

                # Add optional entries to EXCLUDE_DIRS
                for dirvar in config.get("exclude_dirs", ["home/*", "data/user/*"]):
                    EXCLUDE_DIRS.append(dirvar)

                rsync_filter = ' '.join(
                    '--exclude=' + SOURCE + d for d in EXCLUDE_DIRS)
                rsync = subprocess.Popen("rsync --verbose --archive --no-D --acls "
                                     "--hard-links --xattrs {rsync_filter} "
                                     "{src}* {dst}".format(src=SOURCE,
                                                           dst=DEST, rsync_filter=rsync_filter),
                                     shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                while rsync.poll() is None:
                    line = str(rsync.stdout.readline().decode(
                        "utf-8").replace("\n", ""))
                    if not line:  # still copying the previous file, just wait
                        time.sleep(0.1)
                    else:
                        self.our_current = min(self.our_current + 1, self.our_total)
                        self.update_progress(_("Copying /%s") % line)
                log(_("rsync exited with return code: %s") % str(rsync.poll()))
            elif config.get("use_unsquashfs",True)  and 0 == os.system("which unsquashfs &>/dev/null"):
                pwd = os.getcwd()
                os.chdir("/target")
                self.update_progress(_("Extracting rootfs."),pulse=True)
                run("unsquashfs /dev/loop0")
                run("mv /target/squashfs-root/* /target")
                run("rm -rf /target/squashfs-root")
                os.chdir(pwd)
            else:
                cp = subprocess.Popen("cp -prvf {src}* {dst}".format(src=SOURCE, dst=DEST),
                                     shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                while cp.poll() is None:
                    line = str(cp.stdout.readline().decode(
                        "utf-8")).split("'")[1]
                    if not line:  # still copying the previous file, just wait
                        time.sleep(0.1)
                    else:
                        self.our_current = min(self.our_current + 1, self.our_total)
                        self.update_progress(_("Copying /%s") % line)
        # Custom commands
        self.do_hook_commands("post_rsync_hook")

        # Steps:
        self.our_total = 12
        self.our_current = 0
        # chroot
        log(" --> Chrooting")
        self.update_progress(_("Entering the system ..."))
        run("mount --bind /dev/ /target/dev/")
        run("mount --bind /dev/shm /target/dev/shm")
        run("mount --bind /dev/pts /target/dev/pts")
        run("mount --bind /sys/ /target/sys/")
        run("mount --bind /proc/ /target/proc/")
        run("mount --bind /run/ /target/run/")
        if os.path.exists("/sys/firmware/efi"):
            run("mount --bind /sys/firmware/efi/efivars /target/sys/firmware/efi/efivars")
        run("mv /target/etc/resolv.conf /target/etc/resolv.conf.bk")
        run("cp -f /etc/resolv.conf /target/etc/resolv.conf")

        if config.get("netinstall",False):
            cmd = config.package_manager("install_package",pkgs)
            self.run_and_update("chroot /target {}".format(cmd))

        kernelversion = subprocess.getoutput("uname -r")
        if os.path.exists("/lib/modules/{0}/vmlinuz".format(kernelversion)):
            run(
                "cp /lib/modules/{0}/vmlinuz /target/boot/vmlinuz-{0}".format(kernelversion))

        # add new user
        log(" --> Adding new user")
        self.our_current += 1
        try:
            for cmd in config.distro["run_before_user_creation"]:
                run("chroot||"+cmd)
        except:
            err("This action not supported for your distribution.")
        self.update_progress(_("Adding new user to the system"))
        # TODO: support encryption

        run('chroot||useradd -m -s {shell} -c \"{realname}\" {username}'.format(
            shell=config.get("using_shell", "/bin/bash"), realname=self.setup.real_name,
            username=self.setup.username))

        # Add user to additional groups
        for group in config.get("additional_user_groups", ["audio", "video", "netdev"]):
            run("chroot||usermod -aG {} {}".format(group, self.setup.username))

        if (run("which chpasswd &>/dev/null") == 0) and config.get("use_chpasswd", True):
            fp = open("/target/tmp/.passwd", "w")
            fp.write(self.setup.username + ":" + self.setup.password1 + "\n")
            if config.get("set_root_password", True):
                fp.write("root:" + self.setup.password1 + "\n")
            fp.close()
            run("chroot||cat /tmp/.passwd | chpasswd")
            run("chroot||rm -f /tmp/.passwd")
        else:
            run("chroot||echo -e \"{0}\\n{0}\\n\" | passwd {1}".format(
                self.setup.password1, self.setup.username))
            if config.get("set_root_password", True):
                run("chroot||echo -e \"{0}\\n{0}\\n\" | passwd".format(
                    self.setup.password1))

        self.our_current += 1
        # Set autologin for user if they so elected
        if self.setup.autologin:
            # Auto Login Groups
            for i in config.display_manager["set_autologin"]:
                run(i.replace("{user}",self.setup.username))

        # /etc/fstab, mtab and crypttab
        self.our_current += 1
        self.update_progress(_("Writing filesystem mount information to /etc/fstab"))
        self.write_fstab()
Пример #20
0
    def finish_installation(self):
        # Steps:
        self.our_total = 12
        self.our_current = 4

        # write host+hostname infos
        log(" --> Writing hostname")
        self.our_current += 1
        self.update_progress(_("Setting hostname"))
        hostnamefh = open("/target/etc/hostname", "w")
        hostnamefh.write("%s\n" % self.setup.hostname)
        hostnamefh.close()
        hostsfh = open("/target/etc/hosts", "w")
        hostsfh.write("127.0.0.1\tlocalhost\n")
        hostsfh.write("127.0.1.1\t%s\n" % self.setup.hostname)
        hostsfh.write(
            "# The following lines are desirable for IPv6 capable hosts\n")
        hostsfh.write("::1     localhost ip6-localhost ip6-loopback\n")
        hostsfh.write("fe00::0 ip6-localnet\n")
        hostsfh.write("ff00::0 ip6-mcastprefix\n")
        hostsfh.write("ff02::1 ip6-allnodes\n")
        hostsfh.write("ff02::2 ip6-allrouters\n")
        hostsfh.write("ff02::3 ip6-allhosts\n")
        # Append hosts file from branding
        if os.path.isfile("./branding/hosts"):
            f = open("./branding/hosts","r").readlines()
            for line in f:
                hostsfh.write(line)
        hostsfh.close()

        # set the locale
        log(" --> Setting the locale")
        self.our_current += 1
        self.update_progress(_("Setting locale"))
        run("echo \"%s.UTF-8 UTF-8\" >> /target/etc/locale.gen" %
            self.setup.language)
        run("chroot||locale-gen")
        run("echo \"\" > /target/etc/default/locale")
        run("chroot||localectl set-locale LANG=\"%s.UTF-8\"" %
            self.setup.language)
        run("chroot||localectl set-locale LANG=%s.UTF-8" % self.setup.language)
        open("/target/etc/locale.conf", "w").write("LANG=%s.UTF-8" %
                                                   self.setup.language)
        # set the locale for gentoo / sulin
        if os.path.exists("/target/etc/env.d"):
            l = open("/target/etc/env.d/20language", "w")
            l.write("LANG={}.UTF-8".format(self.setup.language))
            l.write("LC_ALL={}.UTF-8".format(self.setup.language))
            l.flush()
            l.close()
            run("chroot||env-update")

        # set the timezone
        log(" --> Setting the timezone")
        self.our_current += 1
        self.update_progress(_("Setting timezone"))
        run("echo \"%s\" > /target/etc/timezone" % self.setup.timezone)
        run("rm -f /target/etc/localtime")
        run("ln -s /usr/share/zoneinfo/%s /target/etc/localtime" %
            self.setup.timezone)

        # Keyboard settings X11
        if not self.setup.keyboard_variant:
            self.setup.keyboard_variant = ""
        self.update_progress(("Settings X11 keyboard options"))
        newconsolefh = None
        if os.path.exists("/target/etc/X11/xorg.conf.d"):
            newconsolefh = open(
                "/target/etc/X11/xorg.conf.d/10-keyboard.conf", "w")
        elif os.path.exists("/target/usr/share/X11/xorg.conf.d/"):
            newconsolefh = open(
                "/target/usr/share/X11/xorg.conf.d/10-keyboard.conf", "w")
        if newconsolefh:
            newconsolefh.write('Section "InputClass"\n')
            newconsolefh.write('Identifier "system-keyboard"\n')
            newconsolefh.write('MatchIsKeyboard "on"\n')
            newconsolefh.write('Option "XkbLayout" "{}"\n'.format(
                self.setup.keyboard_layout))
            newconsolefh.write('Option "XkbModel" "{}"\n'.format(
                self.setup.keyboard_model))
            newconsolefh.write('Option "XkbVariant" "{}"\n'.format(
                self.setup.keyboard_variant))
            if "," in self.setup.keyboard_layout:
                newconsolefh.write('Option "XkbOptions" "grp:ctrls_toggle"\n')
            newconsolefh.write('EndSection\n')
            newconsolefh.close()

        # set the keyboard options..
        log(" --> Setting the keyboard")
        self.our_current += 1
        self.update_progress(_("Setting keyboard options"))
        if os.path.exists("/target/etc/default/console-setup"):
            consolefh = open("/target/etc/default/console-setup", "r")
            newconsolefh = open("/target/etc/default/console-setup.new", "w")
            for line in consolefh:
                line = line.rstrip("\r\n")
                if(line.startswith("XKBMODEL=")):
                    newconsolefh.write("XKBMODEL=\"%s\"\n" %
                                       self.setup.keyboard_model)
                elif(line.startswith("XKBLAYOUT=")):
                    newconsolefh.write("XKBLAYOUT=\"%s\"\n" %
                                       self.setup.keyboard_layout)
                elif(line.startswith("XKBVARIANT=") and self.setup.keyboard_variant != ""):
                    newconsolefh.write("XKBVARIANT=\"%s\"\n" %
                                       self.setup.keyboard_variant)
                else:
                    newconsolefh.write("%s\n" % line)
            consolefh.close()
            newconsolefh.close()
            run("chroot||rm /etc/default/console-setup")
            run("chroot||mv /etc/default/console-setup.new /etc/default/console-setup")

        # lfs like systems uses vconsole.conf (systemd)
        if os.path.exists("/target/etc/vconsole.conf"):
            consolefh = open("/target/etc/vconsole.conf", "r")
            newconsolefh = open("/target/etc/vconsole.conf.new", "w")
            for line in consolefh:
                line = line.rstrip("\r\n")
                if(line.startswith("KEYMAP=")):
                    if(self.setup.keyboard_variant != ""):
                        newconsolefh.write(
                            "KEYMAP=\"{0}-{1}\"\n".format(self.setup.keyboard_layout, self.setup.keyboard_variant))
                    else:
                        newconsolefh.write("KEYMAP=\"{0}\"\n".format(
                            self.setup.keyboard_layout))
                else:
                    newconsolefh.write("%s\n" % line)
            consolefh.close()
            newconsolefh.close()
            run("chroot||rm /etc/vconsole.conf")
            run("chroot||mv /etc/vconsole.conf.new /etc/vconsole.conf")

        # debian like systems uses this (systemd)
        if os.path.exists("/target/etc/default/keyboard"):
            consolefh = open("/target/etc/default/keyboard", "r")
            newconsolefh = open("/target/etc/default/keyboard.new", "w")
            for line in consolefh:
                line = line.rstrip("\r\n")
                if(line.startswith("XKBMODEL=")):
                    newconsolefh.write("XKBMODEL=\"%s\"\n" %
                                       self.setup.keyboard_model)
                elif(line.startswith("XKBLAYOUT=")):
                    newconsolefh.write("XKBLAYOUT=\"%s\"\n" %
                                       self.setup.keyboard_layout)
                elif(line.startswith("XKBVARIANT=") and self.setup.keyboard_variant != ""):
                    newconsolefh.write("XKBVARIANT=\"%s\"\n" %
                                       self.setup.keyboard_variant)
                elif(line.startswith("XKBOPTIONS=")):
                    newconsolefh.write("XKBOPTIONS=grp:ctrls_toggle")
                else:
                    newconsolefh.write("%s\n" % line)
            consolefh.close()
            newconsolefh.close()
            run("chroot||rm /etc/default/keyboard")
            run("chroot||mv /etc/default/keyboard.new /etc/default/keyboard")

        # Keyboard settings openrc
        if os.path.exists("/target/etc/conf.d/keymaps"):
            newconsolefh = open("/target/etc/conf.d/keymaps", "w")
            if not self.setup.keyboard_layout:
                self.setup.keyboard_layout = "en"
            newconsolefh.write("keymap=\"{}{}\"\n".format(
                self.setup.keyboard_layout, self.setup.keyboard_variant))
            newconsolefh.close()

        # remove pacman
        self.update_progress(_("Clearing package manager"),True)
        log(" --> Clearing package manager")
        log(config.get("remove_packages", ["17g-installer"]))
        run("chroot||yes | {}".format(config.package_manager(
            "remove_package_with_unusing_deps", config.get("remove_packages", ["17g-installer"]))))

        if self.setup.luks:
            with open("/target/etc/default/grub.d/61_live-installer.cfg", "w") as f:
                f.write("#! /bin/sh\n")
                f.write("set -e\n\n")
                f.write('GRUB_CMDLINE_LINUX="cryptdevice=%s:lvmlmde root=/dev/mapper/lvmlmde-root resume=/dev/mapper/lvmlmde-swap"\n' %
                        self.auto_root_physical_partition)
            run("chroot||echo \"power/disk = shutdown\" >> /etc/sysfs.d/local.conf")

        # recreate initramfs (needed in case of skip_mount also, to include things like mdadm/dm-crypt/etc in case its needed to boot a custom install)
        log(" --> Configuring Initramfs")
        self.our_current += 1
        self.update_progress(_("Generating initramfs"),pulse=True)

        for command in config.update_initramfs():
            run("chroot||"+command)
        self.update_progress(_("Preparing bootloader installation"),pulse=True)
        try:
            grub_prepare_commands = config.distro["grub_prepare"]
            for command in grub_prepare_commands:
                run(command)
        except:
            err("Grub prepare process not available for your distribution!")

        # install GRUB bootloader (EFI & Legacy)
        log(" --> Configuring Grub")
        self.our_current += 1
        if(self.setup.grub_device is not None):
            self.update_progress(_("Installing bootloader"),pulse=True)
            log(" --> Running grub-install")

            if os.path.exists("/sys/firmware/efi"):
                grub_cmd = config.distro["grub_installation_efi"]
                run(grub_cmd.replace("{disk}", self.setup.grub_device))
            else:
                grub_cmd = config.distro["grub_installation_legacy"]
                run(grub_cmd.replace("{disk}", self.setup.grub_device))

            # fix not add windows grub entry
            run("chroot||grub-mkconfig -o /boot/grub/grub.cfg")
            self.update_progress(_("Configuring bootloader"),pulse=True)
            self.do_configure_grub()
            grub_retries = 0
            while (not self.do_check_grub()):
                self.do_configure_grub()
                grub_retries = grub_retries + 1
                if grub_retries >= 5:
                    self.error_message(message=_(
                        "WARNING: The grub bootloader was not configured properly! You need to configure it manually."))
                    break

        # Custom commands
        self.do_hook_commands("post_install_hook")

        # now unmount it
        log(" --> Unmounting partitions")
        run("umount -lf /target/dev/shm")
        run("umount -lf /target/dev/pts")
        if os.path.exists("/sys/firmware/efi"):
            run("umount -lf /target/sys/firmware/efi/")
        if self.setup.gptonefi:
            run("umount -lf /target/boot/efi")
            run("umount -lf /target/media/cdrom")
        run("umount -lf /target/boot")
        run("umount -lf /target/dev/")
        run("umount -lf /target/sys/")
        run("umount -lf /target/proc/")
        run("umount -lf /target/run/")
        run("rm -f /target/etc/resolv.conf")
        run("mv /target/etc/resolv.conf.bk /target/etc/resolv.conf")
        for partition in self.setup.partitions:
            if(partition.mount_as is not None and partition.mount_as != "" and partition.mount_as != "/" and partition.mount_as != "swap"):
                self.do_unmount("/target" + partition.mount_as)
        self.do_unmount("/target")
        self.do_unmount("/source")

        self.update_progress(_("Installation finished"),done=True)
        log(" --> All done")
Пример #21
0
def main():
    args = runner_arguments()
    logger.account_info(args)
    start_time = datetime.datetime.now().replace(microsecond=0)
    logger.info('Template runner start time: [{}]'.format(start_time))

    # Create the blob client, for use in obtaining references to
    # blob storage containers and uploading files to containers.
    blob_client = azureblob.BlockBlobService(
        account_name=args.StorageAccountName,
        account_key=args.StorageAccountKey)

    # Create a batch account using AAD
    batch_client = create_batch_client(args)

    # Create a keyvault client using AAD
    keyvault_client_with_url = create_keyvault_client(args)

    # Clean up any storage container that is older than a 7 days old.
    utils.cleanup_old_resources(blob_client)

    repository_branch_name = args.RepositoryBranchName
    if repository_branch_name == "current":
        repository_branch_name = Repository('../').head.shorthand

    logger.info('Pulling resource files from the branch: {}'.format(
        repository_branch_name))

    try:
        images_refs = []  # type: List[utils.ImageReference]
        with open(args.TestConfig) as f:
            try:
                template = json.load(f)
            except ValueError as e:
                logger.err(
                    "Failed to read test config file due to the following error",
                    e)
                raise e

            for jobSetting in template["tests"]:
                application_licenses = None
                if 'applicationLicense' in jobSetting:
                    application_licenses = jobSetting["applicationLicense"]

                _job_managers.append(
                    job_manager.JobManager(
                        jobSetting["template"], jobSetting["poolTemplate"],
                        jobSetting["parameters"], keyvault_client_with_url,
                        jobSetting["expectedOutput"], application_licenses,
                        repository_branch_name))

            for image in template["images"]:
                images_refs.append(
                    utils.ImageReference(image["osType"], image["offer"],
                                         image["version"]))

        run_job_manager_tests(blob_client, batch_client, images_refs,
                              args.VMImageURL)

    except batchmodels.BatchErrorException as err:
        utils.print_batch_exception(err)
        raise
    finally:
        # Delete all the jobs and containers needed for the job
        # Reties any jobs that failed

        if args.CleanUpResources:
            utils.execute_parallel_jobmanagers("retry", _job_managers,
                                               batch_client, blob_client,
                                               _timeout / 2)
            utils.execute_parallel_jobmanagers("delete_resources",
                                               _job_managers, batch_client,
                                               blob_client)
            utils.execute_parallel_jobmanagers("delete_pool", _job_managers,
                                               batch_client)
        end_time = datetime.datetime.now().replace(microsecond=0)
        logger.print_result(_job_managers)
        logger.export_result(_job_managers, (end_time - start_time))
    logger.info('Sample end: {}'.format(end_time))
    logger.info('Elapsed time: {}'.format(end_time - start_time))
Пример #22
0
def get_content():

  filename = datetime.datetime.today().strftime("%Y-%m-%d")
  product_urls = []
  errors = []

  for brand in BRANDS:
    target_url = 'https://www.ssense.com/en-ca/men/designers/%s.json' % brand
    
    src = requestwrap.get_json(target_url)
    src_page = int(src['meta']['page'])
    max_page = int(src['meta']['total_pages'])
    
    if src_page <= max_page:
      for obj in src['products']:
        product_urls.append(obj['url'])
        
      # get the next page if paginated
      src_page += 1
      src = requestwrap.get_json(target_url + "?page=%s" % src_page)
    
  
  with open("/tmp/%s.csv" % filename, 'w') as csvfile:
    fieldnames = ['isSaleEnabled', 'isSaleSoon', 'isCaptchaEnabled', 'isSkuCaptchaProtected',
                  'productSku', 'productName', 'productGender', 
                  'productComposition', 'productCategory', 'productOrigin', 
                  'productInStock', 'productBrand', 'productRegPrice',
                  'productSalePrice', 'productDiscPrice', 'productCurrency',
                  'productIsUniSize', 'size', 'sizeSku', 'sizeInStock', 'dateRun']
  
    writer = csv.writer(csvfile, dialect='excel')
    # Write header
    writer.writerow(fieldnames)
  
    for url in product_urls:
      src = requestwrap.get_json("https://www.ssense.com/en-ca%s.json" % url)
      
      try:
        isSaleEnabled         = src['context']['isSaleEnabled']
        isSaleSoon            = src['context']['isSaleSoon']
        isCaptchaEnabled      = src['context']['isCaptchaEnabled']
        isSkuCaptchaProtected = src['context']['isSkuCaptchaProtected']
    
        productSku            = src['product']['sku']
        productName           = src['product']['name']
        productGender         = src['product']['gender']
        productComposition    = src['product']['composition']
        productCategory       = src['product']['category']['name']
        productOrigin         = src['product']['countryOfOrigin']
        productInStock        = src['product']['inStock']
        productBrand          = src['product']['brand']['name']
        productRegPrice       = src['product']['price']['regular']
        productSalePrice      = src['product']['price']['sale']
        productDiscPrice      = src['product']['price']['discount']
        productCurrency       = src['product']['price']['currency']
        productIsUniSize      = src['product']['isUniSize']
    
    
    
        productSizes       = []
        productSizeSkus    = []
        productSizeInStock = []
        for sizeObj in src['product']['sizes']:
          productSizes.append(sizeObj['name'])
          productSizeSkus.append(sizeObj['sku'])
          productSizeInStock.append(str(sizeObj['inStock']))

        # convert product(Sizes|SizeSkus|SizeInStock) to strings delimited by spaces
        productSizes       = " ".join(productSizes)
        productSizeSkus    = " ".join(productSizeSkus)
        productSizeInStock = " ".join(productSizeInStock)
    
        writer.writerow([isSaleEnabled, isSaleSoon, isCaptchaEnabled, isSkuCaptchaProtected,
                      productSku, productName, productGender, 
                      productComposition, productCategory, productOrigin, 
                      productInStock, productBrand, productRegPrice,
                      productSalePrice, productDiscPrice, productCurrency,
                      productIsUniSize, productSizes, productSizeSkus, productSizeInStock, filename])
      except KeyError:
        errors.append("ERROR: could not retrieve product details for %s" %s url)
        continue

  for e in errors:
    logger.err(e)

  return filename
Пример #23
0
def font(file):
    try:
        pyglet.font.add_file(file)
    except FileNotFoundError:
        log.err(f"Failed to load font: {file}")
Пример #24
0
            return kernel_vars[key]
        if key in live:
            return live[key]
        if key in main:
            return main[key]
    except:
        return default
    return default


# Distribution
if (get("distribution", "auto") == "auto"):
    for distro_system in glob("configs/distribution/*"):
        distro = load_config(distro_system)
        if not distro:
            err("Failed to load: " + distro_system)
        elif "check_this_dir" in distro and os.path.exists(
                distro["check_this_dir"]):
            break
else:
    distro = load_config("configs/distribution/{}.yaml".format(
        main["distribution"]))

# Initramfs system
if (get("initramfs_system", "auto") == "auto"):
    for initramfs_system in glob("configs/initramfs_systems/*"):
        initramfs = load_config(initramfs_system)
        if not initramfs:
            err("Failed to load: " + initramfs_system)
        elif "check_this_dir" in initramfs and os.path.exists(
                initramfs["check_this_dir"]):
Пример #25
0
    def start_installation(self):

        # mount the media location.
        log(" --> Installation started")
        if (not os.path.exists("/target")):
            os.mkdir("/target")
        if (not os.path.exists("/source")):
            os.mkdir("/source")

        run("umount -lf /target/dev/shm")
        run("umount -lf /target/dev/pts")
        run("umount -lf /target/dev/")
        run("umount -lf /target/sys/")
        run("umount -lf /target/proc/")
        run("umount -lf /target/run/")

        self.mount_source()

        if self.setup.automated:
            self.create_partitions()
        else:
            self.format_partitions()
            self.mount_partitions()

        # Custom commands
        self.do_pre_install_commands()

        # Transfer the files
        SOURCE = "/source/"
        DEST = "/target/"
        EXCLUDE_DIRS = "home/* dev/* proc/* sys/* tmp/* run/* mnt/* media/* lost+found source target".split(
        )

        # Add optional entries to EXCLUDE_DIRS
        for dirvar in config.get("exclude_dirs", ["/home"]):
            EXCLUDE_DIRS.append(dirvar)

        self.our_current = 0
        # (Valid) assumption: num-of-files-to-copy ~= num-of-used-inodes-on-/
        self.our_total = int(
            subprocess.getoutput(
                "df --inodes /{src} | awk 'END{{ print $3 }}'".format(
                    src=SOURCE.strip('/'))))
        log(" --> Copying {} files".format(self.our_total))
        rsync_filter = ' '.join('--exclude=' + SOURCE + d
                                for d in EXCLUDE_DIRS)
        rsync = subprocess.Popen("rsync --verbose --archive --no-D --acls "
                                 "--hard-links --xattrs {rsync_filter} "
                                 "{src}* {dst}".format(
                                     src=SOURCE,
                                     dst=DEST,
                                     rsync_filter=rsync_filter),
                                 shell=True,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)
        while rsync.poll() is None:
            line = str(rsync.stdout.readline().decode("utf-8").replace(
                "\n", ""))
            if not line:  # still copying the previous file, just wait
                time.sleep(0.1)
            else:
                self.our_current = min(self.our_current + 1, self.our_total)
                self.update_progress(_("Copying /%s") % line)
        log(_("rsync exited with return code: %s") % str(rsync.poll()))

        # Steps:
        self.our_total = 12
        self.our_current = 0
        # chroot
        log(" --> Chrooting")
        self.update_progress(_("Entering the system ..."))
        run("mount --bind /dev/ /target/dev/")
        run("mount --bind /dev/shm /target/dev/shm")
        run("mount --bind /dev/pts /target/dev/pts")
        run("mount --bind /sys/ /target/sys/")
        run("mount --bind /proc/ /target/proc/")
        run("mount --bind /run/ /target/run/")
        if os.path.exists("/sys/firmware/efi"):
            run("mount --bind /sys/firmware/efi/ /target/sys/firmware/efi/")
        run("mv /target/etc/resolv.conf /target/etc/resolv.conf.bk")
        run("cp -f /etc/resolv.conf /target/etc/resolv.conf")

        kernelversion = subprocess.getoutput("uname -r")
        if os.path.exists("/lib/modules/{0}/vmlinuz".format(kernelversion)):
            run("cp /lib/modules/{0}/vmlinuz /target/boot/vmlinuz-{0}".format(
                kernelversion))

        # add new user
        log(" --> Adding new user")
        self.our_current += 1
        try:
            for cmd in config.distro["run_before_user_creation"]:
                run("chroot||" + cmd)
        except:
            err("This action not supported for your distribution.")
        self.update_progress(_("Adding new user to the system"))
        # TODO: support encryption

        run('chroot||useradd -m -s {shell} -c \"{realname}\" {username}'.
            format(shell=config.get("using_shell", "/bin/bash"),
                   realname=self.setup.real_name,
                   username=self.setup.username))

        # Add user to additional groups
        for group in config.get("additional_user_groups",
                                ["audio", "video", "netdev"]):
            run("chroot||usermod -aG {} {}".format(group, self.setup.username))

        if (run("which chpasswd &>/dev/null") == 0) and config.get(
                "use_chpasswd", True):
            fp = open("/target/tmp/.passwd", "w")
            fp.write(self.setup.username + ":" + self.setup.password1 + "\n")
            if config.get("set_root_password", True):
                fp.write("root:" + self.setup.password1 + "\n")
            fp.close()
            run("chroot||cat /tmp/.passwd | chpasswd")
            run("chroot||rm -f /tmp/.passwd")
        else:
            run("chroot||echo -e \"{0}\\n{0}\\n\" | passwd {1}".format(
                self.setup.password1, self.setup.username))
            if config.get("set_root_password", True):
                run("chroot||echo -e \"{0}\\n{0}\\n\" | passwd".format(
                    self.setup.password1))

        self.our_current += 1
        self.update_progress(_("Applying login settings"))
        # Set LightDM to show user list by default
        if config.get("list_users_when_auto_login", True):
            run(r"chroot||sed -i -r 's/^#?(greeter-hide-users)\s*=.*/\1=false/' /etc/lightdm/lightdm.conf"
                )
        else:
            run(r"chroot||sed -i -r 's/^#?(greeter-hide-users)\s*=.*/\1=true/' /etc/lightdm/lightdm.conf"
                )

        # Set autologin for user if they so elected
        if self.setup.autologin:
            # LightDM and Auto Login Groups
            run("""chroot||groupadd -r autologin && gpasswd -a {user} autologin
                                  && groupadd -r nopasswdlogin & & gpasswd -a {user} nopasswdlogin"""
                .format(user=self.setup.username))
            run(r"chroot||sed -i -r 's/^#?(autologin-user)\s*=.*/\1={user}/' /etc/lightdm/lightdm.conf"
                .format(user=self.setup.username))

        # /etc/fstab, mtab and crypttab
        self.our_current += 1
        self.update_progress(
            _("Writing filesystem mount information to /etc/fstab"))
        self.write_fstab()
Пример #26
0
if __name__ == '__main__':
    # Initialization
    pyglet.resource.path.append('.')
    load.font("assets/minecraft/Minecraftia.ttf")
    #
    window = Game(width=856,
                  height=482,
                  caption=f"Minecraft 2D v{version}",
                  resizable=True)
    pyglet.clock.schedule_interval(window.update, window.frame_rate)
    pyglet.clock.schedule_interval(window.cal_fps, 0.5)
    from Scenes.Initialization import Initialization

    window.set_minimum_size(128, 128)
    window.set_icon(
        load.image("assets/minecraft/textures/blocks/crafting_table_side.png"))
    window.scene = Initialization(window)
    try:
        pyglet.app.run()
    except Exception as e:
        log.err(f"Fatal Error: {e}")
        raise e

    log.info("Stopping Game")

    window.run = False
    log.info("Cleaning swap folder")
    for name in os.listdir("swap"):
        os.remove(f"swap/{name}")
    log.info("Game is stopped")
Пример #27
0
#!/usr/bin/env python3
from cmd import call_command, set_up_arguments, parser
from logger import err

if __name__ == '__main__':

    set_up_arguments()
    args = parser.parse_args()

    to_format = args.INT_TO_BC
    to_dump = args.FILE
    to_op = args.INT_TO_OP
    from_loc = args.FROM
    to_loc = args.TO

    no_args = not to_format and not to_dump and not to_op

    if no_args:
        err("Missing arguments! Use [-h] for more information.")

    if to_format:
        call_command('-b', to_format)
    elif to_op:
        call_command('-op', to_op)
    elif to_dump:
        call_command('-f', [(to_dump, from_loc, to_loc)])