def setUp(self): # create temporary directory for the source files: self.sourcedir = tempfile.mkdtemp(prefix='sources') self.linkdir = tempfile.mkdtemp(prefix='links') os.chdir(self.sourcedir) logging.info("\nTestReplacingLinkSourceAndTarget: sourcedir: " + self.sourcedir + " and linkdir: " + self.linkdir) # create set of test files: self.create_source_file(self.SOURCEFILE1) self.create_source_file(self.SOURCEFILE2) self.create_source_file(self.SOURCEFILE3) self.create_source_file(self.SOURCEFILE3) self.create_source_file(self.SOURCEFILE4) # create symbolic links: self.create_link_file(self.SOURCEFILE1, self.LINKFILE1) self.create_link_file(self.SOURCEFILE2, self.LINKFILE2) self.create_link_file(self.SOURCEFILE3, self.LINKFILE3) self.create_link_file(self.SOURCEFILE4, self.LINKFILE4) if platform.system() != 'Windows': os.sync()
def delete_item(item_path): failed = 0 while os.path.exists(item_path): try: shutil.rmtree(item_path) except Exception as e: print("TryDelete failed!") print("Item: ", item_path) print("Stat: ", os.stat(item_path)) failed += 1 if failed > 20: print("Deletion failed!") return # You need an explicit sync call or the load_zips call can sometimes miss the new files. # Yes, this was actually an issue. os.sync() time.sleep(0.1 * failed) # Force the destructors to run in case we have the handle open still, somehow. gc.collect() os.stat(item_path) if not os.path.exists(item_path): return
def doio(blks) : os.remove(file) os.sync() print ("remove done, blocks",blks) t= t2.counter() f = open(file,'wb') for i in range(blks): f.write(buf) f.close() t= t2.counter() - t print ("all write",t,"us") t= t2.counter() f = open(file,'rb') for i in range(blks) : a=f.read(512) f.close() t= t2.counter() - t print ("all read",t,"us") # verify last block errs=0 for i in range(len(a)): if a[i] != i%256 : errs += 1 print ("errs",errs) return
def choose_wifi(dialog_title='TiLDA'): with dialogs.WaitingMessage(text='Scanning for networks...', title=dialog_title): visible_aps = nic().list_aps() visible_aps.sort(key=lambda x:x['rssi'], reverse=True) visible_ap_names = [] # We'll get one result for each AP, so filter dupes for ap in visible_aps: if ap['ssid'] not in visible_ap_names: visible_ap_names.append(ap['ssid']) visible_aps = None ssid = dialogs.prompt_option( visible_ap_names, text='Choose wifi network', title=dialog_title ) key = dialogs.prompt_text("Enter wifi key (blank if none)", width = 310, height = 220) if ssid: with open("wifi.json", "wt") as file: if key: conn_details = {"ssid": ssid, "pw": key} else: conn_details = {"ssid": ssid} file.write(json.dumps(conn_details)) os.sync() # We can't connect after scanning for some bizarre reason, so we reset instead pyb.hard_reset()
def sync(): """Sync and drop kernel caches""" log.debug("Syncing and dropping kernel caches""") KernelDropCaches.drop() os.sync() # Wait a bit for all the dirty blocks to be written onto disk time.sleep(3)
def refresh_token(self): LOG.debug('Refreshing token') if identity_lock.acquire(blocking=False): try: data = self.send({ "path": "auth/token", "headers": { "Authorization": "Bearer " + self.identity.refresh } }) IdentityManager.save(data, lock=False) LOG.debug('Saved credentials') except HTTPError as e: if e.response.status_code == 401: LOG.error('Could not refresh token, invalid refresh code.') else: raise finally: identity_lock.release() else: # Someone is updating the identity wait for release with identity_lock: LOG.debug('Refresh is already in progress, waiting until done') time.sleep(1.2) os.sync() self.identity = IdentityManager.load(lock=False) LOG.debug('new credentials loaded')
def download_to(self, target, timeout=90): start_time = time.time() if not self.socket: raise OSError("Invalid response socket state. Has the content already been consumed?") try: if "Content-Length" in self.headers: remaining = int(self.headers["Content-Length"]) elif "content-length" in self.headers: remaining = int(self.headers["content-length"]) else: raise Exception("No Content-Length") with open(target, 'wb') as f: f.write(self.content_so_far) remaining -= len(self.content_so_far) del self.content_so_far while remaining > 0: buf = self.socket.recv(BUFFER_SIZE) f.write(buf) remaining -= len(buf) if (time.time() - start_time) > timeout: raise Exception("HTTP request timeout") f.flush() os.sync() finally: self.close()
def _sync(self): LOG.debug(_("Flush file system buffers")) if hasattr(os, 'sync'): os.sync() else: import ctypes libc = ctype.CDLL("libc.so.6") libc.sync()
def flush(self): """Writes changes to flash""" if self.dirty: with open(self.filename, "wt") as file: file.write(json.dumps(self.data)) file.flush() os.sync() self.dirty = False
def sync(): try: getattr(os, 'sync') except Exception: import sh os.sync = sh.sync finally: os.sync()
def sync(): """Sync and drop kernel caches""" runCmd2('bitbake -m', ignore_status=True) log.debug("Syncing and dropping kernel caches""") KernelDropCaches.drop() os.sync() # Wait a bit for all the dirty blocks to be written onto disk time.sleep(3)
def inner(*args): self = args[0] path = args[1] overlay_path = path+".empty" mount_path = path+".mount" work_path = path+".work" print(_("Sanitary umount, to ensure that there aren't old mounting points")) self.run_external_program("umount {:s}".format(mount_path)) print(_("Mounting overlayfs for {:s} at {:s}, using {:s} for new files").format(path,mount_path,overlay_path)) # remove data inside destination path shutil.rmtree(overlay_path, ignore_errors=True) shutil.rmtree(mount_path, ignore_errors=True) shutil.rmtree(work_path, ignore_errors=True) try: os.mkdir(overlay_path) except: pass try: os.mkdir(mount_path) except: pass try: os.mkdir(work_path) except: pass if (0 != self.run_external_program('mount -t overlay -o rw,lowerdir="{:s}",upperdir="{:s}",workdir="{:s}" overlay "{:s}"'.format(path,overlay_path,work_path,mount_path))): return True # error!!! args2 = [] for a in range(len(args)): if a == 1: args2.append(mount_path) else: args2.append(args[a]) try: retval = func(*args2) except: retval = true self.run_external_program("umount {:s}".format(mount_path)) if (not retval): print(_("Mixing file systems")) self.merge_overlay(path,overlay_path) shutil.rmtree(overlay_path, ignore_errors=True) shutil.rmtree(mount_path, ignore_errors=True) shutil.rmtree(work_path, ignore_errors=True) os.sync() # sync disks return retval
def flush_fs_caches(): os.sync() try: with open('/proc/sys/vm/drop_caches', 'w') as handle: handle.write('3\n') print('-- Flushed fs cache.') except IOError: print('!! You need to be root to flush the fs cache.') sys.exit(-1)
def run(self, distro, image): # from now on we always cleanup ourselves after finishing signal.signal(signal.SIGINT, sig_handler) setup_device(image) wipe_device() self.create_partitions() self.mount_partitions() d = next(d for d in self.DISTROS if d.name == distro) # set rootfs location and start installing d.rootfs = self.part_mount['root'] log.info('Updating distro database') d.update_database() log.info('Bootstrap distro \'{}\''.format(d.long_name)) d.bootstrap() log.info('Mounting kernel partitions') self.mount_kernel_partitions() log.info('Bootstrap distro (phase 2) \'{}\''.format(d.long_name)) d.bootstrap_phase2() log.info('Setting up locale') d.setup_locale() log.info('Installing bootloader') d.install_bootloader() log.info('Installing kernel') d.install_kernel() log.info('Customizing image') d.customize_image() log.info('Setting up network') d.setup_network() # finish installation log.info('Finishing installation') os.sync() uid = os.getenv('SUDO_UID', '') guid = os.getenv('SUDO_GUID', '') if os.path.isfile(image) and uid != '': uid = int(uid) if guid == '': guid = None else: guid = int(guid) shutil.chown(image, uid, guid) return 0
def sync_os(): try: if hasattr(os, 'sync'): os.sync() else: import ctypes libc = ctypes.CDLL("libc.so.6") libc.sync() except: check_call(['sync'], shell=True)
def unmount(self, button=None): folder = self.win.choose_folder.get_file() try: if folder is not None and folder.has_prefix(Gio.File.new_for_path("/media")): mount_point = folder.find_enclosing_mount().get_default_location().get_path() subprocess.check_call(["umount", mount_point]) except (subprocess.CalledProcessError, Error) as ex: # pylint: disable=E0712 self.show_notification( "Speicherkarte konnte nicht sicher entfernt werden", "", "dialog-error") print(ex, file=sys.stderr) finally: os.sync()
def check_files(fn1, fn2): ld = listdir('/fram') if len(ld) > 2: for f in ld: #ignore config file if f !='config': fullname = '/fram/' + f if fullname != fn1 and fullname != fn2: # file not current so move cp(fullname, '/sd/logs/') remove(fullname) sync()
def check_environment(): from mod.settings import (LV2_PEDALBOARDS_DIR, DEFAULT_PEDALBOARD, DEFAULT_PEDALBOARD_COPY, DATA_DIR, DOWNLOAD_TMP_DIR, KEYS_PATH, BANKS_JSON_FILE, FAVORITES_JSON_FILE, UPDATE_CC_FIRMWARE_FILE, UPDATE_MOD_OS_FILE, CAPTURE_PATH, PLAYBACK_PATH) # create temp dirs if not os.path.exists(DOWNLOAD_TMP_DIR): os.makedirs(DOWNLOAD_TMP_DIR) # remove temp files for path in (CAPTURE_PATH, PLAYBACK_PATH, UPDATE_CC_FIRMWARE_FILE): if os.path.exists(path): os.remove(path) # check RW access if os.path.exists(DATA_DIR): if not os.access(DATA_DIR, os.W_OK): print("ERROR: No write access to data dir '%s'" % DATA_DIR) return False else: try: os.makedirs(DATA_DIR) except OSError: print("ERROR: Cannot create data dir '%s'" % DATA_DIR) return False # create needed dirs and files if not os.path.exists(KEYS_PATH): os.makedirs(KEYS_PATH) if not os.path.exists(LV2_PEDALBOARDS_DIR): os.makedirs(LV2_PEDALBOARDS_DIR) if os.path.exists(DEFAULT_PEDALBOARD_COPY) and not os.path.exists(DEFAULT_PEDALBOARD): shutil.copytree(DEFAULT_PEDALBOARD_COPY, DEFAULT_PEDALBOARD) if not os.path.exists(BANKS_JSON_FILE): with open(BANKS_JSON_FILE, 'w') as fh: fh.write("[]") if not os.path.exists(FAVORITES_JSON_FILE): with open(FAVORITES_JSON_FILE, 'w') as fh: fh.write("[]") # remove previous update file if os.path.exists(UPDATE_MOD_OS_FILE): os.remove(UPDATE_MOD_OS_FILE) os.sync() return True
def __exit__(self, type_, value, traceback_): if not self.is_relevant: return os.sync() # This should not be strictly necessary time.sleep(1) # Yikes, mount always fails without this sleep(). try: mount_point = UDISKS.mount(self.usb_disk) config.add_remounted(self.usb_disk) self.exit_callback(details(self.usb_disk)) except dbus.exceptions.DBusException as e: raise MountError(e) gen.log("Mounted %s" % (self.usb_disk))
def register(cfg): """ Register build outputs with the local Configuration Management System. """ build_id = cfg['build_id'] dirpath_branch_cms = cfg['paths']['dirpath_branch_cms'] dirpath_build_cms = os.path.join(dirpath_branch_cms, build_id) da.util.ensure_dir_exists(dirpath_build_cms) os.sync() # Archive results. archive_plan = { 'index': 'copy', 'log': 'copy', '1report': 'copy', } dirpath_branch_tmp = cfg['paths']['dirpath_branch_tmp'] for name in sorted(os.listdir(dirpath_branch_tmp)): if name not in archive_plan: continue action = archive_plan[name] if action == 'copy': dirpath_src = os.path.join(dirpath_branch_tmp, name) dirpath_dst = os.path.join(dirpath_build_cms, name) if os.path.isdir(dirpath_src): shutil.copytree(dirpath_src, dirpath_dst) else: shutil.copyfile(dirpath_src, dirpath_dst) # Set the expiration date on the CMS. build_time = cfg['timestamp']['datetime_utc'] expiration = datetime.timedelta( days = cfg['options']['cms_expiration_days']) da.expiration.set_expiration_date( dirpath_build_cms = dirpath_build_cms, identifier = 'cms', current_time = build_time, timedelta_expiration = expiration) # Delete expired builds. if cfg['options']['enable_cms_delete_old_builds']: rootpath_cms = cfg['paths']['rootpath_cms'] for dirpath_prev_build in _gen_all_previous_builds(rootpath_cms): if da.expiration.has_expired( dirpath_build_cms = dirpath_prev_build, time_now = build_time): shutil.rmtree(dirpath_prev_build)
def write(self): """ Write the bootloader configuration and install the bootloader. """ if self.skip_bootloader: return if self.update_only: self.update() return self.write_config() os.sync() self.stage2_device.format.sync(root=util.getTargetPhysicalRoot()) self.install()
def safeFileCopy(src, dst, sync=True): # src must be a file if (not os.path.isfile(src)) and (not os.path.islink(src)): log.error("safeFileCopy: Can't copy source as " + src + " is not a handled file.") return False # Make sure dst is either non-existent or a file (which we overwrite) if os.path.exists(dst): if os.path.isfile(dst): log.warning("safeFileCopy: Destination file " + dst + " already exists. Will overwrite.") elif os.path.isdir(dst): log.error("safeFileCopy: Destination target " + dst + " is a directory.") return False else: log.error("safeFileCopy: Destination target " + dst + " is unknown.") return False # Copy file to dst.tmp if not os.path.isdir(os.path.dirname(dst)): try: os.makedirs(os.path.dirname(dst)) except: log.error("safeFileCopy: Failed to create directories structure for destination " + dst + ".") return False if os.path.islink(src): linkto = os.readlink(src) os.symlink(linkto, dst + ".tmp") else: with open(src, 'rb') as srcfd, open(dst + ".tmp", "wb") as dsttmpfd: try: shutil.copyfileobj(srcfd, dsttmpfd) except Exception as s: log.error("safeFileCopy: Failed to copy " + src + " to " + dst + ".tmp .") log.error(str(s)) return False shutil.copymode(src, dst + ".tmp") if sync: os.fsync(dsttmpfd) # Rename and sync filesystem to disk os.rename(dst + ".tmp", dst) if sync: # # Make sure the write operation is durable - avoid data loss dirfd = os.open(os.path.dirname(dst), os.O_DIRECTORY) os.fsync(dirfd) os.close(dirfd) os.sync() return True
def write(self): """ Write the bootloader configuration and install the bootloader. """ if self.skip_bootloader: # pylint: disable=no-member return if self.update_only: # pylint: disable=no-member self.update() return try: os.sync() self.stage2_device.format.sync(root=util.getTargetPhysicalRoot()) # pylint: disable=no-member self.install() finally: self.write_config() # pylint: disable=no-member
def Show_Pass(): global PPID global DEBUG_MODE global ROOT_DIR global LOG_DIR now = datetime.datetime.now() print_green(PASS_BANNER) Log("PASS") shutil.move(os.path.join(ROOT_DIR, "%s.txt"%PPID), os.path.join(LOG_DIR, "%s_PASS.txt"%(PPID))) os.sync() print_green("Press Power Button to Shutdown UUT!!") Unmount_Folder()
def setUp(self): # create temporary directory: self.tempdir = tempfile.mkdtemp() os.chdir(self.tempdir) print("\nTestFileWithoutTags: temporary directory: " + self.tempdir) # create set of test files: self.create_tmp_file(self.testfilename) # double-check set-up: self.assertTrue(self.file_exists(self.testfilename)) if platform.system() != 'Windows': os.sync()
def __enter__(self): if not self.is_relevant: return self.assert_no_access() try: gen.log("Unmounting %s" % self.usb_disk) os.sync() # This is needed because UDISK.unmount() can timeout. UDISKS.unmount(self.usb_disk) except dbus.exceptions.DBusException as e: gen.log("Unmount of %s has failed." % self.usb_disk) # This may get the partition mounted. Don't call! # self.exit_callback(details(self.usb_disk)) raise UnmountError(e) gen.log("Unmounted %s" % self.usb_disk) return self
def generate(self,path): """ Ensures that the base system, to create a CHROOT environment, exists """ # Create all, first, in a temporal folder tmp_path = path+".tmp" shutil.rmtree(tmp_path, ignore_errors=True) os.makedirs(tmp_path) if self.distro_type == "debian": server = "http://http.debian.net/debian/" else: server = "http://archive.ubuntu.com/ubuntu/" command = "debootstrap --variant=buildd --arch {:s} {:s} {:s} {:s}".format(self.architecture,self.distro_name,tmp_path,server) if (0 != self.run_external_program(command)): return True # error!!! f = open(os.path.join(tmp_path,"etc","apt","sources.list"),"w") if (self.distro_type == "debian"): # Add contrib and non-free to the list of packages sources if DEBIAN f.write("deb http://ftp.debian.org/debian/ {:s} main contrib non-free\n".format(self.distro_name)) else: # Add restricted, universe and multiverse if UBUNTU f.write("deb http://archive.ubuntu.com/ubuntu/ {:s} main restricted universe multiverse\n".format(self.distro_name)) f.close() command = 'apt clean' if (0 != self.run_chroot(tmp_path,command)): return True # error!!! command = 'apt update' if (0 != self.run_chroot(tmp_path,command)): return True # error!!! command = 'apt install meson ninja-build -y' if (0 != self.run_chroot(tmp_path,command)): return True # error!!! os.sync() os.rename(tmp_path,path) # rename the folder to the definitive name os.sync() shutil.rmtree(tmp_path, ignore_errors=True) return False # no error
def choose_wifi(dialog_title='TiLDA'): filtered_aps = [] with dialogs.WaitingMessage(text='Scanning for networks...', title=dialog_title): visible_aps = nic().list_aps() visible_aps.sort(key=lambda x:x['rssi'], reverse=True) # We'll get one result for each AP, so filter dupes for ap in visible_aps: title = ap['ssid'] security = get_security_level(ap) if security: title = title + ' (%s)' % security ap = { 'title': title, 'ssid': ap['ssid'], 'security': security, } if ap['ssid'] not in [ a['ssid'] for a in filtered_aps ]: filtered_aps.append(ap) del visible_aps ap = dialogs.prompt_option( filtered_aps, text='Choose wifi network', title=dialog_title ) if ap: key = None if ap['security'] != 0: # Backward compat if ap['security'] == None: ap['security'] = 'wifi' key = dialogs.prompt_text( "Enter %s key" % ap['security'], width = 310, height = 220 ) with open("wifi.json", "wt") as file: if key: conn_details = {"ssid": ap['ssid'], "pw": key} else: conn_details = {"ssid": ap['ssid']} file.write(json.dumps(conn_details)) os.sync() # We can't connect after scanning for some bizarre reason, so we reset instead pyb.hard_reset()
def test_etag_sync(tmpdir): fpath = os.path.join(str(tmpdir), 'foo') file_ = open(fpath, 'w') file_.write('foo') file_.close() os.sync() old_etag = vdir.get_etag_from_file(fpath) file_ = open(fpath, 'w') file_.write('foo') file_.close() new_etag = vdir.get_etag_from_file(fpath) assert old_etag != new_etag
def save_off(self, announce=True, reply=print): """Turn off automatic world saves, then force-save once. Optional arguments: announce -- Whether to announce in-game that saves are being disabled. reply -- This function is called with human-readable progress updates. Defaults to the built-in print function. """ if self.status(): reply('Minecraft is running... suspending saves') if announce: self.say('Server backup starting. Server going readonly...') self.command('save-off') self.command('save-all') time.sleep(10) os.sync() else: reply('Minecraft is not running. Not suspending saves.')
def create_ps3(dest, game_id, game_title, icon0, pic0, pic1, cue_files, cu2_files, img_files, mem_cards, aea_files, magic_word, resolution): print('Create PS3 PKG for', game_title) if verbose else None p = popstation() p.verbose = verbose p.game_id = game_id p.game_title = game_title #p.icon0 = icon0 #p.pic1 = pic1 p.complevel = 0 p.magic_word = magic_word if len(aea_files): p.aea = aea_files for i in range(len(img_files)): f = img_files[i] toc = None #toc = p.get_toc_from_ccd(f) # ps3 do not like these tocs if not toc: print('Need to create a TOC') if verbose else None toc = get_toc_from_cu2(cu2_files[i]) print('Add image', f) if verbose else None p.add_img((f, toc)) # create directory structure f = game_id print('GameID', f) try: os.mkdir(f) except: True sfo = { 'ANALOG_MODE': { 'data_fmt': 1028, 'data': 1}, 'ATTRIBUTE': { 'data_fmt': 1028, 'data': 2}, 'BOOTABLE': { 'data_fmt': 1028, 'data': 1}, 'CATEGORY': { 'data_fmt': 516, 'data_max_len': 4, 'data': '1P'}, 'PARENTAL_LEVEL': { 'data_fmt': 1028, 'data': 3}, 'PS3_SYSTEM_VER': { 'data_fmt': 516, 'data_max_len': 8, 'data': '01.7000'}, 'RESOLUTION': { 'data_fmt': 1028, 'data': resolution}, 'SOUND_FORMAT': { 'data_fmt': 1028, 'data': 1}, 'TITLE': { 'data_fmt': 516, 'data_max_len': 128, 'data': game_title}, 'TITLE_ID': { 'data_fmt': 516, 'data_max_len': 16, 'data': game_id}, 'VERSION': { 'data_fmt': 516, 'data_max_len': 8, 'data': '01.00'} } with open(f + '/PARAM.SFO', 'wb') as of: of.write(GenerateSFO(sfo)) temp_files.append(f + '/PARAM.SFO') image = icon0.resize((320, 176), Image.BILINEAR) i = io.BytesIO() image.save(f + '/ICON0.PNG', format='PNG') temp_files.append(f + '/ICON0.PNG') image = pic0.resize((1000, 560), Image.NEAREST) i = io.BytesIO() image.save(f + '/PIC0.PNG', format='PNG') temp_files.append(f + '/PIC0.PNG') image = pic1.resize((1920, 1080), Image.NEAREST) i = io.BytesIO() image.save(f + '/PIC1.PNG', format='PNG') temp_files.append(f + '/PIC1.PNG') image = pic1.resize((310, 250), Image.NEAREST) i = io.BytesIO() image.save(f + '/PIC2.PNG', format='PNG') temp_files.append(f + '/PIC2.PNG') with open('PS3LOGO.DAT', 'rb') as i: with open(f + '/PS3LOGO.DAT', 'wb') as o: o.write(i.read()) temp_files.append(f + '/PS3LOGO.DAT') f = game_id + '/USRDIR' try: os.mkdir(f) except: True _cfg = bytes([ 0x1c, 0x00, 0x00, 0x00, 0x50, 0x53, 0x31, 0x45, 0x6d, 0x75, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x46, 0x69, 0x6c, 0x65, 0x00, 0xe3, 0xb7, 0xeb, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xbb, 0xfa, 0xe2, 0x1b, 0x10, 0x00, 0x00, 0x00, 0x64, 0x69, 0x73, 0x63, 0x5f, 0x6e, 0x6f, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0xd1, 0x5b, 0xf8 ]) with open(f + '/CONFIG', 'wb') as o: o.write(_cfg) temp_files.append(f + '/CONFIG') f = game_id + '/USRDIR/CONTENT' try: os.mkdir(f) except: True p.eboot = game_id + '/USRDIR/CONTENT/EBOOT.PBP' p.iso_bin_dat = game_id + '/USRDIR/ISO.BIN.DAT' try: os.unlink(p.iso_bin_dat) except: True print('Create EBOOT.PBP at', p.eboot) p.create_pbp() temp_files.append(p.eboot) temp_files.append(p.iso_bin_dat) try: os.sync() except: True # sign the ISO.BIN.DAT print('Signing', p.iso_bin_dat) subprocess.call(['python3', './sign3.py', p.iso_bin_dat]) # # USRDIR/SAVEDATA # f = game_id + '/USRDIR/SAVEDATA' try: os.mkdir(f) except: True image = icon0.resize((80,80), Image.BILINEAR) i = io.BytesIO() image.save(f + '/ICON0.PNG', format='PNG') temp_files.append(f + '/ICON0.PNG') if len(mem_cards) < 1: create_blank_mc(f + '/SCEVMC0.VMP') if len(mem_cards) < 2: create_blank_mc(f + '/SCEVMC1.VMP') idx = 0 for mc in mem_cards: mf = f + ('/SCEVMC%d.VMP' % idx) with open(mf, 'wb') as of: print('Installing MemoryCard as', mf) of.write(encode_vmp(mc)) idx = idx + 1 temp_files.append(f + '/SCEVMC0.VMP') temp_files.append(f + '/SCEVMC1.VMP') sfo = { 'CATEGORY': { 'data_fmt': 516, 'data_max_len': 4, 'data': 'MS'}, 'PARENTAL_LEVEL': { 'data_fmt': 1028, 'data': 1}, 'SAVEDATA_DETAIL': { 'data_fmt': 516, 'data_max_len': 4, 'data': ''}, 'SAVEDATA_DIRECTORY': { 'data_fmt': 516, 'data_max_len': 4, 'data': game_id}, 'SAVEDATA_FILE_LIST': { 'data_fmt': 4, 'data_max_len': 3168, 'data': str(bytes(3168))}, 'SAVEDATA_TITLE': { 'data_fmt': 516, 'data_max_len': 128, 'data': ''}, 'TITLE': { 'data_fmt': 516, 'data_max_len': 128, 'data': game_title}, 'SAVEDATA_PARAMS': { 'data_fmt': 4, 'data_max_len': 128, 'data': str(b"A\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xda\xdaC4\x1br\xc2\xede\xa1/k'D\xc6\x11(\xcf\xc8\xb7(\xb8tG+*f\x85L\nm\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8a\xfa,\xa1\xe7+mA\xc5m.\x9a\xba\xbct\xb0")} } with open(f + '/PARAM.SFO', 'wb') as of: of.write(GenerateSFO(sfo)) temp_files.append(f + '/PARAM.SFO') # # Create ISO.BIN.EDAT # print('Create ISO.BIN.EDAT') pack('%s/USRDIR/ISO.BIN.DAT' % game_id, '%s/USRDIR/ISO.BIN.EDAT' % game_id, 'UP9000-%s_00-0000000000000001' % game_id) temp_files.append('%s/USRDIR/ISO.BIN.EDAT' % game_id) # # Create PS3 PKG # print('Create PKG') subprocess.call(['python3', 'PSL1GHT/tools/ps3py/pkg.py', '-c', 'UP9000-%s_00-0000000000000001' % game_id, game_id, dest]) temp_files.append(game_id + '/USRDIR/CONTENT') temp_files.append(game_id + '/USRDIR/SAVEDATA') temp_files.append(game_id + '/USRDIR') temp_files.append(game_id) print('Finished.', dest, 'created')
def _set_jupyter_password(self, password): self._get_db().put('DEVICE_JUPYTER_PASSWORD', password) os.sync()
def manager_thread() -> None: if EON: Process(name="autoshutdownd", target=launcher, args=("selfdrive.autoshutdownd", "autoshutdownd")).start() system("am startservice com.neokii.optool/.MainService") Process(name="road_speed_limiter", target=launcher, args=("selfdrive.road_speed_limiter", "road_speed_limiter")).start() cloudlog.bind(daemon="manager") cloudlog.info("manager start") cloudlog.info({"environ": os.environ}) params = Params() ignore: List[str] = [] if params.get("DongleId", encoding='utf8') in (None, UNREGISTERED_DONGLE_ID): ignore += ["manage_athenad", "uploader"] if os.getenv("NOBOARD") is not None: ignore.append("pandad") ignore += [x for x in os.getenv("BLOCK", "").split(",") if len(x) > 0] ensure_running(managed_processes.values(), started=False, not_run=ignore) started_prev = False sm = messaging.SubMaster(['deviceState']) pm = messaging.PubMaster(['managerState']) while True: sm.update() not_run = ignore[:] started = sm['deviceState'].started driverview = params.get_bool("IsDriverViewEnabled") ensure_running(managed_processes.values(), started, driverview, not_run) # trigger an update after going offroad if started_prev and not started and 'updated' in managed_processes: os.sync() managed_processes['updated'].signal(signal.SIGHUP) started_prev = started running = ' '.join( "%s%s\u001b[0m" % ("\u001b[32m" if p.proc.is_alive() else "\u001b[31m", p.name) for p in managed_processes.values() if p.proc) print(running) cloudlog.debug(running) # send managerState msg = messaging.new_message('managerState') msg.managerState.processes = [ p.get_process_state_msg() for p in managed_processes.values() ] pm.send('managerState', msg) # Exit main loop when uninstall/shutdown/reboot is needed shutdown = False for param in ("DoUninstall", "DoShutdown", "DoReboot"): if params.get_bool(param): shutdown = True params.put("LastManagerExitReason", param) cloudlog.warning(f"Shutting down manager - {param} set") if shutdown: break
NEW_SERIAL_NUMBER = "SB-R4" from opentrons import robot import os robot.comment(f"Setting serial number to {NEW_SERIAL_NUMBER}.") if not robot.is_simulating(): with open("/var/serial", "w") as serial_number_file: serial_number_file.write(NEW_SERIAL_NUMBER + "\n") with open("/etc/machine-info", "w") as serial_number_file: serial_number_file.write(f"DEPLOYMENT=production\nPRETTY_HOSTNAME={NEW_SERIAL_NUMBER}\n") with open("/etc/hostname", "w") as serial_number_file: serial_number_file.write(NEW_SERIAL_NUMBER + "\n") os.sync() robot.comment("Done.")
def executeCommands(self, command): ''' This is where we will actually be executing the commands :param command: the command we want to execute :return: Nothing ''' logging.debug("Execute Command: %s", command) usb = USB() if command == 'remove_usb': logging.debug("In remove usb page") if usb.isUsbPresent(): # check to see if usb is inserted logging.debug("USB still present") self.display.showRemoveUsbPage( ) # tell them to remove it if so self.display.pageStack = 'removeUsb' # let handleButtonPress know to repeat self.command_to_reference = 'remove_usb' # let executeCommands know what we want else: # if they were good and removed USB logging.debug("USB removed") self.display.pageStack = 'success' # let out handleButtonPress know self.display.showSuccessPage() # display our success page if command == 'copy_from_usb': logging.debug("copy from USB") x = ord("a") while (not usb.isUsbPresent('/dev/sd' + chr(x) + "1") and x < ord("k")): # check to see if usb is inserted x += 1 if x == ord("k"): self.display.showNoUsbPage( ) # if not, alert as this is required self.display.pageStack = 'error' return # cycle back to menu dev = '/dev/sd' + chr(x) + '1' with open('/usr/local/connectbox/PauseMount', 'w') as fp: fp.write(" ") time.sleep(2) self.pageStack = 'wait' # Dont allow the display to turn off self.display.showWaitPage("Checking Space") logging.debug("Using location " + dev + " as media copy location") if usb.getMount(dev) == '/media/usb0': logging.debug( "Moving /media/usb0 to /media/usb11 be able to copy") if not os.path.exists( '/media/usb11' ): # check that usb11 exsists to be able to move the mount os.mkdir('/media/usb11') # make the directory if not usb.moveMount( usb.getDev(dev), dev, '/media/usb11'): # see if our remount was successful self.display.showErrorPage( "Moving Mount") # if not generate error page and exit self.display.pageStack = 'error' try: os.remove('/usr/local/connectbox/PauseMount') except: pass return logging.debug("Preparing to check space of source " + (usb.getMount(dev))) (d, s) = usb.checkSpace(usb.getMount( dev)) # verify that source is smaller than destination logging.debug("space checked source : " + str(s) + ", destination : " + str(d) + " device " + dev) if s > d: logging.debug( "There is not enough space we will call an error on " + dev) self.display.showNoSpacePage( 1, dev) # if not, alert as this is a problem self.display.pageStack = 'error' if usb.getMount(dev) == '/media/usb11': logging.debug( "since we moved the moount we want /media/usb0 back") usb.moveMount(dev, '/media/usb11', '/media/usb0') try: os.remove('/media/usb11') except: pass try: os.remove('/usr/local/connectbox/PauseMount') except: pass return a = usb.getMount(dev) logging.debug("starting to do the copy with device " + a) self.display.showWaitPage("Copying Files") if not usb.copyFiles(a): # see if we copied successfully logging.debug("failed the copy. display an error page") self.display.showErrorPage( "Failed Copy") # if not generate error page and exit self.display.pageStack = 'error' try: os.remove('/usr/local/connectbox/PauseMount') except: pass return logging.debug("Finished all usb keys") logging.debug("Ok now we want to remove all the usb keys") curDev = '/dev/sda1' x = ord('a') while (not usb.isUsbPresent(curDev)) and x < ord("k"): logging.debug("is key " + curDev + " present? " + str(usb.isUsbPresent(curDev))) x += 1 curDev = '/dev/sd' + chr(x) + '1' while usb.isUsbPresent(curDev) and x < ord("k"): self.display.showRemoveUsbPage() #show the remove usb page self.display.pageStack = 'removeUsb' #show we removed the usb key self.command_to_reference = 'remove_usb' time.sleep(1) #Wait a second for the removeal while (not usb.isUsbPresent(curDev)) and x < ord("k"): x += 1 # lets look at the next one curDev = '/dev/sd' + chr(x) + '1' #create the next curdev # We finished the umounts self.display.pageStack = 'success' self.display.showSuccessPage() logging.debug("Success page now deleting the PauseMount file") try: os.remove('/usr/local/connectbox/PauseMount') except: pass self.display.pageStack = 'success' # if the usb was removed self.display.showSuccessPage() # display success page os.sync() return elif command == 'erase_folder': file_exists = False # in regards to README.txt file if usb.isUsbPresent(): self.display.pageStack = 'error' self.display.showRemoveUsbPage() return if os.path.isfile('/media/usb0/README.txt' ): # keep the default README if possible file_exists = True subprocess.call( ['cp', '/media/usb0/README.txt', '/tmp/README.txt']) logging.debug("README.txt moved") for file_object in os.listdir('/media/usb0'): file_object_path = os.path.join('/media/usb0', file_object) if os.path.isfile(file_object_path): os.unlink(file_object_path) else: shutil.rmtree(file_object_path) logging.debug("FILES NUKED!!!") if file_exists: subprocess.call( ['mv', '/tmp/README.txt', '/media/usb0/README.txt']) # move back logging.debug("README.txt returned") logging.debug("Life is good!") self.display.pageStack = 'success' self.display.showSuccessPage() elif command == 'copy_to_usb': logging.debug("got to copy to usb code") self.display.showConfirmPage() #We found at least one key x = ord('a') dev = '/dev/sd' + chr(x) + '1' self.display.showInsertUsbPage() #tell them to inert new keys while (not usb.isUsbPresent(dev)) and x < ord('k'): x += 1 dev = '/dev/sd' + chr(x) + '1' if x == ord('k'): x = ord('a') dev = '/dev/sd' + chr(x) + '1' if x == ord('k'): x = ord('a') self.display.pageStack = 'confirm' self.display.showConfirmPage() time.sleep(1) with open('/usr/local/connectbox/PauseMount', 'w') as fp: pass fp.close() time.sleep(2) self.display.pageStack = 'wait' self.display.showWaitPage("Checking Sizes") logging.debug("we have found at least one usb to copy to: " + dev) x = ord('a') dev = '/dev/sd' + chr(x) + '1' y = 0 logging.debug("were ready to start size check") while x < ord('k'): if usb.getMount( dev ) == '/media/usb0': # if the key is mounted on '/media/usb0' then we have to move it. logging.debug( "Moving /media/usb0 to /media/usb11 be able to copy") y += 1 if not os.path.exists( '/media/usb11' ): # check that usb11 exsists to be able to move the mount os.mkdir('/media/usb11') # make the directory if not usb.moveMount( dev, '/media/usb0', '/media/usb11' ): # see if our remount was successful self.display.showErrorPage( "Moving Mount" ) # if not generate error page and exit self.display.pageStack = 'error' try: os.remove('/usr/local/connectbox/PauseMount') except: pass return if usb.getMount(dev) != "": y += 1 x += 1 dev = '/dev/sd' + chr(x) + '1' x = ord('a') dev = '/dev/sd' + chr(x) + '1' while x < ord( 'k' ) and y > 0: #While we know we have a usb key lets check the sizes if usb.getMount(dev) != "": zz = usb.getMount(dev) logging.debug( "getting the size for source /media/usb0 and destination " + zz) (d, s) = usb.checkSpace( '/media/usb0', zz) # verify that source is smaller than destination logging.debug("Space of Destination is : " + str(d) + " , Source: " + str(s) + " at: " + dev) if d < s: #if destination free is less than source we don't have enough space logging.info("source exceeds destination at" + zz) y -= 1 while usb.isUsbPresent(dev): logging.info( "we found we don't have enough sapce on usb key " + dev) self.display.showNoSpacePage( 2, dev) #alert that there is a problem self.display.pageStack = 'remove_usb' #remove this usb self.command_to_reference = 'remove_usb' #let execute commands know what we want time.sleep(1) #wait a second usb.unmount(zz) #Make sure we unmount that device. usb.unmount(dev) #Make sure we unmount the mount point if zz[len(zz) - 1] != '0': # as long as its not /media/usb0 os.system( 'rm -r ' + z ) #Make sure we remove that directory since PauseMount is set else: logging.debug( "Space of Desitinationis ok for source to copy to " + zz) else: #we have a key but it is not mounted if usb.isUsbPresent( dev): #Hmm USB is present but not mounted. z = ord(dev[len(dev) - 2]) - ord( 'a' ) #get the base number of the /dev/sdX1 device that it should be not the ordinate if z == 0: z == ord('1') #I don't want to mount as usb0 else: z += ord('0') while usb.isUsbPresent('/media/usb' + chr(z)) and z < ord(':'): z += 1 #Find a mount that isn't there if z < ord(':'): os.system('mkdir /media/usb' + chr(z)) #Make the directory if (not usb.mount(dev, '/media/usb' + chr(z))): self.disiplay.showErrorPage( "Directory Creation") self.display.pageStack = 'error' try: os.remove( '/usr/local/connectbox/PauseMount') except: pass return x -= 1 #decrement so we can recheck this mount else: self.disiplay.showErrorPage("") self.display.pageStack = 'error' try: os.remove('/usr/local/connectbox/PauseMount') except: pass return x += 1 dev = '/dev/sd' + chr(x) + '1' logging.info( "we passed size checks so we are going on to copy now") # we think we have keys to work with if we go forward from here. l = [] y = [0, 0, 0] self.display.showWaitPage("Copying Now") x = ord('a') dev = '/dev/sd' + chr(x) + "1" logging.info("Ready to start the copies") while x < ord('k'): if usb.isUsbPresent(dev): #find the first usb key y[0] = subprocess.Popen(("/usr/bin/cp -r /media/usb0/* " + usb.getMount(dev) + "/"), shell=True) y[1] = usb.getMount(dev) + "/" y[2] = y[0].pid + 1 l.append(y) logging.info("started copy from /media/usb0/* to " + y[1] + " as pid: " + str(y[2])) x += 1 dev = '/dev/sd' + chr(x) + '1' # x = 0 # y = [0,0,0] # while y in l: # if y[0] == 0 or x ==1: # self.display.showErrorPage("Failed on /media/usb"+y[1]) # if not generate error page and exit # self.display.pageStack = 'error' # logging.info("ok we failed to copy to "+y[1]+" at mount point "+(usb.getMount("/media/usb"+y[1]))) # x =1 # try: # os.kill(y[2],-9) # except: # pass # if usb.isUsbPresent('/media/usb11') and (usb.getDev('/media/usb11')+"/") == y[1]: # os.command('unmount '+usb.getDev('/media/usb11')) # os.command('unmount /media/usb11') # os.command('rmdir -f /media/usb11') # logging.debug("we failed on the move of /media/usb11 -> /media/usb0") # self.disiplay.showErrorPage("Failed on /media/usb11 remount") # self.display.pageStack = 'error' # y[1] = "" #null this event # if x ==1: # try: os.remove('/usr/local/connectbox/PauseMount') # except: # pass # return # Ok we started all the copyies now we need to check for closure of the copy logging.info("Starting end of copy testing, lenth of l is " + str(len(l)) + "value of: " + str(l)) y = [0, 0, 0] yy = 0 x = 0 w = subprocess.Popen(("/usr/bin/ps -ae | /usr/bin/grep cp"), shell=True, stdout=subprocess.PIPE, universal_newlines=True) wo, errs = w.communicate() w.kill() logging.info("ps is showing " + wo) logging.info("l is showing" + str(l)) while x == 0: w = subprocess.Popen(("/usr/bin/ps -ae | /usr/bin/grep cp"), shell=True, stdout=subprocess.PIPE, universal_newlines=True) wo, errs = w.communicate() w.kill() while yy < len(l): y = l[yy] logging.info("y values are: " + str(y[0]) + ", " + str(y[1]) + ", " + str(y[2])) if y[2] != 0: logging.info("testing for " + str(y[2])) if (str(y[2]) not in wo) or (((wo.split(str( y[2]))[1]).split("<")[1]).split(">")[0] == 'defunct'): logging.info( "We finished the copy from /media/usb0 to " + y[1]) try: os.kill(y[2], -9) os.kill(y[2] - 1, -9) except: pass y[2] = 0 y[1] = "" y[0] = 0 l[yy] = y x = 1 else: x = 0 logging.info("testing failed") yy += 1 yy = 0 if x == 1: while y in l: if y[2] != 0: x = 0 logging.info("looping on x=" + str(x)) os.sync() logging.info("Ok now we want to remove all the usb keys") curDev = '/dev/sda1' x = ord('a') while (not usb.isUsbPresent(curDev)) and x < ord("k"): logging.debug("is key " + curDev + " present? " + str(usb.isUsbPresent(curDev))) x += 1 curDev = '/dev/sd' + chr(x) + '1' while usb.isUsbPresent(curDev) and x < ord("k"): self.display.showRemoveUsbPage() #show the remove usb page self.display.pageStack = 'removeUsb' #show we removed the usb key self.command_to_reference = 'remove_usb' time.sleep(3) #Wait a second for the removeal while (not usb.isUsbPresent(curDev)) and x < ord("k"): x += 1 # lets look at the next one curDev = '/dev/sd' + chr(x) + '1' #create the next curdev # We finished the umounts self.display.pageStack = 'success' self.display.showSuccessPage() logging.debug("Success page now deleting the PauseMount file") try: os.remove('/usr/local/connectbox/PauseMount') except: pass self.display.pageStack = 'success' # if the usb was removed self.display.showSuccessPage() # display success page os.sync() return
def get_model_file(name, tag=None, root=os.path.join("~", ".ncnn", "models")): r"""Return location for the pretrained on local file system. This function will download from online model zoo when model cannot be found or has mismatch. The root directory will be created if it doesn't exist. Parameters ---------- name : str Name of the model. root : str, default '~/.ncnn/models' Location for keeping the model parameters. Returns ------- file_path Path to the requested pretrained model file. """ if "NCNN_HOME" in os.environ: root = os.path.join(os.environ["NCNN_HOME"], "models") use_tag = isinstance(tag, str) if use_tag: file_name = "{name}-{short_hash}".format(name=name, short_hash=tag) else: file_name = "{name}".format(name=name) root = os.path.expanduser(root) params_path = os.path.join(root, file_name) lockfile = os.path.join(root, file_name + ".lock") if use_tag: sha1_hash = tag else: sha1_hash = _model_sha1[name] if not os.path.exists(root): os.makedirs(root) with portalocker.Lock( lockfile, timeout=int(os.environ.get("NCNN_MODEL_LOCK_TIMEOUT", 300)) ): if os.path.exists(params_path): if check_sha1(params_path, sha1_hash): return params_path else: logging.warning( "Hash mismatch in the content of model file '%s' detected. " "Downloading again.", params_path, ) else: logging.info("Model file not found. Downloading.") zip_file_path = os.path.join(root, file_name) if file_name in _split_model_bins: file_name_parts = [ "%s.part%02d" % (file_name, i + 1) for i in range(_split_model_bins[file_name]) ] for file_name_part in file_name_parts: file_path = os.path.join(root, file_name_part) repo_url = os.environ.get("NCNN_REPO", github_repo_url) if repo_url[-1] != "/": repo_url = repo_url + "/" download( _url_format.format(repo_url=repo_url, file_name=file_name_part), path=file_path, overwrite=True, ) merge_file(root, file_name_parts, zip_file_path) else: repo_url = os.environ.get("NCNN_REPO", github_repo_url) if repo_url[-1] != "/": repo_url = repo_url + "/" download( _url_format.format(repo_url=repo_url, file_name=file_name), path=zip_file_path, overwrite=True, ) if zip_file_path.endswith(".zip"): with zipfile.ZipFile(zip_file_path) as zf: zf.extractall(root) os.remove(zip_file_path) # Make sure we write the model file on networked filesystems try: os.sync() except AttributeError: pass if check_sha1(params_path, sha1_hash): return params_path else: raise ValueError("Downloaded file has different hash. Please try again.")
def get_model_file(name, tag=None, root=os.path.join('~', '.ncnn', 'models')): r"""Return location for the pretrained on local file system. This function will download from online model zoo when model cannot be found or has mismatch. The root directory will be created if it doesn't exist. Parameters ---------- name : str Name of the model. root : str, default '~/.ncnn/models' Location for keeping the model parameters. Returns ------- file_path Path to the requested pretrained model file. """ if 'NCNN_HOME' in os.environ: root = os.path.join(os.environ['NCNN_HOME'], 'models') use_tag = isinstance(tag, str) if use_tag: file_name = '{name}-{short_hash}'.format(name=name, short_hash=tag) else: file_name = '{name}'.format(name=name) root = os.path.expanduser(root) params_path = os.path.join(root, file_name) lockfile = os.path.join(root, file_name + '.lock') if use_tag: sha1_hash = tag else: sha1_hash = _model_sha1[name] if not os.path.exists(root): os.makedirs(root) with portalocker.Lock(lockfile, timeout=int( os.environ.get('NCNN_MODEL_LOCK_TIMEOUT', 300))): if os.path.exists(params_path): if check_sha1(params_path, sha1_hash): return params_path else: logging.warning( "Hash mismatch in the content of model file '%s' detected. " "Downloading again.", params_path) else: logging.info('Model file not found. Downloading.') zip_file_path = os.path.join(root, file_name) repo_url = os.environ.get('NCNN_REPO', apache_repo_url) if repo_url[-1] != '/': repo_url = repo_url + '/' download(_url_format.format(repo_url=repo_url, file_name=file_name), path=zip_file_path, overwrite=True) if zip_file_path.endswith(".zip"): with zipfile.ZipFile(zip_file_path) as zf: zf.extractall(root) os.remove(zip_file_path) # Make sure we write the model file on networked filesystems try: os.sync() except AttributeError: pass if check_sha1(params_path, sha1_hash): return params_path else: raise ValueError( 'Downloaded file has different hash. Please try again.')
def update_mbusb_cfg_file(iso_link, usb_uuid, usb_mount, distro): """ Update main multibootusb suslinux.cfg file after distro is installed. :return: """ if platform.system() == 'Linux': os.sync() log('Updating multibootusb config file...') sys_cfg_file = os.path.join(usb_mount, "multibootusb", "syslinux.cfg") install_dir = os.path.join(usb_mount, "multibootusb", iso_basename(iso_link)) if os.path.exists(sys_cfg_file): if distro == "hbcd": if os.path.exists( os.path.join(usb_mount, "multibootusb", "menu.lst")): _config_file = os.path.join(usb_mount, "multibootusb", "menu.lst") config_file = open(_config_file, "w") string = re.sub( r'/HBCD', '/multibootusb/' + iso_basename(iso_link) + '/HBCD', _config_file) config_file.write(string) config_file.close() with open(sys_cfg_file, "a") as f: f.write("#start " + iso_basename(config.image_path) + "\n") f.write("LABEL " + iso_basename(config.image_path) + "\n") f.write("MENU LABEL " + iso_basename(config.image_path) + "\n") f.write("BOOT " + '/multibootusb/' + iso_basename(iso_link) + '/' + isolinux_bin_dir(iso_link).replace("\\", "/") + '/' + distro + '.bs' + "\n") f.write("#end " + iso_basename(config.image_path) + "\n") elif distro == "Windows": if os.path.exists(sys_cfg_file): config_file = open(sys_cfg_file, "a") config_file.write("#start " + iso_basename(iso_link) + "\n") config_file.write("LABEL " + iso_basename(iso_link) + "\n") config_file.write("MENU LABEL " + iso_basename(iso_link) + "\n") config_file.write("KERNEL chain.c32 hd0 1 ntldr=/bootmgr" + "\n") config_file.write("#end " + iso_basename(iso_link) + "\n") config_file.close() elif distro == 'f4ubcd': if os.path.exists(sys_cfg_file): config_file = open(sys_cfg_file, "a") config_file.write("#start " + iso_basename(iso_link) + "\n") config_file.write("LABEL " + iso_basename(iso_link) + "\n") config_file.write("MENU LABEL " + iso_basename(iso_link) + "\n") config_file.write("KERNEL grub.exe" + "\n") config_file.write('APPEND --config-file=/multibootusb/' + iso_basename(config.image_path) + '/menu.lst' + "\n") config_file.write("#end " + iso_basename(iso_link) + "\n") config_file.close() elif distro == 'kaspersky': if os.path.exists(sys_cfg_file): config_file = open(sys_cfg_file, "a") config_file.write("#start " + iso_basename(iso_link) + "\n") config_file.write("LABEL " + iso_basename(iso_link) + "\n") config_file.write("MENU LABEL " + iso_basename(iso_link) + "\n") config_file.write("CONFIG " + '/multibootusb/' + iso_basename(config.image_path) + '/kaspersky.cfg' + "\n") config_file.write("#end " + iso_basename(iso_link) + "\n") config_file.close() elif distro == 'grub4dos': update_menu_lst() elif distro == 'grub4dos_iso': update_grub4dos_iso_menu() else: config_file = open(sys_cfg_file, "a") config_file.write("#start " + iso_basename(iso_link) + "\n") config_file.write("LABEL " + iso_basename(iso_link) + "\n") config_file.write("MENU LABEL " + iso_basename(iso_link) + "\n") if distro == "salix-live": if os.path.exists( os.path.join(config.usb_mount, 'multibootusb', iso_basename(iso_link), 'boot', 'grub2-linux.img')): config_file.write("LINUX " + '/multibootusb/' + iso_basename(iso_link) + '/boot/grub2-linux.img' + "\n") else: config_file.write( "BOOT " + '/multibootusb/' + iso_basename(iso_link) + '/' + isolinux_bin_dir(iso_link).replace("\\", "/") + '/' + distro + '.bs' + "\n") elif distro == "pclinuxos": config_file.write("kernel " + '/multibootusb/' + iso_basename(iso_link) + '/isolinux/vmlinuz' + "\n") config_file.write( "append livecd=livecd root=/dev/rd/3 acpi=on vga=788 keyb=us vmalloc=256M nokmsboot " "fromusb root=UUID=" + usb_uuid + " bootfromiso=/multibootusb/" + iso_basename(iso_link) + "/" + iso_name(iso_link) + " initrd=/multibootusb/" + iso_basename(iso_link) + '/isolinux/initrd.gz' + "\n") elif distro == "memtest": config_file.write("kernel " + '/multibootusb/' + iso_basename(iso_link) + '/BOOT/MEMTEST.IMG\n') elif distro == "sgrubd2" or config.distro == 'raw_iso': config_file.write("LINUX memdisk\n") config_file.write("INITRD " + "/multibootusb/" + iso_basename(iso_link) + '/' + iso_name(iso_link) + '\n') config_file.write("APPEND iso\n") elif distro == 'ReactOS': config_file.write("COM32 mboot.c32" + '\n') config_file.write("APPEND /loader/setupldr.sys" + '\n') elif distro == 'pc-unlocker': config_file.write("kernel ../ldntldr" + '\n') config_file.write("append initrd=../ntldr" + '\n') elif distro == 'pc-tool': config_file.write( menus.pc_tool_config(syslinux=True, grub=False)) elif distro == 'grub2only': config_file.write(menus.grub2only()) elif distro == 'memdisk_iso': print(menus.memdisk_iso_cfg(syslinux=True, grub=False)) config_file.write( menus.memdisk_iso_cfg(syslinux=True, grub=False)) elif distro == 'memdisk_img': config_file.write( menus.memdisk_img_cfg(syslinux=True, grub=False)) else: if isolinux_bin_exist(config.image_path) is True: if distro == "generic": distro_syslinux_install_dir = isolinux_bin_dir( iso_link) if isolinux_bin_dir(iso_link) != "/": distro_sys_install_bs = os.path.join( usb_mount, isolinux_bin_dir( iso_link)) + '/' + distro + '.bs' else: distro_sys_install_bs = '/' + distro + '.bs' else: distro_syslinux_install_dir = install_dir distro_syslinux_install_dir = distro_syslinux_install_dir.replace( usb_mount, '') distro_sys_install_bs = distro_syslinux_install_dir + '/' + isolinux_bin_dir( iso_link) + '/' + distro + '.bs' distro_sys_install_bs = "/" + distro_sys_install_bs.replace( "\\", "/") # Windows path issue. if config.syslinux_version == '3': config_file.write( "CONFIG /multibootusb/" + iso_basename(iso_link) + '/' + isolinux_bin_dir(iso_link).replace("\\", "/") + '/isolinux.cfg\n') config_file.write( "APPEND /multibootusb/" + iso_basename(iso_link) + '/' + isolinux_bin_dir(iso_link).replace("\\", "/") + '\n') config_file.write( "# Delete or comment above two lines using # and remove # from below line if " "you get not a COM module error.\n") config_file.write( "#BOOT " + distro_sys_install_bs.replace("//", "/") + "\n") else: config_file.write( "BOOT " + distro_sys_install_bs.replace("//", "/") + "\n") config_file.write("#end " + iso_basename(iso_link) + "\n") config_file.close() # Update extlinux.cfg file by copying updated syslinux.cfg shutil.copy( os.path.join(usb_mount, 'multibootusb', 'syslinux.cfg'), os.path.join(usb_mount, 'multibootusb', 'extlinux.cfg'))
def createPortable(self, mutex): staggingDir = os.path.join(self.installDir, 'stagging') if not os.path.exists(staggingDir): os.makedirs(staggingDir) try: shutil.copytree(self.appInstallPath, os.path.join(staggingDir, self.programName + '.app'), True) except: pass imageSize = self.dirSize(staggingDir) tmpDmg = os.path.join(self.installDir, self.programName + '_tmp.dmg') volumeName = "{}-portable-{}".format(self.programName, self.programVersion) process = subprocess.Popen(['hdiutil', 'create', '-srcfolder', staggingDir, '-volname', volumeName, '-fs', 'HFS+', '-fsargs', '-c c=64,a=16,e=16', '-format', 'UDRW', '-size', str(math.ceil(imageSize * 1.1)), tmpDmg], stdout=subprocess.PIPE) process.communicate() process = subprocess.Popen(['hdiutil', 'attach', '-readwrite', '-noverify', tmpDmg], stdout=subprocess.PIPE) stdout, stderr = process.communicate() device = '' for line in stdout.split(b'\n'): line = line.strip() if len(line) < 1: continue dev = line.split() if len(dev) > 2: device = dev[0].decode(sys.getdefaultencoding()) break time.sleep(2) volumePath = os.path.join('/Volumes', volumeName) volumeIcon = os.path.join(volumePath, '.VolumeIcon.icns') self.copy(os.path.join(self.rootDir, 'StandAlone/share/icons/webcamoid.icns'), volumeIcon) process = subprocess.Popen(['SetFile', '-c', 'icnC', volumeIcon], stdout=subprocess.PIPE) process.communicate() process = subprocess.Popen(['SetFile', '-a', 'C', volumePath], stdout=subprocess.PIPE) process.communicate() appsShortcut = os.path.join(volumePath, 'Applications') if not os.path.exists(appsShortcut): os.symlink('/Applications', appsShortcut) os.sync() process = subprocess.Popen(['hdiutil', 'detach', device], stdout=subprocess.PIPE) process.communicate() packagePath = \ os.path.join(self.pkgsDir, '{}-portable-{}-{}.dmg'.format(self.programName, self.programVersion, platform.machine())) if not os.path.exists(self.pkgsDir): os.makedirs(self.pkgsDir) if os.path.exists(packagePath): os.remove(packagePath) process = subprocess.Popen(['hdiutil', 'convert', tmpDmg, '-format', 'UDZO', '-imagekey', 'zlib-level=9', '-o', packagePath], stdout=subprocess.PIPE) process.communicate() mutex.acquire() print('Created portable package:') self.printPackageInfo(packagePath) mutex.release()
def download(self): self._s3c.download_from_s3(self.s3_bucket, self.s3_key, self.local_file()) os.sync() # ensure flush to disk
def _sync_local_filesystem(self): if hasattr(os, "sync"): os.sync()
def install_patch(): """ Function to certain distros which uses makeboot.sh script for making bootable usb disk. This is required to make sure that same version (32/64 bit) of modules present is the isolinux directory :return: """ if config.distro == 'debian': if platform.system( ) == 'Linux': # Need to syn under Linux. Otherwise, USB disk becomes random read only. os.sync() iso_cfg_ext_dir = os.path.join(multibootusb_host_dir(), "iso_cfg_ext_dir") isolinux_path = os.path.join(iso_cfg_ext_dir, isolinux_bin_path(config.image_path)) iso_linux_bin_dir = isolinux_bin_dir(config.image_path) config.syslinux_version = isolinux_version(isolinux_path) iso_file_list = iso.iso_file_list(config.image_path) os.path.join(config.usb_mount, "multibootusb", iso_basename(config.image_path), isolinux_bin_dir(config.image_path)) if any("makeboot.sh" in s.lower() for s in iso_file_list): for module in os.listdir( os.path.join(config.usb_mount, "multibootusb", iso_basename(config.image_path), isolinux_bin_dir(config.image_path))): if module.endswith(".c32"): if os.path.exists( os.path.join(config.usb_mount, "multibootusb", iso_basename(config.image_path), isolinux_bin_dir(config.image_path), module)): try: os.remove( os.path.join( config.usb_mount, "multibootusb", iso_basename(config.image_path), isolinux_bin_dir(config.image_path), module)) log("Copying " + module) log((resource_path( os.path.join(multibootusb_host_dir(), "syslinux", "modules", config.syslinux_version, module)), os.path.join( config.usb_mount, "multibootusb", iso_basename(config.image_path), isolinux_bin_dir(config.image_path), module))) shutil.copy( resource_path( os.path.join(multibootusb_host_dir(), "syslinux", "modules", config.syslinux_version, module)), os.path.join( config.usb_mount, "multibootusb", iso_basename(config.image_path), isolinux_bin_dir(config.image_path), module)) except Exception as err: log(err) log("Could not copy " + module) else: log('Patch not required...')
def manager_thread(): cloudlog.info("manager start") cloudlog.info({"environ": os.environ}) # save boot log subprocess.call("./bootlog", cwd=os.path.join(BASEDIR, "selfdrive/loggerd")) params = Params() ignore = [] if params.get("DongleId", encoding='utf8') == UNREGISTERED_DONGLE_ID: ignore += ["manage_athenad", "uploader"] if os.getenv("NOBOARD") is not None: ignore.append("pandad") if os.getenv("BLOCK") is not None: ignore += os.getenv("BLOCK").split(",") ensure_running(managed_processes.values(), started=False, not_run=ignore) started_prev = False sm = messaging.SubMaster(['deviceState']) pm = messaging.PubMaster(['managerState']) while True: sm.update() not_run = ignore[:] if sm['deviceState'].freeSpacePercent < 5: not_run.append("loggerd") started = sm['deviceState'].started driverview = params.get_bool("IsDriverViewEnabled") ensure_running(managed_processes.values(), started, driverview, not_run) # trigger an update after going offroad if started_prev and not started and 'updated' in managed_processes: os.sync() managed_processes['updated'].signal(signal.SIGHUP) started_prev = started running_list = [ "%s%s\u001b[0m" % ("\u001b[32m" if p.proc.is_alive() else "\u001b[31m", p.name) for p in managed_processes.values() if p.proc ] cloudlog.debug(' '.join(running_list)) # send managerState msg = messaging.new_message('managerState') msg.managerState.processes = [ p.get_process_state_msg() for p in managed_processes.values() ] pm.send('managerState', msg) # TODO: let UI handle this # Exit main loop when uninstall is needed if params.get_bool("DoUninstall"): break
# util.upload_forecast_batch(conn, json_io_dict_batch, forecast_filename_batch, project_name, model_name, timezero_date_batch, overwrite = over_write) # except Exception as ex: # return ex return "Pass" # Example Run: python3 ./code/zoltar_scripts/upload_covid19_forecasts_to_zoltar.py if __name__ == '__main__': list_of_model_directories = os.listdir('./data-processed/') output_errors = {} for directory in list_of_model_directories: if "." in directory: continue output = upload_covid_all_forecasts( './data-processed/' + directory + '/', directory) if output != "Pass": output_errors[directory] = output with open('./code/zoltar_scripts/validated_file_db.json', 'w') as fw: json.dump(db, fw, indent=4) # List all files that did not get upload and its error if len(output_errors) > 0: for directory, errors in output_errors.items(): print("\n* ERROR IN '", directory, "'") print(errors) os.sync() # make sure we flush before exiting sys.exit("\n ERRORS FOUND EXITING BUILD...") else: print("✓ no errors")
def train(variable_value_dict=None): if not FLAGS.dataset_dir: raise ValueError( 'You must supply the dataset directory with --dataset_dir') assert not (FLAGS.is_training and FLAGS.cost_saturation) #Chong added print_training_parameters() #Chong added #get the initial value dict, will be used by assign_from_feed_dict function if variable_value_dict is None: if not mvm.is_empty(): #need to change to feed_dict of masking variables? raise NotImplementedError variable_value_dict = mvm.get_variable_name_to_initial_value_dict() #Chong added #if user didn't request to resume an ongoing training stored in train_dir if FLAGS.resume_training_from_train_dir is not True or FLAGS.K_heuristic is not None: #clear the training folder import shutil shutil.rmtree(FLAGS.train_dir, ignore_errors=True) os.sync() tf.logging.set_verbosity(tf.logging.INFO) with tf.Graph().as_default(): ###################### # Config model_deploy# ###################### deploy_config = model_deploy.DeploymentConfig( num_clones=FLAGS.num_clones, clone_on_cpu=FLAGS.clone_on_cpu, replica_id=FLAGS.task, num_replicas=FLAGS.worker_replicas, num_ps_tasks=FLAGS.num_ps_tasks) # Create global_step with tf.device(deploy_config.variables_device()): global_step = tf.train.get_or_create_global_step() ###################### # Select the dataset # ###################### dataset = dataset_factory.get_dataset(FLAGS.dataset_name, FLAGS.dataset_split_name, FLAGS.dataset_dir) #################### # Select the network # #################### network_fn = nets_factory.get_network_fn( FLAGS.model_name, num_classes=(dataset.num_classes - FLAGS.labels_offset), weight_decay=FLAGS.weight_decay, is_training=True) ##################################### # Select the preprocessing function # ##################################### preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name image_preprocessing_fn = preprocessing_factory.get_preprocessing( preprocessing_name, is_training=True) ############################################################## # Create a dataset provider that loads data from the dataset # ############################################################## with tf.device(deploy_config.inputs_device()): provider = slim.dataset_data_provider.DatasetDataProvider( dataset, shuffle=True, num_readers=FLAGS.num_readers, common_queue_capacity=20 * FLAGS.batch_size, common_queue_min=10 * FLAGS.batch_size) [image, label] = provider.get(['image', 'label']) label -= FLAGS.labels_offset train_image_size = FLAGS.train_image_size or network_fn.default_image_size image = image_preprocessing_fn(image, train_image_size, train_image_size) images, labels = tf.train.batch( [image, label], batch_size=FLAGS.batch_size, num_threads=FLAGS.num_preprocessing_threads, capacity=5 * FLAGS.batch_size) labels = slim.one_hot_encoding( labels, dataset.num_classes - FLAGS.labels_offset) batch_queue = slim.prefetch_queue.prefetch_queue( [images, labels], capacity=2 * deploy_config.num_clones) #################### # Define the model # #################### def clone_fn(batch_queue): """Allows data parallelism by creating multiple clones of network_fn.""" with tf.device(deploy_config.inputs_device()): images, labels = batch_queue.dequeue() logits, end_points = network_fn(images) ############################# # Specify the loss function # ############################# if 'AuxLogits' in end_points: tf.losses.softmax_cross_entropy( logits=end_points['AuxLogits'], onehot_labels=labels, label_smoothing=FLAGS.label_smoothing, weights=0.4, scope='aux_loss') tf.losses.softmax_cross_entropy( logits=logits, onehot_labels=labels, label_smoothing=FLAGS.label_smoothing, weights=1.0) return end_points # Gather initial summaries. summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES)) clones = model_deploy.create_clones(deploy_config, clone_fn, [batch_queue]) first_clone_scope = deploy_config.clone_scope(0) # Gather update_ops from the first clone. These contain, for example, # the updates for the batch_norm variables created by network_fn. update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, first_clone_scope) # Add summaries for end_points. end_points = clones[0].outputs for end_point in end_points: x = end_points[end_point] #summaries.add(tf.summary.histogram('activations/' + end_point, x)) #summaries.add(tf.summary.scalar('sparsity/' + end_point, tf.nn.zero_fraction(x))) # Add summaries for losses. for loss in tf.get_collection(tf.GraphKeys.LOSSES, first_clone_scope): summaries.add(tf.summary.scalar('losses/%s' % loss.op.name, loss)) # Add summaries for variables. #for variable in slim.get_model_variables(): #summaries.add(tf.summary.histogram(variable.op.name, variable)) ################################# # Configure the moving averages # ################################# if FLAGS.moving_average_decay: moving_average_variables = slim.get_model_variables() variable_averages = tf.train.ExponentialMovingAverage( FLAGS.moving_average_decay, global_step) else: moving_average_variables, variable_averages = None, None ######################################### # Configure the optimization procedure. # ######################################### with tf.device(deploy_config.optimizer_device()): learning_rate = _configure_learning_rate(dataset.num_samples, global_step) optimizer = _configure_optimizer( learning_rate=(FLAGS.learning_rate if FLAGS.optimizer == 'yellowfin' else learning_rate)) summaries.add(tf.summary.scalar('learning_rate', learning_rate)) if FLAGS.sync_replicas: # If sync_replicas is enabled, the averaging will be done in the chief # queue runner. optimizer = tf.train.SyncReplicasOptimizer( opt=optimizer, replicas_to_aggregate=FLAGS.replicas_to_aggregate, variable_averages=variable_averages, variables_to_average=moving_average_variables, replica_id=tf.constant(FLAGS.task, tf.int32, shape=()), total_num_replicas=FLAGS.worker_replicas) elif FLAGS.moving_average_decay: # Update ops executed locally by trainer. update_ops.append( variable_averages.apply(moving_average_variables)) # Variables to train. variables_to_train = _get_variables_to_train() # and returns a train_tensor and summary_op total_loss, clones_gradients = model_deploy.optimize_clones( clones, optimizer, var_list=variables_to_train) # Add total_loss to summary. summaries.add(tf.summary.scalar('total_loss', total_loss)) # Create gradient updates. grad_updates = optimizer.apply_gradients(clones_gradients, global_step=global_step) update_ops.append(grad_updates) update_op = tf.group(*update_ops) with tf.control_dependencies([update_op]): train_tensor = tf.identity(total_loss, name='train_op') # Add the summaries from the first clone. These contain the summaries # created by model_fn and either optimize_clones() or _gather_clone_loss(). summaries |= set( tf.get_collection(tf.GraphKeys.SUMMARIES, first_clone_scope)) # Merge all summaries together. summary_op = tf.summary.merge(list(summaries), name='summary_op') #session_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=True) session_config = tf.ConfigProto(allow_soft_placement=True) #set saver parameters saver = tf_saver.Saver(max_to_keep=2, write_version=2) ########################### # Kicks off the training. # ########################### slim.learning.train( #my_slim_learning.train( train_tensor, logdir=FLAGS.train_dir, master=FLAGS.master, is_chief=(FLAGS.task == 0), #Chong added #init_op = assign_ops, #init_feed_dict = assign_feed_dict, saver=saver, #Chong added end init_fn=_get_init_fn(variable_value_dict), summary_op=summary_op, number_of_steps=FLAGS.max_number_of_steps, log_every_n_steps=FLAGS.log_every_n_steps, save_summaries_secs=FLAGS.save_summaries_secs, save_interval_secs=FLAGS.save_interval_secs, session_config=session_config, sync_optimizer=optimizer if FLAGS.sync_replicas else None)
def set_up(ctx): cfg = ctx.config() lower_mntroot = cfg.lower_mntroot() lowerdir = cfg.lowerdir() lowerimg = cfg.lowerimg() testdir = cfg.testdir() os.sync() if cfg.testing_none(): try: while system("grep -q 'lower_layer " + cfg.union_mntroot() + "' /proc/mounts" + " && umount " + cfg.union_mntroot()): pass except RuntimeError: pass try: while system("grep -q 'lower_layer " + lower_mntroot + "' /proc/mounts" + " && umount " + lower_mntroot): pass except RuntimeError: pass if cfg.testing_overlayfs(): try: while system("grep -q 'overlay " + cfg.union_mntroot() + "' /proc/mounts" + " && umount " + cfg.union_mntroot()): pass except RuntimeError: pass try: while system("grep -q 'lower_layer " + cfg.base_mntroot() + "' /proc/mounts" + " && umount " + cfg.base_mntroot()): pass except RuntimeError: pass try: while system("grep -q 'lower_layer " + lower_mntroot + "' /proc/mounts" + " && umount " + lower_mntroot): pass except RuntimeError: pass try: # grep filter to catch <lower|upper|N>_layer, in case upper and lower are on same fs # and in case different layers are on different fs while system("grep -q '_layer " + cfg.upper_mntroot() + "/' /proc/mounts" + " && umount " + cfg.upper_mntroot() + "/* 2>/dev/null"): pass except RuntimeError: pass try: # grep filter to catch <low|upp>er_layer, in case upper and lower are on same fs while system("grep -q 'er_layer " + cfg.upper_mntroot() + "' /proc/mounts" + " && umount " + cfg.upper_mntroot()): pass except RuntimeError: pass if cfg.is_samefs() and cfg.testing_overlayfs(): # Create base fs for both lower and upper base_mntroot = cfg.base_mntroot() system("mount " + base_mntroot + " 2>/dev/null" " || mount -t tmpfs lower_layer " + base_mntroot) system("mount --make-private " + base_mntroot) try: os.mkdir(base_mntroot + lower_mntroot) except OSError: pass system("mount -o bind " + base_mntroot + lower_mntroot + " " + lower_mntroot) else: # Create a lower layer to union over system("mount " + lower_mntroot + " 2>/dev/null" " || mount -t tmpfs lower_layer " + lower_mntroot) # Systemd has weird ideas about things system("mount --make-private " + lower_mntroot) # # Create a few test files we can use in the lower layer # try: os.mkdir(lowerdir) except OSError: system("rm -rf " + lowerdir) os.mkdir(lowerdir) pieces = testdir.split("/") del pieces[0] path = "" for i in pieces: path += "/" + i ctx.record_file(path, "d") ctx.set_cwd(testdir) for i in range(100, 130): si = str(i) # Under the test directory, we create a bunch of regular files # containing data called foo100 to foo129: create_file(lowerdir + "/foo" + si, ":xxx:yyy:zzz") rec = ctx.record_file("foo" + si, "r") # Then we create a bunch of direct symlinks to those files to = "../a/foo" + si os.symlink(to, lowerdir + "/direct_sym" + si) rec = ctx.record_file("direct_sym" + si, "s", to, rec) # Then we create a bunch of indirect symlinks to those files to = "direct_sym" + si os.symlink(to, lowerdir + "/indirect_sym" + si) ctx.record_file("indirect_sym" + si, "s", to, rec) # Then we create a bunch symlinks that don't point to extant files to = "no_foo" + si os.symlink(to, lowerdir + "/pointless" + si) rec = ctx.record_file("no_foo" + si, None) ctx.record_file("pointless" + si, "s", to, rec) # We create a bunch of directories, each with an empty file # and a populated subdir os.mkdir(lowerdir + "/dir" + si) rec = ctx.record_file("dir" + si, "d") create_file(lowerdir + "/dir" + si + "/a", "") ctx.record_file("dir" + si + "/a", "f") os.mkdir(lowerdir + "/dir" + si + "/pop") ctx.record_file("dir" + si + "/pop", "d") create_file(lowerdir + "/dir" + si + "/pop/b", ":aaa:bbb:ccc") ctx.record_file("dir" + si + "/pop/b", "f") os.mkdir(lowerdir + "/dir" + si + "/pop/c") ctx.record_file("dir" + si + "/pop/c", "d") # And add direct and indirect symlinks to those to = "../a/dir" + si os.symlink(to, lowerdir + "/direct_dir_sym" + si) rec = ctx.record_file("direct_dir_sym" + si, "s", to, rec) #ctx.record_file("direct_dir_sym" + si + "/a", "f") to = "direct_dir_sym" + si os.symlink(to, lowerdir + "/indirect_dir_sym" + si) ctx.record_file("indirect_dir_sym" + si, "s", to, rec) #ctx.record_file("indirect_dir_sym" + si + "/a", "f") # And a bunch of empty directories os.mkdir(lowerdir + "/empty" + si) ctx.record_file("empty" + si, "d") # Everything above is then owned by the bin user for f in ["foo", "direct_sym", "indirect_sym", "pointless"]: os.lchown(lowerdir + "/" + f + si, 1, 1) # Create some root-owned regular files also create_file(lowerdir + "/rootfile" + si, ":xxx:yyy:zzz") ctx.record_file("rootfile" + si, "r") # Non-existent dir ctx.record_file("no_dir" + si, None) if cfg.is_squashfs(): system("mksquashfs " + lowerdir + " " + lowerimg + " -keep-as-directory > /dev/null") system("mount -o loop,ro " + lowerimg + " " + lower_mntroot) system("mount --make-private " + lower_mntroot) else: # The mount has to be read-only for us to make use of it system("mount -o remount,ro " + lower_mntroot) ctx.note_lower_fs(lowerdir)
def manager_thread(): cloudlog.info("manager start") cloudlog.info({"environ": os.environ}) # save boot log subprocess.call("./bootlog", cwd=os.path.join(BASEDIR, "selfdrive/loggerd")) # start daemon processes for p in daemon_processes: start_daemon_process(p) # start persistent processes for p in persistent_processes: start_managed_process(p) # start offroad if EON: pm_apply_packages('enable') start_offroad() if os.getenv("NOBOARD") is not None: del managed_processes["pandad"] if os.getenv("BLOCK") is not None: for k in os.getenv("BLOCK").split(","): del managed_processes[k] started_prev = False logger_dead = False params = Params() thermal_sock = messaging.sub_sock('thermal') pm = messaging.PubMaster(['managerState']) while 1: msg = messaging.recv_sock(thermal_sock, wait=True) if msg.thermal.freeSpacePercent < 0.05: logger_dead = True if msg.thermal.started: for p in car_started_processes: if p == "loggerd" and logger_dead: kill_managed_process(p) else: start_managed_process(p) else: logger_dead = False driver_view = params.get("IsDriverViewEnabled") == b"1" # TODO: refactor how manager manages processes for p in reversed(car_started_processes): if p not in driver_view_processes or not driver_view: kill_managed_process(p) for p in driver_view_processes: if driver_view: start_managed_process(p) else: kill_managed_process(p) # trigger an update after going offroad if started_prev: os.sync() send_managed_process_signal("updated", signal.SIGHUP) started_prev = msg.thermal.started # check the status of all processes, did any of them die? running_list = [ "%s%s\u001b[0m" % ("\u001b[32m" if running[p].is_alive() else "\u001b[31m", p) for p in running ] cloudlog.debug(' '.join(running_list)) # send managerState states = [] for p in managed_processes: state = log.ManagerState.ProcessState.new_message() state.name = p if p in running: state.running = running[p].is_alive() state.pid = running[p].pid state.exitCode = running[p].exitcode or 0 states.append(state) msg = messaging.new_message('managerState') msg.managerState.processes = states pm.send('managerState', msg) # Exit main loop when uninstall is needed if params.get("DoUninstall", encoding='utf8') == "1": break
def sync(self) -> None: os.sync()
def resize(self, new_size): logging.info('Start resizing image \'' + self.name + '\'') # Image must be unmounted to resize it if self.is_mounted: raise ValueError('Cannot resize a mounted livenet image') # Check livenet size if not isinstance( new_size, str) or int(new_size) < LivenetImage.MIN_LIVENET_SIZE or int( new_size) > LivenetImage.MAX_LIVENET_SIZE: raise ValueError('Invalid livenet size') # Create usefull directories for resizement logging.debug('Executing \'mkdir -p ' + self.MOUNT_DIRECTORY + 'mnt_copy\'') os.makedirs(self.MOUNT_DIRECTORY + 'mnt_copy') logging.debug('Executing \'mkdir -p ' + self.MOUNT_DIRECTORY + 'mnt\'') os.makedirs(self.MOUNT_DIRECTORY + 'mnt') logging.debug('Executing \'mkdir -p ' + self.WORKING_DIRECTORY + 'current\'') os.makedirs(self.WORKING_DIRECTORY + 'current') logging.debug('Executing \'mkdir -p ' + self.WORKING_DIRECTORY + 'copy/squashfs-root/LiveOS/\'') os.makedirs(self.WORKING_DIRECTORY + 'copy/squashfs-root/LiveOS/') # Create a new rootfs image with the new size logging.debug('Executing \'dd if=/dev/zero of=' + self.WORKING_DIRECTORY + 'copy/squashfs-root/LiveOS/rootfs.img bs=1M count=' + new_size + '\'') os.system('dd if=/dev/zero of=' + self.WORKING_DIRECTORY + 'copy/squashfs-root/LiveOS/rootfs.img bs=1M count=' + new_size) # Format the fresh rootfs.img into an xfs system logging.debug('Executing \'mkfs.xfs ' + self.WORKING_DIRECTORY + 'copy/squashfs-root/LiveOS/rootfs.img\'') os.system('mkfs.xfs ' + self.WORKING_DIRECTORY + 'copy/squashfs-root/LiveOS/rootfs.img') # Mount the new rootfs.img on it's mount directory logging.debug('Executing \'mount ' + self.WORKING_DIRECTORY + 'copy/squashfs-root/LiveOS/rootfs.img ' + self.MOUNT_DIRECTORY + 'mnt_copy/\'') os.system('mount ' + self.WORKING_DIRECTORY + 'copy/squashfs-root/LiveOS/rootfs.img ' + self.MOUNT_DIRECTORY + 'mnt_copy/') # Unsquash current image logging.debug('Executing \'unsquashfs -d ' + self.WORKING_DIRECTORY + 'current/squashfs-root ' + self.IMAGE_DIRECTORY + 'squashfs.img\'') os.system('unsquashfs -d ' + self.WORKING_DIRECTORY + 'current/squashfs-root ' + self.IMAGE_DIRECTORY + 'squashfs.img') # Mount current rootfs.img on it's mount directory logging.debug('Executing \'mount ' + self.WORKING_DIRECTORY + 'current/squashfs-root/LiveOS/rootfs.img ' + self.MOUNT_DIRECTORY + 'mnt\'') os.system('mount ' + self.WORKING_DIRECTORY + 'current/squashfs-root/LiveOS/rootfs.img ' + self.MOUNT_DIRECTORY + 'mnt') # Create image.xfsdump from current image logging.debug('Executing \'xfsdump -l 0 -L ' + self.name + ' -M media -f ' + self.WORKING_DIRECTORY + '/current/image.xfsdump ' + self.MOUNT_DIRECTORY + 'mnt\'') os.system('xfsdump -l 0 -L ' + self.name + ' -M media -f ' + self.WORKING_DIRECTORY + '/current/image.xfsdump ' + self.MOUNT_DIRECTORY + 'mnt') # Restore with new sized rootfs.img mountage logging.debug('Executing \'xfsrestore -f ' + self.WORKING_DIRECTORY + 'current/image.xfsdump ' + self.MOUNT_DIRECTORY + 'mnt_copy\'') os.system('xfsrestore -f ' + self.WORKING_DIRECTORY + 'current/image.xfsdump ' + self.MOUNT_DIRECTORY + 'mnt_copy') os.sync() # Umount mnt and mnt_copy logging.debug('Executing \'umount ' + self.MOUNT_DIRECTORY + '*\'') os.system('umount ' + self.MOUNT_DIRECTORY + '*') logging.debug('Executing \'rm -rf ' + self.MOUNT_DIRECTORY + '\'') shutil.rmtree(self.MOUNT_DIRECTORY) # Remove old squashfs logging.debug('Executing \'rm -f ' + self.IMAGE_DIRECTORY + 'squashfs.img\'') os.remove(self.IMAGE_DIRECTORY + 'squashfs.img') # Generate the new squashfs logging.debug('Executing \'mksquashfs ' + self.WORKING_DIRECTORY + 'copy/squashfs-root/ ' + self.IMAGE_DIRECTORY + 'squashfs.img\'') os.system('mksquashfs ' + self.WORKING_DIRECTORY + 'copy/squashfs-root/ ' + self.IMAGE_DIRECTORY + 'squashfs.img') logging.debug('Executing \'rm -rf ' + self.WORKING_DIRECTORY + '\'') shutil.rmtree(self.WORKING_DIRECTORY) # Update image size attribute value self.livenet_size = new_size self.register_image() logging.info('Image has been resized to ' + new_size + 'Mb')
def write_launch_file(app, file = "once.txt"): with open(file, "wt") as file: file.write(app) file.flush() os.sync()
def os_sync(): """Ensure flush to disk""" if not ON_WINDOWS: os.sync()
def uninstall(self): Path("/data/__system_reset__").touch() os.sync() self.reboot()
def install_vyos(self, payload): try: install_config = load(payload) except Exception as err: logger.error("Unable to load YAML file: {}".format(err)) return try: # enable output to the stderr from logging import StreamHandler logger.addHandler(StreamHandler()) # define all variables vyos_version = get_version() if vyos_version.startswith('1.2'): vyos_generation = '1.2' rootfspath = '/lib/live/mount/medium/live/filesystem.squashfs' else: vyos_generation = 'other' rootfspath = '/usr/lib/live/mount/medium/live/filesystem.squashfs' install_drive = install_config['install_drive'] partition_size = install_config.get('partition_size', '') root_drive = '/mnt/wroot' root_read = '/mnt/squashfs' root_install = '/mnt/inst_root' dir_rw = '{}/boot/{}/rw'.format(root_drive, vyos_version) dir_work = '{}/boot/{}/work'.format(root_drive, vyos_version) # find and prepare drive and partition if install_drive == 'auto': regex_lsblk = re.compile(r'^(?P<dev_name>\w+) +(?P<dev_size>\d+) +(?P<dev_type>\w+)( +(?P<part_type>[\w-]+))?$') drive_list = self.run_command('lsblk --bytes --nodeps --list --noheadings --output KNAME,SIZE,TYPE') for device_line in drive_list.splitlines(): found = regex_lsblk.search(device_line) if found: if int(found.group('dev_size')) > 2000000000 and found.group('dev_type') == 'disk': # check if a device is not mounted regex_mount = re.compile(r'^(?P<dev_name>\w+) +(?P<mount_point>/.*)$', re.MULTILINE) drive_details = self.run_command('lsblk --list --noheadings --output KNAME,MOUNTPOINT /dev/{}'.format(found.group('dev_name'))) mount_details = regex_mount.search(drive_details) if mount_details: logger.debug("Skipping {}, {} is mounted on {}".format(found.group('dev_name'), mount_details.group('dev_name'), mount_details.group('mount_point'))) else: install_drive = '/dev/{}'.format(found.group('dev_name')) break if install_drive == 'auto': logger.error("No suitable drive found for installation") return logger.debug("Installing to drive: {}".format(install_drive)) # Detect system type if Path('/sys/firmware/efi/').exists(): sys_type = 'EFI' grub_efi = '--force-extra-removable --efi-directory=/boot/efi --bootloader-id=VyOS --no-uefi-secure-boot ' else: sys_type = 'Non-EFI' grub_efi = '' logger.debug("Detected {} system".format(sys_type)) # Create partitions logger.debug('Clearing current partitions table on {}'.format(install_drive)) with open(install_drive, 'w+b') as drive: drive.seek(0) drive.write(b'0' * 17408) drive.seek(-17408, 2) drive.write(b'0' * 17408) sync() self.run_command('partprobe {}'.format(install_drive)) if vyos_generation == '1.2': if sys_type == 'EFI': logger.debug('Creating EFI-compatible partitions table on {}'.format(install_drive)) self.run_command('sgdisk -n 1:1M:100M {}'.format(install_drive)) self.run_command('sgdisk -t 1:EF00 {}'.format(install_drive)) self.run_command('sgdisk -n 2::{} {}'.format(partition_size, install_drive)) else: logger.debug('Creating BIOS-compatible partitions table on {}'.format(install_drive)) disk_parts = '3,{},L,*\n'.format(partition_size) self.run_command('sfdisk -q {}'.format(install_drive), disk_parts) else: if sys_type == 'EFI': logger.debug('Creating EFI-compatible partitions table on {}'.format(install_drive)) disk_parts = 'label: gpt\n,100M,U,*\n,{},L'.format(partition_size) else: logger.debug('Creating BIOS-compatible partitions table on {}'.format(install_drive)) disk_parts = 'label: dos\n,{},L,*\n'.format(partition_size) self.run_command('sfdisk -q -w always -W always {}'.format(install_drive), disk_parts) # update partitons in kernel self.run_command('partprobe {}'.format(install_drive)) partitions_list = self.run_command('fdisk -l {}'.format(install_drive)) regex_fdisk = re.compile(r'^(?P<dev_name>/dev/\w+) +(\* +)?(\d+ +)+([\d\.]+\D) +(\d+ +)?(?P<part_type>\w+).*$') for partition_line in partitions_list.splitlines(): found = regex_fdisk.search(partition_line) if found: if found.group('part_type') == 'Linux': root_partition = '{}'.format(found.group('dev_name')) logger.debug("Using partition for root: {}".format(root_partition)) self.run_command('mkfs -t ext4 -L persistence {}'.format(root_partition)) if found.group('part_type') == 'EFI': efi_partition = '{}'.format(found.group('dev_name')) logger.debug("Using partition for EFI: {}".format(efi_partition)) self.run_command('mkfs -t fat -n EFI {}'.format(efi_partition)) # creating directories for dir in [root_drive, root_read, root_install]: dirpath = Path(dir) logger.debug("Creating directory: {}".format(dir)) dirpath.mkdir(mode=0o755, parents=True) # mounting root drive logger.debug("Mounting root drive: {}".format(root_drive)) self.run_command('mount {} {}'.format(root_partition, root_drive)) for dir in [dir_rw, dir_work]: dirpath = Path(dir) logger.debug("Creating directory: {}".format(dir)) dirpath.mkdir(mode=0o755, parents=True) if sys_type == 'EFI': Path('/boot/efi').mkdir(mode=0o755, parents=True) self.run_command('mount {} {}'.format(efi_partition, '/boot/efi')) # copy rootfs logger.debug("Copying rootfs: {}/boot/{}/{}.squashfs".format(root_drive, vyos_version, vyos_version)) self.run_command('cp -p {} {}/boot/{}/{}.squashfs'.format(rootfspath, root_drive, vyos_version, vyos_version)) # get list of other files for boot and copy to the installation boot directory boot_files = self.run_command('find /boot -maxdepth 1 -type f -o -type l') for file in boot_files.splitlines(): logger.debug("Copying file: {}".format(file)) self.run_command('cp -dp {} {}/boot/{}/'.format(file, root_drive, vyos_version)) # write persistense.conf logger.debug("Writing '{}/persistence.conf".format(root_drive)) self.write_file('{}/persistence.conf'.format(root_drive), '/ union\n') # mount new rootfs logger.debug("Mounting read-only rootfs: {}".format(root_read)) self.run_command('mount -o loop,ro -t squashfs {}/boot/{}/{}.squashfs {}'.format(root_drive, vyos_version, vyos_version, root_read)) logger.debug("Mounting overlay rootfs: {}".format(root_install)) self.run_command('mount -t overlay -o noatime,upperdir={},lowerdir={},workdir={} overlay {}'.format(dir_rw, root_read, dir_work, root_install)) # copy configuration logger.debug("Copying configuration to: {}/opt/vyatta/etc/config/config.boot".format(root_install)) self.run_command('cp -p /opt/vyatta/etc/config/config.boot {}/opt/vyatta/etc/config/config.boot'.format(root_install)) logger.debug("Copying .vyatta_config to: {}/opt/vyatta/etc/config/.vyatta_config".format(root_install)) self.run_command('cp -p /opt/vyatta/etc/config/.vyatta_config {}/opt/vyatta/etc/config/.vyatta_config'.format(root_install)) # install grub logger.debug("Installing GRUB to {}".format(install_drive)) self.run_command('grub-install --no-floppy --recheck --root-directory={} {}{}'.format(root_drive, grub_efi, install_drive)) # configure GRUB logger.debug("Configuring GRUB") self.run_command('/opt/vyatta/sbin/vyatta-grub-setup -u {} {} '' {}'.format(vyos_version, root_partition, root_drive)) # disable cloud-init if required if install_config.get('cloud_init_disable', False) is True: logger.info("Disabling Cloud-init") self.write_file('{}/etc/cloud/cloud-init.disabled'.format(root_install), 'Disabled by VyOS installer (requested by user)\n') # unmount all fs for dir in [root_install, root_read, root_drive]: logger.debug("Unmounting: {}".format(dir)) self.run_command('umount {}'.format(dir)) # reboot the system if this was requested by config if install_config.get('after_install', None) == 'reboot': logger.info("Rebooting host") self.run_command('systemctl reboot', disconnect=True) if install_config.get('after_install', None) == 'poweroff': logger.info("Powering off host") self.run_command('systemctl poweroff', disconnect=True) except Exception as err: logger.error("Unable to install VyOS: {}".format(err)) return
def main(): if len(sys.argv) != 3: print("Please specify a serial port and filename") exit(1) serial_port = sys.argv[1] filename = sys.argv[2] print("Watching {filename} and uploading board at {serial_port}".format( filename=filename, serial_port=serial_port)) with serial.Serial(serial_port, 115200, timeout=1) as ser: ser_text = io.TextIOWrapper(ser, newline='\n') continue_reading = True assume_in_prompt = False while continue_reading: line = ser_text.readline().strip() print(">>>> {}".format(line)) #if 'Hit any key to stop autoboot:' in line: if 'FEC [PRIME], usb_ether' in line: print("Writing control characters") for i in range(5): ser.write(b'\r\n') time.sleep(0.05) assume_in_prompt = True if '=>' in line or assume_in_prompt: print("found prompt") ser.write(b'\r\n') premount_paths = [ partition.mountpoint for partition in psutil.disk_partitions() ] print("Entering USB Mass Storage Mode") print(ser_text.readline()) ser.write(b'ums 0 mmc 0\r\n') print(ser_text.readline()) # Find USB Device target_partition = None for _ in range(3): postmount_paths = [ partition.mountpoint for partition in psutil.disk_partitions() ] new_paths = list( set(premount_paths) - set(postmount_paths)) for partition in new_paths: for node in os.listdir(partition): if node.endswith('-m4.dtb'): target_partition = partition break if target_partition is not None: break if target_partition is not None: break time.sleep(500) # Wait for udev to mount this thing if target_partition is None: print("Unable to find a target partition") exit(1) print("Copying artifact to target partition {}".format( target_partition)) # Copy files basename = os.path.basename(filename) shutil.copy(filename, os.path.join(target_partition, basename)) os.sync() os.sync() print("Copy complete; exiting UMS") ser.write(b'\x03') # CTRL+C print("Requesting system boot...") ser.write(b'boot\r\n') ser.close()
def _set_labs_auth_code(self, auth_code): self._get_db().put('DEVICE_LABS_AUTH_CODE', auth_code) os.sync()
def manager_thread(): shutdownd = Process(name="shutdownd", target=launcher, args=("selfdrive.shutdownd", )) shutdownd.start() cloudlog.info("manager start") cloudlog.info({"environ": os.environ}) # save boot log subprocess.call(["./loggerd", "--bootlog"], cwd=os.path.join(BASEDIR, "selfdrive/loggerd")) # start daemon processes for p in daemon_processes: start_daemon_process(p) # start persistent processes for p in persistent_processes: start_managed_process(p) # start offroad if EON: pm_apply_packages('enable') start_offroad() if os.getenv("NOBOARD") is None: start_managed_process("pandad") if os.getenv("BLOCK") is not None: for k in os.getenv("BLOCK").split(","): del managed_processes[k] started_prev = False logger_dead = False params = Params() thermal_sock = messaging.sub_sock('thermal') while 1: msg = messaging.recv_sock(thermal_sock, wait=True) if msg.thermal.freeSpace < 0.05: logger_dead = True if msg.thermal.started: for p in car_started_processes: if p == "loggerd" and logger_dead: kill_managed_process(p) else: start_managed_process(p) else: logger_dead = False driver_view = params.get("IsDriverViewEnabled") == b"1" # TODO: refactor how manager manages processes for p in reversed(car_started_processes): if p not in driver_view_processes or not driver_view: kill_managed_process(p) for p in driver_view_processes: if driver_view: start_managed_process(p) else: kill_managed_process(p) # trigger an update after going offroad if started_prev: os.sync() send_managed_process_signal("updated", signal.SIGHUP) started_prev = msg.thermal.started # check the status of all processes, did any of them die? running_list = [ "%s%s\u001b[0m" % ("\u001b[32m" if running[p].is_alive() else "\u001b[31m", p) for p in running ] cloudlog.debug(' '.join(running_list)) # Exit main loop when uninstall is needed if params.get("DoUninstall", encoding='utf8') == "1": break
def install_distro(): """ Install selected ISO to USB disk. :return: """ usb_mount = config.usb_mount install_dir = os.path.join(config.usb_mount, "multibootusb", iso_basename(config.image_path)) _iso_file_list = iso.iso_file_list(config.image_path) if not os.path.exists(os.path.join(usb_mount, "multibootusb")): log("Copying multibootusb directory to " + usb_mount) shutil.copytree( resource_path(os.path.join("data", "tools", "multibootusb")), os.path.join(config.usb_mount, "multibootusb")) if not os.path.exists(install_dir): os.makedirs(install_dir) with open(os.path.join(install_dir, "multibootusb.cfg"), "w") as f: f.write(config.distro) with open(os.path.join(install_dir, "iso_file_list.cfg"), 'w') as f: for file_path in _iso_file_list: f.write(file_path + "\n") log("Installing " + iso_name(config.image_path) + " on " + install_dir) if config.distro == "opensuse": iso.iso_extract_file(config.image_path, install_dir, 'boot') status_text = "Copying ISO..." if platform.system() == "Windows": subprocess.call( ["xcopy", config.image_path, usb_mount], shell=True ) # Have to use xcopy as python file copy is dead slow. elif platform.system() == "Linux": log("Copying " + config.image_path + " to " + usb_mount) shutil.copy(config.image_path, usb_mount) elif config.distro == "Windows" or config.distro == "alpine" or config.distro == 'pc-unlocker'\ or config.distro == 'pc-tool' or config.distro == 'grub2only': log("Extracting iso to " + usb_mount) iso_extract_full(config.image_path, usb_mount) elif config.distro == "trinity-rescue": iso.iso_extract_file(config.image_path, usb_mount, '*trk3') elif config.distro == "ipfire": iso.iso_extract_file(config.image_path, usb_mount, '*.tlz') iso.iso_extract_file(config.image_path, usb_mount, 'distro.img') iso.iso_extract_file(config.image_path, install_dir, 'boot') elif config.distro == "zenwalk": config.status_text = "Copying ISO..." iso.iso_extract_file(config.image_path, install_dir, "kernel") copy_iso(config.image_path, install_dir) elif config.distro == "salix-live": # iso.iso_extract_file(config.image_path, install_dir, "boot") iso.iso_extract_file(config.image_path, install_dir, '*syslinux') iso.iso_extract_file(config.image_path, install_dir, '*menus') iso.iso_extract_file(config.image_path, install_dir, '*vmlinuz') iso.iso_extract_file(config.image_path, install_dir, '*initrd*') iso.iso_extract_file(config.image_path, usb_mount, '*modules') iso.iso_extract_file(config.image_path, usb_mount, '*packages') iso.iso_extract_file(config.image_path, usb_mount, '*optional') iso.iso_extract_file(config.image_path, usb_mount, '*liveboot') #iso.iso_extract_full(config.image_path, usb_mount) config.status_text = "Copying ISO..." copy_iso(config.image_path, install_dir) elif config.distro == 'sgrubd2': copy_iso(config.image_path, install_dir) elif config.distro == 'alt-linux': iso.iso_extract_file(config.image_path, install_dir, '-xr!*rescue') iso.iso_extract_file(config.image_path, config.usb_mount, 'rescue') elif config.distro == "generic": #with open(os.path.join(install_dir, "generic.cfg"), "w") as f: # f.write(os.path.join(isolinux_bin_dir(config.image_path), "generic") + ".bs") iso_extract_full(config.image_path, usb_mount) elif config.distro == 'grub4dos': iso_extract_full(config.image_path, usb_mount) elif config.distro == 'ReactOS': iso_extract_full(config.image_path, usb_mount) elif config.distro == 'grub4dos_iso' or config.distro == 'raw_iso': copy_iso(config.image_path, install_dir) else: iso.iso_extract_full(config.image_path, install_dir) if platform.system() == 'Linux': log('ISO extracted successfully. Sync is in progress...') os.sync() if config.persistence != 0: log('Creating persistence...') config.status_text = 'Creating persistence...' persistence.create_persistence() install_patch()
def init_overlay() -> None: overlay_init_file = Path(os.path.join(BASEDIR, ".overlay_init")) # Re-create the overlay if BASEDIR/.git has changed since we created the overlay if overlay_init_file.is_file(): git_dir_path = os.path.join(BASEDIR, ".git") new_files = run( ["find", git_dir_path, "-newer", str(overlay_init_file)]) if not len(new_files.splitlines()): # A valid overlay already exists return else: cloudlog.info(".git directory changed, recreating overlay") cloudlog.info("preparing new safe staging area") params = Params() params.put_bool("UpdateAvailable", False) set_consistent_flag(False) dismount_overlay() if TICI: run(["sudo", "rm", "-rf", STAGING_ROOT]) if os.path.isdir(STAGING_ROOT): shutil.rmtree(STAGING_ROOT) for dirname in [ STAGING_ROOT, OVERLAY_UPPER, OVERLAY_METADATA, OVERLAY_MERGED ]: os.mkdir(dirname, 0o755) if os.lstat(BASEDIR).st_dev != os.lstat(OVERLAY_MERGED).st_dev: raise RuntimeError( "base and overlay merge directories are on different filesystems; not valid for overlay FS!" ) # Leave a timestamped canary in BASEDIR to check at startup. The device clock # should be correct by the time we get here. If the init file disappears, or # critical mtimes in BASEDIR are newer than .overlay_init, continue.sh can # assume that BASEDIR has used for local development or otherwise modified, # and skips the update activation attempt. consistent_file = Path(os.path.join(BASEDIR, ".overlay_consistent")) if consistent_file.is_file(): consistent_file.unlink() overlay_init_file.touch() os.sync() overlay_opts = f"lowerdir={BASEDIR},upperdir={OVERLAY_UPPER},workdir={OVERLAY_METADATA}" mount_cmd = [ "mount", "-t", "overlay", "-o", overlay_opts, "none", OVERLAY_MERGED ] if TICI: run(["sudo"] + mount_cmd) run(["sudo", "chmod", "755", os.path.join(OVERLAY_METADATA, "work")]) else: run(mount_cmd) git_diff = run(["git", "diff"], OVERLAY_MERGED, low_priority=True) params.put("GitDiff", git_diff) cloudlog.info(f"git diff output:\n{git_diff}")
def create_ps2(dest, game_id, game_title, icon0, pic1, cue_files, cu2_files, img_files): print('Create PS2 VCD for', game_title) if verbose else None print('Install VCD in', dest + '/POPS') try: os.stat(dest + '/POPS') except: raise Exception('No POPS directory found') try: os.stat(dest + '/ART') except: raise Exception('No ART directory found') p = popstation() p.verbose = verbose p.game_id = game_id p.game_title = game_title discs_txt = None vmcdir_txt = None if len(img_files) > 1: for i in range(4): pp = game_id[:4] + '_' + game_id[4:7] + '.' + game_id[7:9] + '.' + game_title pp = pp + '_CD%d.VCD\n' % (i + 1) if not vmcdir_txt: vmcdir_txt = pp[:-5] + '\n' if i >= len(img_files): pp = '\n' if not discs_txt: discs_txt = pp else: discs_txt = discs_txt + pp for i in range(len(img_files)): f = img_files[i] toc = p.get_toc_from_ccd(f) if not toc: print('Need to create a TOC') if verbose else None toc = get_toc_from_cu2(cu2_files[i]) print('Add image', f) if verbose else None p.add_img((f, toc)) print('GameID', game_id, game_title) if verbose else None pp = dest + '/POPS/' + game_id[:4] + '_' + game_id[4:7] + '.' + game_id[7:9] + '.' + game_title if len(img_files) > 1: pp = pp + '_CD%d' % (i + 1) try: os.mkdir(pp) except: True p.vcd = pp + '.VCD' print('Create VCD at', p.vcd) if verbose else None p.create_vcd() try: os.sync() except: True if discs_txt: with open(pp + '/DISCS.TXT', 'w') as f: f.write(discs_txt) if vmcdir_txt: with open(pp + '/VMCDIR.TXT', 'w') as f: f.write(vmcdir_txt) if i == 0: create_blank_mc(pp + '/SLOT0.VMC') create_blank_mc(pp + '/SLOT1.VMC') pp = dest + '/ART/' f = pp + game_id[0:4] + '_' + game_id[4:7] + '.' + game_id[7:9] + '_COV.jpg' image = icon0.resize((200, 200)) image = image.convert('RGB') image.save(f, format='JPEG', quality=100, subsampling=0) f = pp + game_id[0:4] + '_' + game_id[4:7] + '.' + game_id[7:9] + '_BG.jpg' image = pic1.resize((640, 480)) image = image.convert('RGB') image.save(f, format='JPEG', quality=100, subsampling=0)
def callPlot(): global ycIpFid, ycIpNam, ycOpFid global spc2Fid, spc3Fid, vbl2Fid, vbl3Fid global Va, Aa, Ka, Ra, Fa # Approach parms global Vc, Hc, Kc, Rc # Cruise parms global Cw, Iw, Aw, Ww, Pw, Lf, Df, Lr, Dr # Wing/Ailr parms global Ch, Ih, Ah, Wh, Ph, Lh, Dh # Hstab parms global Cv, Iv, Av, Wv, Pv, Lv, Dv # Vstab parms global Mp, Rp, Ap, Np, Xp, Ip, Op, Vp, Cp, Tp # Prop parms global Mb, Xb, Yb, Zb # Ballast parms global Hy, Vy # Solver parms # # Versions in Yasim configuration strings, OrderedDict ## single plot versDict = OrderedDict([ ('YASIM_VERSION_CURRENT', '-vCurr'), ]) ## all versions ##versDict = OrderedDict([ ('YASIM_VERSION_ORIGINAL', '-vOrig'), \ ## ('YASIM_VERSION_32', '-v32'), \ ## ('YASIM_VERSION_CURRENT', '-vCurr'), \ ## ('2017.2', '-v2017-2') ]) ## Iterate through each version in dictionary for versKywd in versDict.keys(): versSfix = versDict[versKywd] vcfgFid = ycIpNam + versSfix + '.xml' vdatFid = ycIpNam + '-dat' + versSfix + '.txt' ## ## # open yasim config file, apply version string ## Tix tix for python 3 if (pythVers < 3): vcfgHndl = open(vcfgFid, 'w', 0) else: vcfgHndl = open(vcfgFid, 'w') # Phase 3 write auto file via yconfig template and subsVbls from Tix with open(ycOpFid, 'r') as ycOpHndl: # step each line in template yasim config file for line in ycOpHndl: if '<airplane mass="' in line: # make an index list of double quotes in the line sepsList = [] sepsChar = '"' lastIndx = 0 while (lastIndx > -1): sepsIndx = line.find(sepsChar, (lastIndx + 1)) if (sepsIndx > 0): sepsList.append(sepsIndx) lastIndx = sepsIndx # if ('version=' in line): linePart = line[:line.find('version=')] sepsIndx = line.find('version=') sepsIndx = line.find('"', (sepsIndx + 1)) sepsIndx = line.find('"', (sepsIndx + 1)) linePart = linePart + 'version="' + versKywd linePart = linePart + line[sepsIndx:] line = linePart else: # Use index list to split line into text and numbers lineMass = line[0:(sepsList[1] + 1)] line = lineMass + 'version="' + versKywd + '">' # Write unchanged/modified line into versioned xml vcfgHndl.write(line) #close and sync files vcfgHndl.flush os.fsync(vcfgHndl.fileno()) vcfgHndl.close # os.sync() ycOpHndl.close ## ## GNUPlot needs specification files: auto create 2 vble gnuplot spec file ##ab apltHndl = open(vbl2Fid, 'w', 0) apltHndl = open(vbl2Fid, 'w') #create common annotation test parsed / menu-altered values, big version: all menu parms commNota = ' set title "' + ycIpNam + versSfix + 'Parms:\\nAp:' + str(Va) \ + ' ' + str(Aa) + ' ' + str(Ka) + ' ' + str(Ra) + ' ' + str(Fa) +'\\n' \ + ' Cr:' + str(Vc) + ' ' + str(Hc) + ' ' + str(Kc) + ' ' \ + str(Rc) + '\\nWi:' + str(Cw) + ' ' + str(Iw) + ' ' + str(Aw) \ + ' ' + str(Ww) + ' ' + str(Pw) + ' ' + str(Lf) + ' ' + str(Df) \ + ' ' + str(Lr) + ' ' + str(Dr) \ + '\\nHs:' + str(Ch) + ' ' + str(Ih) + ' ' + str(Ah) \ + ' ' + str(Wh) + ' ' + str(Ph) + ' ' + str(Lh) + ' ' + str(Dh) \ + '\\nVs:' + str(Cv) + ' ' + str(Iv) + ' ' + str(Av) \ + ' ' + str(Wv) + ' ' + str(Pv) + ' ' + str(Lv) + ' ' + str(Dv) \ + 'Ys:'+ str(Vy) + ' ' + str(Hy) + '" \n' # uncomment a line below to have gnuplot show shortened legend #commNota = ' set title "yasiVers.py ' + ycIpNam + versSfix + ' : ' + str(Vy) + 'kTAS at ' + str(Hy) + 'ft" \n' #commNota = ' set title "' + ycIpNam + ' ' + versSfix + ' : ' + str(Vy) + 'kTAS at ' + str(Hy) + 'ft" \n' commNota = ' set title "' + ycIpNam + versSfix + ' Va:' + str( Va) + ' Aa:' + str(Aa) + ' Vc:' + str(Vc) + ' Cw:' + str( Cw) + ' Aw:' + str(Aw) + ' Ww:' + str(Ww) + '"\n' with open(spc2Fid, 'r') as tplt: plotFlag = 0 for line in tplt: # set flag near end when 'plot' string is found if (' plot ' in line): plotFlag = 1 # find the title line in template config if ('set title' in line): line = commNota if (plotFlag < 1): # Write line into auto.xml apltHndl.write(line) # At EOF append plot lines with proper data file name # line = ' plot "' + vdatFid +'" every ::2 using ' \ # + '1:2 with lines title \'lift\', \\\n' # apltHndl.write(line) # line = ' "' + vdatFid +'" every ::2 using ' \ # + '1:3 with lines title \'drag\'' # apltHndl.write(line) ### ab 2020Jan23 L/D only line = ' plot "' + vdatFid +'" every ::2 using ' \ + '1:4 with lines title \'LvsD\'' apltHndl.write(line) apltHndl.close tplt.close # ## auto create 3 vble gnuplot config file apltHndl = open(vbl3Fid, 'w') with open(spc3Fid, 'r') as tplt: plotFlag = 0 for line in tplt: # find the title line in template config if (' plot ' in line): plotFlag = 1 if ('set title' in line): line = commNota if (plotFlag < 1): # Write line into auto.xml apltHndl.write(line) # At EOF append plot lines with proper data file name line = ' plot "' + vdatFid +'" every ::2 using ' \ + '1:2 with lines title \'lift\', \\\n' apltHndl.write(line) line = ' "' + vdatFid +'" every ::2 using ' \ + '1:3 with lines title \'drag\', \\\n' apltHndl.write(line) line = ' "' + vdatFid +'" every ::2 using ' \ + '1:4 with lines title \'L-D\' \n' apltHndl.write(line) apltHndl.close tplt.close ## python3 Need to open/close file to flush it ? vcfgHndl = open(vcfgFid, 'r') vcfgHndl.close() ## # run yasim external process to show console output command_line = 'yasim ' + vcfgFid + ' -a ' + str(Hy) + ' -s ' + str(Vy) command_line = 'yasim ' + vcfgFid args = shlex.split(command_line) p = subprocess.Popen(args) #p.communicate() #p.wait() # # run yasim external process for auto dataset, name used in .p spec ###ab smite yDatHndl = open(yDatFid, 'w') ###ab smite command_line = 'yasim ' + vcfgFid + ' -g -a ' + str(Hy) + ' -s ' + str(Vy) ###ab smite args = shlex.split(command_line) ###ab smite p = subprocess.run(args, stdout=yDatHndl) ###ab smite yDatHndl.close ###ab smite p.wait() # ## # run yasim external process for saved dataset file vDatHndl = open(vdatFid, 'w') command_line = 'yasim ' + vcfgFid + ' -g -a ' + str(Hy) + ' -s ' + str( Vy) args = shlex.split(command_line) p = subprocess.run(args, stdout=vDatHndl) vDatHndl.close p.wait() ## ## Uncomment for separate Lift, Drag curves # run gnuplot with 2 vble command file to plot dataset #apltHndl = open(vbl2Fid, 'w') #apltHndl.close() #command_line = "gnuplot -p " + vbl2Fid #args = shlex.split(command_line) #DEVNULL = open(os.devnull, 'wb') #p = subprocess.run(args, stdout=DEVNULL, stderr=DEVNULL) #p.communicate() #DEVNULL.close() # # run gnuplot with 3 vble command file to plot dataset apltHndl = open(vbl3Fid, 'w') apltHndl.close() command_line = "gnuplot -p " + vbl3Fid args = shlex.split(command_line) DEVNULL = open(os.devnull, 'wb') p = subprocess.run(args, stdout=DEVNULL, stderr=DEVNULL) #p.communicate() DEVNULL.close()