def install_rust(self): """Download and run the rustup installer.""" import errno import stat import tempfile platform = rust.platform() url = rust.rustup_url(platform) checksum = rust.rustup_hash(platform) if not url or not checksum: print('ERROR: Could not download installer.') sys.exit(1) print('Downloading rustup-init... ', end='') fd, rustup_init = tempfile.mkstemp(prefix=os.path.basename(url)) os.close(fd) try: self.http_download_and_save(url, rustup_init, checksum) mode = os.stat(rustup_init).st_mode os.chmod(rustup_init, mode | stat.S_IRWXU) print('Ok') print('Running rustup-init...') subprocess.check_call([rustup_init, '-y', '--default-toolchain', 'stable', '--default-host', platform, ]) cargo_home, cargo_bin = self.cargo_home() self.print_rust_path_advice(RUST_INSTALL_COMPLETE, cargo_home, cargo_bin) finally: try: os.remove(rustup_init) except OSError as e: if e.errno != errno.ENOENT: raise
def install_dockerbeat(): ''' Installs dockerbeat from resources, with a fallback option to try to fetch over the network, for 1.25.5 hosts''' try: bin_path = resource_get('dockerbeat') except NotImplementedError: # Attempt to fetch and install from configured uri with validation bin_path = download_from_upstream() full_beat_path = '/usr/local/bin/dockerbeat' if not bin_path: status_set('blocked', 'Missing dockerbeat binary') return install(bin_path, full_beat_path) os.chmod(full_beat_path, 0o755) codename = lsb_release()['DISTRIB_CODENAME'] # render the apropriate init systems configuration if codename == 'trusty': render('upstart', '/etc/init/dockerbeat.conf', {}) else: render('systemd', '/etc/systemd/system/dockerbeat.service', {}) set_state('dockerbeat.installed')
def test_validation(self): """ test other validations beside mandatory variables """ self.contents = '\n'.join([ 'name = "pi"', 'version = "3.14"', 'homepage = "http://google.com"', 'description = "test easyconfig"', 'toolchain = {"name":"dummy", "version": "dummy"}', 'stop = "notvalid"', ]) self.prep() eb = EasyConfig(self.eb_file, validate=False, valid_stops=self.all_stops) self.assertErrorRegex(EasyBuildError, r"\w* provided '\w*' is not valid", eb.validate) eb['stop'] = 'patch' # this should now not crash eb.validate() eb['osdependencies'] = ['non-existent-dep'] self.assertErrorRegex(EasyBuildError, "OS dependencies were not found", eb.validate) # dummy toolchain, installversion == version self.assertEqual(eb.get_installversion(), "3.14") os.chmod(self.eb_file, 0000) self.assertErrorRegex(EasyBuildError, "Unexpected IOError", EasyConfig, self.eb_file) os.chmod(self.eb_file, 0755) self.contents += "\nsyntax_error'" self.prep() self.assertErrorRegex(EasyBuildError, "SyntaxError", EasyConfig, self.eb_file)
def create_tempdir(): """Create a directory within the system temp directory used to create temp files.""" try: if os.path.isdir(tempdir): shutil.rmtree(tempdir) os.mkdir(tempdir) # Make sure the directory can be removed by anyone in case the user # runs ST later as another user. os.chmod(tempdir, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) except PermissionError: if sublime.platform() != 'windows': current_user = pwd.getpwuid(os.geteuid())[0] temp_uid = os.stat(tempdir).st_uid temp_user = pwd.getpwuid(temp_uid)[0] message = ( 'The SublimeLinter temp directory:\n\n{0}\n\ncould not be cleared ' 'because it is owned by \'{1}\' and you are logged in as \'{2}\'. ' 'Please use sudo to remove the temp directory from a terminal.' ).format(tempdir, temp_user, current_user) else: message = ( 'The SublimeLinter temp directory ({}) could not be reset ' 'because it belongs to a different user.' ).format(tempdir) sublime.error_message(message) from . import persist persist.debug('temp directory:', tempdir)
def ensure_permissions(path, user, group, permissions, maxdepth=-1): """Ensure permissions for path. If path is a file, apply to file and return. If path is a directory, apply recursively (if required) to directory contents and return. :param user: user name :param group: group name :param permissions: octal permissions :param maxdepth: maximum recursion depth. A negative maxdepth allows infinite recursion and maxdepth=0 means no recursion. :returns: None """ if not os.path.exists(path): log("File '%s' does not exist - cannot set permissions" % (path), level=WARNING) return _user = pwd.getpwnam(user) os.chown(path, _user.pw_uid, grp.getgrnam(group).gr_gid) os.chmod(path, permissions) if maxdepth == 0: log("Max recursion depth reached - skipping further recursion", level=DEBUG) return elif maxdepth > 0: maxdepth -= 1 if os.path.isdir(path): contents = glob.glob("%s/*" % (path)) for c in contents: ensure_permissions(c, user=user, group=group, permissions=permissions, maxdepth=maxdepth)
def filter_and_persist_proprietary_tool_panel_configs( self, tool_configs_to_filter ): """Eliminate all entries in all non-shed-related tool panel configs for all tool config file names in the received tool_configs_to_filter.""" for proprietary_tool_conf in self.proprietary_tool_confs: persist_required = False tree, error_message = xml_util.parse_xml( proprietary_tool_conf ) if tree: root = tree.getroot() for elem in root: if elem.tag == 'tool': # Tools outside of sections. file_path = elem.get( 'file', None ) if file_path: if file_path in tool_configs_to_filter: root.remove( elem ) persist_required = True elif elem.tag == 'section': # Tools contained in a section. for section_elem in elem: if section_elem.tag == 'tool': file_path = section_elem.get( 'file', None ) if file_path: if file_path in tool_configs_to_filter: elem.remove( section_elem ) persist_required = True if persist_required: fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-toolshed-fapptpc" ) tmp_filename = fh.name fh.close() fh = open( tmp_filename, 'wb' ) tree.write( tmp_filename, encoding='utf-8', xml_declaration=True ) fh.close() shutil.move( tmp_filename, os.path.abspath( proprietary_tool_conf ) ) os.chmod( proprietary_tool_conf, 0644 )
def handleRemoveReadonly(func, path, exc): excvalue = exc[1] if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES: os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777 func(path) else: raise
def export_data(self): if not self._path: return temp_path = self._path + '_%s.tmp' % uuid.uuid4().hex try: data = self._data.copy() timers = self._timers.keys() commit_log = copy.copy(self._commit_log) with open(temp_path, 'w') as db_file: os.chmod(temp_path, 0600) export_data = [] for key in data: key_ttl = data[key]['ttl'] key_val = data[key]['val'] key_type = type(key_val).__name__ if key_type == 'set' or key_type == 'deque': key_val = list(key_val) export_data.append((key, key_type, key_ttl, key_val)) db_file.write(json.dumps({ 'ver': 1, 'data': export_data, 'timers': timers, 'commit_log': commit_log, })) os.rename(temp_path, self._path) except: try: os.remove(temp_path) except OSError: pass raise
def execute(script_path, process, outputNames): wrapper_path = ctx.download_resource("scriptWrapper.sh") os.chmod(wrapper_path, 0755) os.chmod(script_path, 0755) on_posix = 'posix' in sys.builtin_module_names env = os.environ.copy() process_env = process.get('env', {}) env.update(process_env) if outputNames is not None: env['EXPECTED_OUTPUTS'] = outputNames command = '{0} {1}'.format(wrapper_path, script_path) else: command = script_path ctx.logger.info('Executing: {0} in env {1}'.format(command, env)) process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, cwd=None, bufsize=1, close_fds=on_posix) return_code = None stdout_consumer = OutputConsumer(process.stdout) stderr_consumer = OutputConsumer(process.stderr) while True: return_code = process.poll() if return_code is not None: break time.sleep(0.1) stdout_consumer.join() stderr_consumer.join() parsed_output = parse_output(stdout_consumer.buffer.getvalue()) if outputNames is not None: outputNameList = outputNames.split(';') for outputName in outputNameList: ctx.logger.info('Ouput name: {0} value : {1}'.format(outputName, parsed_output['outputs'][outputName])) ok_message = "Script {0} executed normally with standard output {1} and error output {2}".format(command, stdout_consumer.buffer.getvalue(), stderr_consumer.buffer.getvalue()) error_message = "Script {0} encountered error with return code {1} and standard output {2}, error output {3}".format(command, return_code, stdout_consumer.buffer.getvalue(), stderr_consumer.buffer.getvalue()) if return_code != 0: ctx.logger.error(error_message) raise NonRecoverableError(error_message) else: ctx.logger.info(ok_message) return parsed_output
def put_files_cache(self): if getattr(self, "cached", None): return None sig = self.signature() ssig = Utils.to_hex(self.uid()) + Utils.to_hex(sig) dname = os.path.join(self.generator.bld.cache_global, ssig) tmpdir = tempfile.mkdtemp(prefix=self.generator.bld.cache_global + os.sep + "waf") try: shutil.rmtree(dname) except: pass try: for node in self.outputs: dest = os.path.join(tmpdir, node.name) shutil.copy2(node.abspath(), dest) except (OSError, IOError): try: shutil.rmtree(tmpdir) except: pass else: try: os.rename(tmpdir, dname) except OSError: try: shutil.rmtree(tmpdir) except: pass else: try: os.chmod(dname, Utils.O755) except: pass
def UnzipFilenameToDir(filename, directory): """Unzip |filename| to |directory|.""" cwd = os.getcwd() if not os.path.isabs(filename): filename = os.path.join(cwd, filename) zf = zipfile.ZipFile(filename) # Make base. if not os.path.isdir(directory): os.mkdir(directory) os.chdir(directory) # Extract files. for info in zf.infolist(): name = info.filename if name.endswith('/'): # dir if not os.path.isdir(name): os.makedirs(name) else: # file directory = os.path.dirname(name) if not os.path.isdir(directory): os.makedirs(directory) out = open(name, 'wb') out.write(zf.read(name)) out.close() # Set permissions. Permission info in external_attr is shifted 16 bits. os.chmod(name, info.external_attr >> 16L) os.chdir(cwd)
def user_login(self, user): """ Called immediately after a user authenticates successfully. Saves session information in the user's directory. Expects *user* to be a dict containing a 'upn' value representing the username or userPrincipalName. e.g. 'user@REALM' or just 'someuser'. Any additional values will be attached to the user object/cookie. """ logging.debug("user_login(%s)" % user['upn']) user.update(additional_attributes(user)) # Make a directory to store this user's settings/files/logs/etc user_dir = os.path.join(self.settings['user_dir'], user['upn']) if not os.path.exists(user_dir): logging.info(_("Creating user directory: %s" % user_dir)) mkdir_p(user_dir) os.chmod(user_dir, 0o700) session_file = os.path.join(user_dir, 'session') session_file_exists = os.path.exists(session_file) if session_file_exists: session_data = open(session_file).read() try: session_info = tornado.escape.json_decode(session_data) except ValueError: # Something wrong with the file session_file_exists = False # Overwrite it below if not session_file_exists: with open(session_file, 'w') as f: # Save it so we can keep track across multiple clients session_info = { 'session': generate_session_id(), } session_info.update(user) session_info_json = tornado.escape.json_encode(session_info) f.write(session_info_json) self.set_secure_cookie( "gateone_user", tornado.escape.json_encode(session_info))
def extract_zip(src, dest): """extract a zip file""" import zipfile if isinstance(src, zipfile.ZipFile): bundle = src else: try: bundle = zipfile.ZipFile(src) except Exception: print "src: %s" % src raise namelist = bundle.namelist() for name in namelist: filename = os.path.realpath(os.path.join(dest, name)) if name.endswith('/'): if not os.path.isdir(filename): os.makedirs(filename) else: path = os.path.dirname(filename) if not os.path.isdir(path): os.makedirs(path) _dest = open(filename, 'wb') _dest.write(bundle.read(name)) _dest.close() mode = bundle.getinfo(name).external_attr >> 16 & 0x1FF os.chmod(filename, mode) bundle.close() return namelist
def recursively_add_write_bit(inputdir): """ Function to walk a directory tree, adding the write it to every file and directory. This is mostly useful right before deleting a tree of files extracted from an ISO, since those were all read-only to begin with. """ for dirpath, dirnames, filenames in os.walk(inputdir): # If the path is a symlink, and it is an absolute symlink, this would # attempt to change the permissions of the *host* file, not the # file that is relative to here. That is no good, and could be a # security problem if Oz is being run as root. We skip all paths that # are symlinks; what they point to will be changed later on. if os.path.islink(dirpath): continue os.chmod(dirpath, os.stat(dirpath).st_mode|stat.S_IWUSR) for name in filenames: fullpath = os.path.join(dirpath, name) # we have the same guard for symlinks as above, for the same reason if os.path.islink(fullpath): continue try: # if there are broken symlinks in the ISO, # then the below might fail. This probably # isn't fatal, so just allow it and go on os.chmod(fullpath, os.stat(fullpath).st_mode|stat.S_IWUSR) except OSError as err: if err.errno != errno.ENOENT: raise
def copy_file (self, infile, outfile, preserve_mode=1, preserve_times=1, link=None, level=1): """Copy a file respecting verbose, dry-run and force flags. (The former two default to whatever is in the Distribution object, and the latter defaults to false for commands that don't define it.)""" i = open(infile) line = i.readline() mode_change = len(line) > 1 and line[:2] == '#!' and os.name == 'posix' adjust = string.rstrip(line) == '#!' and os.name == 'posix' and not self.dry_run if adjust: o = open(outfile, 'wt') o.writelines(['#!%s\n' % sys.executable] + i.readlines()) x = (outfile, 1) else: x = file_util.copy_file(infile, outfile, preserve_mode, preserve_times, not self.force, link, self.verbose >= level, self.dry_run) if mode_change: if self.dry_run: self.announce("changing mode of %s" % outfile) else: mode = (os.stat(outfile)[ST_MODE]) | 0111 self.announce("changing mode of %s to %o" % (outfile, mode)) os.chmod(outfile, mode) return x
def run(self, connection, args=None): connection = super(SshAuthorize, self).run(connection, args) if not self.identity_file: self.logger.debug("No authorisation required.") # idempotency return connection # add the authorization keys to the overlay if 'location' not in self.data['lava-overlay']: raise RuntimeError("Missing lava overlay location") if not os.path.exists(self.data['lava-overlay']['location']): raise RuntimeError("Unable to find overlay location") location = self.data['lava-overlay']['location'] lava_path = os.path.abspath("%s/%s" % (location, self.data['lava_test_results_dir'])) output_file = '%s/%s' % (lava_path, os.path.basename(self.identity_file)) shutil.copyfile(self.identity_file, output_file) shutil.copyfile("%s.pub" % self.identity_file, "%s.pub" % output_file) if not self.active: # secondary connections only return connection self.logger.info("Adding SSH authorisation for %s.pub", os.path.basename(output_file)) user_sshdir = os.path.join(location, 'root', '.ssh') if not os.path.exists(user_sshdir): os.makedirs(user_sshdir, 0755) # if /root/.ssh/authorized_keys exists in the test image it will be overwritten # the key exists in the lava_test_results_dir to allow test writers to work around this # after logging in via the identity_file set here authorize = os.path.join(user_sshdir, 'authorized_keys') self.logger.debug("Copying %s to %s", "%s.pub" % self.identity_file, authorize) shutil.copyfile("%s.pub" % self.identity_file, authorize) os.chmod(authorize, 0600) return connection
def fileopen(name, mode='wb', encoding=None): """ Open a file using mode. Default mode is 'wb' meaning to open the file for writing in binary mode. If encoding is given, I/O to and from the file is transparently encoded using the given encoding. Files opened for writing are chmod()ed to 0600. """ if name == 'stdout': return sys.stdout elif name == 'stderr': return sys.stderr elif name == 'stdin': return sys.stdin else: if encoding is not None: import codecs f = codecs.open(name, mode, encoding) else: f = open(name, mode) if 'w' in mode: os.chmod(name, 0600) return f
def update_prefix(path, new_prefix, placeholder=prefix_placeholder, mode='text'): if on_win and (placeholder != prefix_placeholder) and ('/' in placeholder): # original prefix uses unix-style path separators # replace with unix-style path separators new_prefix = new_prefix.replace('\\', '/') path = os.path.realpath(path) with open(path, 'rb') as fi: data = fi.read() if mode == 'text': new_data = data.replace(placeholder.encode('utf-8'), new_prefix.encode('utf-8')) elif mode == 'binary': new_data = binary_replace(data, placeholder.encode('utf-8'), new_prefix.encode('utf-8')) else: sys.exit("Invalid mode:" % mode) if new_data == data: return st = os.lstat(path) with open(path, 'wb') as fo: fo.write(new_data) os.chmod(path, stat.S_IMODE(st.st_mode))
def symlink(src, target): """ symlink file if possible """ if 'win' in sys.platform: shutil.copy(src, target) os.chmod(target, stat.S_IRWXU) else: os.symlink(src, target)
def removeDir(self, path): try: if os.path.isdir(path): shutil.rmtree(path) except OSError, inst: os.chmod(inst.filename, 0777) self.removeDir(path)
def parametric(lists, yaml): if "global" not in lists: lists["global"] = [] scriptname = "%s/runme.sh" % target f = open(scriptname,'w') f.write("#!/bin/bash\n") # the default filename = "%s/default.ceph.conf" % target writefile(lists, filename) writescript(f, "default", "", filename) for param,value in sorted(yaml.iteritems()): if (isinstance(value, dict)): lc = copy.deepcopy(lists) for k,v in sorted(value.iteritems()): populate(lc.get("global"), k, v) filename = "%s/%s.ceph.conf" % (target, param) writefile(lc, filename) writescript(f, param, "", filename) elif (isinstance(value, list)): for vi in value: lc = copy.deepcopy(lists) populate(lc.get("global"), param, vi) filename = "%s/%s_%s.ceph.conf" % (target, param, vi) writefile(lc, filename) writescript(f, param, vi, filename) else: lc = copy.deepcopy(lists) populate(lc.get("global"), param, value) filename = "%s/%s_%s.ceph.conf" % (target, param, value) writefile(lc, filename) writescript(f, param, value, filename) f.close() os.chmod(scriptname, 0755)
def main(): my_config = get_config() new_synapse_config = generate_configuration( my_config, get_zookeeper_topology(), get_all_namespaces() ) with tempfile.NamedTemporaryFile() as tmp_file: new_synapse_config_path = tmp_file.name with open(new_synapse_config_path, 'w') as fp: json.dump(new_synapse_config, fp, sort_keys=True, indent=4, separators=(',', ': ')) # Match permissions that puppet expects os.chmod(new_synapse_config_path, 0644) # Restart synapse if the config files differ should_restart = not filecmp.cmp(new_synapse_config_path, my_config['config_file']) # Always swap new config file into place. Our monitoring system # checks the config['config_file'] file age to ensure that it is # continually being updated. shutil.copy(new_synapse_config_path, my_config['config_file']) if should_restart: subprocess.check_call(SYNAPSE_RESTART_COMMAND)
def do_addattachment(self, zipname, filename, pagename, author=u"Scripting Subsystem", comment=u""): """ Installs an attachment @param pagename: Page where the file is attached. Or in 2.0, the file itself. @param zipname: Filename of the attachment from the zip file @param filename: Filename of the attachment (just applicable for MoinMoin < 2.0) """ if self.request.user.may.write(pagename): _ = self.request.getText attachments = Page(self.request, pagename).getPagePath("attachments", check_create=1) filename = wikiutil.taintfilename(filename) zipname = wikiutil.taintfilename(zipname) target = os.path.join(attachments, filename) page = PageEditor(self.request, pagename, do_editor_backup=0, uid_override=author) rev = page.current_rev() path = page.getPagePath(check_create=0) if not os.path.exists(target): self._extractToFile(zipname, target) if os.path.exists(target): os.chmod(target, config.umask ) action = 'ATTNEW' edit_logfile_append(self, pagename, path, rev, action, logname='edit-log', comment=u'%(filename)s' % {"filename": filename}, author=author) self.msg += u"%(filename)s attached \n" % {"filename": filename} else: self.msg += u"%(filename)s not attached \n" % {"filename": filename} else: self.msg += u"action add attachment: not enough rights - nothing done \n"
def initializeInitD(ownerName): if (os.path.isdir(initdDirName)): fn = join(RANGER_USERSYNC_HOME, initdProgramName) initdFn = join(initdDirName, initdProgramName) shutil.copy(fn, initdFn) if (ownerName != 'ranger'): f = open(initdFn, 'r') filedata = f.read() f.close() find_str = "LINUX_USER=ranger" replace_str = "LINUX_USER="******"/etc/rc2.d", "/etc/rc3.d", "/etc/rc.d/rc2.d", "/etc/rc.d/rc3.d"] for rcDir in rcDirList: if (os.path.isdir(rcDir)): for prefix in initPrefixList: scriptFn = prefix + initdProgramName scriptName = join(rcDir, scriptFn) if isfile(scriptName) or os.path.islink(scriptName): os.remove(scriptName) os.symlink(initdFn, scriptName) userSyncScriptName = "ranger-usersync-services.sh" localScriptName = os.path.abspath(join(RANGER_USERSYNC_HOME, userSyncScriptName)) ubinScriptName = join("/usr/bin", initdProgramName) if isfile(ubinScriptName) or os.path.islink(ubinScriptName): os.remove(ubinScriptName) os.symlink(localScriptName, ubinScriptName)
def display_immediately(self, plain_text, rich_output): """ Show output immediately. This method is similar to the rich output :meth:`displayhook`, except that it can be invoked at any time. INPUT: Same as :meth:`displayhook`. OUTPUT: This method does not return anything. EXAMPLES:: sage: from sage.repl.rich_output.output_basic import OutputPlainText sage: plain_text = OutputPlainText.example() sage: from sage.repl.rich_output.backend_cell import BackendCell sage: backend = BackendCell() sage: _ = backend.display_immediately(plain_text, plain_text) Example plain text output """ if isinstance(rich_output, OutputPlainText): return {u'text/plain': rich_output.text.get()}, {} if isinstance(rich_output, OutputAsciiArt): return {u'text/plain': rich_output.ascii_art.get()}, {} if isinstance(rich_output, OutputLatex): self.display_html(rich_output.mathjax()) elif isinstance(rich_output, OutputHtml): self.display_html(rich_output.html.get()) elif isinstance(rich_output, OutputImageGif): self.display_file(rich_output.gif.filename(), 'text/image-filename') elif isinstance(rich_output, OutputImageJpg): self.display_file(rich_output.jpg.filename(), 'text/image-filename') elif isinstance(rich_output, OutputImagePdf): self.display_file(rich_output.pdf.filename(), 'text/image-filename') elif isinstance(rich_output, OutputImagePng): self.display_file(rich_output.png.filename(), 'text/image-filename') elif isinstance(rich_output, OutputImageSvg): self.display_file(rich_output.svg.filename(), 'text/image-filename') elif isinstance(rich_output, OutputSceneCanvas3d): self.display_file(rich_output.canvas3d.filename(), 'application/x-canvas3d') elif isinstance(rich_output, OutputSceneJmol): path = tempfile.mkdtemp(suffix=".jmol", dir=".") os.chmod(path, stat.S_IRWXU + stat.S_IXGRP + stat.S_IXOTH) rich_output.scene_zip.save_as(os.path.join(path, 'scene.zip')) rich_output.preview_png.save_as(os.path.join(path, 'preview.png')) display_message({'text/plain': 'application/x-jmol file', 'application/x-jmol': path}) else: raise TypeError('rich_output type not supported, got {0}'.format(rich_output)) return {u'text/plain': None}, {}
def _remove_read_only(func, path, exc): excvalue = exc[1] if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES: os.chmod(path, S_IRWXU| S_IRWXG| S_IRWXO) func(path) else: pass
def __setup_principal(self): dns_principal = "DNS/" + self.fqdn + "@" + self.realm installutils.kadmin_addprinc(dns_principal) # Store the keytab on disk self.fstore.backup_file(paths.NAMED_KEYTAB) installutils.create_keytab(paths.NAMED_KEYTAB, dns_principal) p = self.move_service(dns_principal) if p is None: # the service has already been moved, perhaps we're doing a DNS reinstall dns_principal = DN(('krbprincipalname', dns_principal), ('cn', 'services'), ('cn', 'accounts'), self.suffix) else: dns_principal = p # Make sure access is strictly reserved to the named user pent = pwd.getpwnam(self.named_user) os.chown(paths.NAMED_KEYTAB, pent.pw_uid, pent.pw_gid) os.chmod(paths.NAMED_KEYTAB, 0o400) # modify the principal so that it is marked as an ipa service so that # it can host the memberof attribute, then also add it to the # dnsserver role group, this way the DNS is allowed to perform # DNS Updates dns_group = DN(('cn', 'DNS Servers'), ('cn', 'privileges'), ('cn', 'pbac'), self.suffix) mod = [(ldap.MOD_ADD, 'member', dns_principal)] try: self.admin_conn.modify_s(dns_group, mod) except ldap.TYPE_OR_VALUE_EXISTS: pass except Exception, e: root_logger.critical("Could not modify principal's %s entry: %s" \ % (dns_principal, str(e))) raise
def get_perf(filename, folder): ''' run conlleval.pl perl script to obtain precision/recall and F1 score ''' _conlleval = os.path.join(folder, 'conlleval.pl') if not os.path.isfile(_conlleval): url = 'http://www-etud.iro.umontreal.ca/~mesnilgr/atis/conlleval.pl' download(url, _conlleval) os.chmod(_conlleval, stat.S_IRWXU) # give the execute permissions proc = subprocess.Popen(["perl", _conlleval], stdin=subprocess.PIPE, stdout=subprocess.PIPE) stdout, _ = proc.communicate(''.join(open(filename).readlines()).encode('utf-8')) stdout = stdout.decode('utf-8') out = None for line in stdout.split('\n'): if 'accuracy' in line: out = line.split() break # To help debug if out is None: print(stdout.split('\n')) precision = float(out[6][:-2]) recall = float(out[8][:-2]) f1score = float(out[10]) return {'p': precision, 'r': recall, 'f1': f1score}
def build_file_from_blob(blob, mode, target_path, honor_filemode=True): """Build a file or symlink on disk based on a Git object. :param obj: The git object :param mode: File mode :param target_path: Path to write to :param honor_filemode: An optional flag to honor core.filemode setting in config file, default is core.filemode=True, change executable bit """ if stat.S_ISLNK(mode): # FIXME: This will fail on Windows. What should we do instead? src_path = blob.as_raw_string() try: os.symlink(src_path, target_path) except OSError as e: if e.errno == errno.EEXIST: os.unlink(target_path) os.symlink(src_path, target_path) else: raise else: with open(target_path, 'wb') as f: # Write out file f.write(blob.as_raw_string()) if honor_filemode: os.chmod(target_path, mode)
def remove_directory(path): # In Python2 convert to unicode try: path = unicode(path) except NameError: pass # Note that shutil.rmtree fails if there are any broken symlinks in that # folder, so we use os.walk to traverse the directory tree from the bottom # up as recommended here: # http://stackoverflow.com/a/2656408 for root, dirs, files in os.walk(path, topdown=False): for name in files: filename = os.path.join(root, name) if not os.path.islink(filename): os.chmod(filename, stat.S_IWUSR) os.remove(filename) for name in dirs: dir = os.path.join(root, name) if os.path.islink(dir): os.unlink(dir) else: os.rmdir(dir) os.rmdir(path)
def write_job_script(machine, job, job_set): print(" Writing job script...") job.script = "#!/bin/bash -l\n\n" job.script += "# Notes: " + job_set.notes + "\n\n" if machine == 'eagle': job.script += "#SBATCH -J " + job.name + "\n" job.script += "#SBATCH -o %x.o%j\n" job.script += "#SBATCH -A " + job_set.project_allocation + "\n" job.script += "#SBATCH -t " + str(job.walltime) + "\n" job.script += "#SBATCH -p " + job.queue + "\n" job.script += "#SBATCH -N " + str(job.nodes) + "\n" job.script += "#SBATCH --mail-user="******"\n" job.script += "#SBATCH --mail-type=" + job_set.mail_type + "\n" elif machine == 'cori': job.script += "#SBATCH -J " + job.name + "\n" job.script += "#SBATCH -o %x.o%j\n" job.script += "#SBATCH -A " + job_set.project_allocation + "\n" job.script += "#SBATCH -t " + str(job.walltime) + "\n" job.script += "#SBATCH -q " + job.queue + "\n" job.script += "#SBATCH -N " + str(job.nodes) + "\n" job.script += "#SBATCH --mail-user="******"\n" job.script += "#SBATCH --mail-type=" + job_set.mail_type + "\n" job.script += "#SBATCH -C " + job.mapping + "\n" job.script += "#SBATCH -L SCRATCH\n" if job.knl_core_specialization != '': job.script += ("#SBATCH -S " + str(job.knl_core_specialization) + "\n") elif machine == 'summit': job.script += "#BSUB -J " + job.name + "\n" job.script += "#BSUB -o " + job.name + ".o%J\n" job.script += "#BSUB -P " + job_set.project_allocation + "\n" job.script += "#BSUB -W " + str(job.walltime) + "\n" job.script += "#BSUB -nnodes " + str(job.nodes) + "\n" if job.mapping == '7-ranks-per-gpu': job.script += "#BSUB -alloc_flags \"smt1\"" + "\n" job.script += "#BSUB -alloc_flags \"gpumps\"" + "\n" if job.mapping == '14-ranks-per-cpu': job.script += "#BSUB -alloc_flags \"smt2\"" + "\n" if job.mapping == '42-ranks-per-cpu' or job.mapping == '1-rank-per-gpu': job.script += "#BSUB -alloc_flags \"smt1\"" + "\n" job.script += r""" set -e cmd() { echo "+ $@" eval "$@" } """ if job.mapping == 'skylake' or job.mapping == 'knl' or job.mapping == 'haswell' or job.mapping == '42-ranks-per-cpu' or job.mapping == '14-ranks-per-cpu': job.script += ("echo \"Running with " + str(job.ranks_per_node) + " ranks per node and " + str(job.threads_per_rank) + " threads per rank on " + str(job.nodes) + " nodes for a total of " + str(job.total_ranks) + " ranks and " + str(job.total_threads) + " total threads...\"\n\n") elif job.mapping == '1-rank-per-gpu' or job.mapping == '7-ranks-per-gpu': job.script += ("echo \"Running with " + str(job.ranks_per_node) + " ranks per node and " + str(job.ranks_per_gpu) + " ranks per GPU on " + str(job.nodes) + " nodes for a total of " + str(job.total_ranks) + " ranks and " + str(job.total_gpus) + " total GPUs...\"\n\n") if machine == 'eagle': job.script += "MODULES=modules\n" job.script += "COMPILER=gcc-7.4.0\n" job.script += "cmd \"module purge\"\n" job.script += "cmd \"module unuse ${MODULEPATH}\"\n" job.script += ( "cmd \"module use /nopt/nrel/ecom/hpacf/binaries/${MODULES}\"\n" "cmd \"module use /nopt/nrel/ecom/hpacf/compilers/${MODULES}\"\n" "cmd \"module use /nopt/nrel/ecom/hpacf/utilities/${MODULES}\"\n" "cmd \"module use /nopt/nrel/ecom/hpacf/software/${MODULES}/${COMPILER}\"\n" ) job.script += "cmd \"module load gcc\"\n" job.script += "cmd \"module load python\"\n" if job.compiler == 'gnu': job.script += "cmd \"module load mpich\"\n\n" job.script += ("cmd \"export OMP_NUM_THREADS=" + str(job.threads_per_rank) + "\"\n\n") job.script += ("cmd \"" + job.pre_args + "mpirun -np " + str(job.total_ranks) + " --map-by ppr:" + str(job.ranks_per_node) + ":node:pe=" + str(job.threads_per_rank) + " -bind-to core -x OMP_NUM_THREADS " + str(job.executable) + " " + str(os.path.basename(job.input_file)) + " " + job.post_args + "\"\n") elif job.compiler == 'intel': job.script += "cmd \"module load intel-parallel-studio\"\n\n" job.script += "MY_TMP_DIR=/scratch/${USER}/.tmp\n" job.script += "NODE_LIST=${MY_TMP_DIR}/node_list.${SLURM_JOB_ID}\n" job.script += "cmd \"mkdir -p ${MY_TMP_DIR}\"\n" job.script += "cmd \"scontrol show hostname > ${NODE_LIST}\"\n" job.script += "#cmd \"export I_MPI_DEBUG=5\"\n" job.script += "#cmd \"export I_MPI_FABRIC_LIST=ofa,dapl\"\n" job.script += "#cmd \"export I_MPI_FABRICS=shm:ofa\"\n" job.script += "#cmd \"export I_MPI_FALLBACK=0\"\n" job.script += "cmd \"export I_MPI_PIN=1\"\n" job.script += "cmd \"export I_MPI_PIN_DOMAIN=omp\"\n" job.script += ( "cmd \"export KMP_AFFINITY=compact,granularity=core\"\n" ) job.script += ("cmd \"export OMP_NUM_THREADS=" + str(job.threads_per_rank) + "\"\n\n") job.script += ("cmd \"" + job.pre_args + "mpirun -n " + str(job.total_ranks) + " -genvall -f ${NODE_LIST} -ppn " + str(job.ranks_per_node) + " " + str(job.executable) + " " + str(os.path.basename(job.input_file)) + " " + job.post_args + "\"\n") elif machine == 'cori': job.script += ("cmd \"export OMP_NUM_THREADS=" + str(job.threads_per_rank) + "\"\n") job.script += "cmd \"export OMP_PLACES=threads\"\n" job.script += "cmd \"export OMP_PROC_BIND=spread\"\n" if job.mapping == 'knl': job.script += ( "cmd \"module swap craype-haswell craype-mic-knl || true\"\n" ) if job.mapping == 'knl' and job.nodes > 156: my_exe = "/tmp/pelec.ex" job.script += ("cmd \"sbcast -f -F2 -t 300 --compress=lz4 " + str(job.executable) + " " + my_exe + "\"\n") else: my_exe = job.executable job.script += ("\ncmd \"" + job.pre_args + "srun -n " + str(job.total_ranks) + " -c " + str(job.cores_per_rank) + " --cpu_bind=" + str(job.cpu_bind) + " " + my_exe + " " + str(os.path.basename(job.input_file)) + " " + job.post_args + "\"\n") elif machine == 'summit': job.script += "cmd \"module unload xl\"\n" job.script += "cmd \"module load cuda/10.1.168\"\n" if job.compiler == 'pgi': job.script += "cmd \"module load pgi/19.5\"\n" elif job.compiler == 'gcc': job.script += "cmd \"module load gcc\"\n" if job.mapping == '14-ranks-per-cpu': job.script += "cmd \"export OMP_NUM_THREADS=6\"\n" job.script += ("cmd \"" + job.pre_args + "jsrun --nrs ") if job.mapping == '1-rank-per-gpu': job.script += (str(job.total_ranks) + " --tasks_per_rs " + str(1) + " --cpu_per_rs " + str(1) + " --gpu_per_rs " + str(1) + " --rs_per_host " + str(6) + " --bind packed:1") if job.mapping == '7-ranks-per-gpu': job.script += (str(job.total_ranks / job.ranks_per_gpu) + " --tasks_per_rs " + str(7) + " --cpu_per_rs " + str(7) + " --gpu_per_rs " + str(1) + " --rs_per_host " + str(6) + " --bind packed:1") if job.mapping == '42-ranks-per-cpu': job.script += (str(job.total_ranks) + " --tasks_per_rs " + str(1) + " --cpu_per_rs " + str(1) + " --rs_per_host " + str(42) + " --bind packed:1") if job.mapping == '14-ranks-per-cpu': job.script += (str(job.total_ranks) + " --tasks_per_rs " + str(1) + " --cpu_per_rs " + str(3) + " --rs_per_host " + str(14) + " --bind rs") job.script += (" --latency_priority CPU-CPU" + " --launch_distribution packed " + str(job.executable) + " " + str(os.path.basename(job.input_file)) + " " + job.post_args + "\"\n") # Write job script to file job.script_file = os.path.join(job.path, job.name + '.sh') job_script_file_handle = open(job.script_file, 'w') job_script_file_handle.write(job.script) job_script_file_handle.close() # Make the job script executable st = os.stat(job.script_file) os.chmod(job.script_file, st.st_mode | stat.S_IEXEC)
def chown(self, path, uid, gid): full_path = self._full_path(path) os.chown(full_path, uid, gid) if self._cont_exists(path): cont_path = self._cont_path(path) os.chmod(cont_path, mode)
def chmod(self, path, mode): full_path = self._full_path(path) os.chmod(full_path, mode) if self._cont_exists(path): cont_path = self._cont_path(path) os.chmod(cont_path, mode)
#!/usr/bin/python import os filename = "/etc/passwd" filename1 = "/etc/shadow" filename2 = "/etc/group" filename3 = "/etc/gshadow" filename4 = "/etc/passwd-" filename5 = "/etc/shadow-" filename6 = "/etc/group-" filename7 = "/etc/gshadow-" mask = oct(os.stat(filename).st_mode)[-3:] mask1 = oct(os.stat(filename1).st_mode)[-3:] mask2 = oct(os.stat(filename2).st_mode)[-3:] mask3 = oct(os.stat(filename3).st_mode)[-3:] mask4 = oct(os.stat(filename4).st_mode)[-3:] mask5 = oct(os.stat(filename5).st_mode)[-3:] mask6 = oct(os.stat(filename6).st_mode)[-3:] mask7 = oct(os.stat(filename7).st_mode)[-3:] os.chmod('/etc/passwd', 0o600) os.chmod('/etc/shadow', 0o600) os.chmod('/etc/group', 0o600) os.chmod('/etc/gshadow', 0o600) os.chmod('/etc/passwd-', 0o600) os.chmod('/etc/shadow-', 0o600) os.chmod('/etc/group-', 0o600) os.chmod('/etc/gshadow', 0o600)
def start(): # delete already existing matlab engine for this user try: del(d2d_instances[session['uid']]) except: pass ROUND = int(request.args.get('round')) if ROUND is 0: ROUND = False status = {} status['nFinePoints_min'] = False status['arSimu'] = False nFinePoints = int(request.args.get('nFinePoints')) do_compile = str(request.args.get('compile')) # Set the directory you want to copy from rootDir = os.path.dirname(os.path.join( os.getcwd(), request.args.get('model'))) # save the origin of the copied model originDir = os.path.join(os.getcwd(), request.args.get('model')) if COPY is True: temp_dir = os.path.join(os.getcwd(), 'temp', str(session['uid'])) try: for root, dirs, files in os.walk(temp_dir): for fname in files: full_path = os.path.join(root, fname) os.chmod(full_path, stat.S_IWRITE) remove_tree(temp_dir) except: pass copy_tree(rootDir, temp_dir) rootDir = os.path.join(temp_dir, os.path.basename(request.args.get('model'))) else: rootDir = originDir d2d_instances.update( { session['uid']: {'d2d': pyd2d.d2d(), 'alive': 1, 'model': rootDir, 'origin': originDir, 'nFinePoints': nFinePoints, 'ROUND': ROUND, 'MODEL': 1, 'DSET': 1 } }) load = None if not do_compile.endswith('on'): try: results = os.listdir(os.path.join( os.path.dirname(d2d_instances[session['uid']]['model']), 'Results')) for savename in results: if savename.endswith('_d2d_presenter'): load = savename print("A result has been found and will be loaded.") break if load is None: print("No saved results found, compiling " + "from scratch and save the result. This might take " + "some time. Keep in mind that arSave will only be " + "persistent if COPY is False.") except: pass else: load = False print("The model will be compiled. This might take some time.") d2d_instances[session['uid']]['d2d'].load_model( d2d_instances[session['uid']]['model'], load=load, origin=d2d_instances[session['uid']]['origin']) try: nFinePoints_min = d2d_instances[session['uid']]['d2d'].get( {'d2d_presenter.nFinePoints_min'}, 1, 1, False, 'list')['d2d_presenter.nFinePoints_min'][0][0] except: nFinePoints_min = nFinePoints if nFinePoints_min > nFinePoints: nFinePoints = nFinePoints_min status['nFinePoints_min'] = nFinePoints d2d_instances[session['uid']]['d2d'].set( {'ar.config.nFinePoints': nFinePoints}) d2d_instances[session['uid']]['d2d'].eval("arLink;") status['arSimu'] = d2d_instances[session['uid']]['d2d'].simu() d2d_instances[session['uid']]['d2d'].eval("arChi2;") # thread will shut down the matlab instance on inactivity t = Thread(target=d2d_close_instance, args=(session['uid'], )) t.start() if str(request.args.get('direct_access')).endswith('true'): return redirect("#main_page") else: return jsonify(status=status)
def _save(self, name, content): full_path = self.path(name) # Create any intermediate directories that do not exist. # Note that there is a race between os.path.exists and os.makedirs: # if os.makedirs fails with EEXIST, the directory was created # concurrently, and we can continue normally. Refs #16082. directory = os.path.dirname(full_path) if not os.path.exists(directory): try: os.makedirs(directory) except OSError as e: if e.errno != errno.EEXIST: raise if not os.path.isdir(directory): raise IOError("%s exists and is not a directory." % directory) # There's a potential race condition between get_available_name and # saving the file; it's possible that two threads might return the # same name, at which point all sorts of fun happens. So we need to # try to create the file, but if it already exists we have to go back # to get_available_name() and try again. while True: try: # This file has a file path that we can move. if hasattr(content, 'temporary_file_path'): file_move_safe(content.temporary_file_path(), full_path) content.close() # This is a normal uploadedfile that we can stream. else: # This fun binary flag incantation makes os.open throw an # OSError if the file already exists before we open it. flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0)) # The current umask value is masked out by os.open! fd = os.open(full_path, flags, 0o666) try: locks.lock(fd, locks.LOCK_EX) _file = None for chunk in content.chunks(): if _file is None: mode = 'wb' if isinstance(chunk, bytes) else 'wt' _file = os.fdopen(fd, mode) _file.write(chunk) finally: locks.unlock(fd) if _file is not None: _file.close() else: os.close(fd) except OSError as e: if e.errno == errno.EEXIST: # Ooops, the file exists. We need a new file name. name = self.get_available_name(name) full_path = self.path(name) else: raise else: # OK, the file save worked. Break out of the loop. break if settings.FILE_UPLOAD_PERMISSIONS is not None: os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS) return name
# -*- coding: utf-8 -*- from __future__ import print_function import argparse import os import stat import sys # find the import for catkin's python package - either from source space or from an installed underlay if os.path.exists(os.path.join('/opt/ros/kinetic/share/catkin/cmake', 'catkinConfig.cmake.in')): sys.path.insert(0, os.path.join('/opt/ros/kinetic/share/catkin/cmake', '..', 'python')) try: from catkin.environment_cache import generate_environment_script except ImportError: # search for catkin package in all workspaces and prepend to path for workspace in "/home/afifi/catkin_ws/devel;/opt/ros/kinetic".split(';'): python_path = os.path.join(workspace, 'lib/python2.7/dist-packages') if os.path.isdir(os.path.join(python_path, 'catkin')): sys.path.insert(0, python_path) break from catkin.environment_cache import generate_environment_script code = generate_environment_script('/home/afifi/Graduation_project/SLAM_and_navigation_using_ROS/devel/env.sh') output_filename = '/home/afifi/Graduation_project/SLAM_and_navigation_using_ROS/build/catkin_generated/setup_cached.sh' with open(output_filename, 'w') as f: #print('Generate script for cached setup "%s"' % output_filename) f.write('\n'.join(code)) mode = os.stat(output_filename).st_mode os.chmod(output_filename, mode | stat.S_IXUSR)
def compile_client(war_location, client_location): GWT_VERSION_NUMBER = '2.6.1' external_location = os.path.join(client_location, 'external') gwt_location = os.path.join(external_location,'gwt') if 'GWT_HOME' in os.environ and os.path.exists(os.environ['GWT_HOME']): gwt_location = os.environ['GWT_HOME'] else: gwt_location = gwt_location about_txt = os.path.join(gwt_location, 'about.txt') if os.path.exists(about_txt): first_line = open(about_txt).readlines()[0] if GWT_VERSION_NUMBER in first_line: # Correct version if len(glob.glob(war_location + "/*/*.cache.html")) > 5: print(file=sys.stderr) print("A compiled version of the WebLab-Deusto client was found, and it will ", file=sys.stderr) print("be used. If you have not recently compiled it and you have upgraded or", file=sys.stderr) print("modified anything in the client, you should compile it before doing", file=sys.stderr) print("this. So as to compile it, run:", file=sys.stderr) print("", file=sys.stderr) curpath = os.path.abspath('.') if sys.platform.find('win') == 0: print(" %s> cd %s" % (curpath, client_location), file=sys.stderr) print(" %s> gwtc" % client_location, file=sys.stderr) else: print(" user@machine:%s$ cd %s" % (curpath, client_location), file=sys.stderr) print(" user@machine:%s$ ./gwtc.sh" % client_location, file=sys.stderr) print(file=sys.stderr) print("And then run the setup.py again.", file=sys.stderr) print(file=sys.stderr) return else: print("", file=sys.stderr) print("No WebLab-Deusto compiled client found. Trying to compile it...", file=sys.stderr) else: print("", file=sys.stderr) print("Old GWT version found (required: %s). Trying to compile the client..." % GWT_VERSION_NUMBER, file=sys.stderr) else: print("", file=sys.stderr) print("No GWT version found. Trying to compile the client...", file=sys.stderr) try: subprocess.Popen(["javac", "--help"], shell=False, stdout = subprocess.PIPE, stderr = subprocess.PIPE).wait() except: print("", file=sys.stderr) print("Java compiler not found. Please, install the Java Development Kit (JDK).", file=sys.stderr) print("", file=sys.stderr) print("In Windows, you may need to download it from:", file=sys.stderr) print("", file=sys.stderr) print(" http://java.oracle.com/ ", file=sys.stderr) print("", file=sys.stderr) print("And install it. Once installed, you should add the jdk directory to PATH.", file=sys.stderr) print("This depends on the particular version of Windows, but you can follow ", file=sys.stderr) print("the following instructions to edit the PATH environment variable and add", file=sys.stderr) print("something like C:\\Program Files\\Oracle\\Java\\jdk6\\bin to PATH:", file=sys.stderr) print("", file=sys.stderr) print(" http://docs.oracle.com/javase/tutorial/essential/environment/paths.html", file=sys.stderr) print("", file=sys.stderr) print("In Linux systems, you may install openjdk from your repository and ", file=sys.stderr) print("will be configured.", file=sys.stderr) sys.exit(-1) # # Check that GWT is downloaded. Download it otherwise. # GWT_VERSION = 'gwt-%s' % GWT_VERSION_NUMBER GWT_URL = "http://storage.googleapis.com/gwt-releases/%s.zip" % GWT_VERSION external_location = os.path.join(client_location, 'external') gwt_location = os.path.join(external_location,'gwt') must_download = False if os.path.exists(about_txt): first_line = open(about_txt).readlines()[0] if GWT_VERSION_NUMBER not in first_line: must_download = True else: must_download = True if must_download: print("", file=sys.stderr) print("WebLab-Deusto relies on Google Web Toolkit (GWT). In order to compile ", file=sys.stderr) print("the client, we need to download it. If you had already downloaded it, ", file=sys.stderr) print("place it in: ", file=sys.stderr) print("", file=sys.stderr) print(" %s" % gwt_location, file=sys.stderr) print("", file=sys.stderr) print("or somewhere else and create an environment variable called GWT_HOME pointing", file=sys.stderr) print("to it. Anyway, I will attempt to download it and place it there.", file=sys.stderr) print("", file=sys.stderr) print("Downloading %s..." % GWT_URL, file=sys.stderr) print("(please wait a few minutes; GWT is ~100 MB)", file=sys.stderr) try: os.mkdir(external_location) except (OSError, IOError): pass # Could be already created try: shutil.rmtree(gwt_location) except: pass # TODO: this places in memory the whole file (~100 MB). urllib.urlretrieve? gwt_content = urllib2.urlopen(GWT_URL).read() gwt_fileobj = StringIO.StringIO(gwt_content) del gwt_content print("Downloaded. Extracting...", file=sys.stderr) zf = zipfile.ZipFile(gwt_fileobj) zf.extractall(external_location) del gwt_fileobj, zf shutil.move(os.path.join(external_location, GWT_VERSION), gwt_location) print("Extracted at %s" % gwt_location, file=sys.stderr) # These two directories are amost 200MB, so it's better to remove them try: shutil.rmtree(os.path.join(gwt_location, 'doc')) shutil.rmtree(os.path.join(gwt_location, 'samples')) except (OSError, IOError) as e: print("WARNING: Error trying to remove GWT doc and samples directories: %s" % e, file=sys.stderr) libclient_location = os.path.join(external_location, 'lib-client') VERSION = "3.8.2" JUNIT_URL = "http://downloads.sourceforge.net/project/junit/junit/3.8.2/junit3.8.2.zip" junit_location = os.path.join(libclient_location, 'junit.jar') if not os.path.exists(junit_location): print("", file=sys.stderr) print("WebLab-Deusto relies on JUnit. In order to test the client, we", file=sys.stderr) print("need to download it. If you already downloaded it, place the", file=sys.stderr) print(".jar of the %s version in: " % VERSION, file=sys.stderr) print("", file=sys.stderr) print(" %s" % junit_location, file=sys.stderr) print("", file=sys.stderr) print("I will attempt to download it and place it there.", file=sys.stderr) print("", file=sys.stderr) print("Downloading %s..." % JUNIT_URL, file=sys.stderr) print("(plase wait a few seconds; junit is ~400 KB)", file=sys.stderr) try: os.mkdir(libclient_location) except (OSError, IOError): pass # Could be already created junit_content = urllib2.urlopen(JUNIT_URL).read() junit_fileobj = StringIO.StringIO(junit_content) del junit_content print("Downloaded. Extracting...", file=sys.stderr) zf = zipfile.ZipFile(junit_fileobj) jar_in_zip_name = 'junit%s/junit.jar' % VERSION zf.extract(jar_in_zip_name, libclient_location) del zf shutil.move(os.path.join(libclient_location, jar_in_zip_name.replace('/', os.sep)), junit_location) print("Extracted to %s." % junit_location) # # Check that ant is installed. Download it otherwise. # try: subprocess.Popen(["ant","--help"], shell=False, stdout = subprocess.PIPE, stderr = subprocess.PIPE).wait() except: ant_installed = False else: ant_installed = True ant_location = os.path.join(external_location, 'ant') if not ant_installed and not os.path.exists(ant_location): print("", file=sys.stderr) print("Ant not found. In order to compile the client, Apache Ant is required.", file=sys.stderr) print("You can download and install Apache Ant from its official site:", file=sys.stderr) print("", file=sys.stderr) print(" http://ant.apache.org/", file=sys.stderr) print("", file=sys.stderr) print("And put it in the PATH environment variable so the 'ant' command works.", file=sys.stderr) print("Anyway, I will download it and use the downloaded version.", file=sys.stderr) print("", file=sys.stderr) print("Downloading... (please wait for a few minutes; ant is ~8 MB)", file=sys.stderr) # # Ant does not maintain all the versions in the main repository, but only the last one. First # we need to retrieve the latest version. content = urllib2.urlopen("http://www.apache.org/dist/ant/binaries/").read() version = content.split('-bin.zip')[0].split('ant-')[-1] ANT_FILENAME = 'apache-ant-%s-bin.zip' % version ANT_DIRNAME = 'apache-ant-%s' % version ANT_URL = "http://www.apache.org/dist/ant/binaries/%s" % ANT_FILENAME # TODO: this places in memory the whole file (~24 MB). urllib.urlretrieve? ant_content = urllib2.urlopen(ANT_URL).read() ant_fileobj = StringIO.StringIO(ant_content) del ant_content print("Downloaded. Extracting...", file=sys.stderr) zf = zipfile.ZipFile(ant_fileobj) zf.extractall(external_location) del ant_fileobj, zf shutil.move(os.path.join(external_location, ANT_DIRNAME), ant_location) print("Extracted at %s" % ant_location, file=sys.stderr) curpath = os.path.abspath(os.getcwd()) os.chdir(client_location) try: if ant_installed: ant_name = 'ant' else: ant_name = os.path.join('external','ant','bin','ant') # Required in UNIX, ignored in Linux os.chmod(ant_name, stat.S_IREAD |stat.S_IWRITE | stat.S_IEXEC | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH ) if os.system(ant_name + ' gwtc') != 0: print("ERROR: Could not compile the client!!!", file=sys.stderr) sys.exit(-1) finally: os.chdir(curpath)
def dump(my, field_storage, file_name): web = WebContainer.get_web() ticket = web.get_form_value("transaction_ticket") if not ticket: security = Environment.get_security() ticket = security.get_ticket_key() tmpdir = Environment.get_tmp_dir() subdir = web.get_form_value("subdir") custom_upload_dir = web.get_form_value("upload_dir") if subdir: file_dir = "%s/%s/%s/%s" % (tmpdir, "upload", ticket, subdir) else: file_dir = "%s/%s/%s" % (tmpdir, "upload", ticket) if custom_upload_dir: if subdir: file_dir = "%s/%s"%(file_dir,subdir) else: file_dir = custom_upload_dir ''' If upload method is html5, the action is an empty string. Otherwise, action is either 'create' or 'append'. ''' action = web.get_form_value("action") html5_mode = False if not action: html5_mode = True action = "create" ''' With some recent change done in cherrypy._cpreqbody line 294, we can use the field storage directly on Linux when the file is uploaded in html5 mode. TODO: This shortcut cannot be used with upload_multipart.py ''' path = field_storage.get_path() # Base 64 encoded files are uploaded and decoded in FileUpload base_decode = None if action == "create": if os.name == 'nt': f = field_storage.file else: f = open(path, 'rb') header = f.read(22) f.seek(0) if header.startswith("data:image/png;base64,"): base_decode = True else: base_decode = False if os.name != 'nt': f.close() if html5_mode and file_name and path and not base_decode: ''' example of path: /home/tactic/tactic_temp/temp/tmpTxXIjM example of to_path: /home/tactic/tactic_temp/upload/ XX-dev-2924f964921857bf239acef4f9bcf3bf/miso_ramen.jpg ''' if not os.path.exists(file_dir): os.makedirs(file_dir) basename = os.path.basename(path) to_path = "%s/%s" % (file_dir, file_name) shutil.move(path, to_path) ''' # Close the mkstemp file descriptor fd = field_storage.get_fd() if fd: os.close( fd ) ''' # Because _cpreqbody makes use of mkstemp, the file permissions # are set to 600. This switches to the permissions as defined # by the TACTIC users umask try: current_umask = os.umask(0) os.umask(current_umask) os.chmod(to_path, 0o666 - current_umask) except Exception, e: print "WARNING: ", e return [to_path]
def setUpAuth(cls): """Run before all tests and set up authentication""" authm = QgsApplication.authManager() assert (authm.setMasterPassword('masterpassword', True)) cls.pg_conf = os.path.join(cls.tempfolder, 'postgresql.conf') cls.pg_hba = os.path.join(cls.tempfolder, 'pg_hba.conf') # Client side cls.sslrootcert_path = os.path.join( cls.certsdata_path, 'chains_subissuer-issuer-root_issuer2-root2.pem') cls.sslcert = os.path.join(cls.certsdata_path, 'gerardus_cert.pem') cls.sslkey = os.path.join(cls.certsdata_path, 'gerardus_key.pem') assert os.path.isfile(cls.sslcert) assert os.path.isfile(cls.sslkey) assert os.path.isfile(cls.sslrootcert_path) os.chmod(cls.sslcert, stat.S_IRUSR) os.chmod(cls.sslkey, stat.S_IRUSR) os.chmod(cls.sslrootcert_path, stat.S_IRUSR) cls.auth_config = QgsAuthMethodConfig("PKI-Paths") cls.auth_config.setConfig('certpath', cls.sslcert) cls.auth_config.setConfig('keypath', cls.sslkey) cls.auth_config.setName('test_pki_auth_config') cls.username = '******' cls.sslrootcert = QSslCertificate.fromPath(cls.sslrootcert_path) assert cls.sslrootcert is not None authm.storeCertAuthorities(cls.sslrootcert) authm.rebuildCaCertsCache() authm.rebuildTrustedCaCertsCache() authm.rebuildCertTrustCache() assert (authm.storeAuthenticationConfig(cls.auth_config)[0]) assert cls.auth_config.isValid() # Server side cls.server_cert = os.path.join(cls.certsdata_path, 'localhost_ssl_cert.pem') cls.server_key = os.path.join(cls.certsdata_path, 'localhost_ssl_key.pem') cls.server_rootcert = cls.sslrootcert_path os.chmod(cls.server_cert, stat.S_IRUSR) os.chmod(cls.server_key, stat.S_IRUSR) os.chmod(cls.server_rootcert, stat.S_IRUSR) # Place conf in the data folder with open(cls.pg_conf, 'w+') as f: f.write( QGIS_POSTGRES_CONF_TEMPLATE % { 'port': cls.port, 'tempfolder': cls.tempfolder, 'server_cert': cls.server_cert, 'server_key': cls.server_key, 'sslrootcert_path': cls.sslrootcert_path, }) with open(cls.pg_hba, 'w+') as f: f.write(QGIS_POSTGRES_HBA_TEMPLATE)
def configure_syslog(middleware): systemdataset = middleware.call_sync("systemdataset.config") if not systemdataset["path"] or not use_syslog_dataset(middleware): if os.path.islink("/var/log"): if not os.path.realpath("/var/log"): os.rename( "/var/log", "/var/log." + datetime.now().strftime("%Y%m%d%H%M%S")) else: os.unlink("/var/log") shutil.copytree("/conf/base/var/log", "/var/log") middleware.call_sync("core.reconfigure_logging") return log_path = os.path.join(systemdataset["path"], f"syslog-{systemdataset['uuid']}") if os.path.exists(os.path.join(log_path, "log")): # log directory exists, pick up any new files or # directories and create them. Existing files will be # appended. This is done this way so that ownership and # permissions are always preserved. if not os.path.islink("/var/log"): for item in os.listdir("/var/log"): dst = os.path.join(log_path, "log", item) item = os.path.join("/var/log", item) if os.path.isdir(item): # Pick up any new directories and sync them if not os.path.isdir(dst): shutil.copytree(item, dst) else: # If the file exists already, append to # it, otherwise, copy it over. if os.path.isfile(dst): with open(item, "rb") as f1: with open(dst, "ab") as f2: shutil.copyfileobj(f1, f2) else: shutil.copy(item, dst) else: # This is the first time syslog is going to log to this # directory, so create the log directory and sync files. shutil.copytree("/conf/base/var/log", os.path.join(log_path, "log")) os.chmod(os.path.join(log_path, "log"), 0o755) os.chown(os.path.join(log_path, "log"), 0, 0) subprocess.run( f"/usr/local/bin/rsync -avz /var/log/* {shlex.quote(log_path + '/log/')}", shell=True, stdout=subprocess.DEVNULL) if not os.path.islink("/var/log") or not os.path.realpath("/var/log"): os.rename("/var/log", "/var/log." + datetime.now().strftime("%Y%m%d%H%M%S")) os.symlink(os.path.join(log_path, "log"), "/var/log") # Let's make sure that the permissions for directories/files in /var/log # reflect that of /conf/base/var/log subprocess.run("mtree -c -p /conf/base/var/log | mtree -eu", cwd="/var/log", shell=True, stdout=subprocess.DEVNULL) middleware.call_sync("core.reconfigure_logging")
f.write('--solver="{}" '.format(solver_file)) print(train_src_param) f.write(train_src_param) print(job_dir) print(model_name) if solver_param['solver_mode'] == P.Solver.GPU: f.write('--gpu {}\n'.format(gpus)) #else: # f.write('2>&1 | tee {}\\{}.log\n'.format(job_dir, model_name)) # Copy the python script to job_dir. py_file = os.path.abspath(__file__) shutil.copy(py_file, job_dir) # Run the job. os.chmod(job_file, stat.S_IRWXU) if run_soon: # flag : WillChoi # modified date : 17.09.15 # modify : Added 'os.path.expanduser' to re-present path style to Windows job_file = os.path.abspath(job_file) print(job_file) #subprocess.call(job_file, shell=True) p = subprocess.Popen(["powershell.exe", job_file], stdout=sys.stdout) p. communicate() #p = subprocess.Popen(["powershell.exe", # 'Stop-Transcript'], # stdout=sys.stdout) #p.communicate()
OUT_PATH = create_experiment_folder(c.output_path, c.run_name, args.debug) AUDIO_PATH = os.path.join(OUT_PATH, 'test_audios') c_logger = ConsoleLogger() if args.rank == 0: os.makedirs(AUDIO_PATH, exist_ok=True) new_fields = {} if args.restore_path: new_fields["restore_path"] = args.restore_path new_fields["github_branch"] = get_git_branch() copy_config_file(args.config_path, os.path.join(OUT_PATH, 'config.json'), new_fields) os.chmod(AUDIO_PATH, 0o775) os.chmod(OUT_PATH, 0o775) LOG_DIR = OUT_PATH tb_logger = TensorboardLogger(LOG_DIR, model_name='TTS') # write model desc to tensorboard tb_logger.tb_add_text('model-description', c['run_description'], 0) try: main(args) except KeyboardInterrupt: remove_experiment_folder(OUT_PATH) try: sys.exit(0) except SystemExit:
def setup_batch_beast_fit( num_percore=5, nice=None, overwrite_logfile=True, prefix=None, use_sd=True, nsubs=1, nprocs=1, ): """ Sets up batch files for submission to the 'at' queue on linux (or similar) systems Parameters ---------- num_percore : int (default = 5) number of fitting runs per core nice : int (default = None) set this to an integer (-20 to 20) to prepend a "nice" level to the fitting command overwrite_logfile : boolean (default = True) if True, will overwrite the log file; if False, will append to existing log file prefix : string (default=None) Set this to a string (such as 'source activate astroconda') to prepend to each batch file (use '\n's to make multiple lines) use_sd : boolean (default=True) If True, split runs based on source density (determined by finding matches to datamodel.astfile with SD info) nsubs : int (default=1) number of subgrids used for the physics model nprocs : int (default=1) Number of parallel processes to use when doing the fitting (currently only implemented for subgrids) Returns ------- run_info_dict : dict Dictionary indicating which catalog files have complete modeling, and which job files need to be run """ # before doing ANYTHING, force datamodel to re-import (otherwise, any # changes within this python session will not be loaded!) importlib.reload(datamodel) # check input parameters verify_params.verify_input_format(datamodel) # setup the subdirectory for the batch and log files job_path = datamodel.project + "/fit_batch_jobs/" if not os.path.isdir(job_path): os.mkdir(job_path) log_path = job_path + "logs/" if not os.path.isdir(log_path): os.mkdir(log_path) # get file name lists (to check if they exist and/or need to be resumed) file_dict = create_filenames.create_filenames(use_sd=use_sd, nsubs=nsubs) # - input files photometry_files = file_dict["photometry_files"] # modelsedgrid_files = file_dict['modelsedgrid_files'] # noise_files = file_dict['noise_files'] # - output files stats_files = file_dict["stats_files"] pdf_files = file_dict["pdf_files"] lnp_files = file_dict["lnp_files"] # - total number of files n_files = len(photometry_files) # - other useful info sd_sub_info = file_dict["sd_sub_info"] gridsub_info = file_dict["gridsub_info"] # names of output log files log_files = [] # initialize a variable name (otherwise it got auto-added in the wrong # place and broke the code) pf = None for i in range(n_files): sd_piece = "" if use_sd is True: sd_piece = "_bin" + sd_sub_info[i][0] + "_sub" + sd_sub_info[i][1] gridsub_piece = "" if nsubs > 1: gridsub_piece = "_gridsub" + str(gridsub_info[i]) log_files.append("beast_fit" + sd_piece + gridsub_piece + ".log") # start making the job files! pf_open = False cur_f = 0 cur_total_size = 0.0 j = -1 # keep track of which files are done running run_info_dict = { "phot_file": photometry_files, "done": np.full(n_files, False), "files_to_run": [], } for i, phot_file in enumerate(photometry_files): print("") # check if this is a full run reg_run = False run_done = False if not os.path.isfile(stats_files[i]): reg_run = True print("no stats file") if not os.path.isfile(pdf_files[i]): reg_run = True print("no pdf1d file") if not os.path.isfile(lnp_files[i]): reg_run = True print("no lnp file") # first check if the pdf1d mass spacing is correct if not reg_run: hdulist = fits.open(pdf_files[i]) delta1 = hdulist["M_ini"].data[-1, 1] - hdulist["M_ini"].data[-1, 0] if delta1 > 1.0: # old linear spacing print("pdf1d lin mass spacing - full refitting needed") old_mass_spacing = True else: old_mass_spacing = False print("pdf1d log mass spacing - ok") if old_mass_spacing: run_done = False reg_run = True # now check if the number of results is the same as # the number of observations if not reg_run: # get the observed catalog obs = Table.read(phot_file) # get the fit results catalog t = Table.read(stats_files[i]) # get the number of stars that have been fit (indxs, ) = np.where(t["Pmax"] != 0.0) # get the number of entries in the lnp file f = tables.open_file(lnp_files[i], "r") nlnp = f.root._v_nchildren - 2 f.close() print("# obs, stats, lnp = ", len(obs), len(indxs), nlnp) if (len(indxs) == len(obs)) & (nlnp == len(obs)): # final check, is the pdf1d file correctly populated tot_prob = np.sum(hdulist["M_ini"].data, axis=1) (tindxs, ) = np.where(tot_prob > 0.0) print("# good pdf1d = ", len(tindxs) - 1) if len(tindxs) == (len(obs) + 1): run_done = True if run_done: print(stats_files[i] + " done") run_info_dict["done"][i] = True else: j += 1 if j % num_percore == 0: cur_f += 1 # close previous files if j != 0: pf.close() # slurm needs the job file to be executable os.chmod(joblist_file, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH) print( "total sed_trim size [Gb] = ", cur_total_size / (1024.0 * 1024.0 * 1024.0), ) cur_total_size = 0.0 # open the slurm and param files pf_open = True joblist_file = job_path + "beast_batch_fit_" + str( cur_f) + ".joblist" pf = open(joblist_file, "w") run_info_dict["files_to_run"].append(joblist_file) # write out anything at the beginning of the file if prefix is not None: pf.write(prefix + "\n") # flag for resuming resume_str = "" if reg_run: print(stats_files[i] + " does not exist " + "- adding job as a regular fit job (not resume job)") else: print(stats_files[i] + " not done - adding to continue fitting list (" + str(len(indxs)) + "/" + str(len(t["Pmax"])) + ")") resume_str = "-r" # prepend a `nice` value nice_str = "" if nice is not None: nice_str = "nice -n" + str(int(nice)) + " " # choose whether to append or overwrite log file pipe_str = " > " if not overwrite_logfile: pipe_str = " >> " # set SD+sub option sd_str = "" if use_sd is True: sd_str = ' --choose_sd_sub "{0}" "{1}" '.format( sd_sub_info[i][0], sd_sub_info[i][1]) # set gridsub option gs_str = "" if nsubs > 1: gs_str = " --choose_subgrid {0} ".format(gridsub_info[i]) job_command = (nice_str + "python -m beast.tools.run.run_fitting " + resume_str + sd_str + gs_str + " --nsubs " + str(nsubs) + " --nprocs " + str(nprocs) + pipe_str + log_path + log_files[i]) pf.write(job_command + "\n") if pf_open: pf.close() # slurm needs the job file to be executable os.chmod(joblist_file, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH) # return the info about completed modeling return run_info_dict
def untar_file(filename, location): # type: (str, str) -> None """ Untar the file (with path `filename`) to the destination `location`. All files are written based on system defaults and umask (i.e. permissions are not preserved), except that regular file members with any execute permissions (user, group, or world) have "chmod +x" applied after being written. Note that for windows, any execute changes using os.chmod are no-ops per the python docs. """ ensure_dir(location) if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'): mode = 'r:gz' elif filename.lower().endswith(BZ2_EXTENSIONS): mode = 'r:bz2' elif filename.lower().endswith(XZ_EXTENSIONS): mode = 'r:xz' elif filename.lower().endswith('.tar'): mode = 'r' else: logger.warning( 'Cannot determine compression type for file %s', filename, ) mode = 'r:*' tar = tarfile.open(filename, mode) try: leading = has_leading_dir([member.name for member in tar.getmembers()]) for member in tar.getmembers(): fn = member.name if leading: # https://github.com/python/mypy/issues/1174 fn = split_leading_dir(fn)[1] # type: ignore path = os.path.join(location, fn) if member.isdir(): ensure_dir(path) elif member.issym(): try: # https://github.com/python/typeshed/issues/2673 tar._extract_member(member, path) # type: ignore except Exception as exc: # Some corrupt tar files seem to produce this # (specifically bad symlinks) logger.warning( 'In the tar file %s the member %s is invalid: %s', filename, member.name, exc, ) continue else: try: fp = tar.extractfile(member) except (KeyError, AttributeError) as exc: # Some corrupt tar files seem to produce this # (specifically bad symlinks) logger.warning( 'In the tar file %s the member %s is invalid: %s', filename, member.name, exc, ) continue ensure_dir(os.path.dirname(path)) with open(path, 'wb') as destfp: shutil.copyfileobj(fp, destfp) fp.close() # Update the timestamp (useful for cython compiled files) # https://github.com/python/typeshed/issues/2673 tar.utime(member, path) # type: ignore # member have any execute permissions for user/group/world? if member.mode & 0o111: # make dest file have execute for user/group/world # no-op on windows per python docs os.chmod(path, (0o777 - current_umask() | 0o111)) finally: tar.close()
return True except BaseException: return False # Download binaries for gen_direct_links module, give correct perms if not os.path.exists('bin'): os.mkdir('bin') url1 = 'https://raw.githubusercontent.com/yshalsager/megadown/master/megadown' url2 = 'https://raw.githubusercontent.com/yshalsager/cmrudl.py/master/cmrudl.py' dl1 = Downloader(url=url1, filename="bin/megadown") dl1 = Downloader(url=url1, filename="bin/cmrudl") os.chmod('bin/megadown', 0o755) os.chmod('bin/cmrudl', 0o755) # Global Variables COUNT_MSG = 0 USERS = {} COUNT_PM = {} LASTMSG = {} ENABLE_KILLME = True CMD_HELP = {} AFKREASON = "no reason" ZALG_LIST = [[ "̖", " ̗", " ̘", " ̙",
def install_file(filename, destination, file_mode=0o644, dir_mode=0o755): install_dir = os.path.dirname(destination) if not os.path.isdir(install_dir): os.makedirs(install_dir, dir_mode) copy(filename, destination) os.chmod(destination, file_mode)
def dorw(action, name, exc): os.chmod(name, stat.S_IWRITE) if (os.path.isdir(name)): os.rmdir(name) else: os.remove(name)
def _install_hook(self, name): hook = self._hook_path(name) with open(hook, "w+") as fobj: fobj.write(f"#!/bin/sh\nexec dvc git-hook {name} $@\n") os.chmod(hook, 0o777)
def test_log_config_append_unreadable(self): os.chmod(self.log_config_append, 0) self.config(log_config_append=self.log_config_append) self.assertRaises(log.LogConfigError, log.setup, self.CONF, 'test_log_config_append')
def _fix_filemode(path): mode = stat.S_IMODE(os.stat(path, follow_symlinks=False).st_mode) if mode & 0o4000 or mode & 0o2000: logger.warning('Removing suid/guid from {}'.format(path)) os.chmod(path, mode & 0o1777)
def makesg(profile, sgid, vpcid, source, destin, shell): script = open('%s.sh' % sgid, 'w') cmd = [ 'aws', 'ec2', 'describe-security-groups', '--region=%s' % source, '--output=json', ] if source: cmd.append("--group-id=%s" % (sgid)) else: cmd = [ 'aws', 'ec2', 'describe-security-groups', '--group-id=%s' % sgid, '--output=json', ] if profile: cmd.append('--profile') cmd.append(profile) ap = subprocess.check_output(cmd) data = json.loads(ap.decode('utf-8')) if 'SecurityGroups' not in data: print("Internal error: no SecurityGroups key in data") sys.exit(3) sg1 = data['SecurityGroups'][0] groupName = sg1['GroupName'] #+ '_migrated' groupDesc = sg1['Description'] groupTags = sg1['Tags'] # Sanity check perms = sg1['IpPermissions'] + sg1['IpPermissionsEgress'] for ipp in perms: if 'FromPort' not in ipp: continue if 'IpProtocol' not in ipp: continue if 'IpRanges' not in ipp: continue if 'ToPort' not in ipp: continue if len(ipp['UserIdGroupPairs']) > 0: sys.stderr.write("Warning: ignoring User Id info\n") for ipr in ipp['IpRanges']: for k in ipr.keys(): if k != 'CidrIp' and k != 'Description': sys.stderr.write("Error: Don't know how to handle") sys.stderr.write("key %s in IpRanges\n" % (k)) sys.exit(4) destinations = [] if shell: print("# Commands auto-generated by the copysg.py script", file=script) print(" ", file=script) destinations = regions_vpcs cmd.append("--filters=Name=group-name,Values='%s'" % (groupName)) if destin is not None: filtered = { dest: vpc for dest, vpc in destinations.items() if dest not in [source] and dest == destin } else: filtered = { dest: vpc for dest, vpc in destinations.items() if dest not in [source] } for dest, vpc in filtered.items(): ap = subprocess.check_output(cmd) if ap is not None: delete_cmd = "aws ec2 delete-security-group" create_cmd = "aws ec2 create-security-group --vpc-id=%s" % (vpc) cmd_existing = [ 'aws', 'ec2', 'describe-security-groups', '--region=%s' % dest, "--query=SecurityGroups[].GroupName" ] ap = subprocess.check_output(cmd_existing) sgNames = json.loads(ap) if shell: if ap is not None: if groupName != 'default' and groupName in sgNames: print( "sgout=(`%s --group-name='%s' --region %s --output table`)" % (delete_cmd, groupName, dest), file=script) print('if [ $? != 0 ]; then', file=script) print(' echo "Error: %s failed"' % (delete_cmd), file=script) print(' exit 1', file=script) print('fi', file=script) print( "sgout=(`%s --group-name='%s' --region %s --description='%s' --output table`)" % (create_cmd, groupName, dest, groupDesc), file=script) print('if [ $? != 0 ]; then', file=script) print(' echo "Error: %s failed"' % (create_cmd), file=script) print(' exit 1', file=script) print('fi', file=script) print('if [ "${sgout[6]}" != \'GroupId\' ]; then', file=script) print(' echo "Error: expected \'GroupId\', got ${sgout[6]}"', file=script) print(' exit 1', file=script) print('fi', file=script) print('SGID=${sgout[8]}', file=script) else: if groupName != 'default' and groupName in sgNames: print("%s --group-name='%s' --region %s" % (delete_cmd, groupName, dest), file=script) print( "sgcreate=$(%s --group-name='%s' --region %s --description='%s')" % (create_cmd, groupName, dest, groupDesc), file=script) print( "sgid=$(echo $sgcreate|sed 's/{//g;s/}//g;s/\"//g'|cut -d ':' -f2)", file=script) for ipp in sg1['IpPermissions']: if 'FromPort' not in ipp: continue if 'IpProtocol' not in ipp: continue if 'IpRanges' not in ipp: continue if 'ToPort' not in ipp: continue for ipr in ipp['IpRanges']: cidr = ipr['CidrIp'] auth_cmd = "aws ec2 authorize-security-group-ingress" if shell: print("%s --region %s --group-id=$SGID --protocol='%s'" % (auth_cmd, dest, ipp['IpProtocol']), end=" ", file=script) else: print("%s --region %s --group-name=%s --protocol='%s'" % (auth_cmd, dest, groupName, ipp['IpProtocol']), end=" ", file=script) if ipp['ToPort'] < 0: ipp['ToPort'] = ipp['FromPort'] if ipp['FromPort'] != ipp['ToPort']: print("--port=%s-%s" % (ipp['FromPort'], ipp['ToPort']), end=" ", file=script) else: print("--port=%s" % (ipp['FromPort']), end=" ", file=script) print("--cidr=%s" % (ipr['CidrIp']), file=script) if shell: print('if [ $? != 0 ]; then', file=script) print(' echo "Error: %s failed"' % (auth_cmd), file=script) print(' exit 1', file=script) print('fi', file=script) for ipp in sg1['IpPermissionsEgress']: if 'FromPort' not in ipp: continue if 'IpProtocol' not in ipp: continue if 'IpRanges' not in ipp: continue if 'ToPort' not in ipp: continue for ipr in ipp['IpRanges']: cidr = ipr['CidrIp'] auth_cmd = "aws ec2 authorize-security-group-egress" if shell: print("%s --region %s --group-id=$SGID --protocol='%s'" % (auth_cmd, dest, ipp['IpProtocol']), end=" ", file=script) else: print("%s --region %s --group-id=$SGID --protocol='%s'" % (auth_cmd, dest, ipp['IpProtocol']), end=" ", file=script) if ipp['ToPort'] < 0: # ICMP ToPort was -1 ??? ipp['ToPort'] = ipp['FromPort'] if ipp['FromPort'] != ipp['ToPort']: print("--port=%s-%s" % (ipp['FromPort'], ipp['ToPort']), end=" ", file=script) else: print("--port=%s" % (ipp['FromPort']), end=" ", file=script) print("--cidr=%s" % (ipr['CidrIp']), file=script) if shell: print('if [ $? != 0 ]; then') print(' echo "Error: %s failed"' % (auth_cmd)) print(' exit 1', file=script) print('fi', file=script) for tag in groupTags: if shell: print("aws ec2 create-tags --region %s --resources $SGID" % (dest), end=" ", file=script) print('--tags "Key=%s,Value=%s"' % (tag['Key'], tag['Value']), file=script) else: print("aws ec2 create-tags --region %s --resources $sgid" % (dest), end=" ", file=script) print('--tags "Key=%s,Value=%s"' % (tag['Key'], tag['Value']), file=script) #setting script permissions os.chmod(script.name, 0o755)
def parse_config(): """Parse the configuration file. Parses or creates a configuration file and returns a dictionary with all the configuration variables. """ # Folder that will contain all configfiles config_folder = os.path.expanduser(os.path.join('~', '.pyxtra')) config_file = os.path.join(config_folder, 'config') log_file = os.path.join(config_folder, 'sent.log') config = ConfigParser.ConfigParser() # ConfigParser instance # Create folder if necessary if not os.path.isdir(config_folder): os.mkdir(config_folder) # Read config, write default config file if it doesn't exist yet. if not len(config.read(config_file)): print 'Could not find configuration file. Creating %s.' % config_file # Login data username = raw_input('\nUsername: '******'Enter your password, in case you want to store it in the ' \ 'config file. Warning: Password will be saved in plaintext.' password = getpass.getpass('Password (ENTER to skip): ').strip() # Logging logging_msg = 'Do you want to log all sent SMS to %s?' % log_file logging = 'yes' if yn_choice(logging_msg) else 'no' # Auto send long sms auto_send_long_sms_msg = 'Do you want to ignore warnings for SMS ' \ 'longer than %u characters?' % __xtra_sms_max_length auto_send_long_sms = 'yes' if yn_choice(auto_send_long_sms_msg, 'n') else 'no' # Anticaptcha anticaptcha_msg = 'Do you want to use the anticaptcha service?' anticaptcha = 'yes' if yn_choice(anticaptcha_msg) else 'no' print 'Initial configuration is finished. You may edit your ' + \ 'configuration file at %s.\n' % config_file config.add_section('settings') config.set('settings', 'username', username) config.set('settings', 'password', password) config.set('settings', 'logging', logging) config.set('settings', 'auto_send_long_sms', auto_send_long_sms) config.add_section('captcha') config.set('captcha', 'anticaptcha', anticaptcha) config.set('captcha', 'anticaptcha_max_tries', 3) config.write(open(config_file, 'w')) os.chmod(config_file, stat.S_IREAD | stat.S_IWRITE) else: # Add sections if necessary for s in ['settings', 'captcha']: try: config.add_section(s) except ConfigParser.DuplicateSectionError: pass # Get user data username = config.get('settings', 'username') password = config.get('settings', 'password') # Get logging settings try: logging = config.getboolean('settings', 'logging') except (ValueError, ConfigParser.NoOptionError): logging_msg = 'Do you want to log all sent SMS to %s?' % log_file logging = 'yes' if yn_choice(logging_msg) else 'no' config.set('settings', 'logging', logging) # Get long sms settings try: auto_send_long_sms = config.getboolean('settings', 'auto_send_long_sms') except (ValueError, ConfigParser.NoOptionError): auto_send_long_sms_msg = 'Do you want to ignore warnings for SMS ' \ 'longer than %u characters?' % __xtra_sms_max_length auto_send_long_sms = 'yes' if yn_choice(auto_send_long_sms_msg, 'n') else 'no' config.set('settings', 'auto_send_long_sms', auto_send_long_sms) # Get anticaptcha settings try: anticaptcha = config.getboolean('captcha', 'anticaptcha') except (ValueError, ConfigParser.NoOptionError): anticaptcha_msg = 'Do you want to use the anticaptcha service?' anticaptcha = 'yes' if yn_choice(anticaptcha_msg) else 'no' config.set('captcha', 'anticaptcha', anticaptcha) try: anticaptcha_max_tries = config.getint('captcha', 'anticaptcha_max_tries') except (ValueError, ConfigParser.NoOptionError): config.set('captcha', 'anticaptcha_max_tries', 3) # Write possibly changed configuration file config.write(open(config_file, 'w')) # Set default values if not 'anticaptcha_max_tries' in locals(): anticaptcha_max_tries = 3 if not anticaptcha in ['yes', True]: anticaptcha = False if not auto_send_long_sms in ['yes', True]: auto_send_long_sms = False return {'username': username, 'password': password, 'logging': logging, 'auto_send_long_sms': auto_send_long_sms, 'anticaptcha': anticaptcha, 'anticaptcha_max_tries': anticaptcha_max_tries}
def run(profile, task, inst, dest_dir, ini_fn, section, no_destroy, creds_file='~/.aws/credentials'): ec2_json, app_json, profile_json = 'ec2.json', 'app.json', 'profile.json' if not os.path.exists(task): raise RuntimeError('No such task directory: "%s"' % task) if not os.path.exists(os.path.join(task, inst)): raise RuntimeError('No such instance directory: "%s/%s"' % (task, inst)) if os.path.exists(dest_dir): raise RuntimeError('Destination directory "%s" exists' % dest_dir) os.makedirs(dest_dir) task_json = os.path.join(task, 'task.json') inst_json = os.path.join(task, inst, 'inst.json') app, region, subnet, az, security_group, ami, keypair, bid_price, inst, arch, prof, aws_prof, params = \ load_aws_json(profile_json, app_json, task_json, inst_json, ec2_json, profile=profile) vagrant_args = '' if no_destroy: vagrant_args += ' --no-destroy-on-error' for template_dir in ['scripts', 'vagrant_include']: for base_fn in os.listdir(template_dir): fn = os.path.join(template_dir, base_fn) if os.path.isfile(fn): shutil.copyfile(fn, os.path.join(dest_dir, base_fn)) for base_fn in os.listdir(task): fn = os.path.join(task, base_fn) if os.path.isfile(fn): shutil.copyfile(fn, os.path.join(dest_dir, base_fn)) def _write_exports(export_fh): # TODO: move the credentials file parsing to here; better to do it in the script export_fh.write('#!/bin/bash\n') for k, v in params: export_fh.write('export %s=%s\n' % (k, v)) run_scr = os.path.join(dest_dir, 'run.sh') run_nd_scr = os.path.join(dest_dir, 'run_no_destroy.sh') destroy_scr = os.path.join(dest_dir, 'destroy.sh') ssh_scr = os.path.join(dest_dir, 'ssh.sh') slack_scr = os.path.join(dest_dir, 'slack.py') creds_scr = os.path.join(dest_dir, 'creds.py') with open(run_scr, 'wt') as fh: _write_exports(fh) fh.write('./creds.py && \\\n') fh.write('vagrant up %s 2>&1 | tee vagrant.log && \\\n' % vagrant_args) fh.write('vagrant destroy -f\n') fh.write('rm -f .creds.tmp\n') with open(run_nd_scr, 'wt') as fh: _write_exports(fh) fh.write('./creds.py && \\\n') fh.write('vagrant up %s 2>&1 | tee vagrant.log\n' % vagrant_args) fh.write('rm -f .creds.tmp\n') with open(destroy_scr, 'wt') as fh: _write_exports(fh) fh.write('touch .creds.tmp && vagrant destroy -f\n') fh.write('rm -f .creds.tmp\n') with open(ssh_scr, 'wt') as fh: _write_exports(fh) fh.write('touch .creds.tmp && vagrant ssh\n') fh.write('rm -f .creds.tmp\n') slack_scr_in = 'slack_template.py' if not os.path.exists(slack_scr_in): raise RuntimeError('Cannot find slack template script') with open(slack_scr, 'wt') as fh: with open(slack_scr_in, 'rt') as ifh: for ln in ifh: fh.write(ln) fh.write('\n\nif __name__ == "__main__":\n') fh.write(' do_slack_report("%s", "vagrant.log")\n' % slack_webhook_url(ini_fn, section=section)) creds_scr_in = 'make_creds_template.py' if not os.path.exists(creds_scr_in): raise RuntimeError('Cannot find make_creds template script') with open(creds_scr, 'wt') as fh: with open(creds_scr_in, 'rt') as ifh: for ln in ifh: fh.write(ln) fh.write('\n\nif __name__ == "__main__":\n') fh.write(' make_creds_file("%s", "%s")\n' % (aws_prof, creds_file)) for scr in [ run_scr, run_nd_scr, destroy_scr, ssh_scr, slack_scr, creds_scr ]: os.chmod(scr, 0o700)
def setup_vagrant_pattern(shutit, skel_path, skel_delivery, skel_domain, skel_module_name, skel_shutitfiles, skel_domain_hash, skel_depends, skel_vagrant_num_machines, skel_vagrant_machine_prefix, skel_vagrant_ssh_access, skel_vagrant_docker): # TODO: ability to pass in option values, or take defaults # Gather requirements for multinode vagrant setup: options = [] if skel_vagrant_num_machines is None: options.append({ 'name': 'num_machines', 'question': 'How many machines do you want?', 'value': '3', 'ok_values': [] }) else: num_machines = skel_vagrant_num_machines if skel_vagrant_machine_prefix is None: options.append({ 'name': 'machine_prefix', 'question': 'What do you want to call the machines (eg superserver)?', 'value': 'machine', 'ok_values': [] }) else: machine_prefix = skel_vagrant_machine_prefix if skel_vagrant_ssh_access is None: options.append({ 'name': 'ssh_access', 'question': 'Do you want to have open ssh access between machines (yes or no)?', 'value': 'yes', 'ok_values': ['yes', 'no'] }) else: ssh_access = skel_vagrant_ssh_access if skel_vagrant_docker is None: options.append({ 'name': 'docker', 'question': 'Do you want Docker on the machine (yes or no)?', 'value': 'no', 'ok_values': ['yes', 'no'] }) else: docker = skel_vagrant_docker if len(options) > 0: while True: count = 1 print('') for opt in options: print( str(count) + ': ' + opt['question'] + ' (current: ' + opt['value'] + ')') count += 1 print('') choice = shutit_util.get_input( msg= 'Choose an item to change if you want to change the default. Hit return to continue.\nIf you want to change a config, choose the number: ' ) if choice == '': break else: try: choice = int(choice) except ValueError: print('Bad value, ignoring') continue # off by one choice -= 1 item = options[choice] value = shutit_util.get_input(msg='Input the value: ') if len(item['ok_values']) > 0 and value not in item['ok_values']: print('Bad value, ignoring') continue item['value'] = value for opt in options: if opt['name'] == 'num_machines': num_machines = int(opt['value']) if opt['name'] == 'machine_prefix': machine_prefix = opt['value'] if opt['name'] == 'ssh_access': if opt['value'] == 'no': ssh_access = False elif opt['value'] == 'yes': ssh_access = True else: shutit.fail('Bad value for ssh_access') if opt['name'] == 'docker': if opt['value'] == 'no': docker = False elif opt['value'] == 'yes': docker = True else: shutit.fail('Bad value for docker') num_machines = int(num_machines) # Set up Vagrantfile data for the later machine_stanzas = '' machine_list_code = '''\n\t\t# machines is a dict of dicts containing information about each machine for you to use.\n\t\tmachines = {}''' vagrant_up_section = '' for m in range(1, num_machines + 1): machine_name = machine_prefix + str(m) machine_fqdn = machine_name + '.vagrant.test' # vagrant_image is calculated within the code later machine_stanzas += ( '''\n config.vm.define "''' + machine_name + '''" do |''' + machine_name + '''| ''' + machine_name + """.vm.box = ''' + '"' + vagrant_image + '"' + ''' """ + machine_name + '''.vm.hostname = "''' + machine_fqdn + '''"''' + '''\n config.vm.provider :virtualbox do |vb|\n vb.name = "''' + skel_module_name + '_' + str(m) + '''"\n end end''') # machine_list_code machine_list_code += """\n\t\tmachines.update({'""" + machine_name + """':{'fqdn':'""" + machine_fqdn + """'}})""" machine_list_code += """\n\t\tip = shutit.send_and_get_output('''vagrant landrush ls 2> /dev/null | grep -w ^''' + machines['""" + machine_name + """']['fqdn'] + ''' | awk '{print $2}' ''')""" machine_list_code += """\n\t\tmachines.get('""" + machine_name + """').update({'ip':ip})""" vagrant_up_section += '''\t\ttry: shutit.multisend('vagrant up --provider ' + shutit.cfg['shutit-library.virtualization.virtualization.virtualization']['virt_method'] + " ''' + machine_name + '''",{'assword for':pw,'assword:':pw},timeout=99999) except NameError: shutit.multisend('vagrant up ''' + machine_name + """'""" + ''',{'assword for':pw,'assword:':pw},timeout=99999) if shutit.send_and_get_output("""vagrant status | grep -w ^''' + machine_name + ''' | awk '{print $2}'""") != 'running': shutit.pause_point("machine: ''' + machine_name + ''' appears not to have come up cleanly") ''' vagrant_dir_section_1 = """ if shutit.build['vagrant_run_dir'] is None: shutit.build['vagrant_run_dir'] = os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda:0))) + '/vagrant_run' shutit.build['module_name'] = '""" + skel_module_name + """_' + ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(6)) shutit.build['this_vagrant_run_dir'] = shutit.build['vagrant_run_dir'] + '/' + shutit.build['module_name'] shutit.send(' command rm -rf ' + shutit.build['this_vagrant_run_dir'] + ' && command mkdir -p ' + shutit.build['this_vagrant_run_dir'] + ' && command cd ' + shutit.build['this_vagrant_run_dir'])""" vagrant_dir_section_n = """ shutit.send(' command mkdir -p ' + shutit.build['this_vagrant_run_dir'] + ' && command cd ' + shutit.build['this_vagrant_run_dir'])""" if ssh_access: copy_keys_code = ''' for machine in sorted(machines.keys()): shutit.login(command='vagrant ssh ' + machine) shutit.login(command='sudo su -',password='******') root_password = '******' shutit.install('net-tools') # netstat needed if not shutit.command_available('host'): shutit.install('bind-utils') # host needed shutit.multisend('passwd',{'assword:':root_password}) shutit.send("""sed -i 's/.*PermitRootLogin.*/PermitRootLogin yes/g' /etc/ssh/sshd_config""") shutit.send("""sed -i 's/.*PasswordAuthentication.*/PasswordAuthentication yes/g' /etc/ssh/sshd_config""") shutit.send('service ssh restart || systemctl restart sshd') shutit.multisend('ssh-keygen',{'Enter':'','verwrite':'n'}) shutit.logout() shutit.logout() for machine in sorted(machines.keys()): shutit.login(command='vagrant ssh ' + machine) shutit.login(command='sudo su -',password='******') for copy_to_machine in machines: for item in ('fqdn','ip'): shutit.multisend('ssh-copy-id root@' + machines[copy_to_machine][item],{'assword:':root_password,'ontinue conn':'yes'}) shutit.logout() shutit.logout()''' else: copy_keys_code = '' if docker: docker_code = ''' for machine in sorted(machines.keys()): shutit.login(command='vagrant ssh ' + machine) shutit.login(command='sudo su -',password='******') # Workaround for docker networking issues + landrush. shutit.install('docker') shutit.insert_text('Environment=GODEBUG=netdns=cgo','/lib/systemd/system/docker.service',pattern='.Service.') shutit.send('mkdir -p /etc/docker',note='Create the docker config folder') shutit.send_file('/etc/docker/daemon.json',"""{ "dns": ["8.8.8.8"] }""",note='Use the google dns server rather than the vagrant one. Change to the value you want if this does not work, eg if google dns is blocked.') shutit.send('systemctl daemon-reload && systemctl restart docker')''' else: docker_code = '' get_config_section = ''' def get_config(self, shutit): shutit.get_config(self.module_id,'vagrant_image',default='ubuntu/xenial64') shutit.get_config(self.module_id,'vagrant_provider',default='virtualbox') shutit.get_config(self.module_id,'gui',default='false') shutit.get_config(self.module_id,'memory',default='1024') return True''' shared_imports = '''import random import logging import string import os import inspect''' # Set up files: # .gitignore gitignore_filename = skel_path + '/.gitignore' gitignore_file = open(gitignore_filename, 'w+') gitignore_file.write('''*pyc vagrant_run''') gitignore_file.close() # run.sh runsh_filename = skel_path + '/run.sh' runsh_file = open(runsh_filename, 'w+') runsh_file.write('''#!/bin/bash [[ -z "$SHUTIT" ]] && SHUTIT="$1/shutit" [[ ! -a "$SHUTIT" ]] || [[ -z "$SHUTIT" ]] && SHUTIT="$(which shutit)" if [[ ! -a "$SHUTIT" ]] then echo "Must have shutit on path, eg export PATH=$PATH:/path/to/shutit_dir" exit 1 fi ./destroy_vms.sh $SHUTIT build --echo -d bash -m shutit-library/vagrant -m shutit-library/virtualization "$@" if [[ $? != 0 ]] then exit 1 fi''') runsh_file.close() os.chmod(runsh_filename, 0o755) # destroy_vms.sh destroyvmssh_filename = skel_path + '/destroy_vms.sh' destroyvmssh_file = open(destroyvmssh_filename, 'w+') destroyvmssh_file_contents = '''#!/bin/bash MODULE_NAME=''' + skel_module_name + ''' rm -rf $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/vagrant_run/* if [[ $(command -v VBoxManage) != '' ]] then while true do VBoxManage list runningvms | grep ${MODULE_NAME} | awk '{print $1}' | xargs -IXXX VBoxManage controlvm 'XXX' poweroff && VBoxManage list vms | grep ''' + skel_module_name + ''' | awk '{print $1}' | xargs -IXXX VBoxManage unregistervm 'XXX' --delete # The xargs removes whitespace if [[ $(VBoxManage list vms | grep ${MODULE_NAME} | wc -l | xargs) -eq '0' ]] then break else ps -ef | grep virtualbox | grep ${MODULE_NAME} | awk '{print $2}' | xargs kill sleep 10 fi done fi if [[ $(command -v virsh) ]] && [[ $(kvm-ok 2>&1 | command grep 'can be used') != '' ]] then virsh list | grep ${MODULE_NAME} | awk '{print $1}' | xargs -n1 virsh destroy fi ''' destroyvmssh_file.write(destroyvmssh_file_contents) destroyvmssh_file.close() os.chmod(destroyvmssh_filename, 0o755) # build.cnf os.system('mkdir -p ' + skel_path + '/configs') # git setup os.system('git init') os.system('git submodule init') os.system('git submodule add https://github.com/ianmiell/shutit-library') # User message shutit.log('''# Run:\n\ncd ''' + skel_path + ''' && ./run.sh\n\n# to run.''', transient=True) # CREATE THE MODULE FILE # Handle shutitfiles. If there are no shutitfiles, handle separately. # If there are more than one, you need to treat the first one differently. if skel_shutitfiles: _total = len(skel_shutitfiles) _count = 0 for skel_shutitfile in skel_shutitfiles: _count += 1 module_modifier = '_' + str(_count) new_module_filename = skel_path + '/' + os.path.join( skel_module_name + module_modifier + '.py') shutit.cfg['skeleton']['module_modifier'] = module_modifier (sections, skel_module_id, skel_module_name, _, _) = shutitfile.shutitfile_to_shutit_module( shutit, skel_shutitfile, skel_path, skel_domain, skel_module_name, skel_domain_hash, skel_delivery, skel_depends, _count, _total, module_modifier) shutit.cfg['skeleton']['header_section'] = sections[ 'header_section'] shutit.cfg['skeleton']['config_section'] = sections[ 'config_section'] shutit.cfg['skeleton']['build_section'] = sections['build_section'] shutit.cfg['skeleton']['finalize_section'] = sections[ 'finalize_section'] shutit.cfg['skeleton']['test_section'] = sections['test_section'] shutit.cfg['skeleton']['isinstalled_section'] = sections[ 'isinstalled_section'] shutit.cfg['skeleton']['start_section'] = sections['start_section'] shutit.cfg['skeleton']['stop_section'] = sections['stop_section'] shutit.cfg['skeleton']['final_section'] = sections['final_section'] module_file = open(new_module_filename, 'w+') # We only write out the heavy stuff for vagrant on the first time round if _count == 1: module_file.write( shared_imports + """ """ + shutit.cfg['skeleton']['header_section'] + """ def build(self, shutit): shutit.run_script('''""" + destroyvmssh_file_contents + """''') vagrant_image = shutit.cfg[self.module_id]['vagrant_image'] vagrant_provider = shutit.cfg[self.module_id]['vagrant_provider'] gui = shutit.cfg[self.module_id]['gui'] memory = shutit.cfg[self.module_id]['memory'] """ + vagrant_dir_section_1 + """ if shutit.send_and_get_output('vagrant plugin list | grep landrush') == '': shutit.send('vagrant plugin install landrush') shutit.send('vagrant init ' + vagrant_image) shutit.send_file(shutit.build['this_vagrant_run_dir'] + '/Vagrantfile','''Vagrant.configure("2") do |config| config.landrush.enabled = true config.vm.provider "virtualbox" do |vb| vb.gui = ''' + gui + ''' vb.memory = "''' + memory + '''" end """ + machine_stanzas + """ end''') pw = shutit.get_env_pass() """ + vagrant_up_section + """ """ + machine_list_code + """ """ + copy_keys_code + """ """ + docker_code + """ """ + shutit.cfg['skeleton']['build_section'] + """ shutit.log('''# Vagrantfile created in: ''' + shutit.build['vagrant_run_dir'] + '''\n# Run: cd ''' + shutit.build['vagrant_run_dir'] + ''' && vagrant status && vagrant landrush ls # to get information about your machines' setup.''',add_final_message=True,level=logging.DEBUG) return True """ + get_config_section + """ """ + shutit.cfg['skeleton']['config_section'] + """ return True def test(self, shutit): """ + shutit.cfg['skeleton']['test_section'] + """ return True def finalize(self, shutit): """ + shutit.cfg['skeleton']['finalize_section'] + """ return True def is_installed(self, shutit): """ + shutit.cfg['skeleton']['isinstalled_section'] + """ return False def start(self, shutit): """ + shutit.cfg['skeleton']['start_section'] + """ return True def stop(self, shutit): """ + shutit.cfg['skeleton']['stop_section'] + """ return True def module(): return """ + skel_module_name + module_modifier + """( '""" + skel_module_id + """', """ + skel_domain_hash + """.000""" + str(_count) + """, description='', maintainer='', delivery_methods=['bash'], depends=['""" + skel_depends + """','shutit-library.virtualization.virtualization.virtualization','tk.shutit.vagrant.vagrant.vagrant'] )""") # In the non-first one, we don't have all the setup stuff (but we do have some!) else: module_file.write( shared_imports + """ """ + shutit.cfg['skeleton']['header_section'] + """ def build(self, shutit): shutit.run_script('''""" + destroyvmssh_file_contents + """''') """ + vagrant_dir_section_n + """ """ + machine_list_code + """ """ + shutit.cfg['skeleton']['build_section'] + """ """ + shutit.cfg['skeleton']['config_section'] + """ return True def test(self, shutit): """ + shutit.cfg['skeleton']['test_section'] + """ return True def finalize(self, shutit): """ + shutit.cfg['skeleton']['finalize_section'] + """ return True def is_installed(self, shutit): """ + shutit.cfg['skeleton']['isinstalled_section'] + """ return False def start(self, shutit): """ + shutit.cfg['skeleton']['start_section'] + """ return True def stop(self, shutit): """ + shutit.cfg['skeleton']['stop_section'] + """ return True def module(): return """ + skel_module_name + module_modifier + """( '""" + skel_module_id + """',""" + skel_domain_hash + """.000""" + str(_count) + """, description='', maintainer='', delivery_methods=['bash'], depends=['""" + skel_depends + """','shutit-library.virtualization.virtualization.virtualization','tk.shutit.vagrant.vagrant.vagrant'] )""") module_file.close() # Set up build.cnf build_cnf_filename = skel_path + '/configs/build.cnf' if _count == 1: build_cnf_file = open(build_cnf_filename, 'w+') build_cnf_file.write( '''############################################################################### # PLEASE NOTE: This file should be changed only by the maintainer. # PLEASE NOTE: This file is only sourced if the "shutit build" command is run # and this file is in the relative path: configs/build.cnf # This is to ensure it is only sourced if _this_ module is the # target. ############################################################################### # When this module is the one being built, which modules should be built along with it by default? # This feeds into automated testing of each module. [''' + skel_module_id + '''] shutit.core.module.build:yes''') build_cnf_file.close() else: build_cnf_file = open(build_cnf_filename, 'a') build_cnf_file.write(''' [''' + skel_domain + '''.''' + skel_module_name + module_modifier + '''] shutit.core.module.build:yes''') build_cnf_file.close() os.chmod(build_cnf_filename, 0o400) else: # No shutitfiles to consider, so simpler logic here. shutit.cfg['skeleton'][ 'header_section'] = 'from shutit_module import ShutItModule\n\nclass ' + skel_module_name + '(ShutItModule):\n' new_module_filename = skel_path + '/' + skel_module_name + '.py' module_file = open(new_module_filename, 'w+') module_file.write( shared_imports + """ """ + shutit.cfg['skeleton']['header_section'] + """ def build(self, shutit): shutit.run_script('''""" + destroyvmssh_file_contents + """''') vagrant_image = shutit.cfg[self.module_id]['vagrant_image'] vagrant_provider = shutit.cfg[self.module_id]['vagrant_provider'] gui = shutit.cfg[self.module_id]['gui'] memory = shutit.cfg[self.module_id]['memory'] shutit.build['vagrant_run_dir'] = os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda:0))) + '/vagrant_run' shutit.build['module_name'] = '""" + skel_module_name + """_' + ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(6)) shutit.build['this_vagrant_run_dir'] = shutit.build['vagrant_run_dir'] + '/' + shutit.build['module_name'] shutit.send(' command rm -rf ' + shutit.build['this_vagrant_run_dir'] + ' && command mkdir -p ' + shutit.build['this_vagrant_run_dir'] + ' && command cd ' + shutit.build['this_vagrant_run_dir']) shutit.send('command rm -rf ' + shutit.build['this_vagrant_run_dir'] + ' && command mkdir -p ' + shutit.build['this_vagrant_run_dir'] + ' && command cd ' + shutit.build['this_vagrant_run_dir']) if shutit.send_and_get_output('vagrant plugin list | grep landrush') == '': shutit.send('vagrant plugin install landrush') shutit.send('vagrant init ' + vagrant_image) shutit.send_file(shutit.build['this_vagrant_run_dir'] + '/Vagrantfile','''Vagrant.configure("2") do |config| config.landrush.enabled = true config.vm.provider "virtualbox" do |vb| vb.gui = ''' + gui + ''' vb.memory = "''' + memory + '''" end """ + machine_stanzas + """ end''') pw = shutit.get_env_pass() """ + vagrant_up_section + """ """ + machine_list_code + """ """ + copy_keys_code + """ """ + docker_code + """ shutit.login(command='vagrant ssh ' + sorted(machines.keys())[0]) shutit.login(command='sudo su -',password='******') shutit.logout() shutit.logout() shutit.log('''Vagrantfile created in: ''' + shutit.build['this_vagrant_run_dir'],add_final_message=True,level=logging.DEBUG) shutit.log('''Run:\n\n\tcd ''' + shutit.build['this_vagrant_run_dir'] + ''' && vagrant status && vagrant landrush ls\n\nTo get a picture of what has been set up.''',add_final_message=True,level=logging.DEBUG) return True """ + get_config_section + """ def test(self, shutit): return True def finalize(self, shutit): return True def is_installed(self, shutit): return False def start(self, shutit): return True def stop(self, shutit): return True def module(): return """ + skel_module_name + """( '""" + skel_domain + '''.''' + skel_module_name + """', """ + skel_domain_hash + """.0001, description='', maintainer='', delivery_methods=['bash'], depends=['""" + skel_depends + """','shutit-library.virtualization.virtualization.virtualization','tk.shutit.vagrant.vagrant.vagrant'] )""") module_file.close() build_cnf_filename = skel_path + '/configs/build.cnf' build_cnf_file = open(build_cnf_filename, 'w+') build_cnf_file.write( '''############################################################################### # PLEASE NOTE: This file should be changed only by the maintainer. # PLEASE NOTE: This file is only sourced if the "shutit build" command is run # and this file is in the relative path: configs/build.cnf # This is to ensure it is only sourced if _this_ module is the # target. ############################################################################### # When this module is the one being built, which modules should be built along with it by default? # This feeds into automated testing of each module. [''' + skel_domain + '''.''' + skel_module_name + '''] shutit.core.module.build:yes''') build_cnf_file.close() os.chmod(build_cnf_filename, 0o400)
def change_mode(files: list): for file in files: mode = 0o664 if not file.endswith("/pycket") else 0o755 print("changing mode of %s to %s" % (file, oct(mode)[2:])) chmod(file, mode)
def interp_SMB_ICESat2(base_dir, FILE, model_version, CROSSOVERS=False, GZIP=False, VERBOSE=False, MODE=0o775): # read data from input file print('{0} -->'.format(os.path.basename(FILE))) if VERBOSE else None # Open the HDF5 file for reading fileID = h5py.File(FILE, 'r') # output data directory ddir = os.path.dirname(FILE) # extract parameters from ICESat-2 ATLAS HDF5 file name rx = re.compile(r'(processed_)?(ATL\d{2})_(\d{4})(\d{2})_(\d{2})(\d{2})_' r'(\d{3})_(\d{2})(.*?).h5$') SUB,PRD,TRK,GRAN,SCYC,ECYC,RL,VERS,AUX = rx.findall(FILE).pop() # get projection and region name based on granule REGION,proj4_params = set_projection(GRAN) # determine main model group from region and model_version MODEL, = [key for key,val in models[REGION].items() if model_version in val] # keyword arguments for all models KWARGS = dict(SIGMA=1.5, FILL_VALUE=np.nan) # set model specific parameters if (MODEL == 'MAR'): match_object=re.match(r'(MARv\d+\.\d+(.\d+)?)',model_version) MAR_VERSION=match_object.group(0) MAR_REGION=dict(GL='Greenland',AA='Antarctic')[REGION] # model subdirectories SUBDIRECTORY=dict(AA={}, GL={}) SUBDIRECTORY['GL']['MARv3.9-ERA']=['ERA_1958-2018_10km','daily_10km'] SUBDIRECTORY['GL']['MARv3.10-ERA']=['ERA_1958-2019-15km','daily_15km'] SUBDIRECTORY['GL']['MARv3.11-NCEP']=['NCEP1_1948-2020_20km','daily_20km'] SUBDIRECTORY['GL']['MARv3.11-ERA']=['ERA_1958-2019-15km','daily_15km'] SUBDIRECTORY['GL']['MARv3.11.2-ERA-6km']=['6km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.2-ERA-7.5km']=['7.5km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.2-ERA-10km']=['10km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.2-ERA-15km']=['15km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.2-ERA-20km']=['20km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.2-NCEP-20km']=['20km_NCEP1'] SUBDIRECTORY['GL']['MARv3.11.5-ERA-6km']=['6km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.5-ERA-10km']=['10km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.5-ERA-15km']=['15km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.5-ERA-20km']=['20km_ERA5'] MAR_MODEL=SUBDIRECTORY[REGION][model_version] DIRECTORY=os.path.join(base_dir,'MAR',MAR_VERSION,MAR_REGION,*MAR_MODEL) # keyword arguments for variable coordinates MAR_KWARGS=dict(AA={}, GL={}) MAR_KWARGS['GL']['MARv3.9-ERA'] = dict(XNAME='X10_153',YNAME='Y21_288') MAR_KWARGS['GL']['MARv3.10-ERA'] = dict(XNAME='X10_105',YNAME='Y21_199') MAR_KWARGS['GL']['MARv3.11-NCEP'] = dict(XNAME='X12_84',YNAME='Y21_155') MAR_KWARGS['GL']['MARv3.11-ERA'] = dict(XNAME='X10_105',YNAME='Y21_199') MAR_KWARGS['GL']['MARv3.11.2-ERA-6km'] = dict(XNAME='X12_251',YNAME='Y20_465') MAR_KWARGS['GL']['MARv3.11.2-ERA-7.5km'] = dict(XNAME='X12_203',YNAME='Y20_377') MAR_KWARGS['GL']['MARv3.11.2-ERA-10km'] = dict(XNAME='X10_153',YNAME='Y21_288') MAR_KWARGS['GL']['MARv3.11.2-ERA-15km'] = dict(XNAME='X10_105',YNAME='Y21_199') MAR_KWARGS['GL']['MARv3.11.2-ERA-20km'] = dict(XNAME='X12_84',YNAME='Y21_155') MAR_KWARGS['GL']['MARv3.11.2-NCEP-20km'] = dict(XNAME='X12_84',YNAME='Y21_155') MAR_KWARGS['GL']['MARv3.11.5-ERA-6km'] = dict(XNAME='X12_251',YNAME='Y20_465') MAR_KWARGS['GL']['MARv3.11.5-ERA-10km'] = dict(XNAME='X10_153',YNAME='Y21_288') MAR_KWARGS['GL']['MARv3.11.5-ERA-15km'] = dict(XNAME='X10_105',YNAME='Y21_199') MAR_KWARGS['GL']['MARv3.11.5-ERA-20km'] = dict(XNAME='X12_84',YNAME='Y21_155') KWARGS.update(MAR_KWARGS[REGION][model_version]) # netCDF4 variable names for direct fields VARIABLES = ['SMB','ZN6','ZN4','ZN5'] # output variable keys for both direct and derived fields KEYS = ['SMB','zsurf','zfirn','zmelt','zsmb','zaccum'] # HDF5 longname and description attributes for each variable LONGNAME = {} LONGNAME['SMB'] = "Cumulative SMB" LONGNAME['zsurf'] = "Height" LONGNAME['zfirn'] = "Compaction" LONGNAME['zmelt'] = "Surface Melt" LONGNAME['zsmb'] = "Surface Mass Balance" LONGNAME['zaccum'] = "Surface Accumulation" DESCRIPTION = {} DESCRIPTION['SMB'] = "Cumulative Surface Mass Balance" DESCRIPTION['zsurf'] = "Snow Height Change" DESCRIPTION['zfirn'] = "Snow Height Change due to Compaction" DESCRIPTION['zmelt'] = "Snow Height Change due to Surface Melt" DESCRIPTION['zsmb'] = "Snow Height Change due to Surface Mass Balance" DESCRIPTION['zaccum'] = "Snow Height Change due to Surface Accumulation" elif (MODEL == 'RACMO'): RACMO_VERSION,RACMO_MODEL=model_version.split('-') # netCDF4 variable names VARIABLES = ['hgtsrf'] # output variable keys KEYS = ['zsurf'] # HDF5 longname attributes for each variable LONGNAME = {} LONGNAME['zsurf'] = "Height" DESCRIPTION = {} DESCRIPTION['zsurf'] = "Snow Height Change" elif (MODEL == 'MERRA2-hybrid'): # regular expression pattern for extracting version merra2_regex = re.compile(r'GSFC-fdm-((v\d+)(\.\d+)?)$') # get MERRA-2 version and major version MERRA2_VERSION = merra2_regex.match(model_version).group(1) # MERRA-2 hybrid directory DIRECTORY=os.path.join(base_dir,'MERRA2_hybrid',MERRA2_VERSION) # MERRA-2 region name from ATL11 region MERRA2_REGION = dict(AA='ais',GL='gris')[REGION] # keyword arguments for MERRA-2 interpolation programs if MERRA2_VERSION in ('v0','v1','v1.0'): KWARGS['VERSION'] = merra2_regex.match(model_version).group(2) # netCDF4 variable names VARIABLES = ['FAC','cum_smb_anomaly','height'] # add additional Greenland variables if (MERRA2_REGION == 'gris'): VARIABLES.append('runoff_anomaly') else: KWARGS['VERSION'] = MERRA2_VERSION.replace('.','_') # netCDF4 variable names VARIABLES = ['FAC','SMB_a','h_a'] # add additional Greenland variables if (MERRA2_REGION == 'gris'): VARIABLES.append('Me_a') # use compressed files KWARGS['GZIP'] = GZIP # output variable keys KEYS = ['zsurf','zfirn','zsmb','zmelt'] # HDF5 longname and description attributes for each variable LONGNAME = {} LONGNAME['zsurf'] = "Height" LONGNAME['zfirn'] = "Compaction" LONGNAME['zsmb'] = "Surface Mass Balance" LONGNAME['zmelt'] = "Surface Melt" DESCRIPTION = {} DESCRIPTION['zsurf'] = "Snow Height Change" DESCRIPTION['zfirn'] = "Snow Height Change due to Compaction" DESCRIPTION['zsmb'] = "Snow Height Change due to Surface Mass Balance" DESCRIPTION['zmelt'] = "Snow Height Change due to Surface Melt" # pyproj transformer for converting from latitude/longitude # into polar stereographic coordinates crs1 = pyproj.CRS.from_string("epsg:{0:d}".format(4326)) crs2 = pyproj.CRS.from_string(proj4_params) transformer = pyproj.Transformer.from_crs(crs1, crs2, always_xy=True) # read each input beam pair within the file IS2_atl11_pairs = [] for ptx in [k for k in fileID.keys() if bool(re.match(r'pt\d',k))]: # check if subsetted beam contains reference points try: fileID[ptx]['ref_pt'] except KeyError: pass else: IS2_atl11_pairs.append(ptx) # copy variables for outputting to HDF5 file IS2_atl11_corr = {} IS2_atl11_fill = {} IS2_atl11_dims = {} IS2_atl11_corr_attrs = {} # number of GPS seconds between the GPS epoch (1980-01-06T00:00:00Z UTC) # and ATLAS Standard Data Product (SDP) epoch (2018-01-01T00:00:00Z UTC) # Add this value to delta time parameters to compute full gps_seconds IS2_atl11_corr['ancillary_data'] = {} IS2_atl11_corr_attrs['ancillary_data'] = {} for key in ['atlas_sdp_gps_epoch']: # get each HDF5 variable IS2_atl11_corr['ancillary_data'][key] = fileID['ancillary_data'][key][:] # Getting attributes of group and included variables IS2_atl11_corr_attrs['ancillary_data'][key] = {} for att_name,att_val in fileID['ancillary_data'][key].attrs.items(): IS2_atl11_corr_attrs['ancillary_data'][key][att_name] = att_val # HDF5 group name for across-track data XT = 'crossing_track_data' # for each input beam pair within the file for ptx in sorted(IS2_atl11_pairs): # output data dictionaries for beam IS2_atl11_corr[ptx] = dict(cycle_stats=collections.OrderedDict(), crossing_track_data=collections.OrderedDict()) IS2_atl11_fill[ptx] = dict(cycle_stats={},crossing_track_data={}) IS2_atl11_dims[ptx] = dict(cycle_stats={},crossing_track_data={}) IS2_atl11_corr_attrs[ptx] = dict(cycle_stats={},crossing_track_data={}) # extract along-track and across-track variables ref_pt = {} latitude = {} longitude = {} delta_time = {} groups = ['AT'] # dictionary with output variables OUTPUT = {} # number of average segments and number of included cycles # fill_value for invalid heights and corrections fv = fileID[ptx]['h_corr'].attrs['_FillValue'] # shape of along-track data n_points,n_cycles = fileID[ptx]['delta_time'][:].shape # along-track (AT) reference point, latitude, longitude and time ref_pt['AT'] = fileID[ptx]['ref_pt'][:].copy() latitude['AT'] = np.ma.array(fileID[ptx]['latitude'][:], fill_value=fileID[ptx]['latitude'].attrs['_FillValue']) latitude['AT'].mask = (latitude['AT'] == latitude['AT'].fill_value) longitude['AT'] = np.ma.array(fileID[ptx]['longitude'][:], fill_value=fileID[ptx]['longitude'].attrs['_FillValue']) longitude['AT'].mask = (longitude['AT'] == longitude['AT'].fill_value) delta_time['AT'] = np.ma.array(fileID[ptx]['delta_time'][:], fill_value=fileID[ptx]['delta_time'].attrs['_FillValue']) delta_time['AT'].mask = (delta_time['AT'] == delta_time['AT'].fill_value) # allocate for output height for along-track data OUTPUT['AT'] = {} for key in KEYS: OUTPUT['AT'][key] = np.ma.empty((n_points,n_cycles),fill_value=fv) OUTPUT['AT'][key].mask = np.ones((n_points,n_cycles),dtype=bool) OUTPUT['AT'][key].interpolation = np.zeros((n_points,n_cycles),dtype=np.uint8) # if running ATL11 crossovers if CROSSOVERS: # add to group groups.append('XT') # shape of across-track data n_cross, = fileID[ptx][XT]['delta_time'].shape # across-track (XT) reference point, latitude, longitude and time ref_pt['XT'] = fileID[ptx][XT]['ref_pt'][:].copy() latitude['XT'] = np.ma.array(fileID[ptx][XT]['latitude'][:], fill_value=fileID[ptx][XT]['latitude'].attrs['_FillValue']) latitude['XT'].mask = (latitude['XT'] == latitude['XT'].fill_value) longitude['XT'] = np.ma.array(fileID[ptx][XT]['longitude'][:], fill_value=fileID[ptx][XT]['longitude'].attrs['_FillValue']) latitude['XT'].mask = (latitude['XT'] == longitude['XT'].fill_value) delta_time['XT'] = np.ma.array(fileID[ptx][XT]['delta_time'][:], fill_value=fileID[ptx][XT]['delta_time'].attrs['_FillValue']) delta_time['XT'].mask = (delta_time['XT'] == delta_time['XT'].fill_value) # allocate for output height for across-track data OUTPUT['XT'] = {} for key in KEYS: OUTPUT['XT'][key] = np.ma.empty((n_cross),fill_value=fv) OUTPUT['XT'][key].mask = np.ones((n_cross),dtype=bool) OUTPUT['XT'][key].interpolation = np.zeros((n_cross),dtype=np.uint8) # extract lat/lon and convert to polar stereographic X,Y = transformer.transform(longitude['AT'],longitude['AT']) # for each valid cycle of ICESat-2 ATL11 data for c in range(n_cycles): # find valid elevations for cycle valid = np.logical_not(delta_time['AT'].mask[:,c]) i, = np.nonzero(valid) # convert time from ATLAS SDP to date in decimal-years tdec = convert_delta_time(delta_time['AT'][i,c])['decimal'] if (MODEL == 'MAR') and np.any(valid): # read and interpolate daily MAR outputs for key,var in zip(KEYS,VARIABLES): OUT = SMBcorr.interpolate_mar_daily(DIRECTORY, proj4_params, MAR_VERSION, tdec, X[i], Y[i], VARIABLE=var, **KWARGS) # set attributes to output for iteration OUTPUT['AT'][key].data[i,c] = np.copy(OUT.data) OUTPUT['AT'][key].mask[i,c] = np.copy(OUT.mask) OUTPUT['AT'][key].interpolation[i,c] = np.copy(OUT.interpolation) # calculate derived fields OUTPUT['AT']['zsmb'].data[i,c] = OUTPUT['AT']['zsurf'].data[i,c] - \ OUTPUT['AT']['zfirn'].data[i,c] OUTPUT['AT']['zsmb'].mask[i,c] = OUTPUT['AT']['zsurf'].mask[i,c] | \ OUTPUT['AT']['zfirn'].mask[i,c] OUTPUT['AT']['zaccum'].data[i,c] = OUTPUT['AT']['zsurf'].data[i,c] - \ OUTPUT['AT']['zfirn'].data[i,c] - OUTPUT['AT']['zmelt'].data OUTPUT['AT']['zaccum'].mask[i,c] = OUTPUT['AT']['zsurf'].mask[i,c] | \ OUTPUT['AT']['zfirn'].mask[i,c] | OUTPUT['AT']['zmelt'].mask[i,c] elif (MODEL == 'RACMO') and np.any(valid): # read and interpolate daily RACMO outputs for key,var in zip(KEYS,VARIABLES): OUT = SMBcorr.interpolate_racmo_daily(base_dir, proj4_params, RACMO_MODEL, tdec, X[i], Y[i], VARIABLE=var, **KWARGS) # set attributes to output for iteration OUTPUT['AT'][key].data[i,c] = np.copy(OUT.data) OUTPUT['AT'][key].mask[i,c] = np.copy(OUT.mask) OUTPUT['AT'][key].interpolation[i,c] = np.copy(OUT.interpolation) elif (MODEL == 'MERRA2-hybrid') and np.any(valid): # read and interpolate 5-day MERRA2-Hybrid outputs for key,var in zip(KEYS,VARIABLES): OUT = SMBcorr.interpolate_merra_hybrid(DIRECTORY, proj4_params, MERRA2_REGION, tdec, X[i], Y[i], VARIABLE=var, **KWARGS) # set attributes to output for iteration OUTPUT['AT'][key].data[i,c] = np.copy(OUT.data) OUTPUT['AT'][key].mask[i,c] = np.copy(OUT.mask) OUTPUT['AT'][key].interpolation[i,c] = np.copy(OUT.interpolation) #-- if interpolating to ATL11 crossover locations if CROSSOVERS: # extract lat/lon and convert to polar stereographic X,Y = transformer.transform(longitude['XT'],longitude['XT']) # find valid elevations for cycle valid = np.logical_not(delta_time['XT'].mask[:]) i, = np.nonzero(valid) # convert time from ATLAS SDP to date in decimal-years tdec = convert_delta_time(delta_time['XT'][i])['decimal'] if (MODEL == 'MAR') and np.any(valid): # read and interpolate daily MAR outputs for key,var in zip(KEYS,VARIABLES): OUT = SMBcorr.interpolate_mar_daily(DIRECTORY, proj4_params, MAR_VERSION, tdec, X[i], Y[i], VARIABLE=var, **KWARGS) # set attributes to output for iteration OUTPUT['XT'][key].data[i] = np.copy(OUT.data) OUTPUT['XT'][key].mask[i] = np.copy(OUT.mask) OUTPUT['XT'][key].interpolation[i] = np.copy(OUT.interpolation) # calculate derived fields OUTPUT['XT']['zsmb'].data[i] = OUTPUT['XT']['zsurf'].data[i] - \ OUTPUT['XT']['zfirn'].data[i] OUTPUT['XT']['zsmb'].mask[i] = OUTPUT['XT']['zsurf'].mask[i] | \ OUTPUT['XT']['zfirn'].mask[i] OUTPUT['XT']['zaccum'].data[i] = OUTPUT['XT']['zsurf'].data[i] - \ OUTPUT['XT']['zfirn'].data[i] - OUTPUT['AT']['zmelt'].data[i] OUTPUT['XT']['zaccum'].mask[i] = OUTPUT['XT']['zsurf'].mask[i] | \ OUTPUT['XT']['zfirn'].mask[i] | OUTPUT['XT']['zmelt'].mask[i] elif (MODEL == 'RACMO') and np.any(valid): # read and interpolate daily RACMO outputs for key,var in zip(KEYS,VARIABLES): OUT = SMBcorr.interpolate_racmo_daily(base_dir, proj4_params, RACMO_MODEL, tdec, X[i], Y[i], VARIABLE=var, **KWARGS) # set attributes to output for iteration OUTPUT['XT'][key].data[i] = np.copy(OUT.data) OUTPUT['XT'][key].mask[i] = np.copy(OUT.mask) OUTPUT['XT'][key].interpolation[i] = np.copy(OUT.interpolation) elif (MODEL == 'MERRA2-hybrid') and np.any(valid): # read and interpolate 5-day MERRA2-Hybrid outputs for key,var in zip(KEYS,VARIABLES): OUT = SMBcorr.interpolate_merra_hybrid(DIRECTORY, proj4_params, MERRA2_REGION, tdec, X[i], Y[i], VARIABLE=var, **KWARGS) # set attributes to output for iteration OUTPUT['XT'][key].data[i] = np.copy(OUT.data) OUTPUT['XT'][key].mask[i] = np.copy(OUT.mask) OUTPUT['XT'][key].interpolation[i] = np.copy(OUT.interpolation) # group attributes for beam IS2_atl11_corr_attrs[ptx]['description'] = ('Contains the primary science parameters ' 'for this data set') IS2_atl11_corr_attrs[ptx]['beam_pair'] = fileID[ptx].attrs['beam_pair'] IS2_atl11_corr_attrs[ptx]['ReferenceGroundTrack'] = fileID[ptx].attrs['ReferenceGroundTrack'] IS2_atl11_corr_attrs[ptx]['first_cycle'] = fileID[ptx].attrs['first_cycle'] IS2_atl11_corr_attrs[ptx]['last_cycle'] = fileID[ptx].attrs['last_cycle'] IS2_atl11_corr_attrs[ptx]['equatorial_radius'] = fileID[ptx].attrs['equatorial_radius'] IS2_atl11_corr_attrs[ptx]['polar_radius'] = fileID[ptx].attrs['polar_radius'] # geolocation, time and reference point # reference point IS2_atl11_corr[ptx]['ref_pt'] = ref_pt['AT'].copy() IS2_atl11_fill[ptx]['ref_pt'] = None IS2_atl11_dims[ptx]['ref_pt'] = None IS2_atl11_corr_attrs[ptx]['ref_pt'] = collections.OrderedDict() IS2_atl11_corr_attrs[ptx]['ref_pt']['units'] = "1" IS2_atl11_corr_attrs[ptx]['ref_pt']['contentType'] = "referenceInformation" IS2_atl11_corr_attrs[ptx]['ref_pt']['long_name'] = "Reference point number" IS2_atl11_corr_attrs[ptx]['ref_pt']['source'] = "ATL06" IS2_atl11_corr_attrs[ptx]['ref_pt']['description'] = ("The reference point is the " "7 digit segment_id number corresponding to the center of the ATL06 data used " "for each ATL11 point. These are sequential, starting with 1 for the first " "segment after an ascending equatorial crossing node.") IS2_atl11_corr_attrs[ptx]['ref_pt']['coordinates'] = \ "delta_time latitude longitude" # cycle_number IS2_atl11_corr[ptx]['cycle_number'] = fileID[ptx]['cycle_number'][:].copy() IS2_atl11_fill[ptx]['cycle_number'] = None IS2_atl11_dims[ptx]['cycle_number'] = None IS2_atl11_corr_attrs[ptx]['cycle_number'] = collections.OrderedDict() IS2_atl11_corr_attrs[ptx]['cycle_number']['units'] = "1" IS2_atl11_corr_attrs[ptx]['cycle_number']['long_name'] = "Orbital cycle number" IS2_atl11_corr_attrs[ptx]['cycle_number']['source'] = "ATL06" IS2_atl11_corr_attrs[ptx]['cycle_number']['description'] = ("Number of 91-day periods " "that have elapsed since ICESat-2 entered the science orbit. Each of the 1,387 " "reference ground track (RGTs) is targeted in the polar regions once " "every 91 days.") # delta time IS2_atl11_corr[ptx]['delta_time'] = delta_time['AT'].copy() IS2_atl11_fill[ptx]['delta_time'] = delta_time['AT'].fill_value IS2_atl11_dims[ptx]['delta_time'] = ['ref_pt','cycle_number'] IS2_atl11_corr_attrs[ptx]['delta_time'] = collections.OrderedDict() IS2_atl11_corr_attrs[ptx]['delta_time']['units'] = "seconds since 2018-01-01" IS2_atl11_corr_attrs[ptx]['delta_time']['long_name'] = "Elapsed GPS seconds" IS2_atl11_corr_attrs[ptx]['delta_time']['standard_name'] = "time" IS2_atl11_corr_attrs[ptx]['delta_time']['calendar'] = "standard" IS2_atl11_corr_attrs[ptx]['delta_time']['source'] = "ATL06" IS2_atl11_corr_attrs[ptx]['delta_time']['description'] = ("Number of GPS " "seconds since the ATLAS SDP epoch. The ATLAS Standard Data Products (SDP) epoch offset " "is defined within /ancillary_data/atlas_sdp_gps_epoch as the number of GPS seconds " "between the GPS epoch (1980-01-06T00:00:00.000000Z UTC) and the ATLAS SDP epoch. By " "adding the offset contained within atlas_sdp_gps_epoch to delta time parameters, the " "time in gps_seconds relative to the GPS epoch can be computed.") IS2_atl11_corr_attrs[ptx]['delta_time']['coordinates'] = \ "ref_pt cycle_number latitude longitude" # latitude IS2_atl11_corr[ptx]['latitude'] = latitude['AT'].copy() IS2_atl11_fill[ptx]['latitude'] = latitude['AT'].fill_value IS2_atl11_dims[ptx]['latitude'] = ['ref_pt'] IS2_atl11_corr_attrs[ptx]['latitude'] = collections.OrderedDict() IS2_atl11_corr_attrs[ptx]['latitude']['units'] = "degrees_north" IS2_atl11_corr_attrs[ptx]['latitude']['contentType'] = "physicalMeasurement" IS2_atl11_corr_attrs[ptx]['latitude']['long_name'] = "Latitude" IS2_atl11_corr_attrs[ptx]['latitude']['standard_name'] = "latitude" IS2_atl11_corr_attrs[ptx]['latitude']['source'] = "ATL06" IS2_atl11_corr_attrs[ptx]['latitude']['description'] = ("Center latitude of " "selected segments") IS2_atl11_corr_attrs[ptx]['latitude']['valid_min'] = -90.0 IS2_atl11_corr_attrs[ptx]['latitude']['valid_max'] = 90.0 IS2_atl11_corr_attrs[ptx]['latitude']['coordinates'] = \ "ref_pt delta_time longitude" # longitude IS2_atl11_corr[ptx]['longitude'] = longitude['AT'].copy() IS2_atl11_fill[ptx]['longitude'] = longitude['AT'].fill_value IS2_atl11_dims[ptx]['longitude'] = ['ref_pt'] IS2_atl11_corr_attrs[ptx]['longitude'] = collections.OrderedDict() IS2_atl11_corr_attrs[ptx]['longitude']['units'] = "degrees_east" IS2_atl11_corr_attrs[ptx]['longitude']['contentType'] = "physicalMeasurement" IS2_atl11_corr_attrs[ptx]['longitude']['long_name'] = "Longitude" IS2_atl11_corr_attrs[ptx]['longitude']['standard_name'] = "longitude" IS2_atl11_corr_attrs[ptx]['longitude']['source'] = "ATL06" IS2_atl11_corr_attrs[ptx]['longitude']['description'] = ("Center longitude of " "selected segments") IS2_atl11_corr_attrs[ptx]['longitude']['valid_min'] = -180.0 IS2_atl11_corr_attrs[ptx]['longitude']['valid_max'] = 180.0 IS2_atl11_corr_attrs[ptx]['longitude']['coordinates'] = \ "ref_pt delta_time latitude" # cycle statistics variables IS2_atl11_corr_attrs[ptx]['cycle_stats']['Description'] = ("The cycle_stats subgroup " "contains summary information about segments for each reference point, including " "the uncorrected mean heights for reference surfaces, blowing snow and cloud " "indicators, and geolocation and height misfit statistics.") IS2_atl11_corr_attrs[ptx]['cycle_stats']['data_rate'] = ("Data within this group " "are stored at the average segment rate.") # for each along-track dataset for key,val in OUTPUT['AT'].items(): # add to output IS2_atl11_corr[ptx]['cycle_stats'][key] = val.copy() IS2_atl11_fill[ptx]['cycle_stats'][key] = val.fill_value IS2_atl11_dims[ptx]['cycle_stats'][key] = ['ref_pt','cycle_number'] IS2_atl11_corr_attrs[ptx]['cycle_stats'][key] = collections.OrderedDict() IS2_atl11_corr_attrs[ptx]['cycle_stats'][key]['units'] = "meters" IS2_atl11_corr_attrs[ptx]['cycle_stats'][key]['contentType'] = "referenceInformation" IS2_atl11_corr_attrs[ptx]['cycle_stats'][key]['long_name'] = LONGNAME[key] IS2_atl11_corr_attrs[ptx]['cycle_stats'][key]['description'] = DESCRIPTION[key] IS2_atl11_corr_attrs[ptx]['cycle_stats'][key]['source'] = MODEL IS2_atl11_corr_attrs[ptx]['cycle_stats'][key]['reference'] = model_version IS2_atl11_corr_attrs[ptx]['cycle_stats'][key]['coordinates'] = \ "../ref_pt ../cycle_number ../delta_time ../latitude ../longitude" # if crossover measurements were calculated if CROSSOVERS: # crossing track variables IS2_atl11_corr_attrs[ptx][XT]['Description'] = ("The crossing_track_data " "subgroup contains elevation data at crossover locations. These are " "locations where two ICESat-2 pair tracks cross, so data are available " "from both the datum track, for which the granule was generated, and " "from the crossing track.") IS2_atl11_corr_attrs[ptx][XT]['data_rate'] = ("Data within this group are " "stored at the average segment rate.") # reference point IS2_atl11_corr[ptx][XT]['ref_pt'] = ref_pt['XT'].copy() IS2_atl11_fill[ptx][XT]['ref_pt'] = None IS2_atl11_dims[ptx][XT]['ref_pt'] = None IS2_atl11_corr_attrs[ptx][XT]['ref_pt'] = collections.OrderedDict() IS2_atl11_corr_attrs[ptx][XT]['ref_pt']['units'] = "1" IS2_atl11_corr_attrs[ptx][XT]['ref_pt']['contentType'] = "referenceInformation" IS2_atl11_corr_attrs[ptx][XT]['ref_pt']['long_name'] = ("fit center reference point number, " "segment_id") IS2_atl11_corr_attrs[ptx][XT]['ref_pt']['source'] = "derived, ATL11 algorithm" IS2_atl11_corr_attrs[ptx][XT]['ref_pt']['description'] = ("The reference-point number of the " "fit center for the datum track. The reference point is the 7 digit segment_id number " "corresponding to the center of the ATL06 data used for each ATL11 point. These are " "sequential, starting with 1 for the first segment after an ascending equatorial " "crossing node.") IS2_atl11_corr_attrs[ptx][XT]['ref_pt']['coordinates'] = \ "delta_time latitude longitude" # reference ground track of the crossing track IS2_atl11_corr[ptx][XT]['rgt'] = fileID[ptx][XT]['rgt'][:].copy() IS2_atl11_fill[ptx][XT]['rgt'] = fileID[ptx][XT]['rgt'].attrs['_FillValue'] IS2_atl11_dims[ptx][XT]['rgt'] = None IS2_atl11_corr_attrs[ptx][XT]['rgt'] = collections.OrderedDict() IS2_atl11_corr_attrs[ptx][XT]['rgt']['units'] = "1" IS2_atl11_corr_attrs[ptx][XT]['rgt']['contentType'] = "referenceInformation" IS2_atl11_corr_attrs[ptx][XT]['rgt']['long_name'] = "crossover reference ground track" IS2_atl11_corr_attrs[ptx][XT]['rgt']['source'] = "ATL06" IS2_atl11_corr_attrs[ptx][XT]['rgt']['description'] = "The RGT number for the crossing data." IS2_atl11_corr_attrs[ptx][XT]['rgt']['coordinates'] = \ "ref_pt delta_time latitude longitude" # cycle_number of the crossing track IS2_atl11_corr[ptx][XT]['cycle_number'] = fileID[ptx][XT]['cycle_number'][:].copy() IS2_atl11_fill[ptx][XT]['cycle_number'] = fileID[ptx][XT]['cycle_number'].attrs['_FillValue'] IS2_atl11_dims[ptx][XT]['cycle_number'] = None IS2_atl11_corr_attrs[ptx][XT]['cycle_number'] = collections.OrderedDict() IS2_atl11_corr_attrs[ptx][XT]['cycle_number']['units'] = "1" IS2_atl11_corr_attrs[ptx][XT]['cycle_number']['long_name'] = "crossover cycle number" IS2_atl11_corr_attrs[ptx][XT]['cycle_number']['source'] = "ATL06" IS2_atl11_corr_attrs[ptx][XT]['cycle_number']['description'] = ("Cycle number for the " "crossing data. Number of 91-day periods that have elapsed since ICESat-2 entered " "the science orbit. Each of the 1,387 reference ground track (RGTs) is targeted " "in the polar regions once every 91 days.") # delta time of the crossing track IS2_atl11_corr[ptx][XT]['delta_time'] = delta_time['XT'].copy() IS2_atl11_fill[ptx][XT]['delta_time'] = delta_time['XT'].fill_value IS2_atl11_dims[ptx][XT]['delta_time'] = ['ref_pt'] IS2_atl11_corr_attrs[ptx][XT]['delta_time'] = {} IS2_atl11_corr_attrs[ptx][XT]['delta_time']['units'] = "seconds since 2018-01-01" IS2_atl11_corr_attrs[ptx][XT]['delta_time']['long_name'] = "Elapsed GPS seconds" IS2_atl11_corr_attrs[ptx][XT]['delta_time']['standard_name'] = "time" IS2_atl11_corr_attrs[ptx][XT]['delta_time']['calendar'] = "standard" IS2_atl11_corr_attrs[ptx][XT]['delta_time']['source'] = "ATL06" IS2_atl11_corr_attrs[ptx][XT]['delta_time']['description'] = ("Number of GPS " "seconds since the ATLAS SDP epoch. The ATLAS Standard Data Products (SDP) epoch offset " "is defined within /ancillary_data/atlas_sdp_gps_epoch as the number of GPS seconds " "between the GPS epoch (1980-01-06T00:00:00.000000Z UTC) and the ATLAS SDP epoch. By " "adding the offset contained within atlas_sdp_gps_epoch to delta time parameters, the " "time in gps_seconds relative to the GPS epoch can be computed.") IS2_atl11_corr_attrs[ptx]['delta_time']['coordinates'] = \ "ref_pt latitude longitude" # latitude of the crossover measurement IS2_atl11_corr[ptx][XT]['latitude'] = latitude['XT'].copy() IS2_atl11_fill[ptx][XT]['latitude'] = latitude['XT'].fill_value IS2_atl11_dims[ptx][XT]['latitude'] = ['ref_pt'] IS2_atl11_corr_attrs[ptx][XT]['latitude'] = collections.OrderedDict() IS2_atl11_corr_attrs[ptx][XT]['latitude']['units'] = "degrees_north" IS2_atl11_corr_attrs[ptx][XT]['latitude']['contentType'] = "physicalMeasurement" IS2_atl11_corr_attrs[ptx][XT]['latitude']['long_name'] = "crossover latitude" IS2_atl11_corr_attrs[ptx][XT]['latitude']['standard_name'] = "latitude" IS2_atl11_corr_attrs[ptx][XT]['latitude']['source'] = "ATL06" IS2_atl11_corr_attrs[ptx][XT]['latitude']['description'] = ("Center latitude of " "selected segments") IS2_atl11_corr_attrs[ptx][XT]['latitude']['valid_min'] = -90.0 IS2_atl11_corr_attrs[ptx][XT]['latitude']['valid_max'] = 90.0 IS2_atl11_corr_attrs[ptx][XT]['latitude']['coordinates'] = \ "ref_pt delta_time longitude" # longitude of the crossover measurement IS2_atl11_corr[ptx][XT]['longitude'] = longitude['XT'].copy() IS2_atl11_fill[ptx][XT]['longitude'] = longitude['XT'].fill_value IS2_atl11_dims[ptx][XT]['longitude'] = ['ref_pt'] IS2_atl11_corr_attrs[ptx][XT]['longitude'] = collections.OrderedDict() IS2_atl11_corr_attrs[ptx][XT]['longitude']['units'] = "degrees_east" IS2_atl11_corr_attrs[ptx][XT]['longitude']['contentType'] = "physicalMeasurement" IS2_atl11_corr_attrs[ptx][XT]['longitude']['long_name'] = "crossover longitude" IS2_atl11_corr_attrs[ptx][XT]['longitude']['standard_name'] = "longitude" IS2_atl11_corr_attrs[ptx][XT]['longitude']['source'] = "ATL06" IS2_atl11_corr_attrs[ptx][XT]['longitude']['description'] = ("Center longitude of " "selected segments") IS2_atl11_corr_attrs[ptx][XT]['longitude']['valid_min'] = -180.0 IS2_atl11_corr_attrs[ptx][XT]['longitude']['valid_max'] = 180.0 IS2_atl11_corr_attrs[ptx][XT]['longitude']['coordinates'] = \ "ref_pt delta_time latitude" # for each crossover dataset for key,val in OUTPUT['XT'].items(): # add to output IS2_atl11_corr[ptx][XT][key] = val.copy() IS2_atl11_fill[ptx][XT][key] = val.fill_value IS2_atl11_dims[ptx][XT][key] = ['ref_pt'] IS2_atl11_corr_attrs[ptx][XT][key] = collections.OrderedDict() IS2_atl11_corr_attrs[ptx][XT][key]['units'] = "meters" IS2_atl11_corr_attrs[ptx][XT][key]['contentType'] = "referenceInformation" IS2_atl11_corr_attrs[ptx][XT][key]['long_name'] = LONGNAME[key] IS2_atl11_corr_attrs[ptx][XT][key]['description'] = DESCRIPTION[key] IS2_atl11_corr_attrs[ptx][XT][key]['source'] = MODEL IS2_atl11_corr_attrs[ptx][XT][key]['reference'] = model_version IS2_atl11_corr_attrs[ptx][XT][key]['coordinates'] = \ "ref_pt delta_time latitude longitude" # output HDF5 files with interpolated surface mass balance data args = (PRD,model_version,TRK,GRAN,SCYC,ECYC,RL,VERS,AUX) file_format = '{0}_{1}_{2}{3}_{4}{5}_{6}_{7}{8}.h5' # print file information print('\t{0}'.format(file_format.format(*args))) if VERBOSE else None HDF5_ATL11_corr_write(IS2_atl11_corr, IS2_atl11_corr_attrs, CLOBBER=True, INPUT=os.path.basename(FILE), CROSSOVERS=CROSSOVERS, FILL_VALUE=IS2_atl11_fill, DIMENSIONS=IS2_atl11_dims, FILENAME=os.path.join(ddir,file_format.format(*args))) # change the permissions mode os.chmod(os.path.join(ddir,file_format.format(*args)), MODE)
def send_sms(browser, receiver, logging=False, auto_send_long_sms=False, message=None): """Send SMS. Query for message and send SMS. """ # Get message text while not message: message = raw_input('Message: ').strip() message = unicode(message, sys.stdin.encoding).encode('utf-8') # To utf-8 count = len(message) if count > __xtra_sms_max_length: if auto_send_long_sms or yn_choice('Message is %u characters long and ' \ 'will be split into several SMS. ' \ 'Do you want to send it anyway?' % count): i = 0 while i < count: send_sms(browser, receiver, logging, auto_send_long_sms, message[i:i + __xtra_sms_max_length - 1]) i += __xtra_sms_max_length - 1 return # Actually send the SMS url = 'https://xtrazone.sso.bluewin.ch/index.php/20,53,ajax,,,283/' \ '?route=%2Fmessaging%2Foutbox%2Fsendmobilemsg' data = {'attachmentId': '', 'attachments': '', 'messagebody': message, 'receiversnames': receiver, 'recipients': '[]', } browser.open(url, urllib.urlencode(data)) resp = json.loads(browser.response().read()) try: valid_headlines = ( u'Verarbeitung erfolgreich', u'Traitement réussi.', u'Elaborazione riuscita.', ) if (resp['content']['headline'] not in valid_headlines or resp['content']['isError'] != False): raise RuntimeError('Unknown error sending SMS.') except TypeError: # Something went wrong. if __tracebacks: print resp if 'Auf diese Inhalte kannst Du nicht zugreifen' in resp['content']: print 'Session has expired. Reconnecting...' cfg = parse_config() login(browser, cfg['username'], cfg['password'], cfg['anticaptcha'], cfg['anticaptcha_max_tries']) send_sms(browser, receiver, logging, auto_send_long_sms, message) else: raise RuntimeError('Unknown error sending SMS.') else: print 'SMS sent successfully.' # If desired, log SMS if logging: pyxtra_folder = os.path.expanduser(os.path.join('~', '.pyxtra')) log_file = os.path.join(pyxtra_folder, 'sent.log') log_file_created = not os.path.exists(log_file) f = open(log_file, 'a') # Open file for appending print >> f, 'Date: %s' % datetime.now().strftime('%d.%m.%Y %H:%M:%S') print >> f, 'Receivers: %s' % receiver print >> f, 'Message: %s\n' % message f.close() if log_file_created: os.chmod(log_file, stat.S_IREAD | stat.S_IWRITE)
def set_permissive_permissions(filename): os.chmod(filename, stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH | \ stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | \ stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP)
def runImpl(self, flavor, device, pyVersion, args): result = TestRunResult() result.succeeded = True # Preparation for pattern-based test cases if len(self.testCases) > 0: # Locating and reading baseline file baselineFile = self.findBaselineFile(flavor, device) if baselineFile != None: with open(baselineFile, "r") as f: baseline = f.read().split("\n") if args.verbose: six.print_("Baseline: " + baselineFile) # Before running the test, pre-creating TestCaseRunResult object for each test case # and compute filtered lines from baseline file. # Note: some test cases might fail at this time if baseline and/or patterns are inconsistent if not args.update_baseline: for testCase in self.testCases: testCaseRunResult = testCase.processBaseline(baseline) if not testCaseRunResult.succeeded: result.succeeded = False result.testCaseRunResults.append(testCaseRunResult) else: if not args.create_baseline: return TestRunResult.fatalError("Baseline file sanity check", "Can't find baseline file") # preparing run directory pyVersionLabel = "_{0}".format(pyVersion) if pyVersion else '' runDir = os.path.join(args.run_dir, "{0}_{1}@{2}_{3}{4}".format(self.suite, self.name, flavor, device, pyVersionLabel)) runDir = cygpath(runDir) if not os.path.isdir(runDir): os.makedirs(runDir) # preparing environment for the test script os.environ["TEST_FLAVOR"] = flavor os.environ["TEST_DEVICE"] = device os.environ["TEST_BUILD_LOCATION"] = args.build_location if windows: if args.build_sku == "cpu": os.environ["TEST_CNTK_BINARY"] = os.path.join(args.build_location, (flavor + "_CpuOnly"), "cntk.exe") else: os.environ["TEST_CNTK_BINARY"] = os.path.join(args.build_location, flavor, "cntk.exe") os.environ["MPI_BINARY"] = os.path.join(os.environ["MSMPI_BIN"], "mpiexec.exe") else: tempPath = os.path.join(args.build_location, args.build_sku, flavor, "bin", "cntk") if not os.path.isfile(tempPath): for bsku in ["/build/gpu/", "/build/cpu/", "/build/1bitsgd/"]: if tempPath.find(bsku) >= 0: tempPath = tempPath.replace(bsku, "/build/") break os.environ["TEST_CNTK_BINARY"] = tempPath os.environ["MPI_BINARY"] = "mpiexec" os.environ["TEST_1BIT_SGD"] = ("1" if args.build_sku == "1bitsgd" else "0") if not os.path.exists(os.environ["TEST_CNTK_BINARY"]): raise ValueError("the cntk executable does not exist at path '%s'"%os.environ["TEST_CNTK_BINARY"]) os.environ["TEST_BIN_DIR"] = os.path.dirname(os.environ["TEST_CNTK_BINARY"]) os.environ["TEST_DIR"] = self.testDir os.environ["TEST_DATA_DIR"] = self.dataDir os.environ["TEST_RUN_DIR"] = runDir # WORKAROUND: changing current dir to the dataDir so relative paths in SCP files work as expected os.chdir(self.dataDir) # Running test script #TODO:port this properly to windows # Writing standard output to the file and to the console (if --verbose) logFile = os.path.join(runDir, "output.txt") allLines = [] if args.verbose: six.print_(self.fullName + ":>" + logFile) with open(logFile, "w") as output: if not windows: testScript = self.testDir + "/run-test" st = os.stat(testScript) os.chmod(testScript, st.st_mode | stat.S_IEXEC | stat.S_IXOTH) cmdLine = ["bash", "-c", self.testDir + "/run-test 2>&1"] process = subprocess.Popen(cmdLine, stdout=subprocess.PIPE) while True: line = process.stdout.readline() if not line: break if len(line)>0 and line[-1]=='\n': line=line[:len(line)-1] if args.verbose: # TODO find a better way if sys.version_info.major < 3: six.print_(self.fullName + ": " + line) else: six.print_(self.fullName + ": " + line.decode('utf-8').rstrip()) if args.dry_run: print (line) continue six.print_(line, file=output) allLines.append(line) output.flush() for testCaseRunResult in result.testCaseRunResults: testCaseRunResult.testCase.processLine(line, testCaseRunResult, args.verbose) exitCode = process.wait() success = True # saving log file path, so it can be reported later result.logFile = logFile # checking exit code if exitCode != 0: if args.dry_run: six.print_("[SKIPPED]") return result else: return TestRunResult.fatalError("Exit code must be 0", "==> got exit code {0} when running: {1}".format(exitCode, " ".join(cmdLine)), logFile = logFile) # finalizing verification - need to check whether we have any unmatched lines for testCaseRunResult in result.testCaseRunResults: testCaseRunResult.testCase.finalize(testCaseRunResult) if not testCaseRunResult.succeeded: result.succeeded = False if len(self.testCases)>0 and (args.update_baseline or (baselineFile == None)) and result.succeeded: # When running in --update-baseline or --create-baseline mode # verifying that new output is successfully matching every pattern in the testcases.yml # If this is not the case then baseline update/create will be rejected for testCase in self.testCases: testCaseRunResult = testCase.processBaseline(allLines) if not testCaseRunResult.succeeded: result.succeeded = False result.testCaseRunResults.append(testCaseRunResult) if baselineFile == None: baselineFile = self.newBaselineFilePath(device) if result.succeeded: if args.verbose: if args.update_baseline: six.print_("Updating baseline file " + baselineFile) else: six.print_("Creating baseline file " + baselineFile) with open(baselineFile, "w") as f: f.write("\n".join(allLines)) return result