def get_info(self): logging.debug("Checking for StockPickr User %s", self.user) sql = "select uid from spr_user where name=%s;" self.id = self.db.query(sql, self.user, one=True) if not self.id: self.db.insert("spr_user", {"name": self.user}) self.id = self.db.last_insert_id()
def extract(vm, remote_path, dest_dir): """ Extract the autotest .tar.bz2 file on the guest, ensuring the final destination path will be dest_dir. :param vm: VM object :param remote_path: Remote file path :param dest_dir: Destination dir for the contents """ basename = os.path.basename(remote_path) logging.debug("Extracting %s on VM %s", basename, vm.name) session.cmd("rm -rf %s" % dest_dir, timeout=240) dirname = os.path.dirname(remote_path) session.cmd("cd %s" % dirname) session.cmd("mkdir -p %s" % os.path.dirname(dest_dir)) e_cmd = "tar xjvf %s -C %s" % (basename, os.path.dirname(dest_dir)) output = session.cmd(e_cmd, timeout=240) autotest_dirname = "" for line in output.splitlines()[1:]: autotest_dirname = line.split("/")[0] break if autotest_dirname != os.path.basename(dest_dir): session.cmd("cd %s" % os.path.dirname(dest_dir)) session.cmd("mv %s %s" % (autotest_dirname, os.path.basename(dest_dir)))
def tearDownModule(): global vtgate_server logging.debug("in tearDownModule") if utils.options.skip_teardown: return logging.debug("Tearing down the servers and setup") utils.vtgate_kill(vtgate_server) tablet.kill_tablets([shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]) teardown_procs = [shard_0_master.teardown_mysql(), shard_0_replica.teardown_mysql(), shard_1_master.teardown_mysql(), shard_1_replica.teardown_mysql(), ] utils.wait_procs(teardown_procs, raise_on_error=False) environment.topo_server_teardown() utils.kill_sub_processes() utils.remove_tmp_files() shard_0_master.remove_tree() shard_0_replica.remove_tree() shard_1_master.remove_tree() shard_1_replica.remove_tree()
def shell(self): """Performs the 'shell' command.""" lib_jars = [] if self.flags.jars is not None: lib_jars.extend(self.flags.jars.split(",")) env = dict(self.env) express_cp = self.express.get_classpath(lib_jars=lib_jars) env["EXPRESS_CP"] = ":".join(express_cp) dist_cache_paths = self.express.list_paths_for_dist_cache(lib_jars=lib_jars) env["TMPJARS"] = ",".join(dist_cache_paths) # EXPRESS_MODE environment variable must be a path to the mode specific scala script: script_name = self._MODE_SCRIPT.get(self.flags.mode) assert (script_name is not None), ("Invalid Express mode: %r." % self.flags.mode) script_path = os.path.join(self.express.home_dir, "bin", script_name) assert os.path.isfile(script_path), ("Script not found: %r" % script_path) env["EXPRESS_MODE"] = script_path # express shell binary needs to be in the same directory as this script shell_path = os.path.join(self.express.home_dir, "bin", "express-shell") assert os.path.isfile(shell_path), ("Shell not found: %r" % shell_path) cmd = [shell_path] + list(self.args) logging.debug( "Launching the express shell using the command:\n%s", " \\\n\t".join(map(repr, cmd))) proc = subprocess.Popen(cmd, env=env) try: return proc.wait() except subprocess.SubprocessError: proc.kill()
def get(self, version_id): self.check_version_id(version_id) cache_key = version_id + str(self.__class__) page = self.cache.get(cache_key) if self.cache_bust or not page: logging.debug('Cache miss with key: %s' % cache_key) retrieved_data = EmailTemplate.fetch_all(self.data_sources[version_id]) trail_blocks = deduplication.build_unique_trailblocks(retrieved_data, self.priority_list[version_id], excluded=self.exclude_from_deduplication()) today = datetime.datetime.now() date = today.strftime('%A %d %b %Y') template_name = self.template_names[version_id] + '.html' template = self.resolve_template(template_name) ads = {} if hasattr(self, 'ad_tag') and self.ad_tag: ad_fetcher = AdFetcher(self.ad_tag) for name, type in self.ad_config.iteritems(): ads[name] = ad_fetcher.fetch_type(type) page = template.render(ads=ads, date=date, data=self.additional_template_data(), **trail_blocks) self.cache.add(cache_key, page, 300) else: logging.debug('Cache hit with key: %s' % cache_key) self.response.out.write(page)
def schema_shell(self): """Performs the 'schema-shell' command.""" schema_shell_home = self.env.get(SCHEMA_SHELL_HOME) assert (schema_shell_home is not None), \ ("Environment variable undefined: %r" % SCHEMA_SHELL_HOME) assert os.path.isdir(schema_shell_home), \ ("Invalid home directory for KijiSchema shell: %r" % schema_shell_home) schema_shell_script = os.path.join(schema_shell_home, "bin", "kiji-schema-shell") assert os.path.isfile(schema_shell_script), \ ("KijiSchema shell not found: %r" % schema_shell_script) env = dict(self.env) classpath = env.get(KIJI_CLASSPATH, "").split(":") + list(self.express.get_classpath()) env[KIJI_CLASSPATH] = ":".join(classpath) java_opts = env.get("JAVA_OPTS", "") # FIXME: I cannot find any trace of the Java system property "express.tmpjars"! # java_opts += (" -Dexpress.tmpjars=%s" % ???) # Relevant for KijiSchema 1.1 only and will be removed in Express 3.0: java_opts += " -Dorg.kiji.schema.impl.AvroCellEncoder.SCHEMA_VALIDATION=DISABLED" env["JAVA_OPTS"] = java_opts cmd = [schema_shell_script] logging.debug("Launching kiji-schema shell with:\n%s\with KIJI_CLASSPATH:\n%s", " \\\n\t".join(map(repr, cmd)), "\n".join(map(tab_indent, classpath))) logging.debug("Computed KIJI_CLASSPATH:") proc = subprocess.Popen(cmd, env=env) try: return proc.wait() except subprocess.SubProcessError: proc.kill()
def setup_logging(log_level): """Initializes the logging system. Args: log_level: Logging level. """ global _LOGGING_INITIALIZED if _LOGGING_INITIALIZED: logging.debug("setup_logging: logging system already initialized") return log_formatter = logging.Formatter( fmt="%(asctime)s %(levelname)s %(filename)s:%(lineno)s : %(message)s", ) # Override the log date formatter to include the time zone: def format_time(record, datefmt=None): time_tuple = time.localtime(record.created) tz_name = time.tzname[time_tuple.tm_isdst] return "%(date_time)s-%(millis)03d-%(tz_name)s" % dict( date_time=time.strftime("%Y%m%d-%H%M%S", time_tuple), millis=record.msecs, tz_name=tz_name, ) log_formatter.formatTime = format_time logging.root.handlers.clear() logging.root.setLevel(log_level) console_handler = logging.StreamHandler() console_handler.setFormatter(log_formatter) console_handler.setLevel(log_level) logging.root.addHandler(console_handler) _LOGGING_INITIALIZED = True
def jar(self): """Performs the 'jar' command.""" class_name = getattr(self.flags, "class") if (class_name is None) and (len(self.args) > 0): class_name = self.pop_args_head() assert (class_name is not None), ("No class name specified with [--class=]<class>.") lib_jars = [] if self.flags.jars is not None: lib_jars.extend(self.flags.jars) classpath = list(self.express.get_classpath(lib_jars=lib_jars)) java_opts = [] if self.flags.java_opts is not None: java_opts = [self.flags.java_opts] user_args = list(self.args) logging.info("Running java class %r with parameters: %r", class_name, user_args) cmd = [ "java", # This property is only needed in kiji-schema v1.1 : "-Dorg.kiji.schema.impl.AvroCellEncoder.SCHEMA_VALIDATION=DISABLED", ] + java_opts + [ "-classpath", ":".join(classpath), class_name, ] + user_args logging.debug("Running command:\n%s\n", " \\\n\t".join(map(repr, cmd))) return subprocess.call(cmd)
def process_request(self, request): # SESSION_KEY = '_auth_user_id' # BACKEND_SESSION_KEY = '_auth_user_backend' token = request.GET.get('token', False) token_id = request.GET.get('id', False) timestamp = request.GET.get('timestamp', False) if token and token_id and timestamp: logging.debug('SSO: user %s login attempt via SSO in with timestamp %s and token %s \n' % ( token_id, timestamp, token)) if self.check_token(token, token_id, timestamp): # everything passed, authenticate user logging.debug('SSO: user %s token and timestamp pass \n' % token_id) logging.debug('SSO: Attempting to authenticate as %s \n' % token_id) try: user = self.authenticate(token_id) except Exception as e: logging.debug('SSO: user %s does not exist, trying to create \n' % token_id) rooibos_LDAP.new_account_from_ldap(token_id) if user.username == token_id: try: # THIS WAS THE KEY TO IT WORKING user.backend = settings.SSO_BACKEND login(request, user) logging.debug('SSO: process_request - user.backend = %s' % user.backend) #return None #logging.debug(user.last_login) # logging.debug(request.session['_auth_user_id']) # logging.debug(request.session['_auth_user_backend']) except Exception: raise
def getContestData(contestCode, expiryTime = None, writeInFile = None): logging.debug("In getContestData("+contestCode+')') expiryTime, writeInFile = getGlobals(expiryTime, writeInFile) data = {} if expiryTime > 0: data = checkInFile('contest/' + contestCode, expiryTime) if data is not None: return data else: data = {} URL = "https://www.codechef.com/api/contests/" + contestCode data = json.loads(requests.get(URL, headers={'User-Agent': 'Mozilla/5.0'}).text) #Make start_time and end_time keys directly in data data['start_time'] = data['time']['start'] data['end_time'] = data['time']['end'] #Removing unnecessary keys. keysToRemove = ['problems_data','time','problemsstats', 'user', 'announcements', 'rules', 'autoRefresh', 'banner', 'todos'] data = removeKeys(data, keysToRemove) #From here too. for contest in data['problems']: data['problems'][contest] = removeKeys(data['problems'][contest], ['status_url','submit_url','problem_url','allow_submission']) if writeInFile: writeToFile('contest/' + contestCode, data) logging.debug("getContestData() = " + json.dumps(data, indent = 4)) return data
def _LoadFromFile(self): """Read the version file and set the version components""" with open(self.version_file, 'r') as version_fh: for line in version_fh: if not line.strip(): continue match = self.FindValue('CHROME_BRANCH', line) if match: self.chrome_branch = match logging.debug('Set the Chrome branch number to:%s', self.chrome_branch) continue match = self.FindValue('CHROMEOS_BUILD', line) if match: self.build_number = match logging.debug('Set the build version to:%s', self.build_number) continue match = self.FindValue('CHROMEOS_BRANCH', line) if match: self.branch_build_number = match logging.debug('Set the branch version to:%s', self.branch_build_number) continue match = self.FindValue('CHROMEOS_PATCH', line) if match: self.patch_number = match logging.debug('Set the patch version to:%s', self.patch_number) continue logging.debug(self.VersionString())
def PublishManifest(self, manifest, version, build_id=None): """Publishes the manifest as the manifest for the version to others. Args: manifest: Path to manifest file to publish. version: Manifest version string, e.g. 6102.0.0-rc4 build_id: Optional integer giving build_id of the build that is publishing this manifest. If specified and non-negative, build_id will be included in the commit message. """ # Note: This commit message is used by master.cfg for figuring out when to # trigger slave builders. commit_message = 'Automatic: Start %s %s %s' % (self.build_names[0], self.branch, version) if build_id is not None and build_id >= 0: commit_message += '\nCrOS-Build-Id: %s' % build_id logging.info('Publishing build spec for: %s', version) logging.info('Publishing with commit message: %s', commit_message) logging.debug('Manifest contents below.\n%s', osutils.ReadFile(manifest)) # Copy the manifest into the manifest repository. spec_file = '%s.xml' % os.path.join(self.all_specs_dir, version) osutils.SafeMakedirs(os.path.dirname(spec_file)) shutil.copyfile(manifest, spec_file) # Actually push the manifest. self.PushSpecChanges(commit_message)
def __init__(self, openid_, issued, attrs=None, sreg_=None): logging.debug('init janrain openid object') self.openid = openid_ self.issued = issued self.attrs = attrs or {} self.sreg = sreg_ or {} self.is_iname = (xri.identifierScheme(openid_) == 'XRI')
def run_and_check_vcpupin(vm, vm_ref, vcpu, cpu_list, options): """ Run the vcpupin command and then check the result. """ if vm_ref == "name": vm_ref = vm.name elif vm_ref == "uuid": vm_ref = vm.get_uuid() # Execute virsh vcpupin command. cmdResult = virsh.vcpupin(vm_ref, vcpu, cpu_list, options, debug=True) if cmdResult.exit_status: if not status_error: # Command fail and it is in positive case. raise error.TestFail(cmdResult) else: # Command fail and it is in negative case. return else: if status_error: # Command success and it is in negative case. raise error.TestFail(cmdResult) else: # Command success and it is in positive case. # "--config" will take effect after VM destroyed. pid = None vcpu_pid = None if options == "--config": virsh.destroy(vm.name) else: pid = vm.get_pid() logging.debug("vcpus_pid: %s", vm.get_vcpus_pid()) vcpu_pid = vm.get_vcpus_pid()[vcpu] # Check the result of vcpupin command. check_vcpupin(vm.name, vcpu, cpu_list, pid, vcpu_pid)
def test_restart(self): """test_restart tests that when starting a second vttablet with the same configuration as another one, it will kill the previous process and take over listening on the socket. If vttablet listens to other ports (like gRPC), this feature will break. We believe it is not widely used, so we're OK with this for now. (container based installations usually handle tablet restarts by using a different set of servers, and do not rely on this feature at all). """ if environment.topo_server().flavor() != 'zookeeper': logging.info("Skipping this test in non-github tree") return if tablet_62344.grpc_enabled(): logging.info("Skipping this test as second gRPC port interferes") return utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) # create the database so vttablets start, as it is serving tablet_62344.create_db('vt_test_keyspace') tablet_62344.init_tablet('master', 'test_keyspace', '0') proc1 = tablet_62344.start_vttablet() proc2 = tablet_62344.start_vttablet() for timeout in xrange(20): logging.debug("Sleeping waiting for first process to die") time.sleep(1.0) proc1.poll() if proc1.returncode is not None: break if proc1.returncode is None: self.fail("proc1 still running") tablet_62344.kill_vttablet()
def graph_user(user, depth=0): logging.debug("Searching for %s", user) logging.debug("At depth %d", depth) followers = following = [] result = [x.value for x in db.query(select_user % user)] if result: result = result.pop() following = result['following'] followers = result['followers'] if not GH.has_node(user): logging.debug("Adding %s to graph", user) GH.add_node(user) for follower in followers: if not GH.has_node(follower): GH.add_node(follower) logging.debug("Adding %s to graph", follower) if depth < max_depth: graph_user(follower, depth + 1) GH.add_edge(follower, user, {'weight': 2}) for follow in following: if not GH.has_node(follow): GH.add_node(follow) logging.debug("Adding %s to graph", follow) if depth < max_depth: graph_user(follow, depth + 1) if GH.has_edge(follow, user): GH[follow][user]['weight'] += 1 else: GH.add_edge(user, follow, {'weight': 1})
def execute(self, email_models): logging.debug("In Destiny::execute()") if not email_models: return emails_id = [] destinations = {} for destiny in self._plugins.keys(): destinations.setdefault(destiny, email_models[-1].get(destiny)) emails_id.append(email_models[-1].email_id()) for email_model in email_models[:-1]: for destiny in self._plugins.keys(): d_tables = destinations.get(destiny).get("tables") for d_table in d_tables: for k, v in d_table.iteritems(): m_tables = email_model.get(destiny).get("tables") for m_table in m_tables: if k in m_table: d_table.setdefault(k, []).extend(m_table[k]) emails_id.append(email_model.email_id()) for destiny, models in destinations.iteritems(): for forward in self._plugins.get(destiny): try: forward.execute(models) except Exception, e: logging.error("!! Error-execute: %s" % (str(e),)) logging.info("Add emails in queure error: %s" % str(emails_id)) for email_id in emails_id: self.add_email_error(email_id) continue
def refetch(source): """Refetch the author's URLs and look for new or updated syndication links that might not have been there the first time we looked. Args: source: models.Source subclass. Changes to property values (e.g. domains, domain_urls, last_syndication_url) are stored in source.updates; they should be updated transactionally later. Return: a dict of syndicated_url to a list of new models.SyndicatedPosts """ if not source.updates: source.updates = {} logging.debug('attempting to refetch h-feed for %s', source.label()) results = {} for url in _get_author_urls(source): results.update(_process_author(source, url, refetch=True)) now = util.now_fn() logging.debug('updating source last_hfeed_fetch %s', now) source.updates['last_hfeed_fetch'] = now return results
def call(self, function, params=None): self.requestPerMinute += 1 now = datetime.utcnow() if self.requestPerMinute >= self.requestLimit: waittime = 60 - now.second logging.warning("Limit for request per minute exceeded. Waiting for: {0} sec.".format(waittime)) time.sleep(waittime) now = datetime.utcnow() if self.checkMinute != now.minute: self.requestPerMinute = 0 self.checkMinute = now.minute payload = '' try: p = "" if not params else '?' + "&".join( ["{key}={value}".format(key=k, value=v) for (k, v) in params.iteritems()]) url = "{base}.{func}{params}".format(base=self.baseConfig["url"], func=function, params=p) logging.debug("{0} {1} API call:{2}".format(self.checkMinute, self.requestPerMinute, url)) request = urllib2.Request(url, None, self.baseConfig["headers"]) stream = urllib2.urlopen(request) payload = stream.read() data = json.loads(payload) if isinstance(data, dict) and 'ruid' in data: logging.error('Api call failed with error: {0} Code: {1}'.format(data['message'], data['code'])) return None return data except Exception as e: logging.error('Error: {0} Context: {1}'.format(e, payload)) return None
def make_parser(): """ Construct the command line parser """ logging.info("Constructing parser") description = "Store and retrieve snippets of text" parser = argparse.ArgumentParser(description=description) subparsers = parser.add_subparsers(help="Available commands") # Subparser for the put command logging.debug("Constructing put subparser") put_parser = subparsers.add_parser("put", help="Store a snippet") put_parser.add_argument("name", help="The name of the snippet") put_parser.add_argument("snippet", help="The snippet text") put_parser.add_argument("filename", default="snippets.csv", nargs="?", help="The snippet filename") put_parser.set_defaults(command="put") # Subparser for the get command logging.debug("Constructing get subparser") put_parser = subparsers.add_parser("get", help="Retrieve a snippet") put_parser.add_argument("name", help="The name of the snippet") put_parser.add_argument("filename", default="snippets.csv", nargs="?", help="The snippet filename") put_parser.set_defaults(command="get") return parser
def upsert_prelude_project(self,path=None,filename=None,uuid=None,version=None,nclips=None): cursor=self.conn.cursor() self.conn.commit() #if uuid is None: # raise DataError("You need to pass a valid uuid") #does the project entry already exist? If so leave it (avoiding database bloat) cursor.execute("SELECT id FROM prelude_projects WHERE filename=%s AND filepath=%s", (filename, path, )) result = cursor.fetchone() if result is not None: logging.debug("Prelude project {0}/{1} already exists in database, not touching it".format(path, filename)) return result[0] try: sqlcmd = """insert into prelude_projects (filepath,filename,uuid,version,clips,lastseen) values (%s,%s,%s,%s,%s,now()) returning id""" cursor.execute(sqlcmd,(path,filename,uuid,version,nclips)) except psycopg2.IntegrityError as e: #if we violate unique keys, try to update on filename self.conn.rollback() try: sqlcmd = """update prelude_projects set filepath=%s, filename=%s, uuid=%s, version=%s, clips=%s, lastseen=now() where filepath=%s and filename=%s returning id""" cursor.execute(sqlcmd,(path,filename,uuid,version,nclips,path,filename)) except psycopg2.IntegrityError as e: #if that causes a violation, try to update on uuid self.conn.rollback() sqlcmd = """update prelude_projects set filepath=%s, filename=%s, uuid=%s, version=%s, clips=%s, lastseen=now() where uuid=%s returning id""" cursor.execute(sqlcmd,(path,filename,uuid,version,nclips,uuid)) self.conn.commit() result=cursor.fetchone() return result[0] #return id of inserted row
def execute(self): targets = self.get_targets(self.is_non_synthetic_python_target) with self.invalidated(targets=targets) as invalidation_check: if not invalidation_check.invalid_vts: logging.debug(self.NOOP_MSG_HAS_TARGET_BUT_NO_SOURCE) return invalid_tgts = [vt.target for vt in invalidation_check.invalid_vts] sources = self._calculate_isortable_python_sources(invalid_tgts) if not sources: logging.debug(self.NOOP_MSG_HAS_TARGET_BUT_NO_SOURCE) return isort = self.context.products.get_data(IsortPrep.Isort) args = self.get_passthru_args() + sources # NB: We execute isort out of process to avoid unwanted side-effects from importing it: # https://github.com/timothycrosley/isort/issues/456 with pushd(get_buildroot()): workunit_factory = functools.partial(self.context.new_workunit, name='run-isort', labels=[WorkUnitLabel.TOOL, WorkUnitLabel.LINT]) cmdline, exit_code = isort.run(workunit_factory, args) if exit_code != 0: raise TaskError('{} ... exited non-zero ({}).'.format(cmdline, exit_code), exit_code=exit_code)
def __public_response(self, messages): message = None for m in messages: pretty_message = "%s [%s %s] %s" % (m.id, m.created_at, m.user.screen_name, m.text) logging.info("found public message: %s" % pretty_message) if not self.__analyzer.should_respond(m): logging.info("not responding") continue response = TwitterResponseAccessor.get_by_message_id(str(m.id)) if not response: message = m break else: logging.debug("found response to public message %s" % m.id) sent_message = None if message: # TODO: search for username also username = message.user.screen_name parsed_tweet = parse_tweet(message.text) plain_tweet = parsed_tweet.plain_text speaker = self.__select_speaker() sources, mix = Mixer(speaker).mix_response(plain_tweet, min_results=1, max_length=130-len(username)) response_text = "@%s %s" % (username, mix) logging.info("responding to public message %s: %s" % (message.id, response_text)) sent_message = self.__twitter.PostUpdate(response_text, message.id) TwitterResponseAccessor.create(str(message.id), response_id=str(sent_message.id), user=username, tweet_type=TwitterResponse.MENTION) self.__reporter.posted(response_text) return sent_message
def act(self, force_act=False, action=None, skip_responses=False): """ returns: (action, response) tuple. response type depends on the action that was performed. """ if not force_act: config = ConfigurationAccessor.get_or_create() if config and (config.is_tweeting is not None) and (not safe_int(config.is_tweeting)): logging.debug("config.is_tweeting is False; hiding") return () result = [] responded = False if not skip_responses: try: direct, response = self.respond() if (direct or response): # a response to a direct message or mention was generated responded = True if direct: result.append(direct.AsDict()) if response: result.append(response.AsDict()) except Exception, e: logging.error(e)
def test_rollback(self): return try: vtgate_conn = get_connection() count = 10 vtgate_conn.begin() vtgate_conn._execute( "delete from vt_insert_test", {}, KEYSPACE_NAME, 'master', keyranges=[self.keyrange]) kid_list = shard_kid_map[shard_names[self.shard_index]] for x in xrange(count): keyspace_id = kid_list[count%len(kid_list)] vtgate_conn._execute( "insert into vt_insert_test (msg, keyspace_id) values (%(msg)s, %(keyspace_id)s)", {'msg': 'test %s' % x, 'keyspace_id': keyspace_id}, KEYSPACE_NAME, 'master', keyspace_ids=[pack_kid(keyspace_id)]) vtgate_conn.commit() vtgate_conn.begin() vtgate_conn._execute( "delete from vt_insert_test", {}, KEYSPACE_NAME, 'master', keyranges=[self.keyrange]) vtgate_conn.rollback() results, rowcount = vtgate_conn._execute( "select * from vt_insert_test", {}, KEYSPACE_NAME, 'master', keyranges=[self.keyrange])[:2] logging.debug("ROLLBACK TEST rowcount %d count %d" % (rowcount, count)) self.assertEqual(rowcount, count, "Fetched rows(%d) != inserted rows(%d), rollback didn't work" % (rowcount, count)) except Exception, e: logging.debug("Write failed with error %s" % str(e)) raise
def restore_zk(filename): """ Restores Zookeeper data from a fixed file in the local FS. Args: filename: A str, the path to the temporary Zookeeper backup file. """ handle = kazoo.client.KazooClient(hosts=ZK_DEFAULT_HOST) handle.start() with open(filename, 'r') as f: for line in f.readlines(): pair = json.loads(line) path = pair.keys()[0] value = pair.values()[0].decode('base64') try: handle.create(path, bytes(value), makepath=True) logging.debug("Created '{0}'".format(path)) except kazoo.exceptions.NodeExistsError: try: handle.set(path, bytes(value)) logging.debug("Updated '{0}'".format(path)) except kazoo.exceptions.BadArgumentsError: logging.warning("BadArgumentsError for path '{0}'".format(path)) except kazoo.exceptions.NoNodeError: logging.warning("NoNodeError for path '{0}'. Parent nodes are " "missing".format(path)) except kazoo.exceptions.ZookeeperError: logging.warning("ZookeeperError for path '{0}'".format(path)) handle.stop()
def process_users(users_file, keytab_path): with open(users_file) as csvfile: r = csv.DictReader(csvfile, delimiter=',') for row in r: if row['Name'] == '#': continue d = { 'username': row['Name'], 'description': row['Description'], 'first': row['Name'], 'last': row['Name'], 'realm': REALM, 'ipa_server': IPA_SERVER, 'path_prefix': keytab_path } if not user_exists(d['username']): create_user(d) # When a user is created in IPA, his password autoexpires. # Need to explicitly login or set the expiration to a later date so that the keytab # can be retrieved and used later change_krb_expiration(d['username']) create_keytab(d) for g in BASE_GROUPS: add_group(d['username'], g) else: logging.debug('User %s already exists' % d['username'])
def apply(self, ui): logging.debug(self.movement) if self.movement == 'last': ui.current_buffer.focus_last() ui.update() else: MoveCommand.apply(self, ui)
def traits(args): """ %prog traits directory Make HTML page that reports eye and skin color. """ p = OptionParser(traits.__doc__) opts, args = p.parse_args(args) if len(args) < 1: sys.exit(not p.print_help()) samples = [] for folder in args: targets = iglob(folder, "*-traits.json") if not targets: continue filename = targets[0] js = json.load(open(filename)) js["skin_rgb"] = make_rgb( js["traits"]["skin-color"]["L"], js["traits"]["skin-color"]["A"], js["traits"]["skin-color"]["B"]) js["eye_rgb"] = make_rgb( js["traits"]["eye-color"]["L"], js["traits"]["eye-color"]["A"], js["traits"]["eye-color"]["B"]) samples.append(js) template = Template(traits_template) fw = open("report.html", "w") print >> fw, template.render(samples=samples) logging.debug("Report written to `{}`".format(fw.name)) fw.close()
def upsert_edit_project(self,filepath,filename,uuid,version,desc=None,opens_with=None): cursor = self.conn.cursor() matches=re.search(u'(\.[^\.]+)$',filename) file_xtn="" if matches is not None: file_xtn=str(matches.group(1)) else: raise ArgumentError("Filename %s does not appear to have a file extension" % filename) typenum=self.project_type_for_extension(file_xtn,desc=desc,opens_with=opens_with) #does the project entry already exist? If so leave it (avoiding database bloat) cursor.execute("SELECT id FROM edit_projects WHERE filename=%s AND filepath=%s", (filename, filepath, )) result = cursor.fetchone() if result is not None: logging.debug("Edit project {0}/{1} already exists in database, not touching it".format(filepath, filename)) return result[0] try: cursor.execute("insert into edit_projects (filename,filepath,type,lastseen,valid) values (%s,%s,%s,now(),true) returning id", (filename,filepath,typenum)) except psycopg2.IntegrityError as e: #this is kept in case of race conditions self.conn.rollback() cursor.execute("update edit_projects set lastseen=now(), valid=true where filename=%s and filepath=%s returning id", (filename,filepath)) result = cursor.fetchone() id = result[0] sqlcmd="update edit_projects set uuid=%s, version=%s where id=%s" cursor.execute(sqlcmd, (uuid,version,id)) self.conn.commit() return id
def GetDevices(self, sender, connection): logging.debug("GetDevices") return self.devices
def main(): # Parse arguments and pass through unrecognised args parser = argparse.ArgumentParser( add_help=False, usage='%(prog)s [test_runner.py options] [script options] [scripts]', description=__doc__, epilog=''' Help text and arguments for individual test script:''', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument( '--combinedlogslen', '-c', type=int, default=0, help= 'print a combined log (of length n lines) from all test nodes and test framework to the console on failure.' ) parser.add_argument( '--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') parser.add_argument( '--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.') parser.add_argument( '--extended', action='store_true', help='run the extended test suite in addition to the basic tests') parser.add_argument( '--force', '-f', action='store_true', help= 'run tests even on platforms where they are disabled by default (e.g. windows).' ) parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') parser.add_argument( '--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') parser.add_argument( '--keepcache', '-k', action='store_true', help= 'the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.' ) parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs') parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs") args, unknown_args = parser.parse_known_args() # args to be passed on always start with two dashes; tests are the remaining unknown args tests = [arg for arg in unknown_args if arg[:2] != "--"] passon_args = [arg for arg in unknown_args if arg[:2] == "--"] # Read config generated by configure. config = configparser.ConfigParser() configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" config.read_file(open(configfile)) passon_args.append("--configfile=%s" % configfile) # Set up logging logging_level = logging.INFO if args.quiet else logging.DEBUG logging.basicConfig(format='%(message)s', level=logging_level) # Create base test directory tmpdir = "%s/underline_test_runner_%s" % ( args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S")) os.makedirs(tmpdir) logging.debug("Temporary test directory at %s" % tmpdir) enable_wallet = config["components"].getboolean("ENABLE_WALLET") enable_utils = config["components"].getboolean("ENABLE_UTILS") enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") if config["environment"]["EXEEXT"] == ".exe" and not args.force: # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 print( "Tests currently disabled on Windows by default. Use --force option to enable" ) sys.exit(0) if not (enable_wallet and enable_utils and enable_bitcoind): print( "No functional tests to run. Wallet, utils, and underlined must all be enabled" ) print( "Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make" ) sys.exit(0) # Build list of tests if tests: # Individual tests have been specified. Run specified tests that exist # in the ALL_SCRIPTS list. Accept the name with or without .py extension. tests = [re.sub("\.py$", "", t) + ".py" for t in tests] test_list = [] for t in tests: if t in ALL_SCRIPTS: test_list.append(t) else: print("{}WARNING!{} Test '{}' not found in full test list.". format(BOLD[1], BOLD[0], t)) else: # No individual tests have been specified. # Run all base tests, and optionally run extended tests. test_list = BASE_SCRIPTS if args.extended: # place the EXTENDED_SCRIPTS first since the three longest ones # are there and the list is shorter test_list = EXTENDED_SCRIPTS + test_list # Remove the test cases that the user has explicitly asked to exclude. if args.exclude: tests_excl = [ re.sub("\.py$", "", t) + ".py" for t in args.exclude.split(',') ] for exclude_test in tests_excl: if exclude_test in test_list: test_list.remove(exclude_test) else: print("{}WARNING!{} Test '{}' not found in current test list.". format(BOLD[1], BOLD[0], exclude_test)) if not test_list: print( "No valid test scripts specified. Check that your test is in one " "of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests" ) sys.exit(0) if args.help: # Print help for test_runner.py, then print help of the first script (with args removed) and exit. parser.print_help() subprocess.check_call([(config["environment"]["SRCDIR"] + '/test/functional/' + test_list[0].split()[0])] + ['-h']) sys.exit(0) check_script_list(config["environment"]["SRCDIR"]) check_script_prefixes() if not args.keepcache: shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True) run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args, args.combinedlogslen)
def extract_arguments(self, method_arg): i = 0 args = [] isArray = False while i < len(method_arg): # class or interface if method_arg[i] == "L": if method_arg[i - 1] == "[": isArray = True # check array if isArray: args.append('[L' + method_arg[i + 1:method_arg.find(';')] + ';') isArray = False else: args.append(method_arg[i + 1:method_arg.find(';')]) logging.debug(' ' + method_arg[i + 1:method_arg.find(';')]) i = method_arg.find(';') + 1 method_arg = method_arg.replace(';', ' ', 1) continue # Int if method_arg[i] == "I": if method_arg[i - 1] == "[": isArray = True if isArray: args.append('[I') isArray = False else: args.append('int') # Boolean if method_arg[i] == "Z": if method_arg[i - 1] == "[": isArray = True if isArray: args.append('[Z') isArray = False else: args.append('boolean') # Float if method_arg[i] == "F": if method_arg[i - 1] == "[": isArray = True if isArray: args.append('[F') isArray = False else: args.append('float') # Long if method_arg[i] == "J": if method_arg[i - 1] == "[": isArray = True if isArray: args.append('[J') isArray = False else: args.append('long') # Double if method_arg[i] == "D": if method_arg[i - 1] == "[": isArray = True if isArray: args.append('[D') isArray = False else: args.append('double') # Char if method_arg[i] == "C": if method_arg[i - 1] == "[": isArray = True if isArray: args.append('[C') isArray = False else: args.append('char') # Byte if method_arg[i] == "B": if method_arg[i - 1] == "[": isArray = True if isArray: args.append('[B') isArray = False else: args.append('byte') # Short if method_arg[i] == "S": if method_arg[i - 1] == "[": isArray = True if isArray: args.append('[S') isArray = False else: args.append('short') i += 1 return args
def create_javascript(self, smali_file_path): javascript = "Java.perform(function() {\n" with open(smali_file_path, 'r') as smali_file: # JavaScript logging.debug(' ' + smali_file_path + '\n') # read by line smali_file_lines = smali_file.readlines() for smali_file_line in smali_file_lines: # find class name if smali_file_line.split(' ')[0] == ".class": # get full class name smali_file_line = smali_file_line.split(' ') smali_file_line.reverse() full_class_name = smali_file_line[0][1:] full_class_name = full_class_name[:len(full_class_name) - 2].replace('/', '.') logging.debug(" full_class_name: " + full_class_name) # get class name class_name = full_class_name.split('.') class_name.reverse() class_name = class_name[0] logging.debug(" class_name: " + class_name) # declare class javascript += ' var ' + class_name + ' = Java.use(\'' + full_class_name + '\');\n\n' continue # find .method if smali_file_line.split(' ')[0] == ".method": # constructor hook if 'constructor' in smali_file_line and \ not 'static' in smali_file_line: args = smali_file_line.split(' ').pop() args = args[args.find('(') + 1:args.rfind(')')] args_list = self.extract_arguments(args) #print(args_list) args_len = len(args_list) args_string = '' args_quota_added = [] for i in range(args_len): # replace / to . args_list[i] = args_list[i].replace('/', '.') args_quota_added.append('\'' + args_list[i] + '\'') # arg string create ex) arg0,arg1 if i == args_len - 1: args_string += 'arg' + str(i) else: args_string += 'arg' + str(i) + ',' javascript += ' ' + class_name + '.$init.overload(' + ','.join( args_quota_added ) + ').implementation = function(' + args_string + ') {\n' javascript += ' console.log(\'[Constructor] ' + full_class_name + '(' + ','.join( args_list) + ')\');\n' for i in range(args_len): javascript += ' console.log(\' [arg' + str( i) + '] \' + arg' + str(i) + ');\n' javascript += ' return this.$init(' + args_string + ');\n' javascript += ' };\n\n' # method hook elif not 'constructor' in smali_file_line: logging.debug("") method_line_list = smali_file_line.split(' ') method_line_list.reverse() method_line = method_line_list[0] # extract method method_name = method_line.split('(')[0] method_arg = method_line.split('(')[1].split(')')[0] method_ret_type = method_line.split(')')[1].split( '\n')[0] logging.debug(" method_name: " + method_name) logging.debug(" method_arg: " + method_arg) logging.debug(" method_ret_type: " + method_ret_type) # args extract if len(method_arg) == 0: # args is not exist javascript += ' ' + class_name + '.' + method_name + '.overload().implementation = function(){\n' javascript += ' console.log(\'[Method] ' + full_class_name + '.' + method_name + '()\');\n' # create retval javascript += ' var retval = this.' + method_name + '();\n' # logging.info return if method_ret_type != 'V': javascript += ' console.log(\' [ret] \' + retval);\n' # return method javascript += ' return retval;\n' javascript += ' };\n\n' continue else: # args exist args_list = self.extract_arguments(method_arg) logging.debug(" args_list: " + str(args_list)) args_string = '' args_len = len(args_list) args_quota_added = [] for i in range(args_len): # replace / to . args_list[i] = args_list[i].replace('/', '.') args_quota_added.append('\'' + args_list[i] + '\'') # arg string create args_string += 'arg' + str(i) # if last arg if i != args_len - 1: args_string += ',' javascript += ' ' + class_name + '.' + method_name + '.overload(' + ','.join( args_quota_added ) + ').implementation = function(' + args_string + '){\n' # print hook method name javascript += ' console.log(\'[Method] ' + full_class_name + '.' + method_name + '(' + ','.join( args_list) + ')\');\n' # print args for i in range(args_len): javascript += ' console.log(\' [arg' + str( i) + '] \' + arg' + str(i) + ');\n' # create retval javascript += ' var retval = this.' + method_name + '(' + args_string + ');\n' # print ret if method_ret_type != 'V': javascript += ' console.log(\' [ret] \' + retval);\n' # return method javascript += ' return retval;\n' javascript += ' };\n\n' javascript += "});\n" return javascript
sys.exit(1) ssrcObj = Proxy(("PYRO:{}@{}:{}").format( SSRC.DAEMON.PYRO_OBJECT_ID, SSRC.DAEMON.PYRO_HOST, SSRC.DAEMON.PYRO_PORT)) try: if(options.shutdown): ssrcObj.Exit() elif(options.status): print("{:.2f}".format(ssrcObj.Status())) else: if(options.in_band): if(options.hot): ssrcObj.Adjust(SSRC.TemperatureState.WARM) logging.debug( "Adjust down 1% to {:.2f}%".format(ssrcObj.Status())) elif(options.cold): ssrcObj.Adjust(SSRC.TemperatureState.COOL) logging.debug( "Adjust up 0.25% to {:.2f}%".format(ssrcObj.Status())) else: ssrcObj.Adjust(SSRC.TemperatureState.PERFECT) logging.debug( "Stay at {:.2f}%".format(ssrcObj.Status())) else: if(options.hot): ssrcObj.Adjust(SSRC.TemperatureState.HOT) logging.debug( "Adjust down 20% to {:.2f}%".format(ssrcObj.Status())) elif(options.cold): ssrcObj.Adjust(SSRC.TemperatureState.COLD)
import openpyxl, logging logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') from openpyxl.styles import Font from openpyxl.utils import get_column_letter wb = openpyxl.Workbook() logging.debug(wb.worksheets) wb.create_sheet(title='Tables',index=0) logging.debug(wb.worksheets) sheet = wb.worksheets[0] #First row Frozen and BOLD fontObj = Font(name='Times New Roman', bold=True) for colNum in range(1,10): sheet.column_dimensions[get_column_letter(colNum)].width = 15 sheet.cell(row=1, column=colNum).font = fontObj sheet.cell(row=1,column=colNum).value = "Number = " + str(colNum) sheet = wb.active sheet.freeze_panes = 'a2' #Rest of the rows (tables upto 20) for colNum in range(1,sheet.max_column+1): for rowNum in range(2,22): sheet.cell(row = rowNum, column = colNum).value = (rowNum-1)*colNum #merge last row sheet.merge_cells('a23:i23') #SUM function for all the tables for colNum in range(1,10): colAlpha = get_column_letter(colNum)
def _SubMain(log_level, tool_prefix, output_directory): logging.basicConfig( level=int(log_level), format='nm: %(levelname).1s %(relativeCreated)6d %(message)s') bulk_analyzer = _BulkObjectFileAnalyzerWorker(tool_prefix, output_directory) while True: payload_len = int(sys.stdin.read(8) or '0', 16) if not payload_len: logging.debug('nm bulk subprocess received eof.') break paths = sys.stdin.read(payload_len).split('\x01') bulk_analyzer.AnalyzePaths(paths) bulk_analyzer.Close() paths_by_name = bulk_analyzer.Get() encoded_keys, encoded_values = concurrent.EncodeDictOfLists(paths_by_name) try: sys.stdout.write('%08x' % len(encoded_keys)) sys.stdout.write(encoded_keys) sys.stdout.write(encoded_values) except IOError, e: # Parent process exited. if e.errno == errno.EPIPE: sys.exit(1) logging.debug('nm bulk subprocess finished.') if __name__ == '__main__': _SubMain(*sys.argv[1:])
def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=[], combined_logs_len=0): # Warn if bitcoind is already running (unix only) try: if subprocess.check_output(["pidof", "underlined"]) is not None: print( "%sWARNING!%s There is already a underlined process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0])) except (OSError, subprocess.SubprocessError): pass # Warn if there is a cache directory cache_dir = "%s/test/cache" % build_dir if os.path.isdir(cache_dir): print( "%sWARNING!%s There is a cache directory here: %s. If tests fail unexpectedly, try deleting the cache directory." % (BOLD[1], BOLD[0], cache_dir)) #Set env vars if "LITECOIND" not in os.environ: os.environ["LITECOIND"] = build_dir + '/src/underlined' + exeext os.environ["LITECOINCLI"] = build_dir + '/src/underline-cli' + exeext tests_dir = src_dir + '/test/functional/' flags = ["--srcdir={}/src".format(build_dir)] + args flags.append("--cachedir=%s" % cache_dir) if enable_coverage: coverage = RPCCoverage() flags.append(coverage.flag) logging.debug("Initializing coverage directory at %s" % coverage.dir) else: coverage = None if len(test_list) > 1 and jobs > 1: # Populate cache try: subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir]) except subprocess.CalledProcessError as e: sys.stdout.buffer.write(e.output) raise #Run Tests job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags) time0 = time.time() test_results = [] max_len_name = len(max(test_list, key=len)) for _ in range(len(test_list)): test_result, testdir, stdout, stderr = job_queue.get_next() test_results.append(test_result) if test_result.status == "Passed": logging.debug( "\n%s%s%s passed, Duration: %s s" % (BOLD[1], test_result.name, BOLD[0], test_result.time)) elif test_result.status == "Skipped": logging.debug("\n%s%s%s skipped" % (BOLD[1], test_result.name, BOLD[0])) else: print("\n%s%s%s failed, Duration: %s s\n" % (BOLD[1], test_result.name, BOLD[0], test_result.time)) print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n') print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n') if combined_logs_len and os.path.isdir(testdir): # Print the final `combinedlogslen` lines of the combined logs print('{}Combine the logs and print the last {} lines ...{}'. format(BOLD[1], combined_logs_len, BOLD[0])) print('\n============') print('{}Combined log for {}:{}'.format( BOLD[1], testdir, BOLD[0])) print('============\n') combined_logs, _ = subprocess.Popen( [ os.path.join(tests_dir, 'combine_logs.py'), '-c', testdir ], universal_newlines=True, stdout=subprocess.PIPE).communicate() print("\n".join( deque(combined_logs.splitlines(), combined_logs_len))) print_results(test_results, max_len_name, (int(time.time() - time0))) if coverage: coverage.report_rpc_coverage() logging.debug("Cleaning up coverage data") coverage.cleanup() # Clear up the temp directory if all subdirectories are gone if not os.listdir(tmpdir): os.rmdir(tmpdir) all_passed = all( map(lambda test_result: test_result.was_successful, test_results)) sys.exit(not all_passed)
def init(args): encoding = args.get('--encoding') extra_ignore_dirs = args.get('--ignore') follow_links = not args.get('--no-follow-links') input_path = args['<path>'] if input_path is None: input_path = os.path.abspath(os.curdir) if extra_ignore_dirs: extra_ignore_dirs = extra_ignore_dirs.split(',') candidates = get_all_imports(input_path, encoding=encoding, extra_ignore_dirs=extra_ignore_dirs, follow_links=follow_links) candidates = get_pkg_names(candidates) logging.debug("Found imports: " + ", ".join(candidates)) pypi_server = "https://pypi.python.org/pypi/" proxy = None if args["--pypi-server"]: pypi_server = args["--pypi-server"] if args["--proxy"]: proxy = {'http': args["--proxy"], 'https': args["--proxy"]} if args["--use-local"]: logging.debug( "Getting package information ONLY from local installation.") imports = get_import_local(candidates, encoding=encoding) else: logging.debug("Getting packages information from Local/PyPI") local = get_import_local(candidates, encoding=encoding) # Get packages that were not found locally difference = [x for x in candidates if x.lower() not in [z['name'].lower() for z in local]] imports = local + get_imports_info(difference, proxy=proxy, pypi_server=pypi_server) path = (args["--savepath"] if args["--savepath"] else os.path.join(input_path, "requirements.txt")) if args["--diff"]: diff(args["--diff"], imports) return if args["--clean"]: clean(args["--clean"], imports) return if (not args["--print"] and not args["--savepath"] and not args["--force"] and os.path.exists(path)): logging.warning("Requirements.txt already exists, " "use --force to overwrite it") return if args.get('--no-pin'): imports = [{'name': item["name"], 'version': ''} for item in imports] if args["--print"]: output_requirements(imports) logging.info("Successfully output requirements") else: generate_requirements_file(path, imports) logging.info("Successfully saved requirements file in " + path)
def get_all_imports( path, encoding=None, extra_ignore_dirs=None, follow_links=True): imports = set() raw_imports = set() candidates = [] ignore_errors = False ignore_dirs = [".hg", ".svn", ".git", ".tox", "__pycache__", "env", "venv"] if extra_ignore_dirs: ignore_dirs_parsed = [] for e in extra_ignore_dirs: ignore_dirs_parsed.append(os.path.basename(os.path.realpath(e))) ignore_dirs.extend(ignore_dirs_parsed) walk = os.walk(path, followlinks=follow_links) for root, dirs, files in walk: dirs[:] = [d for d in dirs if d not in ignore_dirs] candidates.append(os.path.basename(root)) files = [fn for fn in files if os.path.splitext(fn)[1] == ".py"] candidates += [os.path.splitext(fn)[0] for fn in files] for file_name in files: file_name = os.path.join(root, file_name) with open_func(file_name, "r", encoding=encoding) as f: contents = f.read() try: tree = ast.parse(contents) for node in ast.walk(tree): if isinstance(node, ast.Import): for subnode in node.names: raw_imports.add(subnode.name) elif isinstance(node, ast.ImportFrom): raw_imports.add(node.module) except Exception as exc: if ignore_errors: traceback.print_exc(exc) logging.warn("Failed on file: %s" % file_name) continue else: logging.error("Failed on file: %s" % file_name) raise exc # Clean up imports for name in [n for n in raw_imports if n]: # Sanity check: Name could have been None if the import # statement was as ``from . import X`` # Cleanup: We only want to first part of the import. # Ex: from django.conf --> django.conf. But we only want django # as an import. cleaned_name, _, _ = name.partition('.') imports.add(cleaned_name) packages = imports - (set(candidates) & imports) logging.debug('Found packages: {0}'.format(packages)) with open(join("stdlib"), "r") as f: data = {x.strip() for x in f} data = {x for x in data if x not in py2_exclude} if py2 else data return list(packages - data)
#! python3 # mapIt.py - Launches a map in the browser using an address from the # command line or clipboard. import logging logging.disable(logging.CRITICAL) # comment to enable debug logging output logging.basicConfig(level=logging.DEBUG, format='%(asctime)s- %(levelname)s- %(message)s') import webbrowser, sys, pyperclip logging.debug('sys.argv variable is (%s)' % sys.argv) if len(sys.argv) > 1: # Get address from command line address = ' '.join(sys.argv[1:]) logging.debug('Address entered is (%s)' % address) else: # Get address from clipboard address = pyperclip.paste() logging.debug('Address copied from clipboard is (%s)' % address) webbrowser.open('https://www.google.com/maps/place/' + address)
if v not in d: print('%s not key in map file %s' % (v,sMapFile)) dMapper[k] = d[v] if v in d else dMapper[k] return dMapper,dMasterPtrn def ArgParser(): parser = argparse.ArgumentParser(description='This code for converting ctm alignment file to praat TextGrid file', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('ctm_dir', help='The path to the directory contains ctm files', type=str) parser.add_argument('phones', help='The path to the phones file that mapping between int to symbol', type=str) parser.add_argument('out_dir', help='The path to store TextGrid files', type=str) parser.add_argument('-l', '--lexicon', help='The lexicon file mapping between words tp phoneme sequence', dest='lexicon', type=str, default='') parser.add_argument('-m', '--phMap', help='Mapping phoneme symbols to other symbols (i.e samba to ipa)', dest='phMap', type=str, default='') return parser.parse_args() if __name__ == '__main__': args = ArgParser() sCtmDir, sPhonesFile, sOutDir = args.ctm_dir, args.phones, args.out_dir sPhoneMapFile = args.phMap #Regular expression to remove the position suffix from the key of dictionary ptrn_pos = re.compile('_[BIES]') dMapper,rPtrn = prepare_mapper(((sPhonesFile,None),(sPhoneMapFile,ptrn_pos))) log.info("Mapper have been created from %s" % ' '.join((sPhonesFile,sPhoneMapFile))) lCtmFiles = glob.glob(join(sCtmDir,'*.ctm')) log.info("%d cmt files found in %s" % (len(lCtmFiles),sCtmDir)) dAlign = defaultdict(list) for fCtm in lCtmFiles: log.debug("Now processing %s" % fCtm) loadCMT(fCtm,dAlign) Generate_TxtGrid(sOutDir, dAlign, dMapper=dMapper, cSpkrIdDil='-', sSuffix = '',bWordTier = False)
frame = "fk5" if results.pos: ra, dec = results.pos pos = SkyCoord([ra] * u.degree, [dec] * u.degree, frame=frame) log.info(os.path.join(datadir, "Halpha_error.fits")) sm = SM( ha_file=os.path.join(datadir, "Halpha_map.fits"), err_file=os.path.join(datadir, "Halpha_error.fits"), nu=nu, log=log, d=d, v=v, ) if results.halpha: logging.debug(sm.get_halpha(pos)) val, err = sm.get_halpha(pos) print("Halpha: ", val, "(Rayleighs)") print("err_Halpha: ", err, "(Rayleighs)") if results.xi: val, err = sm.get_xi(pos) print("xi: ", val) print("err_xi: ", err) if results.sm: val, err = sm.get_sm(pos) print("sm: ", val, "kpc m^{-20/3}") print("err_sm: ", err, "kpc m^{-20/3}") if results.m: val, err = sm.get_m(pos) print("m: ", val * 100, "%") print("err_m: ", err * 100, "%")
def __init__(self): self.__dbpool = psycopg2.pool.ThreadedConnectionPool(self.__minconn, self.__maxconn, database=self.__database, user=self.__dbuser, password=self.__password, host=self.__host, port=self.__port) logging.debug("Opened database Pool successfully")
def parse_response(response, debug=False): data = response.json() if debug: logging.debug('REQUEST: %s' % response.request.url) logging.debug('REQUEST_HEADERS: %s' % response.request.headers) logging.debug('REQUEST_CONTENT: %s' % response.request.body) logging.debug('RESPONSE: %s' % response.content) logging.debug('RESP_HEADERS: %s' % response.headers) logging.debug('RESP_CODE: %s' % response.status_code) return data
def printing(self): print("inside the metaDB print function") for eachMetaRow in self.tagsList: logging.debug("%s, %s, %s, %s, %s ", eachMetaRow.timestamp, eachMetaRow.filename, eachMetaRow.startline, eachMetaRow.endline, eachMetaRow.keywords)
def run_process(args): """Process qrc files.""" import inspect import qrainbowstyle.palette as source palettes = [] for name, obj in inspect.getmembers(source): if inspect.isclass(obj) and issubclass( obj, source.BasePalette) and obj is not source.BasePalette: palettes.append(obj) logging.debug("Found palettes: " + str(palettes)) for palette in palettes: palette_name = str(palette.__name__) logging.debug("Generating files for: " + palette_name) # create directory for every style in palette.py os.chdir(STYLES_PATH) os.makedirs(palette_name, exist_ok=True) os.chdir(os.path.join(STYLES_PATH, palette_name)) open("__init__.py", "w+").close() output_dir = os.path.join(STYLES_PATH, palette_name) # get paths to output directories for this palette images_dir = os.path.join(output_dir, 'images') rc_dir = os.path.join(output_dir, 'rc') # qss_dir = os.path.join(output_dir, 'qss') # create directories os.makedirs(images_dir, exist_ok=True) os.makedirs(rc_dir, exist_ok=True) # os.makedirs(qss_dir, exist_ok=True) qrc_filepath = os.path.join(output_dir, QRC_FILE) qss_filepath = os.path.join(output_dir, QSS_FILE) # variables_scss_filepath = os.path.join(qss_dir, VARIABLES_SCSS_FILE) # Create palette and resources png images logging.debug('Generating palette image ...') create_palette_image(palette=palette, path=images_dir) logging.debug('Generating images ...') create_images(palette=palette, rc_path=rc_dir) logging.debug("Generating images for titlebar buttons") create_titlebar_images(rc_path=rc_dir, palette=palette) logging.debug('Generating qrc ...') generate_qrc_file(rc_path=rc_dir, qrc_path=qrc_filepath) logging.debug('Converting .qrc to _rc.py and/or .rcc ...') for qrc_file in glob.glob('*.qrc'): # get name without extension filename = os.path.splitext(qrc_file)[0] logging.debug(filename + '...') ext = '_rc.py' ext_c = '.rcc' # Create variables SCSS files and compile SCSS files to QSS logging.debug('Compiling SCSS/SASS files to QSS ...') create_qss(palette=palette, qss_filepath=qss_filepath) # creating names py_file_pyqt5 = 'pyqt5_' + filename + ext py_file_pyqt = 'pyqt_' + filename + ext py_file_pyside = 'pyside_' + filename + ext py_file_pyside2 = 'pyside2_' + filename + ext py_file_qtpy = '' + filename + ext py_file_pyqtgraph = 'pyqtgraph_' + filename + ext # append palette used to generate this file used_palette = "\nfrom qrainbowstyle.palette import " + palette.__name__ + "\npalette = " + palette.__name__ + "\n" # calling external commands if args.create in ['pyqt', 'pyqtgraph', 'all']: logging.debug("Compiling for PyQt4 ...") try: call(['pyrcc4', '-py3', qrc_file, '-o', py_file_pyqt]) with open(py_file_pyqt, "a+") as f: f.write(used_palette) except FileNotFoundError: logging.debug("You must install pyrcc4") if args.create in ['pyqt5', 'qtpy', 'all']: logging.debug("Compiling for PyQt5 ...") try: call(['pyrcc5', qrc_file, '-o', py_file_pyqt5]) with open(py_file_pyqt5, "a+") as f: f.write(used_palette) except FileNotFoundError: logging.debug("You must install pyrcc5") if args.create in ['pyside', 'all']: logging.debug("Compiling for PySide ...") try: call( ['pyside-rcc', '-py3', qrc_file, '-o', py_file_pyside]) with open(py_file_pyside, "a+") as f: f.write(used_palette) except FileNotFoundError: logging.debug("You must install pyside-rcc") if args.create in ['pyside2', 'all']: logging.debug("Compiling for PySide 2...") try: call([ 'pyside2-rcc', '-py3', qrc_file, '-o', py_file_pyside2 ]) with open(py_file_pyside2, "a+") as f: f.write(used_palette) except FileNotFoundError: logging.debug("You must install pyside2-rcc") if args.create in ['qtpy', 'all']: logging.debug("Compiling for QtPy ...") # special case - qtpy - syntax is PyQt5 with open(py_file_pyqt5, 'r') as file: filedata = file.read() # replace the target string filedata = filedata.replace('from PyQt5', 'from qtpy') with open(py_file_qtpy, 'w+') as file: # write the file out again file.write(filedata) if args.create not in ['pyqt5']: os.remove(py_file_pyqt5) if args.create in ['pyqtgraph', 'all']: logging.debug("Compiling for PyQtGraph ...") # special case - pyqtgraph - syntax is PyQt4 with open(py_file_pyqt, 'r') as file: filedata = file.read() # replace the target string filedata = filedata.replace('from PyQt4', 'from pyqtgraph.Qt') with open(py_file_pyqtgraph, 'w+') as file: # write the file out again file.write(filedata)
def createFile(self, subtitle): '''pass the URL of the sub and the file it matches, will unzip it and return the path to the created file''' subpage = subtitle["page"] page = urllib2.urlopen(subpage) soup = BeautifulSoup(page) dlhref = soup.find("div", {"class": "download"}).find("a")["href"] subtitle["link"] = "http://subscene.com" + dlhref.split('"')[7] format = "zip" archivefilename = subtitle["filename"].rsplit(".", 1)[0] + '.' + format self.downloadFile(subtitle["link"], archivefilename) subtitlefilename = None if zipfile.is_zipfile(archivefilename): logging.debug("Unzipping file " + archivefilename) zf = zipfile.ZipFile(archivefilename, "r") for el in zf.infolist(): extension = el.orig_filename.rsplit(".", 1)[1] if extension in ("srt", "sub", "txt"): subtitlefilename = srtbasefilename + "." + extension outfile = open(subtitlefilename, "wb") outfile.write(zf.read(el.orig_filename)) outfile.flush() outfile.close() else: logging.info("File %s does not seem to be valid " % el.orig_filename) # Deleting the zip file zf.close() os.remove(archivefilename) return subtitlefilename elif archivefilename.endswith('.rar'): logging.warn( 'Rar is not really supported yet. Trying to call unrar') import subprocess try: args = ['unrar', 'lb', archivefilename] output = subprocess.Popen( args, stdout=subprocess.PIPE).communicate()[0] for el in output.splitlines(): extension = el.rsplit(".", 1)[1] if extension in ("srt", "sub"): args = [ 'unrar', 'e', archivefilename, el, os.path.dirname(archivefilename) ] subprocess.Popen(args) tmpsubtitlefilename = os.path.join( os.path.dirname(archivefilename), el) subtitlefilename = os.path.join( os.path.dirname(archivefilename), srtbasefilename + "." + extension) if os.path.exists(tmpsubtitlefilename): # rename it to match the file os.rename(tmpsubtitlefilename, subtitlefilename) # exit return subtitlefilename except OSError, e: logging.error("Execution failed: %s" % e) return None
def collectWMBSInfo(self): """ Fetches WMBS job information. In addition to WMBS, also collects RunJob info from BossAir :return: dict with the number of jobs in each status """ logging.info("Getting wmbs job info ...") results = {} start = int(time.time()) # first retrieve the site thresholds results['thresholds'] = self.wmagentDB.getJobSlotInfo() logging.debug("Running and pending site thresholds: %s", results['thresholds']) # now fetch the amount of jobs in each state and the amount of created # jobs grouped by task results.update(self.wmagentDB.getAgentMonitoring()) end = int(time.time()) #adding total query time results["total_query_time"] = end - start logging.debug("Total number of jobs in WMBS sorted by status: %s", results['wmbsCountByState']) logging.debug("Total number of 'created' jobs in WMBS sorted by type: %s", results['wmbsCreatedTypeCount']) logging.debug("Total number of 'executing' jobs in WMBS sorted by type: %s", results['wmbsExecutingTypeCount']) logging.debug("Total number of active jobs in BossAir sorted by status: %s", results['activeRunJobByStatus']) logging.debug("Total number of complete jobs in BossAir sorted by status: %s", results['completeRunJobByStatus']) logging.debug("Available slots thresholds to pull work from GQ to LQ: %s", results['thresholdsGQ2LQ']) logging.debug("List of jobs pending for each site, sorted by priority: %s", results['sitePendCountByPrio']) return results
def on_modified(self, event): """Handle file system events.""" if event.src_path.endswith('.qss'): run_process(self.args) logging.debug('\n')
def setStatusFile(api:to_api.API) -> bool: """ Attempts to set the status file according to this server's reported status in Traffic Ops. .. warning:: This will create the directory '/opt/ORTstatus' if it does not exist, and may delete files there without warning! :param api: A :class:`traffic_ops_ort.to_api.API` object to use when interacting with Traffic Ops :returns: whether or not the status file could be set properly """ global STATUS_FILE_DIR from .configuration import MODE, Modes from . import utils logging.info("Setting status file") if not isinstance(MODE, Modes): logging.error("MODE is not set to a valid Mode (from traffic_ops_ort.configuration.Modes)!") return False try: myStatus = api.getMyStatus() except ConnectionError as e: logging.error("Failed to set status file - Traffic Ops connection failed") return False if not os.path.isdir(STATUS_FILE_DIR): logging.warning("status directory does not exist, creating...") doMakeDir = MODE is not Modes.REPORT # Check for user confirmation if in 'INTERACTIVE' mode if doMakeDir and (MODE is not Modes.INTERACTIVE or\ utils.getYesNoResponse("Create status directory '%s'?" % STATUS_FILE_DIR, default='Y')): try: os.makedirs(STATUS_FILE_DIR) return False except OSError as e: logging.error("Failed to create status directory '%s' - %s", STATUS_FILE_DIR, e) logging.debug("%s", e, exc_info=True, stack_info=True) return False else: try: deleteOldStatusFiles(myStatus, api) except ConnectionError as e: logging.error("Failed to delete old status files - Traffic Ops connection failed.") logging.debug("%s", e, exc_info=True, stack_info=True) return False except OSError as e: logging.error("Failed to delete old status files - %s", e) logging.debug("%s", e, exc_info=True, stack_info=True) return False fname = os.path.join(STATUS_FILE_DIR, myStatus) if not os.path.isfile(fname): logging.info("File '%s' to be created", fname) if MODE is not Modes.REPORT and\ (MODE is not Modes.INTERACTIVE or utils.getYesNoResponse("Create file '%s'?", 'y')): try: with open(fname, 'x'): pass except OSError as e: logging.error("Failed to create status file - %s", e) logging.debug("%s", e, exc_info=True, stack_info=True) return False return True
def run() -> int: """ This function is the entrypoint into the script's main flow from :func:`traffic_ops_ort.doMain` It runs the appropriate actions depending on the run mode :returns: an exit code for the script """ from . import configuration, utils, services try: api = to_api.API(configuration.USERNAME, configuration.PASSWORD, configuration.TO_HOST, configuration.HOSTNAME[0], configuration.TO_PORT, configuration.VERIFY, configuration.TO_USE_SSL) except (LoginError, OperationError) as e: logging.critical("Failed to authenticate with Traffic Ops") logging.error(e) logging.debug("%r", e, exc_info=True, stack_info=True) return 1 # If this is just a revalidation, then we can exit if there's no revalidation pending if configuration.MODE == configuration.Modes.REVALIDATE: try: updateRequired = revalidateState(api) except ORTException as e: logging.debug("%r", e, exc_info=True, stack_info=True) return 2 if not updateRequired: logging.info("No revalidation pending") return 0 logging.info("in REVALIDATE mode; skipping package/service processing") # In all other cases, we check for an update to the Delivery Service and apply any found # changes else: try: updateRequired = syncDSState(api) except ORTException as e: logging.debug("%r", e, exc_info=True, stack_info=True) return 2 # Bail on failures - unless this script is BADASS! if not setStatusFile(api): if configuration.MODE is not configuration.Modes.BADASS: logging.critical("Failed to set status as specified by Traffic Ops") return 2 logging.warning("Failed to set status but we're BADASS, so moving on.") logging.info("\nProcessing Packages...") if not processPackages(api): logging.critical("Failed to process packages") if configuration.MODE is not configuration.Modes.BADASS: return 2 logging.warning("Package processing failed but we're BADASS, so attempting to move on") logging.info("Done.\n") logging.info("\nProcessing Services...") if not processServices(api): logging.critical("Failed to process services.") if configuration.MODE is not configuration.Modes.BADASS: return 2 logging.warning("Service processing failed but we're BADASS, so attempting to move on") logging.info("Done.\n") # All modes process configuration files logging.info("\nProcessing Configuration Files...") if not processConfigurationFiles(api): logging.critical("Failed to process configuration files.") return 2 logging.info("Done.\n") if updateRequired: if configuration.MODE is not configuration.Modes.INTERACTIVE or\ utils.getYesNoResponse("Update Traffic Ops?", default='Y'): logging.info("\nUpdating Traffic Ops...") api.updateTrafficOps() logging.info("Done.\n") else: logging.warning("Traffic Ops was not notified of changes. You should do this manually.") return 0 logging.info("Traffic Ops update not necessary") if services.NEEDED_RELOADS and not services.doReloads(): logging.critical("Failed to reload all configuration changes") return 2 return 0
def ground(problem, plagent, exp_signs=None): domain = problem.domain actions = domain.actions.values() predicates = domain.predicates.values() constraints = problem.constraints # Objects objects = problem.objects objects.update(domain.constants) logging.debug('Objects:\n%s' % objects) # Remove old type_map if exp_signs: objects = _update_exp_signs(exp_signs, objects) # Create a map from types to objects type_map = _create_type_map(objects) logging.debug("Type to object map:\n%s" % type_map) # Create type subtype map subtype_map = _create_subtype(domain.types) obj_signifs = {} # add agents, because on significances they have a inner link to I and They signs obj_means = {} # Check logic in exp if exp_signs: signs = exp_signs I_sign = signs['I'] They_sign = signs['They'] obj_means[I_sign] = I_sign.meanings[1] obj_signifs[I_sign] = I_sign.significances[1] obj_means[They_sign] = They_sign.meanings[1] obj_signifs[They_sign] = They_sign.significances[1] signs['situation'] = exp_signs['situation'] else: signs = {} I_sign = Sign("I") obj_means[I_sign] = I_sign.add_meaning() obj_signifs[I_sign] = I_sign.add_significance() signs[I_sign.name] = I_sign They_sign = Sign("They") obj_means[They_sign] = They_sign.add_meaning() obj_signifs[They_sign] = They_sign.add_significance() signs[They_sign.name] = They_sign signs['situation'] = Sign('situation') for obj in objects: obj_sign = Sign(obj) obj_signifs[obj_sign] = obj_sign.add_significance() signs[obj] = obj_sign if obj_sign.name == plagent: connector = obj_signifs[obj_sign].add_feature(obj_signifs[I_sign], zero_out=True) I_sign.add_out_significance(connector) obj_means[obj_sign] = obj_sign.add_meaning() for tp, objects in type_map.items(): if exp_signs: tp_sign = signs[tp.name] else: tp_sign = Sign(tp.name) for obj in objects: obj_signif = obj_signifs[signs[obj]] tp_signif = tp_sign.add_significance() connector = tp_signif.add_feature(obj_signif, zero_out=True) signs[obj].add_out_significance(connector) if not exp_signs: signs[tp.name] = tp_sign others = set() for id in range(1, len(signs['agent'].significances)+1): other_ag = signs['agent'].significances[id].get_signs() if signs[plagent] not in other_ag: others |= other_ag for subagent in others: if subagent.name != plagent: if not They_sign in subagent.significances[1].get_signs(): signif = obj_signifs[They_sign] if signif.is_empty(): They_signif = signif else: They_signif = They_sign.add_significance() connector = subagent.significances[1].add_feature(They_signif, zero_out=True) They_sign.add_out_significance(connector) obj_means[subagent] = subagent.add_meaning() if not exp_signs: updated_predicates = _update_predicates(predicates, actions) signify_predicates(predicates, updated_predicates, signs, subtype_map, domain.constants) signify_actions(actions, constraints, signs, plagent, obj_means, obj_signifs) signify_connection(signs) start_situation, pms = _define_situation('*start*', problem.initial_state, signs, 'image') goal_situation, pms = _define_situation('*finish*', problem.goal, signs, 'image') # if problem.name.startswith("blocks"): # list_signs = task_signs(problem) # _expand_situation_ma_blocks(goal_situation, signs, pms, list_signs) # For task return MaPlanningTask(problem.name, signs, start_situation, goal_situation)
def start_logger(): logging.debug("message 1 message 2")
def combine(self): """ Function for combining multiple coco datasets """ self.resim_dir = os.path.join(self.res_dir, 'combination', 'images') self.resann_dir = os.path.join(self.res_dir, 'combination', 'annotations') # Create directories for combination results and clear the previous ones # The exist_ok is for dealing with combination folder # TODO: Can be done better if os.path.exists(self.resim_dir) is False: os.makedirs(self.resim_dir, exist_ok=True) else: shutil.rmtree(self.resim_dir) os.makedirs(self.resim_dir, exist_ok=True) if os.path.exists(self.resann_dir) is False: os.makedirs(self.resann_dir, exist_ok=True) else: shutil.rmtree(self.resann_dir) os.makedirs(self.resann_dir, exist_ok=True) # Combine images print("Merging image dirs") im_dirs = [os.path.join(self.img_dir, folder) for folder in self.imgfolders] imext = [".png", ".jpg"] logging.debug("Combining Images...") for imdir in tqdm(im_dirs): ims = [i for i in os.listdir(imdir) if i[-4:].lower() in imext] for im in ims: shutil.copyfile(os.path.join(imdir, im), os.path.join(self.resim_dir, im)) # Combine annotations cann = {'images': [], 'annotations': [], 'info': None, 'licenses': None, 'categories': None} logging.debug("Combining Annotations...") dst_ann = os.path.join(self.resann_dir, 'combined.json') print("Merging annotations") for j in tqdm(self.jsonfiles): with open(os.path.join(self.ann_dir, j)) as a: c = json.load(a) ind = self.jsonfiles.index(j) cocofile = self.annfiles[ind] # Check if this is the 1st annotation. # If it is, continue else modify current annotation if ind == 0: cann['images'] = cann['images'] + c['images'] cann['annotations'] = cann['annotations'] + c['annotations'] if 'info' in list(c.keys()): cann['info'] = c['info'] if 'licenses' in list(c.keys()): cann['licenses'] = c['licenses'] cann['categories'] = c['categories'] last_imid = cann['images'][-1]['id'] last_annid = cann['annotations'][-1]['id'] logging.debug("String Ids detected. Converting to int") # If last imid or last_annid is a str, convert it to int if isinstance(last_imid, str): id_dict = {} # Change image id in images field for i, im in enumerate(cann['images']): id_dict[im['id']] = i im['id'] = i if isinstance(last_annid, str): # Change annotation id & image id in annotations field for i, im in enumerate(cann['annotations']): im['id'] = i if isinstance(last_imid, str): im['image_id'] = id_dict[im['image_id']] last_imid = cann['images'][-1]['id'] last_annid = cann['annotations'][-1]['id'] else: new_imids = [(last_imid + i + 1) for i in sorted(list(cocofile.imgs.keys()))] new_annids = [(last_annid + i + 1) for i in sorted(list(cocofile.anns.keys()))] def modify_ids(jf, imids, annids): id_dict = {} for img, newimid in zip(jf['images'], imids): id_dict[img['id']] = newimid img['id'] = newimid for ann, newannid in zip(jf['annotations'], annids): ann['id'] = newannid ann['image_id'] = id_dict[ann['image_id']] return jf c = modify_ids(c, new_imids, new_annids) cann['images'] = cann['images'] + c['images'] cann['annotations'] = cann['annotations'] + c['annotations'] if 'info' in list(c.keys()): cann['info'] = c['info'] if 'licenses' in list(c.keys()): cann['licenses'] = c['licenses'] cann['categories'] = c['categories'] last_imid = cann['images'][-1]['id'] last_annid = cann['annotations'][-1]['id'] with open(dst_ann, 'w') as aw: json.dump(cann, aw)
def watch_cb(name): if name == '': logging.debug('%s went offline' % sender) self.devices.remove(wrap) wrap.remove_from_connection() watcher.cancel()
def run(test, params, env): """ Test qmp event notification, this case will: 1) Start VM with qmp enable. 2) Connect to qmp port then run qmp_capabilities command. 3) Initiate the qmp command defined in config (qmp_cmd) 4) Verify that qmp command works as designed. :param test: QEMU test object :param params: Dictionary with the test parameters :param env: Dictionary with test environmen. """ def check_result(qmp_o, output=None, exception_list=""): """ Check test result with difference way accoriding to result_check. result_check = equal, will compare cmd_return_value with qmp command output. result_check = contain, will try to find cmd_return_value in qmp command output. result_check = m_equal_q, will compare key value in monitor command output and qmp command output. result_check = m_in_q, will try to find monitor command output's key value in qmp command output. result_check = m_format_q, will try to match the output's format with check pattern. :param qmp_o: output from pre_cmd, qmp_cmd or post_cmd. :param o: output from pre_cmd, qmp_cmd or post_cmd or an execpt :param exception_list: element no need check. result set in config file. """ if result_check == "equal": value = output if value != str(qmp_o): raise exceptions.TestFail("QMP command return value does not match " "the expect result. Expect result: '%s'\n" "Actual result: '%s'" % (value, qmp_o)) elif result_check == "contain": values = output.split(';') for value in values: if value in exception_list: continue if value.strip() not in str(qmp_o): raise exceptions.TestFail("QMP command output does not contain " "expect result. Expect result: '%s'\n" "Actual result: '%s'" % (value, qmp_o)) elif result_check == "not_contain": values = output.split(';') for value in values: if value in exception_list: continue if value in str(qmp_o): raise exceptions.TestFail("QMP command output contains unexpect" " result. Unexpect result: '%s'\n" "Actual result: '%s'" % (value, qmp_o)) elif result_check == "m_equal_q": msg = "QMP command ouput is not equal to in human monitor command." msg += "\nQMP command output: '%s'" % qmp_o msg += "\nHuman command output: '%s'" % output res = output.splitlines(True) if type(qmp_o) != type(res): len_o = 1 else: len_o = len(qmp_o) if len(res) != len_o: if res[0].startswith(' '): raise exceptions.TestFail("Human command starts with ' ', " "there is probably some garbage in " "the output.\n" + msg) res_tmp = [] #(qemu)info block in RHEL7 divided into 3 lines for line in res: if not line.startswith(' '): res_tmp.append(line) else: res_tmp[-1] += line res = res_tmp if len(res) != len_o: raise exceptions.TestFail(msg) re_str = r'([^ \t\n\r\f\v=]*)=([^ \t\n\r\f\v=]*)' for i in range(len(res)): if qmp_cmd == "query-version": version = qmp_o['qemu'] version = "%s.%s.%s" % (version['major'], version['minor'], version['micro']) package = qmp_o['package'] re_str = r"([0-9]+\.[0-9]+\.[0-9]+)\s*(\(\S*\))?" hmp_version, hmp_package = re.findall(re_str, res[i])[0] if not hmp_package: hmp_package = package hmp_package = hmp_package.strip() package = package.strip() hmp_version = hmp_version.strip() if version != hmp_version or package != hmp_package: raise exceptions.TestFail(msg) else: matches = re.findall(re_str, res[i]) for key, val in matches: if key in exception_list: continue if '0x' in val: val = long(val, 16) val_str = str(bin(val)) com_str = "" for p in range(3, len(val_str)): if val_str[p] == '1': com_str += '0' else: com_str += '1' com_str = "0b" + com_str value = eval(com_str) + 1 if val_str[2] == '1': value = -value if value != qmp_o[i][key]: msg += "\nValue in human monitor: '%s'" % value msg += "\nValue in qmp: '%s'" % qmp_o[i][key] raise exceptions.TestFail(msg) elif qmp_cmd == "query-block": cmp_str = "u'%s': u'%s'" % (key, val) cmp_s = "u'%s': %s" % (key, val) if '0' == val: cmp_str_b = "u'%s': False" % key elif '1' == val: cmp_str_b = "u'%s': True" % key else: cmp_str_b = cmp_str if (cmp_str not in str(qmp_o[i]) and cmp_str_b not in str(qmp_o[i]) and cmp_s not in str(qmp_o[i])): msg += ("\nCan not find '%s', '%s' or '%s' in " " QMP command output." % (cmp_s, cmp_str_b, cmp_str)) raise exceptions.TestFail(msg) elif qmp_cmd == "query-balloon": if (int(val) * 1024 * 1024 != qmp_o[key] and val not in str(qmp_o[key])): msg += ("\n'%s' is not in QMP command output" % val) raise exceptions.TestFail(msg) else: if (val not in str(qmp_o[i][key]) and str(bool(int(val))) not in str(qmp_o[i][key])): msg += ("\n'%s' is not in QMP command output" % val) raise exceptions.TestFail(msg) elif result_check == "m_in_q": res = output.splitlines(True) msg = "Key value from human monitor command is not in" msg += "QMP command output.\nQMP command output: '%s'" % qmp_o msg += "\nHuman monitor command output '%s'" % output for i in range(len(res)): params = res[i].rstrip().split() for param in params: if param.rstrip() in exception_list: continue try: str_o = str(qmp_o.values()) except AttributeError: str_o = qmp_o if param.rstrip() not in str(str_o): msg += "\nKey value is '%s'" % param.rstrip() raise error.TestFail(msg) elif result_check == "m_format_q": match_flag = True for i in qmp_o: if output is None: raise exceptions.TestError("QMP output pattern is missing") if re.match(output.strip(), str(i)) is None: match_flag = False if not match_flag: msg = "Output does not match the pattern: '%s'" % output raise exceptions.TestFail(msg) qemu_binary = utils_misc.get_qemu_binary(params) if not utils_misc.qemu_has_option("qmp", qemu_binary): raise exceptions.TestSkipError("Host qemu does not support qmp.") vm = env.get_vm(params["main_vm"]) vm.verify_alive() session = vm.wait_for_login(timeout=int(params.get("login_timeout", 360))) module = params.get("modprobe_module") if module: logging.info("modprobe the module %s", module) session.cmd("modprobe %s" % module) qmp_ports = vm.get_monitors_by_type('qmp') if qmp_ports: qmp_port = qmp_ports[0] else: raise exceptions.TestError("Incorrect configuration, no QMP monitor found.") hmp_ports = vm.get_monitors_by_type('human') if hmp_ports: hmp_port = hmp_ports[0] else: raise exceptions.TestError("Incorrect configuration, no QMP monitor found.") callback = {"host_cmd": lambda cmd: process.system_output(cmd, shell=True), "guest_cmd": session.get_command_output, "monitor_cmd": hmp_port.send_args_cmd, "qmp_cmd": qmp_port.send_args_cmd} def send_cmd(cmd): """ Helper to execute command on ssh/host/monitor """ if cmd_type in callback.keys(): return callback[cmd_type](cmd) else: raise exceptions.TestError("cmd_type is not supported") pre_cmd = params.get("pre_cmd") qmp_cmd = params.get("qmp_cmd") cmd_type = params.get("event_cmd_type") post_cmd = params.get("post_cmd") result_check = params.get("cmd_result_check") cmd_return_value = params.get("cmd_return_value") exception_list = params.get("exception_list", "") # Pre command if pre_cmd is not None: logging.info("Run prepare command '%s'.", pre_cmd) pre_o = send_cmd(pre_cmd) logging.debug("Pre-command: '%s'\n Output: '%s'", pre_cmd, pre_o) try: # Testing command logging.info("Run qmp command '%s'.", qmp_cmd) output = qmp_port.send_args_cmd(qmp_cmd) logging.debug("QMP command: '%s' \n Output: '%s'", qmp_cmd, output) except qemu_monitor.QMPCmdError, err: if params.get("negative_test") == 'yes': logging.debug("Negative QMP command: '%s'\n output:'%s'", qmp_cmd, err) if params.get("negative_check_pattern"): check_pattern = params.get("negative_check_pattern") if check_pattern not in str(err): raise exceptions.TestFail("'%s' not in exception '%s'" % (check_pattern, err)) else: raise exceptions.TestFail(err)
check_pattern = params.get("negative_check_pattern") if check_pattern not in str(err): raise exceptions.TestFail("'%s' not in exception '%s'" % (check_pattern, err)) else: raise exceptions.TestFail(err) except qemu_monitor.MonitorProtocolError, err: raise exceptions.TestFail(err) except Exception, err: raise exceptions.TestFail(err) # Post command if post_cmd is not None: logging.info("Run post command '%s'.", post_cmd) post_o = send_cmd(post_cmd) logging.debug("Post-command: '%s'\n Output: '%s'", post_cmd, post_o) if result_check is not None: txt = "Verify that qmp command '%s' works as designed." % qmp_cmd logging.info(txt) if result_check == "equal" or result_check == "contain": if qmp_cmd == "query-name": vm_name = params["main_vm"] check_result(output, vm_name, exception_list) elif qmp_cmd == "query-uuid": uuid_input = params["uuid"] check_result(output, uuid_input, exception_list) else: check_result(output, cmd_return_value, exception_list) elif result_check == "m_format_q": check_result(output, cmd_return_value, exception_list)
def create(username, project, zone, machine_type, accelerator_count, accelerator_type, image, nvme_count, ssh_internal_ip, ssh_key_file, cpu_min_platform=None, boot_ssd_size=None): """Create gcloud computing instance. Args: username: the username of the current user project: project name zone: zone of the GCP computing instance machine_type: the machine type used for the instance accelerator_count: the number of pieces of the accelerator to attach to the instance accelerator_type: the specific type of accelerator to attach to the instance image: the name of the image that the disk will be initialized with nvme_count: the number of NVME local SSD devices to attach to the instance ssh_internal_ip: internal ip to use for ssh. ssh_key_file: ssh key file to use to connect to instance. cpu_min_platform: minimum CPU platform to use, if None use default. boot_ssd_size: If set boot disk is changed to SSD and this size(GB) is used. """ instance_name = get_instance_name(username) machine_type = get_machine_type(machine_type, accelerator_count) logging.debug('Creating gcloud computing instance %s', instance_name) cmd = '''gcloud compute instances create {} \ --image={} \ --project={} \ --zone={} \ --machine-type={} \ --maintenance-policy=TERMINATE \ '''.format(instance_name, image, project, zone, machine_type) if boot_ssd_size: cmd += '--boot-disk-size={}GB --boot-disk-type=pd-ssd '.format( boot_ssd_size) if accelerator_count > 0: cmd += '--accelerator=count={},type={} '.format( accelerator_count, accelerator_type) if cpu_min_platform: cmd += '--min-cpu-platform="{}" '.format(cpu_min_platform) for _ in range(nvme_count): cmd += '--local-ssd=interface=NVME ' run_command(cmd, is_from_user=True) logging.info('Successfully created gcloud computing instance %s ' 'with %s accelerator.\n', instance_name, accelerator_count) ssh_prefix = _ssh_prefix(project, zone, ssh_internal_ip, ssh_key_file) # Wait until we can ssh to the newly created computing instance cmd = '{} --strict-host-key-checking=no --command="exit" {}'.format( ssh_prefix, instance_name) ssh_remaining_retries = 12 ssh_error = None while ssh_remaining_retries > 0: ssh_remaining_retries -= 1 try: run_command(cmd, is_from_user=False) ssh_error = None except Exception as error: # pylint: disable=broad-except ssh_error = error if ssh_remaining_retries: logging.info('Cannot ssh to the computing instance. ' 'Try again after 5 seconds') time.sleep(5) else: logging.error('Cannot ssh to the computing instance after ' '60 seconds due to error:\n%s', str(ssh_error)) if ssh_error: logging.info('Run the commands below manually after ssh into the computing ' 'instance:\n' 'git clone https://github.com/tensorflow/benchmarks.git\n' 'sudo usermod -a -G docker $USER\n') else: cmd = '{} --command="git clone {}" {}'.format( ssh_prefix, 'https://github.com/tensorflow/benchmarks.git', instance_name) run_command(cmd, is_from_user=True) logging.info('Successfully checked-out PerfZero code on the ' 'computing instance\n') cmd = '{} --command="sudo usermod -a -G docker $USER" {}'.format( ssh_prefix, instance_name) run_command(cmd, is_from_user=True) logging.info('Successfully added user to the docker group\n') cmd = '{} {} -- -L 6006:127.0.0.1:6006'.format(ssh_prefix, instance_name) logging.info('Run the command below to ssh to the instance together with ' 'port forwarding for tensorboard:\n%s\n', cmd)
def proxy_call(self, cb): if self.suspended: logging.debug('The service is suspended, delay the call') self.callbacks += [cb] else: cb()