Example #1
1
 def get_info(self):
     logging.debug("Checking for StockPickr User %s", self.user)
     sql = "select uid from spr_user where name=%s;"
     self.id = self.db.query(sql, self.user, one=True)
     if not self.id:
         self.db.insert("spr_user", {"name": self.user})
         self.id = self.db.last_insert_id()
    def get(self, version_id):
        self.check_version_id(version_id)

        cache_key = version_id + str(self.__class__)
        page = self.cache.get(cache_key)

        if self.cache_bust or not page:
            logging.debug('Cache miss with key: %s' % cache_key)
            retrieved_data = EmailTemplate.fetch_all(self.data_sources[version_id])
            trail_blocks = deduplication.build_unique_trailblocks(retrieved_data,
                self.priority_list[version_id],
                excluded=self.exclude_from_deduplication())
            today = datetime.datetime.now()
            date = today.strftime('%A %d %b %Y')

            template_name = self.template_names[version_id] + '.html'
            template = self.resolve_template(template_name)

            ads = {}

            if hasattr(self, 'ad_tag') and self.ad_tag:
                ad_fetcher = AdFetcher(self.ad_tag)
                for name, type in self.ad_config.iteritems():
                    ads[name] = ad_fetcher.fetch_type(type)

            page = template.render(ads=ads, date=date, data=self.additional_template_data(), **trail_blocks)
            self.cache.add(cache_key, page, 300)
        else:
            logging.debug('Cache hit with key: %s' % cache_key)

        self.response.out.write(page)
Example #3
0
 def test_rollback(self):
   return
   try:
     vtgate_conn = get_connection()
     count = 10
     vtgate_conn.begin()
     vtgate_conn._execute(
         "delete from vt_insert_test", {},
         KEYSPACE_NAME, 'master',
         keyranges=[self.keyrange])
     kid_list = shard_kid_map[shard_names[self.shard_index]]
     for x in xrange(count):
       keyspace_id = kid_list[count%len(kid_list)]
       vtgate_conn._execute(
           "insert into vt_insert_test (msg, keyspace_id) values (%(msg)s, %(keyspace_id)s)",
           {'msg': 'test %s' % x, 'keyspace_id': keyspace_id},
           KEYSPACE_NAME, 'master', keyspace_ids=[pack_kid(keyspace_id)])
     vtgate_conn.commit()
     vtgate_conn.begin()
     vtgate_conn._execute(
         "delete from vt_insert_test", {},
         KEYSPACE_NAME, 'master',
         keyranges=[self.keyrange])
     vtgate_conn.rollback()
     results, rowcount = vtgate_conn._execute(
         "select * from vt_insert_test", {},
         KEYSPACE_NAME, 'master',
         keyranges=[self.keyrange])[:2]
     logging.debug("ROLLBACK TEST rowcount %d count %d" % (rowcount, count))
     self.assertEqual(rowcount, count, "Fetched rows(%d) != inserted rows(%d), rollback didn't work" % (rowcount, count))
   except Exception, e:
     logging.debug("Write failed with error %s" % str(e))
     raise
Example #4
0
    def schema_shell(self):
        """Performs the 'schema-shell' command."""
        schema_shell_home = self.env.get(SCHEMA_SHELL_HOME)
        assert (schema_shell_home is not None), \
            ("Environment variable undefined: %r" % SCHEMA_SHELL_HOME)
        assert os.path.isdir(schema_shell_home), \
            ("Invalid home directory for KijiSchema shell: %r" % schema_shell_home)
        schema_shell_script = os.path.join(schema_shell_home, "bin", "kiji-schema-shell")
        assert os.path.isfile(schema_shell_script), \
            ("KijiSchema shell not found: %r" % schema_shell_script)

        env = dict(self.env)

        classpath = env.get(KIJI_CLASSPATH, "").split(":") + list(self.express.get_classpath())
        env[KIJI_CLASSPATH] = ":".join(classpath)

        java_opts = env.get("JAVA_OPTS", "")
        # FIXME: I cannot find any trace of the Java system property "express.tmpjars"!
        # java_opts += (" -Dexpress.tmpjars=%s" % ???)

        # Relevant for KijiSchema 1.1 only and will be removed in Express 3.0:
        java_opts += " -Dorg.kiji.schema.impl.AvroCellEncoder.SCHEMA_VALIDATION=DISABLED"
        env["JAVA_OPTS"] = java_opts

        cmd = [schema_shell_script]
        logging.debug("Launching kiji-schema shell with:\n%s\with KIJI_CLASSPATH:\n%s",
                      " \\\n\t".join(map(repr, cmd)), "\n".join(map(tab_indent, classpath)))
        logging.debug("Computed KIJI_CLASSPATH:")
        proc = subprocess.Popen(cmd, env=env)
        try:
            return proc.wait()
        except subprocess.SubProcessError:
            proc.kill()
Example #5
0
    def jar(self):
        """Performs the 'jar' command."""
        class_name = getattr(self.flags, "class")
        if (class_name is None) and (len(self.args) > 0):
            class_name = self.pop_args_head()
        assert (class_name is not None), ("No class name specified with [--class=]<class>.")

        lib_jars = []
        if self.flags.jars is not None:
            lib_jars.extend(self.flags.jars)
        classpath = list(self.express.get_classpath(lib_jars=lib_jars))

        java_opts = []
        if self.flags.java_opts is not None:
            java_opts = [self.flags.java_opts]

        user_args = list(self.args)
        logging.info("Running java class %r with parameters: %r", class_name, user_args)

        cmd = [
            "java",
            # This property is only needed in kiji-schema v1.1 :
            "-Dorg.kiji.schema.impl.AvroCellEncoder.SCHEMA_VALIDATION=DISABLED",
        ] + java_opts + [
            "-classpath", ":".join(classpath),
            class_name,
        ] + user_args

        logging.debug("Running command:\n%s\n", " \\\n\t".join(map(repr, cmd)))
        return subprocess.call(cmd)
Example #6
0
def getContestData(contestCode, expiryTime = None, writeInFile = None):

    logging.debug("In getContestData("+contestCode+')')
    expiryTime, writeInFile = getGlobals(expiryTime, writeInFile)

    data = {}

    if expiryTime > 0:
        data = checkInFile('contest/' + contestCode, expiryTime)
        if data is not None:
            return data
        else:
            data = {}

    URL = "https://www.codechef.com/api/contests/" + contestCode

    data = json.loads(requests.get(URL, headers={'User-Agent': 'Mozilla/5.0'}).text)

    #Make start_time and end_time keys directly in data
    data['start_time'] = data['time']['start']
    data['end_time'] = data['time']['end']

    #Removing unnecessary keys.
    keysToRemove = ['problems_data','time','problemsstats', 'user', 'announcements', 'rules', 'autoRefresh', 'banner', 'todos']
    data = removeKeys(data, keysToRemove)

    #From here too.
    for contest in data['problems']:
        data['problems'][contest] = removeKeys(data['problems'][contest], ['status_url','submit_url','problem_url','allow_submission'])

    if writeInFile:
        writeToFile('contest/' + contestCode, data)

    logging.debug("getContestData() = " + json.dumps(data, indent = 4))
    return data
  def PublishManifest(self, manifest, version, build_id=None):
    """Publishes the manifest as the manifest for the version to others.

    Args:
      manifest: Path to manifest file to publish.
      version: Manifest version string, e.g. 6102.0.0-rc4
      build_id: Optional integer giving build_id of the build that is
                publishing this manifest. If specified and non-negative,
                build_id will be included in the commit message.
    """
    # Note: This commit message is used by master.cfg for figuring out when to
    #       trigger slave builders.
    commit_message = 'Automatic: Start %s %s %s' % (self.build_names[0],
                                                    self.branch, version)
    if build_id is not None and build_id >= 0:
      commit_message += '\nCrOS-Build-Id: %s' % build_id

    logging.info('Publishing build spec for: %s', version)
    logging.info('Publishing with commit message: %s', commit_message)
    logging.debug('Manifest contents below.\n%s', osutils.ReadFile(manifest))

    # Copy the manifest into the manifest repository.
    spec_file = '%s.xml' % os.path.join(self.all_specs_dir, version)
    osutils.SafeMakedirs(os.path.dirname(spec_file))

    shutil.copyfile(manifest, spec_file)

    # Actually push the manifest.
    self.PushSpecChanges(commit_message)
Example #8
0
 def run_and_check_vcpupin(vm, vm_ref, vcpu, cpu_list, options):
     """
     Run the vcpupin command and then check the result.
     """
     if vm_ref == "name":
         vm_ref = vm.name
     elif vm_ref == "uuid":
         vm_ref = vm.get_uuid()
     # Execute virsh vcpupin command.
     cmdResult = virsh.vcpupin(vm_ref, vcpu, cpu_list, options, debug=True)
     if cmdResult.exit_status:
         if not status_error:
             # Command fail and it is in positive case.
             raise error.TestFail(cmdResult)
         else:
             # Command fail and it is in negative case.
             return
     else:
         if status_error:
             # Command success and it is in negative case.
             raise error.TestFail(cmdResult)
         else:
             # Command success and it is in positive case.
             # "--config" will take effect after VM destroyed.
             pid = None
             vcpu_pid = None
             if options == "--config":
                 virsh.destroy(vm.name)
             else:
                 pid = vm.get_pid()
                 logging.debug("vcpus_pid: %s", vm.get_vcpus_pid())
                 vcpu_pid = vm.get_vcpus_pid()[vcpu]
             # Check the result of vcpupin command.
             check_vcpupin(vm.name, vcpu, cpu_list, pid, vcpu_pid)
Example #9
0
    def upsert_edit_project(self,filepath,filename,uuid,version,desc=None,opens_with=None):
        cursor = self.conn.cursor()

        matches=re.search(u'(\.[^\.]+)$',filename)
        file_xtn=""
        if matches is not None:
            file_xtn=str(matches.group(1))
        else:
            raise ArgumentError("Filename %s does not appear to have a file extension" % filename)

        typenum=self.project_type_for_extension(file_xtn,desc=desc,opens_with=opens_with)
        #does the project entry already exist? If so leave it (avoiding database bloat)
        cursor.execute("SELECT id FROM edit_projects WHERE filename=%s AND filepath=%s", (filename, filepath, ))
        result = cursor.fetchone()
        if result is not None:
            logging.debug("Edit project {0}/{1} already exists in database, not touching it".format(filepath, filename))
            return result[0]

        try:
            cursor.execute("insert into edit_projects (filename,filepath,type,lastseen,valid) values (%s,%s,%s,now(),true) returning id", (filename,filepath,typenum))
        except psycopg2.IntegrityError as e: #this is kept in case of race conditions
            self.conn.rollback()
            cursor.execute("update edit_projects set lastseen=now(), valid=true where filename=%s and filepath=%s returning id", (filename,filepath))

        result = cursor.fetchone()
        id = result[0]

        sqlcmd="update edit_projects set uuid=%s, version=%s where id=%s"
        cursor.execute(sqlcmd, (uuid,version,id))
        self.conn.commit()
        return id
Example #10
0
    def __public_response(self, messages):
        message = None
        
        for m in messages:
            pretty_message = "%s [%s %s] %s" % (m.id, m.created_at, m.user.screen_name, m.text)
            logging.info("found public message: %s" % pretty_message)
            if not self.__analyzer.should_respond(m):
                logging.info("not responding")
                continue
                
            response = TwitterResponseAccessor.get_by_message_id(str(m.id))
            if not response:
                message = m
                break
            else:
                logging.debug("found response to public message %s" % m.id)
        
        sent_message = None
        if message:
            # TODO: search for username also
            username = message.user.screen_name
            parsed_tweet = parse_tweet(message.text)
            plain_tweet = parsed_tweet.plain_text
            speaker = self.__select_speaker()
            sources, mix = Mixer(speaker).mix_response(plain_tweet, min_results=1, max_length=130-len(username))
            response_text = "@%s %s" % (username, mix)
            logging.info("responding to public message %s: %s" % (message.id, response_text))
            
            sent_message = self.__twitter.PostUpdate(response_text, message.id)
            TwitterResponseAccessor.create(str(message.id), response_id=str(sent_message.id), user=username, tweet_type=TwitterResponse.MENTION) 
            self.__reporter.posted(response_text)

        return sent_message
Example #11
0
 def act(self, force_act=False, action=None, skip_responses=False):
     """
     returns:
         (action, response) tuple.  response type depends on the action that was performed.
     """        
     if not force_act:
         config = ConfigurationAccessor.get_or_create()
         if config and (config.is_tweeting is not None) and (not safe_int(config.is_tweeting)):
             logging.debug("config.is_tweeting is False; hiding")
             return ()
     
     result = []
     responded = False
     if not skip_responses:
         try:
             direct, response = self.respond()
             if (direct or response):
                 # a response to a direct message or mention was generated
                 responded = True
                 if direct:
                     result.append(direct.AsDict())
                 if response:
                     result.append(response.AsDict())
         except Exception, e:
             logging.error(e)
Example #12
0
def process_users(users_file, keytab_path):

    with open(users_file) as csvfile:
        r = csv.DictReader(csvfile, delimiter=',')
        for row in r:
            if row['Name'] == '#':
                continue
            d = {
                'username': row['Name'],
                'description': row['Description'],
                'first': row['Name'],
                'last': row['Name'],
                'realm': REALM,
                'ipa_server': IPA_SERVER,
                'path_prefix': keytab_path
            }

            if not user_exists(d['username']):
                create_user(d)
                # When a user is created in IPA, his password autoexpires.
                # Need to explicitly login or set the expiration to a later date so that the keytab
                # can be retrieved and used later
                change_krb_expiration(d['username'])                    
                create_keytab(d)
                for g in BASE_GROUPS:
                    add_group(d['username'], g)
            else:
                logging.debug('User %s already exists' % d['username'])
Example #13
0
def restore_zk(filename):
  """ Restores Zookeeper data from a fixed file in the local FS.

  Args:
    filename: A str, the path to the temporary Zookeeper backup file.
  """
  handle = kazoo.client.KazooClient(hosts=ZK_DEFAULT_HOST)
  handle.start()
  with open(filename, 'r') as f:
    for line in f.readlines():
      pair = json.loads(line)
      path = pair.keys()[0]
      value = pair.values()[0].decode('base64')
      try:
        handle.create(path, bytes(value), makepath=True)
        logging.debug("Created '{0}'".format(path))
      except kazoo.exceptions.NodeExistsError:
        try:
          handle.set(path, bytes(value))
          logging.debug("Updated '{0}'".format(path))
        except kazoo.exceptions.BadArgumentsError:
          logging.warning("BadArgumentsError for path '{0}'".format(path))
      except kazoo.exceptions.NoNodeError:
        logging.warning("NoNodeError for path '{0}'. Parent nodes are "
          "missing".format(path))
      except kazoo.exceptions.ZookeeperError:
        logging.warning("ZookeeperError for path '{0}'".format(path))
  handle.stop()
Example #14
0
File: age.py Project: xuanblo/jcvi
def traits(args):
    """
    %prog traits directory

    Make HTML page that reports eye and skin color.
    """
    p = OptionParser(traits.__doc__)
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    samples = []
    for folder in args:
        targets = iglob(folder, "*-traits.json")
        if not targets:
            continue
        filename = targets[0]
        js = json.load(open(filename))
        js["skin_rgb"] = make_rgb(
            js["traits"]["skin-color"]["L"],
            js["traits"]["skin-color"]["A"],
            js["traits"]["skin-color"]["B"])
        js["eye_rgb"] = make_rgb(
            js["traits"]["eye-color"]["L"],
            js["traits"]["eye-color"]["A"],
            js["traits"]["eye-color"]["B"])
        samples.append(js)

    template = Template(traits_template)
    fw = open("report.html", "w")
    print >> fw, template.render(samples=samples)
    logging.debug("Report written to `{}`".format(fw.name))
    fw.close()
Example #15
0
 def apply(self, ui):
     logging.debug(self.movement)
     if self.movement == 'last':
         ui.current_buffer.focus_last()
         ui.update()
     else:
         MoveCommand.apply(self, ui)
Example #16
0
def graph_user(user, depth=0):
    logging.debug("Searching for %s", user)
    logging.debug("At depth %d", depth)
    followers = following = []
    result = [x.value for x in db.query(select_user % user)]

    if result:
        result = result.pop()
        following = result['following']
        followers = result['followers']

    if not GH.has_node(user):
        logging.debug("Adding %s to graph", user)
        GH.add_node(user)

    for follower in followers:
        if not GH.has_node(follower):
            GH.add_node(follower)
            logging.debug("Adding %s to graph", follower)
            if depth < max_depth:
                graph_user(follower, depth + 1)
        GH.add_edge(follower, user, {'weight': 2})

    for follow in following:
        if not GH.has_node(follow):
            GH.add_node(follow)
            logging.debug("Adding %s to graph", follow)
            if depth < max_depth:
                graph_user(follow, depth + 1)

        if GH.has_edge(follow, user):
            GH[follow][user]['weight'] += 1
        else:
            GH.add_edge(user, follow, {'weight': 1})
Example #17
0
 def __init__(self, openid_, issued, attrs=None, sreg_=None):
     logging.debug('init janrain openid object')
     self.openid = openid_
     self.issued = issued
     self.attrs = attrs or {}
     self.sreg = sreg_ or {}
     self.is_iname = (xri.identifierScheme(openid_) == 'XRI')
Example #18
0
    def upsert_prelude_project(self,path=None,filename=None,uuid=None,version=None,nclips=None):
        cursor=self.conn.cursor()
        self.conn.commit()

        #if uuid is None:
        #    raise DataError("You need to pass a valid uuid")

        #does the project entry already exist? If so leave it (avoiding database bloat)
        cursor.execute("SELECT id FROM prelude_projects WHERE filename=%s AND filepath=%s", (filename, path, ))
        result = cursor.fetchone()
        if result is not None:
            logging.debug("Prelude project {0}/{1} already exists in database, not touching it".format(path, filename))
            return result[0]

        try:
            sqlcmd = """insert into prelude_projects (filepath,filename,uuid,version,clips,lastseen)
                        values (%s,%s,%s,%s,%s,now()) returning id"""
            cursor.execute(sqlcmd,(path,filename,uuid,version,nclips))
        except psycopg2.IntegrityError as e: #if we violate unique keys, try to update on filename
            self.conn.rollback()
            try:
                sqlcmd = """update prelude_projects set filepath=%s, filename=%s, uuid=%s, version=%s, clips=%s, lastseen=now()
                            where filepath=%s and filename=%s returning id"""
                cursor.execute(sqlcmd,(path,filename,uuid,version,nclips,path,filename))
            except psycopg2.IntegrityError as e: #if that causes a violation, try to update on uuid
                self.conn.rollback()
                sqlcmd = """update prelude_projects set filepath=%s, filename=%s, uuid=%s, version=%s, clips=%s, lastseen=now()
                            where uuid=%s returning id"""
                cursor.execute(sqlcmd,(path,filename,uuid,version,nclips,uuid))

        self.conn.commit()
        result=cursor.fetchone()
        return result[0]    #return id of inserted row
  def _LoadFromFile(self):
    """Read the version file and set the version components"""
    with open(self.version_file, 'r') as version_fh:
      for line in version_fh:
        if not line.strip():
          continue

        match = self.FindValue('CHROME_BRANCH', line)
        if match:
          self.chrome_branch = match
          logging.debug('Set the Chrome branch number to:%s',
                        self.chrome_branch)
          continue

        match = self.FindValue('CHROMEOS_BUILD', line)
        if match:
          self.build_number = match
          logging.debug('Set the build version to:%s', self.build_number)
          continue

        match = self.FindValue('CHROMEOS_BRANCH', line)
        if match:
          self.branch_build_number = match
          logging.debug('Set the branch version to:%s',
                        self.branch_build_number)
          continue

        match = self.FindValue('CHROMEOS_PATCH', line)
        if match:
          self.patch_number = match
          logging.debug('Set the patch version to:%s', self.patch_number)
          continue

    logging.debug(self.VersionString())
Example #20
0
  def execute(self):
    targets = self.get_targets(self.is_non_synthetic_python_target)
    with self.invalidated(targets=targets) as invalidation_check:
      if not invalidation_check.invalid_vts:
        logging.debug(self.NOOP_MSG_HAS_TARGET_BUT_NO_SOURCE)
        return

      invalid_tgts = [vt.target for vt in invalidation_check.invalid_vts]
      sources = self._calculate_isortable_python_sources(invalid_tgts)
      if not sources:
        logging.debug(self.NOOP_MSG_HAS_TARGET_BUT_NO_SOURCE)
        return

      isort = self.context.products.get_data(IsortPrep.Isort)
      args = self.get_passthru_args() + sources

      # NB: We execute isort out of process to avoid unwanted side-effects from importing it:
      #   https://github.com/timothycrosley/isort/issues/456
      with pushd(get_buildroot()):
        workunit_factory = functools.partial(self.context.new_workunit,
                                             name='run-isort',
                                             labels=[WorkUnitLabel.TOOL, WorkUnitLabel.LINT])
        cmdline, exit_code = isort.run(workunit_factory, args)
        if exit_code != 0:
          raise TaskError('{} ... exited non-zero ({}).'.format(cmdline, exit_code),
                          exit_code=exit_code)
Example #21
0
    def process_request(self, request):
        # SESSION_KEY = '_auth_user_id'
        # BACKEND_SESSION_KEY = '_auth_user_backend'
        token = request.GET.get('token', False)
        token_id = request.GET.get('id', False)
        timestamp = request.GET.get('timestamp', False)
        if token and token_id and timestamp:
            logging.debug('SSO: user %s login attempt via SSO in with timestamp %s and token %s \n' % (
                token_id, timestamp, token))
            if self.check_token(token, token_id, timestamp):
                # everything passed, authenticate user
                logging.debug('SSO: user %s token and timestamp pass \n' % token_id)
                logging.debug('SSO: Attempting to authenticate as %s \n' % token_id)
                try:
                    user = self.authenticate(token_id)
                except Exception as e:
                    logging.debug('SSO: user %s does not exist, trying to create \n' % token_id)
                    rooibos_LDAP.new_account_from_ldap(token_id)

                if user.username == token_id:
                    try:
                        # THIS WAS THE KEY TO IT WORKING
                        user.backend = settings.SSO_BACKEND
                        login(request, user)
                        logging.debug('SSO: process_request - user.backend = %s' % user.backend)
                        #return None
                        #logging.debug(user.last_login)
                        # logging.debug(request.session['_auth_user_id'])
                        # logging.debug(request.session['_auth_user_backend'])
                    except Exception:
                        raise
Example #22
0
    def call(self, function, params=None):
        self.requestPerMinute += 1
        now = datetime.utcnow()

        if self.requestPerMinute >= self.requestLimit:
            waittime = 60 - now.second
            logging.warning("Limit for request per minute exceeded. Waiting for: {0} sec.".format(waittime))
            time.sleep(waittime)
            now = datetime.utcnow()

        if self.checkMinute != now.minute:
            self.requestPerMinute = 0
            self.checkMinute = now.minute

        payload = ''
        try:
            p = "" if not params else '?' + "&".join(
                ["{key}={value}".format(key=k, value=v) for (k, v) in params.iteritems()])
            url = "{base}.{func}{params}".format(base=self.baseConfig["url"], func=function, params=p)
            logging.debug("{0} {1} API call:{2}".format(self.checkMinute, self.requestPerMinute, url))
            request = urllib2.Request(url, None, self.baseConfig["headers"])
            stream = urllib2.urlopen(request)
            payload = stream.read()
            data = json.loads(payload)
            if isinstance(data, dict) and 'ruid' in data:
                logging.error('Api call failed with error: {0} Code: {1}'.format(data['message'], data['code']))
                return None
            return data

        except Exception as e:
            logging.error('Error: {0} Context: {1}'.format(e, payload))
            return None
Example #23
0
def setup_logging(log_level):
  """Initializes the logging system.

  Args:
    log_level: Logging level.
  """
  global _LOGGING_INITIALIZED
  if _LOGGING_INITIALIZED:
    logging.debug("setup_logging: logging system already initialized")
    return

  log_formatter = logging.Formatter(
      fmt="%(asctime)s %(levelname)s %(filename)s:%(lineno)s : %(message)s",
  )

  # Override the log date formatter to include the time zone:
  def format_time(record, datefmt=None):
    time_tuple = time.localtime(record.created)
    tz_name = time.tzname[time_tuple.tm_isdst]
    return "%(date_time)s-%(millis)03d-%(tz_name)s" % dict(
        date_time=time.strftime("%Y%m%d-%H%M%S", time_tuple),
        millis=record.msecs,
        tz_name=tz_name,
    )
  log_formatter.formatTime = format_time

  logging.root.handlers.clear()
  logging.root.setLevel(log_level)

  console_handler = logging.StreamHandler()
  console_handler.setFormatter(log_formatter)
  console_handler.setLevel(log_level)
  logging.root.addHandler(console_handler)

  _LOGGING_INITIALIZED = True
Example #24
0
def make_parser():
    """ Construct the command line parser """
    logging.info("Constructing parser")
    description = "Store and retrieve snippets of text"
    parser = argparse.ArgumentParser(description=description)

    subparsers = parser.add_subparsers(help="Available commands")

    # Subparser for the put command
    logging.debug("Constructing put subparser")
    put_parser = subparsers.add_parser("put", help="Store a snippet")
    put_parser.add_argument("name", help="The name of the snippet")
    put_parser.add_argument("snippet", help="The snippet text")
    put_parser.add_argument("filename", default="snippets.csv", nargs="?",
                            help="The snippet filename")
    put_parser.set_defaults(command="put")

    # Subparser for the get command
    logging.debug("Constructing get subparser")
    put_parser = subparsers.add_parser("get", help="Retrieve a snippet")
    put_parser.add_argument("name", help="The name of the snippet")
    put_parser.add_argument("filename", default="snippets.csv", nargs="?",
                            help="The snippet filename")
    put_parser.set_defaults(command="get")

    return parser
Example #25
0
    def shell(self):
        """Performs the 'shell' command."""
        lib_jars = []
        if self.flags.jars is not None:
            lib_jars.extend(self.flags.jars.split(","))

        env = dict(self.env)

        express_cp = self.express.get_classpath(lib_jars=lib_jars)
        env["EXPRESS_CP"] = ":".join(express_cp)

        dist_cache_paths = self.express.list_paths_for_dist_cache(lib_jars=lib_jars)
        env["TMPJARS"] = ",".join(dist_cache_paths)

        # EXPRESS_MODE environment variable must be a path to the mode specific scala script:
        script_name = self._MODE_SCRIPT.get(self.flags.mode)
        assert (script_name is not None), ("Invalid Express mode: %r." % self.flags.mode)
        script_path = os.path.join(self.express.home_dir, "bin", script_name)
        assert os.path.isfile(script_path), ("Script not found: %r" % script_path)
        env["EXPRESS_MODE"] = script_path

        # express shell binary needs to be in the same directory as this script
        shell_path = os.path.join(self.express.home_dir, "bin", "express-shell")
        assert os.path.isfile(shell_path), ("Shell not found: %r" % shell_path)
        cmd = [shell_path] + list(self.args)
        logging.debug(
            "Launching the express shell using the command:\n%s",
            " \\\n\t".join(map(repr, cmd)))
        proc = subprocess.Popen(cmd, env=env)
        try:
            return proc.wait()
        except subprocess.SubprocessError:
            proc.kill()
Example #26
0
 def execute(self, email_models):
     logging.debug("In Destiny::execute()")
     if not email_models:
         return
     emails_id = []
     destinations = {}
     for destiny in self._plugins.keys():
         destinations.setdefault(destiny, email_models[-1].get(destiny))
         emails_id.append(email_models[-1].email_id())
     for email_model in email_models[:-1]:
         for destiny in self._plugins.keys():
             d_tables = destinations.get(destiny).get("tables")
             for d_table in d_tables:
                 for k, v in d_table.iteritems():
                     m_tables = email_model.get(destiny).get("tables")
                     for m_table in m_tables:
                         if k in m_table:
                             d_table.setdefault(k, []).extend(m_table[k])
         emails_id.append(email_model.email_id())
     for destiny, models in destinations.iteritems():
         for forward in self._plugins.get(destiny):
             try:
                 forward.execute(models)
             except Exception, e:
                 logging.error("!! Error-execute: %s" % (str(e),))
                 logging.info("Add emails in queure error: %s" % str(emails_id))
                 for email_id in emails_id:
                     self.add_email_error(email_id)
                 continue
Example #27
0
def refetch(source):
  """Refetch the author's URLs and look for new or updated syndication
  links that might not have been there the first time we looked.

  Args:
    source: models.Source subclass. Changes to property values (e.g. domains,
      domain_urls, last_syndication_url) are stored in source.updates; they
      should be updated transactionally later.

  Return:
    a dict of syndicated_url to a list of new models.SyndicatedPosts
  """
  if not source.updates:
    source.updates = {}

  logging.debug('attempting to refetch h-feed for %s', source.label())
  results = {}
  for url in _get_author_urls(source):
    results.update(_process_author(source, url, refetch=True))

  now = util.now_fn()
  logging.debug('updating source last_hfeed_fetch %s', now)
  source.updates['last_hfeed_fetch'] = now

  return results
Example #28
0
  def test_restart(self):
    """test_restart tests that when starting a second vttablet with the same
    configuration as another one, it will kill the previous process
    and take over listening on the socket.

    If vttablet listens to other ports (like gRPC), this feature will
    break. We believe it is not widely used, so we're OK with this for now.
    (container based installations usually handle tablet restarts
    by using a different set of servers, and do not rely on this feature
    at all).
    """
    if environment.topo_server().flavor() != 'zookeeper':
      logging.info("Skipping this test in non-github tree")
      return
    if tablet_62344.grpc_enabled():
      logging.info("Skipping this test as second gRPC port interferes")
      return

    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

    # create the database so vttablets start, as it is serving
    tablet_62344.create_db('vt_test_keyspace')

    tablet_62344.init_tablet('master', 'test_keyspace', '0')
    proc1 = tablet_62344.start_vttablet()
    proc2 = tablet_62344.start_vttablet()
    for timeout in xrange(20):
      logging.debug("Sleeping waiting for first process to die")
      time.sleep(1.0)
      proc1.poll()
      if proc1.returncode is not None:
        break
    if proc1.returncode is None:
      self.fail("proc1 still running")
    tablet_62344.kill_vttablet()
Example #29
0
def tearDownModule():
  global vtgate_server
  logging.debug("in tearDownModule")
  if utils.options.skip_teardown:
    return
  logging.debug("Tearing down the servers and setup")
  utils.vtgate_kill(vtgate_server)
  tablet.kill_tablets([shard_0_master, shard_0_replica, shard_1_master,
                       shard_1_replica])
  teardown_procs = [shard_0_master.teardown_mysql(),
                    shard_0_replica.teardown_mysql(),
                    shard_1_master.teardown_mysql(),
                    shard_1_replica.teardown_mysql(),
                   ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server_teardown()

  utils.kill_sub_processes()
  utils.remove_tmp_files()

  shard_0_master.remove_tree()
  shard_0_replica.remove_tree()
  shard_1_master.remove_tree()
  shard_1_replica.remove_tree()
Example #30
0
    def extract(vm, remote_path, dest_dir):
        """
        Extract the autotest .tar.bz2 file on the guest, ensuring the final
        destination path will be dest_dir.

        :param vm: VM object
        :param remote_path: Remote file path
        :param dest_dir: Destination dir for the contents
        """
        basename = os.path.basename(remote_path)
        logging.debug("Extracting %s on VM %s", basename, vm.name)
        session.cmd("rm -rf %s" % dest_dir, timeout=240)
        dirname = os.path.dirname(remote_path)
        session.cmd("cd %s" % dirname)
        session.cmd("mkdir -p %s" % os.path.dirname(dest_dir))
        e_cmd = "tar xjvf %s -C %s" % (basename, os.path.dirname(dest_dir))
        output = session.cmd(e_cmd, timeout=240)
        autotest_dirname = ""
        for line in output.splitlines()[1:]:
            autotest_dirname = line.split("/")[0]
            break
        if autotest_dirname != os.path.basename(dest_dir):
            session.cmd("cd %s" % os.path.dirname(dest_dir))
            session.cmd("mv %s %s" %
                        (autotest_dirname, os.path.basename(dest_dir)))