예제 #1
0
def main():
    while True:
        try:
            utils.streamPrintFlush(
                "\nTarget Url: eg [www.example.org or http://www.h.com] ",
                sys.stderr)
            lineIn, eofState = readFromStream()
            if eofState: break

            baseUrl = lineIn.strip("\n")

            utils.streamPrintFlush(
                "Your extensions separated by '|' eg png|html: ", sys.stderr)

            lineIn, eofState = readFromStream()
            if eofState: break
            extensions = lineIn.strip("\n")

            utils.streamPrintFlush(
                "\nRecursion Depth(a negative depth indicates you want script to go as far): ",
                sys.stderr)

            lineIn, eofState = readFromStream()
            if eofState: break

            rDepth = int(lineIn.strip("\n"))

            formedRegex = utils.extensionify(extensions
                                             or utils.DEFAULT_EXTENSIONS_REGEX)
            extCompile = utils.regexCompile(formedRegex)

        except ValueError:
            utils.streamPrintFlush("Recursion depth must be an integer\n",
                                   sys.stderr)
        except KeyboardInterrupt:
            utils.streamPrintFlush("Ctrl-C applied. Exiting now..\n",
                                   sys.stderr)
            break
        except Exception:
            continue
        else:
            if not baseUrl:
                continue

            if extCompile:
                getFiles(baseUrl, extCompile, rDepth)

    utils.streamPrintFlush("Bye..\n", sys.stderr)
예제 #2
0
def main():
  while True:
    try:
      utils.streamPrintFlush(
        "\nTarget Url: eg [www.example.org or http://www.h.com] ", sys.stderr
      )
      lineIn, eofState = readFromStream()
      if eofState: break

      baseUrl = lineIn.strip("\n")

      utils.streamPrintFlush(
       "Your extensions separated by '|' eg png|html: ", sys.stderr
      )

      lineIn, eofState = readFromStream()
      if eofState: break
      extensions = lineIn.strip("\n")
      
      utils.streamPrintFlush(
        "\nRecursion Depth(a negative depth indicates you want script to go as far): "
      ,sys.stderr)

      lineIn, eofState = readFromStream()
      if eofState: break
      
      rDepth = int(lineIn.strip("\n"))

      formedRegex = utils.extensionify(extensions or utils.DEFAULT_EXTENSIONS_REGEX)
      extCompile = utils.regexCompile(formedRegex)

    except ValueError:
      utils.streamPrintFlush("Recursion depth must be an integer\n", sys.stderr)
    except KeyboardInterrupt:
      utils.streamPrintFlush("Ctrl-C applied. Exiting now..\n",sys.stderr)
      break
    except Exception:
      continue
    else:
      if not baseUrl:
        continue

      if extCompile:
        getFiles(baseUrl, extCompile, rDepth)

  utils.streamPrintFlush("Bye..\n",sys.stderr)
예제 #3
0
파일: shardy.py 프로젝트: odeke-em/crawlers
def main():
  args, options = restDriver.cliParser()

  # Route manager
  router = Router([
      'http://192.168.1.117:8000', 'http://192.168.1.110:8008', 'http://127.0.0.1:8009'
  ])
  while True:
    try:
      utils.streamPrintFlush(
        "\nTarget Url: eg [www.example.org or http://www.h.com] ", sys.stderr
      )
      lineIn, eofState = readFromStream()
      if eofState: break

      if lineIn:
        baseUrl = lineIn.strip("\n")

      else:
        continue

      utils.streamPrintFlush(
       "Your extensions separated by '|' eg png|html: ", sys.stderr
      )

      lineIn, eofState = readFromStream()
      if eofState: break
      extensions = lineIn.strip("\n")
      
      utils.streamPrintFlush(
        "\nRecursion Depth(a negative depth indicates you want script to go as far): ", sys.stderr
      )

      lineIn, eofState = readFromStream()
      if eofState: break

      elif lineIn:
        rDepth = int(lineIn.strip("\n") or 1)
      else:
        rDepth = 1

      formedRegex = utils.extensionify(extensions or utils.DEFAULT_EXTENSIONS_REGEX)
      extCompile = utils.regexCompile(formedRegex)

    except ValueError:
      utils.streamPrintFlush("Recursion depth must be an integer\n", sys.stderr)
    except KeyboardInterrupt:
      utils.streamPrintFlush("Ctrl-C applied. Exiting now..\n", sys.stderr)
      break
    except Exception:
      # TODO: [Informative exceptions]:
      #       + Handle traceback from sys somehow, since using Exception as e won't
      #         is invalid syntax for x <= Python2.5
      print('Generic exception encountered')
      continue
    else:
      if not baseUrl:
        continue

      if extCompile:
        extractFileUrls(baseUrl, extCompile, router, rDepth)

  utils.streamPrintFlush("Bye..\n",sys.stderr)
예제 #4
0
def main():
    args, options = restDriver.cliParser()

    # Route manager
    router = Router([
        'http://192.168.1.117:8000', 'http://192.168.1.110:8008',
        'http://127.0.0.1:8009'
    ])
    while True:
        try:
            utils.streamPrintFlush(
                "\nTarget Url: eg [www.example.org or http://www.h.com] ",
                sys.stderr)
            lineIn, eofState = readFromStream()
            if eofState: break

            if lineIn:
                baseUrl = lineIn.strip("\n")

            else:
                continue

            utils.streamPrintFlush(
                "Your extensions separated by '|' eg png|html: ", sys.stderr)

            lineIn, eofState = readFromStream()
            if eofState: break
            extensions = lineIn.strip("\n")

            utils.streamPrintFlush(
                "\nRecursion Depth(a negative depth indicates you want script to go as far): ",
                sys.stderr)

            lineIn, eofState = readFromStream()
            if eofState: break

            elif lineIn:
                rDepth = int(lineIn.strip("\n") or 1)
            else:
                rDepth = 1

            formedRegex = utils.extensionify(extensions
                                             or utils.DEFAULT_EXTENSIONS_REGEX)
            extCompile = utils.regexCompile(formedRegex)

        except ValueError:
            utils.streamPrintFlush("Recursion depth must be an integer\n",
                                   sys.stderr)
        except KeyboardInterrupt:
            utils.streamPrintFlush("Ctrl-C applied. Exiting now..\n",
                                   sys.stderr)
            break
        except Exception:
            # TODO: [Informative exceptions]:
            #       + Handle traceback from sys somehow, since using Exception as e won't
            #         is invalid syntax for x <= Python2.5
            print('Generic exception encountered')
            continue
        else:
            if not baseUrl:
                continue

            if extCompile:
                extractFileUrls(baseUrl, extCompile, router, rDepth)

    utils.streamPrintFlush("Bye..\n", sys.stderr)