Ejemplo n.º 1
0
def get_files(path):
    """
    Returns a list of files that the service will ingest
    :param path: The path of the files
    :return: An array of file paths
    """
    print path
    required_files = []

    main_path = os.path.realpath(os.path.join(path, 'email-opt-in'))

    # patch main_path to use child directory as we can't use symlink
    if not config.SYMLINK_ENABLED:
        main_path = utils.get_subdir(main_path)

    tmp = os.path.dirname(main_path)
    # get the list of directory paths
    emaildirs = [x[0] for x in os.walk(tmp)]
    # latest dir path in list
    latestdir = emaildirs[len(emaildirs)-1]

    for filename in os.listdir(latestdir):
        if '.csv' in filename:
            required_files.append(os.path.join(latestdir, filename))
    return required_files
Ejemplo n.º 2
0
def get_files(path):
    """
    Returns a list of files that the service will ingest
    :param path: The path of the files
    :return: An array of file paths
    """
    required_files = []
    main_path = os.path.realpath(os.path.join(path, 'database_state', 'latest'))

    # patch main_path to use child directory as we can't use symlink
    if not config.SYMLINK_ENABLED:
        main_path = utils.get_subdir(main_path)

    for filename in os.listdir(main_path):
        extension = os.path.splitext(filename)[1]
        if extension == '.mongo':
            required_files.append(os.path.join(main_path, filename))
    return required_files
Ejemplo n.º 3
0
def get_files(path):
    """
    Returns a list of files that the service will ingest
    :param path: The path of the files
    :return: An array of file paths
    """

    required_files = []
    main_path = os.path.realpath(os.path.join(path, 'database_state', 'latest'))

    # patch main_path to use child directory as we can't use symlink
    if not config.SYMLINK_ENABLED:
        main_path = utils.get_subdir(main_path)

    for subdir in os.listdir(main_path):
        if os.path.isdir(os.path.join(main_path, subdir)):
            required_files.append(os.path.join(main_path, subdir))
    return required_files
Ejemplo n.º 4
0
def get_files(path):
    """
    Returns a list of files that the service will ingest
    :param path: The path of the files
    :return: An array of file paths
    """
    required_files = []
    main_path = os.path.realpath(os.path.join(path, 'database_state',
                                              'latest'))

    # patch main_path to use child directory as we can't use symlink
    if not config.SYMLINK_ENABLED:
        main_path = utils.get_subdir(main_path)

    for filename in os.listdir(main_path):
        extension = os.path.splitext(filename)[1]
        if extension == '.mongo':
            required_files.append(os.path.join(main_path, filename))
    return required_files
Ejemplo n.º 5
0
def get_files(path):
    """
    Returns a list of files that the service will ingest
    :param path: The path of the files
    :return: An array of file paths
    """
    # print path
    required_files = []

    main_path = os.path.realpath(os.path.join(path, 'ga'))

    # patch main_path to use child directory as we can't use symlink
    if not config.SYMLINK_ENABLED:
        main_path = utils.get_subdir(main_path)

    for filename in os.listdir(main_path):
        if '.csv' in filename:
            required_files.append(os.path.join(main_path, filename))
    return required_files
Ejemplo n.º 6
0
def get_files(path):
    """
    Returns a list of files that the service will ingest
    :param path: The path of the files
    :return: An array of file paths
    """
    required_files = []
    main_path = os.path.realpath(os.path.join(path, "database_state", "latest"))

    try:
        # patch main_path to use child directory as we can't use symlink
        if not config.SYMLINK_ENABLED:
            main_path = utils.get_subdir(main_path)

        for filename in os.listdir(main_path):
            extension = os.path.splitext(filename)[1]
            if extension == ".sql":
                required_files.append(os.path.join(main_path, filename))
    except EnvironmentError, e:
        print str(e)
Ejemplo n.º 7
0
def get_files(path):
    """
    Returns a list of files that the service will ingest
    :param path: The path of the files
    :return: An array of file paths
    """
    print path
    required_files = []

    main_path = os.path.realpath(os.path.join(path, 'database_state', 'latest'))

    # patch main_path to use child directory as we can't use symlink
    if not config.SYMLINK_ENABLED:
        main_path = utils.get_subdir(main_path)

    for filename in os.listdir(main_path):
        if filename == config.DBSTATE_PREFIX.lower() + "email_opt_in-prod-analytics.csv":
            required_files.append(os.path.join(main_path, filename))
            break  # only one email file, once found exit the search
    return required_files