Example #1
0
def cleanup_extracted_file():
    log.info('Removing extracted files for file viewer.')
    root = os.path.join(settings.TMP_PATH, 'file_viewer')
    # Local storage uses local time for file modification. S3 uses UTC time.
    now = datetime.utcnow if storage_is_remote() else datetime.now
    for path in private_storage.listdir(root)[0]:
        full = os.path.join(root, path)
        age = now() - private_storage.modified_time(
            os.path.join(full, 'manifest.webapp'))
        if age.total_seconds() > (60 * 60):
            log.debug('Removing extracted files: %s, %dsecs old.' %
                      (full, age.total_seconds()))
            for subroot, dirs, files in walk_storage(full):
                for f in files:
                    private_storage.delete(os.path.join(subroot, f))
            # Nuke out the file and diff caches when the file gets removed.
            id = os.path.basename(path)
            try:
                int(id)
            except ValueError:
                continue

            key = hashlib.md5()
            key.update(str(id))
            cache.delete('%s:memoize:%s:%s' % (settings.CACHE_PREFIX,
                                               'file-viewer', key.hexdigest()))
Example #2
0
def cleanup_extracted_file():
    log.info('Removing extracted files for file viewer.')
    root = os.path.join(settings.TMP_PATH, 'file_viewer')
    # Local storage uses local time for file modification. S3 uses UTC time.
    now = datetime.utcnow if storage_is_remote() else datetime.now
    for path in private_storage.listdir(root)[0]:
        full = os.path.join(root, path)
        age = now() - private_storage.modified_time(
            os.path.join(full, 'manifest.webapp'))
        if age.total_seconds() > (60 * 60):
            log.debug('Removing extracted files: %s, %dsecs old.' %
                      (full, age.total_seconds()))
            for subroot, dirs, files in walk_storage(full):
                for f in files:
                    private_storage.delete(os.path.join(subroot, f))
            # Nuke out the file and diff caches when the file gets removed.
            id = os.path.basename(path)
            try:
                int(id)
            except ValueError:
                continue

            key = hashlib.md5()
            key.update(str(id))
            cache.delete(
                '%s:memoize:%s:%s' %
                (settings.CACHE_PREFIX, 'file-viewer', key.hexdigest()))
Example #3
0
    def _get_files(self):
        all_files, res = [], OrderedDict()

        # Not using os.path.walk so we get just the right order.
        def iterate(path):
            path_dirs, path_files = private_storage.listdir(path)
            for dirname in sorted(path_dirs):
                full = os.path.join(path, dirname)
                all_files.append(full)
                iterate(full)

            for filename in sorted(path_files):
                full = os.path.join(path, filename)
                all_files.append(full)

        iterate(self.dest)

        for path in all_files:
            filename = smart_unicode(os.path.basename(path), errors='replace')
            short = smart_unicode(path[len(self.dest) + 1:], errors='replace')
            mime, encoding = mimetypes.guess_type(filename)
            if not mime and filename == 'manifest.webapp':
                mime = 'application/x-web-app-manifest+json'
            if storage_is_remote():
                # S3 doesn't have directories, so we check for names with this
                # prefix and call it a directory if there are some.
                subdirs, subfiles = private_storage.listdir(path)
                directory = bool(subdirs or subfiles)
            else:
                directory = os.path.isdir(path)

            res[short] = {
                'binary': self._is_binary(mime, path),
                'depth': short.count(os.sep),
                'directory': directory,
                'filename': filename,
                'full': path,
                'md5': get_md5(path) if not directory else '',
                'mimetype': mime or 'application/octet-stream',
                'syntax': self.get_syntax(filename),
                'modified': (
                    time.mktime(
                        private_storage.modified_time(path).timetuple())
                    if not directory else 0),
                'short': short,
                'size': private_storage.size(path) if not directory else 0,
                'truncated': self.truncate(filename),
                'url': reverse('mkt.files.list',
                               args=[self.file.id, 'file', short]),
                'url_serve': reverse('mkt.files.redirect',
                                     args=[self.file.id, short]),
                'version': self.file.version.version,
            }

        return res
Example #4
0
def clean_old_signed(seconds=60 * 60):
    """Clean out apps signed for reviewers."""
    log.info('Removing old apps signed for reviewers')
    root = settings.SIGNED_APPS_REVIEWER_PATH
    # Local storage uses local time for file modification. S3 uses UTC time.
    now = datetime.utcnow if storage_is_remote() else datetime.now
    for nextroot, dirs, files in walk_storage(root, storage=private_storage):
        for fn in files:
            full = os.path.join(nextroot, fn)
            age = now() - private_storage.modified_time(full)
            if age.total_seconds() > seconds:
                log.debug('Removing signed app: %s, %dsecs old.' %
                          (full, age.total_seconds()))
                private_storage.delete(full)
Example #5
0
def clean_old_signed(seconds=60 * 60):
    """Clean out apps signed for reviewers."""
    log.info('Removing old apps signed for reviewers')
    root = settings.SIGNED_APPS_REVIEWER_PATH
    # Local storage uses local time for file modification. S3 uses UTC time.
    now = datetime.utcnow if storage_is_remote() else datetime.now
    for nextroot, dirs, files in walk_storage(root):
        for fn in files:
            full = os.path.join(nextroot, fn)
            age = now() - private_storage.modified_time(full)
            if age.total_seconds() > seconds:
                log.debug('Removing signed app: %s, %dsecs old.' % (
                    full, age.total_seconds()))
                private_storage.delete(full)
Example #6
0
    def _get_files(self):
        all_files, res = [], OrderedDict()

        # Not using os.path.walk so we get just the right order.
        def iterate(path):
            path_dirs, path_files = private_storage.listdir(path)
            for dirname in sorted(path_dirs):
                full = os.path.join(path, dirname)
                all_files.append(full)
                iterate(full)

            for filename in sorted(path_files):
                full = os.path.join(path, filename)
                all_files.append(full)

        iterate(self.dest)

        for path in all_files:
            filename = smart_unicode(os.path.basename(path), errors='replace')
            short = smart_unicode(path[len(self.dest) + 1:], errors='replace')
            mime, encoding = mimetypes.guess_type(filename)
            if not mime and filename == 'manifest.webapp':
                mime = 'application/x-web-app-manifest+json'
            if storage_is_remote():
                # S3 doesn't have directories, so we check for names with this
                # prefix and call it a directory if there are some.
                subdirs, subfiles = private_storage.listdir(path)
                directory = bool(subdirs or subfiles)
            else:
                directory = os.path.isdir(path)

            res[short] = {
                'binary':
                self._is_binary(mime, path),
                'depth':
                short.count(os.sep),
                'directory':
                directory,
                'filename':
                filename,
                'full':
                path,
                'md5':
                get_md5(path) if not directory else '',
                'mimetype':
                mime or 'application/octet-stream',
                'syntax':
                self.get_syntax(filename),
                'modified':
                (time.mktime(private_storage.modified_time(path).timetuple())
                 if not directory else 0),
                'short':
                short,
                'size':
                private_storage.size(path) if not directory else 0,
                'truncated':
                self.truncate(filename),
                'url':
                reverse('mkt.files.list', args=[self.file.id, 'file', short]),
                'url_serve':
                reverse('mkt.files.redirect', args=[self.file.id, short]),
                'version':
                self.file.version.version,
            }

        return res