Beispiel #1
0
    def verify_access(cls, settings):
        log = get_logger_module(cls.__name__)
        # try megadf to check if we can access
        command = cls.build_command_argumetns(command_str="megadf", settings=settings)
        log.debug("Executing command: %s", command)

        try:
            cls.check_call(command)
        except CalledProcessError as e:
            raise DestinationInaccessible("Failed access. Running '%s' result was '%s'", command, e.output)
        log.debug("Access verified to destination mega (command: %s)", command)
Beispiel #2
0
 def create_dest_directories(cls, settings):
     dst_path = cls.to_absoulte_dst_path(settings)
     log = get_logger_module(cls.__name__)
     # we generate each directory from root so we create all that are missing
     splited = dst_path.split(cls.dir_delimiter)
     subdirs = []
     # note: first 2 will be "" and root directory (since absolute path starts with "/<root>")
     for included in xrange(3, len(splited) + 1):
         subdirs.append(cls.dir_delimiter.join(splited[:included]))
     if subdirs:
         command = cls.build_command_argumetns(command_str="megamkdir", settings=settings, extra_args=subdirs)
         log.debug("Executing command: %s", command)
         call(command, start_new_session=True)
Beispiel #3
0
 def do_init(self,
             sender_spec,
             tmp_file_parts_basepath,
             should_split_small_files,
             global_quota):
     super(_CompressorJob, self).do_init()
     self._tmp_file_parts_basepath = tmp_file_parts_basepath
     self._destinations = sender_spec.destinations
     self._current_block = None
     self._block_fragmenter = _BlockFragmenter(sender_spec=sender_spec,
                                               should_split_small_files=should_split_small_files,
                                               global_quota=global_quota)
     self.name = "".join((self.__class__.__name__, '(to ', str(self._destinations), ')'))
     self.log = get_logger_module(self.name)
Beispiel #4
0
import code
import traceback
import signal
import sys

from fcb.utils.log_helper import get_logger_module

_log = get_logger_module("debugging")


# logic taken from http://stackoverflow.com/questions/132058/showing-the-stack-trace-from-a-running-python-application
def attach_debugger(sig, frame):
    """Interrupt running process, and provide a python prompt for
    interactive debugging."""
    d = {'_frame': frame}  # Allow access to frame object.
    d.update(frame.f_globals)  # Unless shadowed by global
    d.update(frame.f_locals)

    i = code.InteractiveConsole(d)
    message = "Signal received : entering python shell.\nTraceback:\n"
    message += ''.join(traceback.format_stack(frame))
    i.interact(message)


# Adapted from http://bzimmer.ziclix.com/2008/12/17/python-thread-dumps/
def gen_stacktraces():
    stacktrace = []
    for threadId, stack in sys._current_frames().items():
        stacktrace.append("\n# ThreadID: %s" % threadId)
        for filename, lineno, name, line in traceback.extract_stack(stack):
            stacktrace.append('File: "%s", line %d, in %s' % (filename, lineno, name))
Beispiel #5
0
import os
import poplib
import re
import tempfile
import time

from sqlalchemy import update, and_
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.sql.expression import select

from fcb.database.helpers import get_session
from fcb.database.schema import FilesContainer, CheckerState, Destination, FilesDestinations
from fcb.processing.models.FileInfo import FileInfo
from fcb.utils.log_helper import get_logger_for, get_logger_module

log = get_logger_module('mail_checker')


# FIXME optimize
class CheckHistoryVerifier(object):
    def __init__(self, mail_dst):
        self._session_resource = get_session()
        self._log = get_logger_for(self)
        self._mail_dst = mail_dst
        self._last_checked_date = None

    def close(self):
        if self._session_resource:
            with self._session_resource as session:
                session.commit()
                session.close()
Beispiel #6
0
 def init(self, max_pending_for_processing):
     self.log = get_logger_module(self.__class__.__name__)
     self._max_pending_for_processing = max_pending_for_processing
     self._cur_available = self._max_pending_for_processing
Beispiel #7
0
 def init(self, next_task=None, *args, **kwargs):
     self.log = get_logger_module(self.__class__.__name__)
     self.next_task(next_task)
     if hasattr(self, "do_init") and isinstance(self.do_init, Callable):
         self.do_init(*args, **kwargs)
Beispiel #8
0
from PIL import Image
import numpy
import math

from fcb.framework.workers import hd_worker_pool
from fcb.framework.workflow.HeavyPipelineTask import HeavyPipelineTask
from fcb.processing.models.FileInfo import FileInfo
from fcb.utils.log_helper import get_logger_module

_log = get_logger_module("ToImage")


# TODO image and array manipulation functions need to be optimized (a lot)

def _determine_dimensions(num_of_pixels):
    """
    Given a number of pixels, determines the largest width and height that define a
      rectangle with such an area
    """
    for x in xrange(int(math.sqrt(num_of_pixels)) + 1, 1, -1):
        if num_of_pixels % x == 0:
            return num_of_pixels // x, x
    return 1, num_of_pixels  # if no better dimensions could be found, use a "line"


def _to_image_array(file_path):
    """
    Converts the file in file_path to a numpy array (matrix) representing an RGB image
    The dimensions of the image are calculated using __determine_dimensions.
    Padding is added provide enough bytes to generate the image (between 1 and 3 bytes can be added).
    """
Beispiel #9
0
import signal
import sys

from fcb.checker import mail, mega
from fcb.checker.settings import Configuration
from fcb.database.helpers import get_session
from fcb.database.helpers import get_db_version
from fcb.utils.log_helper import get_logger_module

log = get_logger_module("mail_checker")


def main():
    # noinspection PyUnresolvedReferences
    import log_configuration

    if len(sys.argv) < 2:
        log.error("Usage: %s <config_file>", sys.argv[0])
        exit(1)

    conf = Configuration(sys.argv[1])

    with get_session() as session:
        db_version = get_db_version(session)
        if db_version != 3:
            log.error("Invalid database version (%d). 3 expected", db_version)
            session.close()
            exit(1)

        session.close()
Beispiel #10
0
import sys

from fcb.database.helpers import get_session
from fcb.database.schema import FilesContainer
from fcb.processing.transformations.Cipher import Cipher
from fcb.processing.transformations import ToImage
from fcb.utils.digest import gen_sha1
from fcb.utils.log_helper import get_logger_module


# noinspection PyUnresolvedReferences
import log_configuration

log = get_logger_module('untransform_file')


def decrypt(in_filename, out_filename, cipher_key_getter):
    key = cipher_key_getter()
    if key is None:
        log.error("No key to decrypt file '%s', will ignore it", in_filename)
    log.debug("Decrypting file '%s' with key '%s'. Resulting file will be called '%s'." %
              (in_filename, key, out_filename))
    Cipher.decrypt_file(key=key, in_filename=in_filename, out_filename=out_filename)
    log.info("File '%s' decrypted to '%s'." % (in_filename, out_filename))


def transform_from_image(in_filename, out_filename):
    ToImage.from_image_to_file(in_filename, out_filename)


def untransform(in_filename, cipher_key_getter):
Beispiel #11
0
from fcb.processing.transformations.ToImage import ToImage
from fcb.sending.debug.FakeSender import FakeSender
from fcb.sending.SentLog import SentLog
from fcb.sending.debug.SlowSender import SlowSender
from fcb.sending.directory.ToDirectorySender import ToDirectorySender
from fcb.sending.mail.MailSender import MailSender
from fcb.sending.mega.MegaSender import MegaSender
from fcb.utils import trickle
from fcb.utils.Settings import Settings, InvalidSettings
from fcb.utils.log_helper import get_logger_module, deep_print


# noinspection PyUnresolvedReferences
import fcb.log_configuration

log = get_logger_module('main')


class App(Component):
    pipeline = Pipeline()

    def init(self, settings, session):
        log.debug(deep_print(settings, "Building pipeline using settings loaded:"))

        # FIXME senders setting should be simpler to handle
        sender_settings = [sender_settings for sender_settings in settings.mail_accounts]
        if settings.dir_dest is not None:
            sender_settings.append(settings.dir_dest)
        if settings.mega_settings is not None:
            sender_settings.append(settings.mega_settings)
Beispiel #12
0
from sqlalchemy import select
from sqlalchemy.orm.util import aliased
from sqlalchemy.sql.expression import exists, and_

from fcb.database.helpers import get_session
from fcb.database.schema import FilesDestinations, Destination, FileFragment, FilesContainer, FilesInContainers, \
    UploadedFile
from fcb.utils.log_helper import get_logger_module


_log = get_logger_module('cleanup_helper')


def delete_unverified_uploads(destinations):
    """
    :param destinations: list of Destination.destination

    For each Destination.destination:
        Deletes all FilesDestinations where the destination is not verified
        Deletes each FilesContainer in the deleted FilesDestinations if not present in a non deleted FilesDestinations
        Deletes each FileFragment if corresponds to a FilesInContainers for a FilesContainer deleted and not in
            a non deleted FilesContainer
        Deletes each UploadedFile if corresponds to a FilesInContainers for a FilesContainer deleted and not in
            a non deleted FilesContainer and/or has no more FileFragment in non deleted FilesContainer
    """
    with get_session() as session:
        # TODO use triggers or cascades to delete relations
        for destination in destinations:
            _log.info("Deleting unverified uploads for destination %s", destination)

            # get unverified FilesDestinations for the configured mail_conf