def verify_access(cls, settings): log = get_logger_module(cls.__name__) # try megadf to check if we can access command = cls.build_command_argumetns(command_str="megadf", settings=settings) log.debug("Executing command: %s", command) try: cls.check_call(command) except CalledProcessError as e: raise DestinationInaccessible("Failed access. Running '%s' result was '%s'", command, e.output) log.debug("Access verified to destination mega (command: %s)", command)
def create_dest_directories(cls, settings): dst_path = cls.to_absoulte_dst_path(settings) log = get_logger_module(cls.__name__) # we generate each directory from root so we create all that are missing splited = dst_path.split(cls.dir_delimiter) subdirs = [] # note: first 2 will be "" and root directory (since absolute path starts with "/<root>") for included in xrange(3, len(splited) + 1): subdirs.append(cls.dir_delimiter.join(splited[:included])) if subdirs: command = cls.build_command_argumetns(command_str="megamkdir", settings=settings, extra_args=subdirs) log.debug("Executing command: %s", command) call(command, start_new_session=True)
def do_init(self, sender_spec, tmp_file_parts_basepath, should_split_small_files, global_quota): super(_CompressorJob, self).do_init() self._tmp_file_parts_basepath = tmp_file_parts_basepath self._destinations = sender_spec.destinations self._current_block = None self._block_fragmenter = _BlockFragmenter(sender_spec=sender_spec, should_split_small_files=should_split_small_files, global_quota=global_quota) self.name = "".join((self.__class__.__name__, '(to ', str(self._destinations), ')')) self.log = get_logger_module(self.name)
import code import traceback import signal import sys from fcb.utils.log_helper import get_logger_module _log = get_logger_module("debugging") # logic taken from http://stackoverflow.com/questions/132058/showing-the-stack-trace-from-a-running-python-application def attach_debugger(sig, frame): """Interrupt running process, and provide a python prompt for interactive debugging.""" d = {'_frame': frame} # Allow access to frame object. d.update(frame.f_globals) # Unless shadowed by global d.update(frame.f_locals) i = code.InteractiveConsole(d) message = "Signal received : entering python shell.\nTraceback:\n" message += ''.join(traceback.format_stack(frame)) i.interact(message) # Adapted from http://bzimmer.ziclix.com/2008/12/17/python-thread-dumps/ def gen_stacktraces(): stacktrace = [] for threadId, stack in sys._current_frames().items(): stacktrace.append("\n# ThreadID: %s" % threadId) for filename, lineno, name, line in traceback.extract_stack(stack): stacktrace.append('File: "%s", line %d, in %s' % (filename, lineno, name))
import os import poplib import re import tempfile import time from sqlalchemy import update, and_ from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.sql.expression import select from fcb.database.helpers import get_session from fcb.database.schema import FilesContainer, CheckerState, Destination, FilesDestinations from fcb.processing.models.FileInfo import FileInfo from fcb.utils.log_helper import get_logger_for, get_logger_module log = get_logger_module('mail_checker') # FIXME optimize class CheckHistoryVerifier(object): def __init__(self, mail_dst): self._session_resource = get_session() self._log = get_logger_for(self) self._mail_dst = mail_dst self._last_checked_date = None def close(self): if self._session_resource: with self._session_resource as session: session.commit() session.close()
def init(self, max_pending_for_processing): self.log = get_logger_module(self.__class__.__name__) self._max_pending_for_processing = max_pending_for_processing self._cur_available = self._max_pending_for_processing
def init(self, next_task=None, *args, **kwargs): self.log = get_logger_module(self.__class__.__name__) self.next_task(next_task) if hasattr(self, "do_init") and isinstance(self.do_init, Callable): self.do_init(*args, **kwargs)
from PIL import Image import numpy import math from fcb.framework.workers import hd_worker_pool from fcb.framework.workflow.HeavyPipelineTask import HeavyPipelineTask from fcb.processing.models.FileInfo import FileInfo from fcb.utils.log_helper import get_logger_module _log = get_logger_module("ToImage") # TODO image and array manipulation functions need to be optimized (a lot) def _determine_dimensions(num_of_pixels): """ Given a number of pixels, determines the largest width and height that define a rectangle with such an area """ for x in xrange(int(math.sqrt(num_of_pixels)) + 1, 1, -1): if num_of_pixels % x == 0: return num_of_pixels // x, x return 1, num_of_pixels # if no better dimensions could be found, use a "line" def _to_image_array(file_path): """ Converts the file in file_path to a numpy array (matrix) representing an RGB image The dimensions of the image are calculated using __determine_dimensions. Padding is added provide enough bytes to generate the image (between 1 and 3 bytes can be added). """
import signal import sys from fcb.checker import mail, mega from fcb.checker.settings import Configuration from fcb.database.helpers import get_session from fcb.database.helpers import get_db_version from fcb.utils.log_helper import get_logger_module log = get_logger_module("mail_checker") def main(): # noinspection PyUnresolvedReferences import log_configuration if len(sys.argv) < 2: log.error("Usage: %s <config_file>", sys.argv[0]) exit(1) conf = Configuration(sys.argv[1]) with get_session() as session: db_version = get_db_version(session) if db_version != 3: log.error("Invalid database version (%d). 3 expected", db_version) session.close() exit(1) session.close()
import sys from fcb.database.helpers import get_session from fcb.database.schema import FilesContainer from fcb.processing.transformations.Cipher import Cipher from fcb.processing.transformations import ToImage from fcb.utils.digest import gen_sha1 from fcb.utils.log_helper import get_logger_module # noinspection PyUnresolvedReferences import log_configuration log = get_logger_module('untransform_file') def decrypt(in_filename, out_filename, cipher_key_getter): key = cipher_key_getter() if key is None: log.error("No key to decrypt file '%s', will ignore it", in_filename) log.debug("Decrypting file '%s' with key '%s'. Resulting file will be called '%s'." % (in_filename, key, out_filename)) Cipher.decrypt_file(key=key, in_filename=in_filename, out_filename=out_filename) log.info("File '%s' decrypted to '%s'." % (in_filename, out_filename)) def transform_from_image(in_filename, out_filename): ToImage.from_image_to_file(in_filename, out_filename) def untransform(in_filename, cipher_key_getter):
from fcb.processing.transformations.ToImage import ToImage from fcb.sending.debug.FakeSender import FakeSender from fcb.sending.SentLog import SentLog from fcb.sending.debug.SlowSender import SlowSender from fcb.sending.directory.ToDirectorySender import ToDirectorySender from fcb.sending.mail.MailSender import MailSender from fcb.sending.mega.MegaSender import MegaSender from fcb.utils import trickle from fcb.utils.Settings import Settings, InvalidSettings from fcb.utils.log_helper import get_logger_module, deep_print # noinspection PyUnresolvedReferences import fcb.log_configuration log = get_logger_module('main') class App(Component): pipeline = Pipeline() def init(self, settings, session): log.debug(deep_print(settings, "Building pipeline using settings loaded:")) # FIXME senders setting should be simpler to handle sender_settings = [sender_settings for sender_settings in settings.mail_accounts] if settings.dir_dest is not None: sender_settings.append(settings.dir_dest) if settings.mega_settings is not None: sender_settings.append(settings.mega_settings)
from sqlalchemy import select from sqlalchemy.orm.util import aliased from sqlalchemy.sql.expression import exists, and_ from fcb.database.helpers import get_session from fcb.database.schema import FilesDestinations, Destination, FileFragment, FilesContainer, FilesInContainers, \ UploadedFile from fcb.utils.log_helper import get_logger_module _log = get_logger_module('cleanup_helper') def delete_unverified_uploads(destinations): """ :param destinations: list of Destination.destination For each Destination.destination: Deletes all FilesDestinations where the destination is not verified Deletes each FilesContainer in the deleted FilesDestinations if not present in a non deleted FilesDestinations Deletes each FileFragment if corresponds to a FilesInContainers for a FilesContainer deleted and not in a non deleted FilesContainer Deletes each UploadedFile if corresponds to a FilesInContainers for a FilesContainer deleted and not in a non deleted FilesContainer and/or has no more FileFragment in non deleted FilesContainer """ with get_session() as session: # TODO use triggers or cascades to delete relations for destination in destinations: _log.info("Deleting unverified uploads for destination %s", destination) # get unverified FilesDestinations for the configured mail_conf