def verify_access(cls, settings):
log = get_logger_module(cls.__name__)
# try megadf to check if we can access
command = cls.build_command_argumetns(command_str="megadf", settings=settings)
log.debug("Executing command: %s", command)
try:
cls.check_call(command)
except CalledProcessError as e:
raise DestinationInaccessible("Failed access. Running '%s' result was '%s'", command, e.output)
log.debug("Access verified to destination mega (command: %s)", command)
def create_dest_directories(cls, settings):
dst_path = cls.to_absoulte_dst_path(settings)
log = get_logger_module(cls.__name__)
# we generate each directory from root so we create all that are missing
splited = dst_path.split(cls.dir_delimiter)
subdirs = []
# note: first 2 will be "" and root directory (since absolute path starts with "/<root>")
for included in xrange(3, len(splited) + 1):
subdirs.append(cls.dir_delimiter.join(splited[:included]))
if subdirs:
command = cls.build_command_argumetns(command_str="megamkdir", settings=settings, extra_args=subdirs)
log.debug("Executing command: %s", command)
call(command, start_new_session=True)
def do_init(self,
sender_spec,
tmp_file_parts_basepath,
should_split_small_files,
global_quota):
super(_CompressorJob, self).do_init()
self._tmp_file_parts_basepath = tmp_file_parts_basepath
self._destinations = sender_spec.destinations
self._current_block = None
self._block_fragmenter = _BlockFragmenter(sender_spec=sender_spec,
should_split_small_files=should_split_small_files,
global_quota=global_quota)
self.name = "".join((self.__class__.__name__, '(to ', str(self._destinations), ')'))
self.log = get_logger_module(self.name)
import code
import traceback
import signal
import sys
from fcb.utils.log_helper import get_logger_module
_log = get_logger_module("debugging")
# logic taken from http://stackoverflow.com/questions/132058/showing-the-stack-trace-from-a-running-python-application
def attach_debugger(sig, frame):
"""Interrupt running process, and provide a python prompt for
interactive debugging."""
d = {'_frame': frame} # Allow access to frame object.
d.update(frame.f_globals) # Unless shadowed by global
d.update(frame.f_locals)
i = code.InteractiveConsole(d)
message = "Signal received : entering python shell.\nTraceback:\n"
message += ''.join(traceback.format_stack(frame))
i.interact(message)
# Adapted from http://bzimmer.ziclix.com/2008/12/17/python-thread-dumps/
def gen_stacktraces():
stacktrace = []
for threadId, stack in sys._current_frames().items():
stacktrace.append("\n# ThreadID: %s" % threadId)
for filename, lineno, name, line in traceback.extract_stack(stack):
stacktrace.append('File: "%s", line %d, in %s' % (filename, lineno, name))
import os
import poplib
import re
import tempfile
import time
from sqlalchemy import update, and_
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.sql.expression import select
from fcb.database.helpers import get_session
from fcb.database.schema import FilesContainer, CheckerState, Destination, FilesDestinations
from fcb.processing.models.FileInfo import FileInfo
from fcb.utils.log_helper import get_logger_for, get_logger_module
log = get_logger_module('mail_checker')
# FIXME optimize
class CheckHistoryVerifier(object):
def __init__(self, mail_dst):
self._session_resource = get_session()
self._log = get_logger_for(self)
self._mail_dst = mail_dst
self._last_checked_date = None
def close(self):
if self._session_resource:
with self._session_resource as session:
session.commit()
session.close()
def init(self, max_pending_for_processing):
self.log = get_logger_module(self.__class__.__name__)
self._max_pending_for_processing = max_pending_for_processing
self._cur_available = self._max_pending_for_processing
def init(self, next_task=None, *args, **kwargs):
self.log = get_logger_module(self.__class__.__name__)
self.next_task(next_task)
if hasattr(self, "do_init") and isinstance(self.do_init, Callable):
self.do_init(*args, **kwargs)
from PIL import Image
import numpy
import math
from fcb.framework.workers import hd_worker_pool
from fcb.framework.workflow.HeavyPipelineTask import HeavyPipelineTask
from fcb.processing.models.FileInfo import FileInfo
from fcb.utils.log_helper import get_logger_module
_log = get_logger_module("ToImage")
# TODO image and array manipulation functions need to be optimized (a lot)
def _determine_dimensions(num_of_pixels):
"""
Given a number of pixels, determines the largest width and height that define a
rectangle with such an area
"""
for x in xrange(int(math.sqrt(num_of_pixels)) + 1, 1, -1):
if num_of_pixels % x == 0:
return num_of_pixels // x, x
return 1, num_of_pixels # if no better dimensions could be found, use a "line"
def _to_image_array(file_path):
"""
Converts the file in file_path to a numpy array (matrix) representing an RGB image
The dimensions of the image are calculated using __determine_dimensions.
Padding is added provide enough bytes to generate the image (between 1 and 3 bytes can be added).
"""
import signal
import sys
from fcb.checker import mail, mega
from fcb.checker.settings import Configuration
from fcb.database.helpers import get_session
from fcb.database.helpers import get_db_version
from fcb.utils.log_helper import get_logger_module
log = get_logger_module("mail_checker")
def main():
# noinspection PyUnresolvedReferences
import log_configuration
if len(sys.argv) < 2:
log.error("Usage: %s <config_file>", sys.argv[0])
exit(1)
conf = Configuration(sys.argv[1])
with get_session() as session:
db_version = get_db_version(session)
if db_version != 3:
log.error("Invalid database version (%d). 3 expected", db_version)
session.close()
exit(1)
session.close()
import sys
from fcb.database.helpers import get_session
from fcb.database.schema import FilesContainer
from fcb.processing.transformations.Cipher import Cipher
from fcb.processing.transformations import ToImage
from fcb.utils.digest import gen_sha1
from fcb.utils.log_helper import get_logger_module
# noinspection PyUnresolvedReferences
import log_configuration
log = get_logger_module('untransform_file')
def decrypt(in_filename, out_filename, cipher_key_getter):
key = cipher_key_getter()
if key is None:
log.error("No key to decrypt file '%s', will ignore it", in_filename)
log.debug("Decrypting file '%s' with key '%s'. Resulting file will be called '%s'." %
(in_filename, key, out_filename))
Cipher.decrypt_file(key=key, in_filename=in_filename, out_filename=out_filename)
log.info("File '%s' decrypted to '%s'." % (in_filename, out_filename))
def transform_from_image(in_filename, out_filename):
ToImage.from_image_to_file(in_filename, out_filename)
def untransform(in_filename, cipher_key_getter):
from fcb.processing.transformations.ToImage import ToImage
from fcb.sending.debug.FakeSender import FakeSender
from fcb.sending.SentLog import SentLog
from fcb.sending.debug.SlowSender import SlowSender
from fcb.sending.directory.ToDirectorySender import ToDirectorySender
from fcb.sending.mail.MailSender import MailSender
from fcb.sending.mega.MegaSender import MegaSender
from fcb.utils import trickle
from fcb.utils.Settings import Settings, InvalidSettings
from fcb.utils.log_helper import get_logger_module, deep_print
# noinspection PyUnresolvedReferences
import fcb.log_configuration
log = get_logger_module('main')
class App(Component):
pipeline = Pipeline()
def init(self, settings, session):
log.debug(deep_print(settings, "Building pipeline using settings loaded:"))
# FIXME senders setting should be simpler to handle
sender_settings = [sender_settings for sender_settings in settings.mail_accounts]
if settings.dir_dest is not None:
sender_settings.append(settings.dir_dest)
if settings.mega_settings is not None:
sender_settings.append(settings.mega_settings)
from sqlalchemy import select
from sqlalchemy.orm.util import aliased
from sqlalchemy.sql.expression import exists, and_
from fcb.database.helpers import get_session
from fcb.database.schema import FilesDestinations, Destination, FileFragment, FilesContainer, FilesInContainers, \
UploadedFile
from fcb.utils.log_helper import get_logger_module
_log = get_logger_module('cleanup_helper')
def delete_unverified_uploads(destinations):
"""
:param destinations: list of Destination.destination
For each Destination.destination:
Deletes all FilesDestinations where the destination is not verified
Deletes each FilesContainer in the deleted FilesDestinations if not present in a non deleted FilesDestinations
Deletes each FileFragment if corresponds to a FilesInContainers for a FilesContainer deleted and not in
a non deleted FilesContainer
Deletes each UploadedFile if corresponds to a FilesInContainers for a FilesContainer deleted and not in
a non deleted FilesContainer and/or has no more FileFragment in non deleted FilesContainer
"""
with get_session() as session:
# TODO use triggers or cascades to delete relations
for destination in destinations:
_log.info("Deleting unverified uploads for destination %s", destination)
# get unverified FilesDestinations for the configured mail_conf