Ejemplo n.º 1
0
def process_sync(sync_id):
    sync = Sync.get(sync_id)
    if not sync:
        return
    if Transfer.find_one({'sync_id': sync['_id'], 'finished': None}):
        set_retry(sync)
        return
    src = get_uris(**sync['src'])
    if not src:
        set_retry(sync)
        return
    dst = get_uris(**sync['dst'])
    if not dst:
        set_retry(sync)
        return
    dst = dst[0]

    parameters = {
        'exclusions': sync.get('exclusions'),
        'delete': sync.get('delete'),
        }
    is_mount = (sync['dst'].get('uuid') or sync['src'].get('uuid')) is not None
    transfer_id = Transfer.add(src, dst, sync_id=sync['_id'],
            parameters=parameters, is_mount=is_mount)
    logger.info('added transfer %s to %s', src, dst)

    sync['transfer_id'] = transfer_id
    sync['processed'] = datetime.utcnow()
    sync['reserved'] = datetime.utcnow() + timedelta(hours=sync['recurrence'])
    Sync.save(sync, safe=True)
Ejemplo n.º 2
0
def create_media():
    data = request.json
    type = data.get('type')
    if not type:
        return jsonify(error='missing type')
    langs = data.get('langs') or []

    if 'id' in data:
        if not data.get('mode'):
            return jsonify(error='missing mode')
        id = ObjectId(data['id'])
        search = _get_object_search(id, type)
        if not search:
            return jsonify(error='%s %s does not exist' % (type, id))
        search['langs'] = langs
        search['mode'] = data['mode']
        search['safe'] = False
        if not Search.add(**search):
            return jsonify(error='failed to create search %s' % search)
        return jsonify(result=True)

    name = data.get('name')
    if not name:
        return jsonify(error='missing name')

    if type == 'url':
        dst = Settings.get_settings('paths')['finished_download']
        try:
            Transfer.add(name, dst)
        except Exception, e:
            return jsonify(error='failed to create transfer: %s' % str(e))
Ejemplo n.º 3
0
class Manager:
    def __init__(self, screen, clock, **kwargs):
        # self.turns = turns()
        self.screen = screen
        self.status = _INTRO
        self.minigames = []
        self.board = Board(screen=screen, clock=clock)
        self.intro = Slideshow(screen=screen, slides=((1800000, "intro.png"),))
        self.minigames.append((Missiles(screen=screen, clock=clock), ["Dodge"]))
        self.minigames.append((WhackAMullet(screen=screen, clock=clock), ["Kill"]))
        self.minigames.append((Freeze(screen=screen, clock=clock), ["Dodge", "Kill", "Jump", "Type"]))
        self.minigames.append((Platformer(clock=clock, screen=screen), ["Jump"]))
        # self.minigames.append(EasyWin(screen))
        self.minigames.append((ClickClack(screen), ["Type"]))
        self.current = self.intro
        self.catcher = None
        self.wait = 7
        self.currWait = 0
        self.counter = 0

    def draw(self, screen):
        self.current.draw()

    def update(self):
        if self.status != _TRANSFER:
            self.catcher = self.current.update(result=self.catcher)
            if self.status == _MINI and self.catcher != None:
                print "switching to tbg"
                self.current.reset()
                self.current = self.board
                self.status = _TBG
                self.currWait = self.wait
            elif self.status == _TBG and self.catcher != None:
                print "switching to transfer"
                self.easywin = EasyWin(self.screen)
                self.status = _TRANSFER
                self.counter += 1
                if self.counter == len(self.minigames):
                    self.counter = 0
                select = self.minigames[self.counter]
                # select = self.minigames[random.randint(0,len(self.minigames)-1)]
                self.current = Transfer(self.screen, select[1][random.randint(0, len(select[1]) - 1)], select[0])
                # self.current = self.minigames[random.randint(0,len(self.minigames)-1)]
                # self.status = _MINI
                self.currWait = self.wait
            elif self.status == _INTRO and self.catcher != None:
                self.catcher = None
                self.current.reset()
                self.current = self.board
                self.status = _TBG
                self.currWait = self.wait
        else:
            # clear queue
            # self.currWait -= 1
            result = self.current.update()
            pygame.event.get()
            if result != None:
                print "switching to mini"
                self.current = result
                self.status = _MINI
Ejemplo n.º 4
0
def run():
    Transfer.remove({'finished': {
            '$lt': datetime.utcnow() - DELTA_OBSOLETE,
            }}, safe=True)

    if Settings.get_settings('transmission').get('active', True):
        clean_torrents()

    if Settings.get_settings('sabnzbd').get('active', True):
        clean_nzbs()
Ejemplo n.º 5
0
def create_transfer():
    data = request.json
    src = data.get('src')
    if not src:
        return jsonify(error='missing src')

    try:
        Transfer.add(src, data.get('dst'), data.get('type'))
    except Exception, e:
        return jsonify(error=str(e))
Ejemplo n.º 6
0
def process_query(query):
    '''Process the query and return the number of results.
    '''
    if urlparse(query).scheme:
        dst = Settings.get_settings('paths')['finished_download']
        try:
            Transfer.add(query, dst)
        except Exception, e:
            logger.error('failed to create transfer for %s: %s', query, str(e))
            return -1
        count = 1
Ejemplo n.º 7
0
def main():
    pm.COMMANDLINE = True
    args = parse_args()

    pwd = None

    try:
        if not args.free:
            if KEYRING:
                pwd = keyring.get_password('pyfilemail', args.username)

            elif pm.NETRC:
                machine = pm._netrc.authenticators(args.username)
                if machine:
                    pwd = machine[2]

                else:
                    pwd = None

            if pwd is None:
                pwd = getpass.getpass('Please enter Filemail password: '******'pyfilemail', args.username, pwd)

        fm_user = User(args.username, password=pwd)

        transfer = Transfer(
            fm_user,
            to=args.to,
            subject=args.subject,
            message=args.message,
            notify=args.notify,
            confirmation=args.confirm,
            days=args.days,
            downloads=args.downloads,
            password=args.password,
            checksum=args.checksum,
            zip_=args.compress
            )

        transfer.add_files(args.payload)

        res = transfer.send()

        if res.status_code == 200:
            msg = '\nTransfer complete!'
            logger.info(msg)

    except KeyboardInterrupt:
        msg = '\nAborted by user!'
        logger.warning(msg)
Ejemplo n.º 8
0
    def _restore_transfers(self, response):
        """Restore transfers from josn retreived Filemail
        :param response: response object from request
        :rtype: ``list`` with :class:`Transfer` objects
        """

        transfers = []
        for transfer_data in response.json()['transfers']:
            transfer = Transfer(self, _restore=True)
            transfer.transfer_info.update(transfer_data)
            transfer.get_files()
            transfers.append(transfer)

        return transfers
Ejemplo n.º 9
0
    def _callback(self, processed, total):
        if not Transfer.find_one({
                '_id': self.transfer['_id'],
                'finished': None,
                }):
            logger.info('aborted sftp transfer %s to %s', self.transfer['src'], self.transfer['dst'])
            sys.exit(1)     # to avoid zombies

        now = time.time()
        if now - self.last_callback < settings.PROGRESS_CALLBACK_DELTA:
            return
        self.last_callback = now
        self.transfer['transferred'] = self.transferred
        self.transfer['info']['name'] = self.name
        Transfer.save(self.transfer, safe=True)
Ejemplo n.º 10
0
Archivo: hmf.py Proyecto: tbs1980/hmf
    def __init__(self, M=None, mf_fit="ST", delta_h=200.0,
                 delta_wrt='mean', cut_fit=True, z2=None, nz=None,
                 delta_c=1.686, mv_scheme="trapz", **kwargs):
        """
        Initializes some parameters      
        """
        if M is None:
            M = np.linspace(10, 15, 501)

        # A list of all available kwargs (sent to Cosmology via Transfer)
        self._cp = ["sigma_8", "n", "w", "cs2_lam", "t_cmb", "y_he", "N_nu",
                    "omegan", "H0", "h", "omegab",
                    "omegac", "omegav", "omegab_h2", "omegac_h2",
                    "force_flat", "default"]

        # Set up a simple dictionary of kwargs which can be later updated
        self._cpdict = {k:v for k, v in kwargs.iteritems() if k in self._cp}

        # Set all given parameters.
        self.mf_fit = mf_fit
        self.M = M
        self.delta_h = delta_h
        self.delta_wrt = delta_wrt
        self.cut_fit = cut_fit
        self.z2 = z2
        self.nz = nz
        self.delta_c = delta_c
        self.transfer = Transfer(**kwargs)
        self.mv_scheme = mv_scheme

        tools.check_kr(self.M[0], self.M[-1], self.cosmo.mean_dens,
                       self.transfer.lnk[0], self.transfer.lnk[-1])
Ejemplo n.º 11
0
    def process(self):
        query = self._get_query()
        dst = Settings.get_settings('paths')['finished_download']

        logger.info('processing %s search "%s"', self.category, query)

        self._search_url()

        for result in results(query,
                category=self.category,
                sort=self.session['sort_results'],
                pages_max=self.session['pages_max'],
                **self._get_filters(query)):
            if not result:
                self.session['nb_errors'] += 1
                continue

            Result.add_result(result, search_id=self._id)

            if not result.auto:
                continue
            if self.safe and not result.safe:
                continue

            if result.get('hash'):
                spec = {'info.hash': result.hash}
            else:
                spec = {'src': result.url}
            if Transfer.find_one(spec):
                continue

            self.session['nb_results'] += 1
            if not self._validate_result(result):
                self.session['nb_pending'] += 1
                continue

            if self.mode == 'inc':
                self._add_next('episode')

            transfer_id = Transfer.add(result.url, dst, type=result.type)
            self.transfers.insert(0, transfer_id)

            self.session['nb_downloads'] += 1
            logger.info('found "%s" on %s (%s)', result.title, result.plugin, result.url)

            if self.mode != 'ever':
                break
Ejemplo n.º 12
0
def manage_torrent(client, hash, dst):
    try:
        torrent = client.get_torrent(hash=hash)
        if not torrent:
            return

        if not client.check_torrent_files(torrent,
                check_unfinished=settings.CHECK_UNFINISHED_TORRENTS):
            invalid_dir = str(Settings.get_settings('paths')['invalid'])
            if torrent.progress == 100 and not client.move_files(torrent, invalid_dir):
                return
            if client.remove_torrent(hash=hash, delete_data=True):
                logger.info('removed invalid torrent "%s" (%s%% done)', torrent.name, int(torrent.progress))
            Transfer.update({'info.hash': hash, 'finished': None},
                    {'$set': {'finished': datetime.utcnow()}}, safe=True)
            return

        if torrent.progress == 100:
            destination = client.get_destination_dir(torrent, dst)
            if not client.move_files(torrent, destination):
                return
            if client.remove_torrent(hash=hash):
                logger.info('moved finished torrent "%s" to %s', torrent.name, dst)
            Transfer.update({'info.hash': hash, 'finished': None},
                    {'$set': {'finished': datetime.utcnow()}}, safe=True)
            return

        torrent_settings = Settings.get_settings('torrent')
        now = datetime.utcnow()

        inactive_delta = torrent_settings['inactive_delta']
        if inactive_delta:
            date = torrent.date_active or torrent.date_added
            if date < now - timedelta(hours=inactive_delta) \
                    and client.remove_torrent(hash=hash, delete_data=True):
                logger.debug('removed inactive torrent "%s": no activity since %s', torrent.name, date)
                return

        added_delta = torrent_settings['added_delta']
        if added_delta:
            date = torrent.date_added
            if date < now - timedelta(hours=added_delta) \
                    and client.remove_torrent(hash=hash, delete_data=True):
                logger.debug('removed obsolete torrent "%s": added %s', torrent.name, date)

    except TransmissionError, e:
        logger.error('torrent client error: %s', str(e))
Ejemplo n.º 13
0
    def process_transfer(self, id):
        transfer = Transfer.find_one({'_id': id, 'queued': None})
        if not transfer:
            return

        try:
            info = get_torrent_client().add_torrent(transfer['src'])
            Transfer.update({'_id': transfer['_id']},
                    {'$set': {
                        'queued': datetime.utcnow(),
                        'info': info,
                    }}, safe=True)
        except TorrentExists, e:
            Transfer.update({'_id': transfer['_id']},
                    {'$set': {'finished': datetime.utcnow()}},
                    safe=True)
            logger.debug('failed to start torrent: %s', str(e))
Ejemplo n.º 14
0
def run():
    running = {}
    for res in Transfer.find({
            'started': {'$ne': None},
            'finished': None,
            }):
        add_running(running, res['type'])

    settings_ = Settings.get_settings('general')
    retry_delta = timedelta(seconds=settings_['retry_delta'])

    for transfer in Transfer.find({
            'finished': None,
            '$or': [
                {'added': None},
                {
                'added': {'$lt': datetime.utcnow() - retry_delta},
                'started': None,
                'tries': {'$lt': settings_['max_tries']},
                },
                ],
            }):
        limit = settings.WORKERS_LIMITS.get(transfer['type'])
        if limit and running.get(transfer['type'], 0) >= limit:
            continue

        factory = get_factory()
        target = '%s.workers.add.process' % settings.PACKAGE_NAME
        if factory.get(target=target,
                args=(transfer['_id'], transfer['type'])):
            continue
        factory.add(target=target,
                args=(transfer['_id'], transfer['type']),
                timeout=settings.PROCESS_TIMEOUT)

        now = datetime.utcnow()
        Transfer.update({'_id': transfer['_id']}, {
                '$set': {'added': now, 'started': now},
                '$inc': {'tries': 1},
                }, safe=True)

        add_running(running, transfer['type'])

        count_str = ' (#%s)' % transfer['tries'] if transfer['tries'] > 1 else ''
        logger.info('started%s %s transfer %s to %s', count_str, transfer['type'], transfer['src'], transfer['dst'])
Ejemplo n.º 15
0
 def audio(self):
     print("audio")
     if len(self.audio_folder) != 0:
         files = Transfer(self.audio_folder)
         self.s.send(str.encode("!list_server_audio_files"))
         self.listen_list(files)
     else:
         message_box = Popups("Error", "Please select audio folder")
         message_box.message_box()
Ejemplo n.º 16
0
 def __init__(self, conf_obj):
     """This class returns an object which has the required 
     configuration values. Please add/delete data from __init__ 
     to suit requirements."""
     self.test = 0
     self.conf_obj = conf_obj
     self.transSftp = Transfer("sftp")
     # for next version this should be changed to
     # ftp_trans.FtpTrans(self.conf_obj)
     self.ftp_obj = ftp_trans.FtpTrans()
     self.runid = self.conf_obj.runid
     self.backup_dir = self.conf_obj.backup_path
     # Change below code to modify logging format
     if not os.path.isdir(self.backup_dir):
         raise error.FnfError(self.backup_dir, "Error: backup dir not found" + self.backup_dir + "; Exiting")
     self.root_dir = self.conf_obj.root_path
     self.runid_dir = os.getcwd()
     self.transCopy = Transfer("copy")
Ejemplo n.º 17
0
    def test_predict(self):
        c = Params()
        c.load_config_file(JSON_PATH)
        cnn = Transfer(c, MODEL_NAME, PATCH_TYPE)
        model = cnn.load_model(mode=0)
        model.compile(optimizer="RMSprop",
                      loss='categorical_crossentropy',
                      metrics=['accuracy'])

        imgCone = ImageCone(c, Open_Slide())

        # 读取数字全扫描切片图像
        tag = imgCone.open_slide("Tumor/Tumor_004.tif", None, "Tumor_004")
        seeds = [(34880, 48960), (36224, 49920), (13312, 57856)]  # C, C, S
        result = cnn.predict(model, imgCone, 20, 256, seeds)
        print(result)

        result = cnn.predict_on_batch(model, imgCone, 20, 256, seeds, 2)
        print(result)
Ejemplo n.º 18
0
    def getEntityByFilter(self, filters):
        """get all entities matching a filter

        Arguments:
            filter {[[array of filters]]} -- [filters for datastore
                                            of the form ["key","operator","val"]]

        Returns:
            [list] -- [list of matching entities]
        """
        query = self._client.query(kind=self._kind)
        for filter in filters:
            query.add_filter(*filter)
        transfers_list = []
        transfers = list(query.fetch())
        for entity in transfers:
            transfer = Transfer(**entity)
            transfer.transferId = entity.key.id
            transfers_list.append(transfer.get_dict())
        return transfers_list
Ejemplo n.º 19
0
    def test_extract_features_for_train_batch(self):
        c = Params()
        c.load_config_file(JSON_PATH)

        # "inception_v3", "densenet121", "densenet169", "densenet201", "resnet50", "inception_resnet_v2",
        #                           "vgg16", "mobilenet_v2"
        MODEL_NAME_set = ["densenet169"]
        SAMPLE_FIlENAME_set = [
            "T_NC_2000_256", "T_NC_4000_256"
        ]  # "T_NC_500_128", , "T_NC_4000_256" "T_NC_2000_256"
        PATCH_TYPE_set = ["2000_256",
                          "4000_256"]  # "500_128", , "4000_256" "2000_256"

        type_set = zip(PATCH_TYPE_set, SAMPLE_FIlENAME_set)

        for p_item, f_item in type_set:
            for net_item in MODEL_NAME_set:

                cnn = Transfer(c, net_item, p_item)
                cnn.extract_features_for_train(f_item, 100)
 def debit_points(self, user, signature, amount=0.0):
     """Debit points from user. Need to verify sufficient points."""
     if self.hosting_node == None:
         return False
     transfer = Transfer(user, signature, amount)
     if Verification.verify_single_transfer(transfer, self.get_balance):
         self.__open_transfers.append(transfer)
         # participants.add(user)
         self.save_data()
         return True
     return False
 def credit_points(self, user, signature, amount=0.0):
     """Credit points to user. No checks required"""
     if self.hosting_node == None:
         return False
     transfer = Transfer(user, signature, amount)
     if not Wallet.verify_transfer(transfer):
         return False
     self.__open_transfers.append(transfer)
     # participants.add(user)
     self.save_data()
     return True
Ejemplo n.º 22
0
def transfer_event_source(request: Request):
    """
    SSE (Server Sent Event) endpoint for transfer events.
    """
    return StreamingResponse(
        Transfer.stream(request),
        media_type='text/event-stream',
        headers={
            'Cache-Control': 'no-cache',
            'X-Accel-Buffering': 'no',
        },
    )
Ejemplo n.º 23
0
    def connect(self):
        self.s = socket.socket()
        self.s.settimeout(5)
        try:
            self.s.connect(self.addr)
        except socket.error:
            return "Server not found. Try again."

        self.trans = Transfer(self.s)
        self.trans.send(self.username.encode())
        response = self.trans.recvData()
        if not response:
            return "Server not responding."
        if response == b"in-use":
            return "Username already in use."
        if response == b"success":
            self.s.settimeout(None)
            self.connected = True
            self.s.ioctl(socket.SIO_KEEPALIVE_VALS, (1, 5000, 3000))
            threading.Thread(target=self.mainThread, daemon=True).start()
            return True
Ejemplo n.º 24
0
    def process_transfer(self, id):
        from mediacore.web.search.plugins.filestube import get_download_urls, FilestubeError

        self.transfer = Transfer.find_one({'_id': id})
        if not self.transfer:
            return

        try:
            src = get_download_urls(self.transfer['src'])
        except FilestubeError, e:
            logger.error('failed to get filestube urls for %s: %s', self.transfer['src'], str(e))
            return
Ejemplo n.º 25
0
class TransferTest(unittest.TestCase):
    def setUp(self):
        sp = MagicMock(spotipy.Spotify())
        self.transfer = Transfer(sp, True)

        self.abbey_road = ['Abbey Road', 'The Beatles']
        self.transfer.get_itunes_album = MagicMock(
            return_value=self.abbey_road)
        self.transfer.curr_album_artist = None

    def test_changed_None_Some(self):
        changed = self.transfer.album_changed()
        self.assertTrue(changed)
        self.assertEqual(self.transfer.get_album_artist(), self.abbey_road)

    def test_changed_Some_Some(self):
        self.transfer.curr_album_artist = ['Revolver', 'The Beatles']

        changed = self.transfer.album_changed()
        self.assertTrue(changed)
        self.assertEqual(self.transfer.get_album_artist(), self.abbey_road)

    def test_changed_None_None(self):
        self.transfer.album_changed = MagicMock(return_value=None)

        changed = self.transfer.album_changed()
        self.assertFalse(changed)

    def test_changed_Some_None(self):
        self.transfer.album_changed = MagicMock(return_value=None)
        self.transfer.curr_album_artist = self.abbey_road

        changed = self.transfer.album_changed()
        self.assertFalse(changed)
Ejemplo n.º 26
0
    def _callback(self, download_total, downloaded, upload_total, uploaded):
        if not Transfer.find_one({
                '_id': self.transfer['_id'],
                'finished': None,
                }):
            self.aborted = True
            return

        now = time.time()
        if now - self.last_callback < settings.PROGRESS_CALLBACK_DELTA:
            return
        self.last_callback = now

        transferred = downloaded or uploaded
        self.transferred_current = transferred
        transferred += self.transferred
        self.transfer['total'] = self.total
        self.transfer['transferred'] = transferred
        total = float(self.transfer['total'])
        self.transfer['progress'] = transferred * 100 / total if total else 0
        self.transfer['info']['name'] = self.name
        Transfer.save(self.transfer, safe=True)
Ejemplo n.º 27
0
    def process_transfer(self, id):
        import media
        from mediacore.web.search.plugins.rutracker import download_torrent, DownloadError

        transfer = Transfer.find_one({'_id': id, 'queued': None})
        if not transfer:
            return

        try:
            data = download_torrent(transfer['src'])
        except DownloadError, e:
            logger.error('failed to start torrent: %s', str(e))
            return
    def load_data(self):
        """Initialize blockchain + open transfers data from a file"""
        try:
            with open("blockchain.txt", mode="r") as f:
                file_content = f.readlines()
                blockchain = json.loads(file_content[0][:-1])
                # OrderedDict
                updated_blockchain = []
                for block in blockchain:
                    converted_transfers = [
                        Transfer(tx["user"], tx["signature"], tx["amount"])
                        for tx in block["transfers"]
                    ]
                    # converted_transfers = [OrderedDict(
                    #     [('user', tx['user']), ('amount', tx['amount'])]) for tx in block['transfers']]
                    updated_block = Block(
                        block["index"],
                        block["previous_hash"],
                        converted_transfers,
                        block["proof"],
                        block["timestamp"],
                    )
                    updated_blockchain.append(updated_block)
                self.__chain = updated_blockchain
                open_transfers = json.loads(file_content[1][:-1])
                # OrderedDict
                updated_transfers = []
                for tx in open_transfers:
                    updated_transfer = Transfer(tx["user"], tx["signature"],
                                                tx["amount"])
                    # updated_transfer = OrderedDict(
                    #     [('user', tx['user']), ('amount', tx['amount'])])
                    updated_transfers.append(updated_transfer)
                self.__open_transfers = updated_transfers
                peer_nodes = json.loads(file_content[2])
                self.__peer_nodes = set(peer_nodes)

        except (IOError, IndexError):
            pass
Ejemplo n.º 29
0
    def process_transfer(self, id):
        self.transfer = Transfer.find_one({'_id': id})
        if not self.transfer:
            return

        src = self.transfer['src']
        dst = self.transfer['dst']
        if not isinstance(src, (tuple, list)):
            src = [src]

        temp_dir = Settings.get_settings('paths')['tmp']
        res = self.process(src, dst, temp_dir)
        if res:
            self.transfer['info']['files'] = res
            self.transfer['transferred'] = self.total
            self.transfer['progress'] = 100
            logger.info('finished http transfer %s to %s', src, dst)
        else:
            self.transfer['started'] = None
            logger.error('failed to process http transfer %s to %s', src, dst)
        self.transfer['finished'] = datetime.utcnow()
        Transfer.save(self.transfer, safe=True)
Ejemplo n.º 30
0
    def process_transfer(self, id):
        from mediacore.web.search.plugins.binsearch import get_nzb, BinsearchError

        transfer = Transfer.find_one({'_id': id, 'queued': None})
        if not transfer:
            return

        qs = parse_qs(urlparse(transfer['src']).query)
        try:
            name = clean(qs['b'][0])
        except KeyError:
            Transfer.update({'_id': transfer['_id']},
                    {'$set': {'finished': datetime.utcnow()}}, safe=True)
            logger.error('failed to get nzb name from %s', transfer['src'])
            return

        try:
            nzb_data = get_nzb(transfer['src'])
        except BinsearchError, e:
            Transfer.update({'_id': transfer['_id']},
                    {'$set': {'started': None}}, safe=True)
            logger.error('failed to get nzb data from %s: %s', transfer['src'], str(e))
            return
Ejemplo n.º 31
0
def manage_nzbs():
    client = get_nzb_client()

    for transfer in Transfer.find({
            'started': {'$ne': None},
            'finished': None,
            'type': 'binsearch',
            }):
        nzb_id = transfer['info'].get('nzo_id')
        if not nzb_id:
            continue
        info = client.get_nzb(nzb_id)
        if not info:
            Transfer.update({'_id': transfer['_id']},
                    {'$set': {'finished': datetime.utcnow()}},
                    safe=True)
        else:
            info['name'] = info.get('filename', transfer['info'].get('name'))
            total = get_float(info.get('mb', 0)) * 1024 ** 2
            Transfer.update({'_id': transfer['_id']},
                    {'$set': {
                        'total': total,
                        'transferred': total - get_float(info.get('mbleft', 0)) * 1024 ** 2,
                        'progress': get_float(info.get('percentage', 0)),
                        'info': info,
                    }}, safe=True)

    paths = Settings.get_settings('paths')

    # Manage queued nzbs
    for nzb in client.list_nzbs():
        transfer = get_nzb_transfer(nzb['nzo_id'])
        if not transfer:
            now = datetime.utcnow()
            Transfer.add(nzb['filename'], str(paths['default']),
                    type='binsearch', added=now, started=now, queued=now,
                    info={'nzo_id': nzb['nzo_id']})
        elif transfer['finished']:
            client.remove_nzb(nzb['nzo_id'])
            logger.info('removed finished nzb "%s" (%s)', nzb['filename'], nzb['nzo_id'])

    # Manage finished nzbs
    for nzb in client.list_nzbs(history=True):
        transfer = get_nzb_transfer(nzb['nzo_id'])
        if nzb['status'] == 'Completed':
            dst = transfer['dst'] if transfer else str(paths['default'])
        elif nzb['status'] == 'Failed':
            dst = str(paths['invalid'])
        else:
            continue
        manage_nzb(client, nzb_id=nzb['nzo_id'], dst=dst)
Ejemplo n.º 32
0
    def __init__(self):
        wx.Frame.__init__(self, None, -1, "Budget ver. 1.0", size=(1100, 650))
        self.Centre(wx.BOTH)
        panel = wx.Panel(self, -1)

        notebook = wx.Notebook(panel)
        notebook.AddPage(Expenses(notebook), "Expenses")
        notebook.AddPage(Income(notebook), "Income")
        notebook.AddPage(Transfer(notebook), "Transfer")
        notebook.AddPage(Accounts(notebook), "Accounts")
        notebook.AddPage(Analysis(notebook), "Analysis")
        notebook.AddPage(Manage(notebook), "Manage")

        sizer = wx.BoxSizer()
        sizer.Add(notebook, 1, wx.EXPAND)
        panel.SetSizer(sizer)
Ejemplo n.º 33
0
    def __init__(self, verbose=False):
        self.classname = "pyUpdater"
        self.version = "0.1"
        self.configdir = "/opt/pyUpdater"
        self.configfile = "conf/pyUpdater.json"
        self.configCustomFile = ""
        self.verbose = verbose
        self.loadConfig()
        logging.basicConfig(filename=self.config['logfile'],
                            level=self.config['loglevel'],
                            format=self.config['logformat'])
        self.preservefile = self.config['whitelist'].split(';')
        self.createTmpDir()

        # INIT transfer module
        self.tr = Tr()
        self.init_transfer()
Ejemplo n.º 34
0
    def process_transfer(self, id):
        self.transfer = Transfer.find_one({'_id': id})
        if not self.transfer:
            return

        src = self.transfer['src']
        dst = self.transfer['dst']
        parameters = self.transfer.get('parameters', {})
        try:
            self.process(src, dst, exclude=parameters.get('exclusions'),
                    delete=parameters.get('delete', True))
            self.transfer['transferred'] = self.transferred
            self.transfer['progress'] = 100
            logger.info('finished ftp transfer %s to %s', src, dst)
        except Exception, e:
            self.transfer['started'] = None
            logger.error('failed to process ftp transfer %s to %s: %s', src, dst, str(e))
    def get_chosen_transfer(self, startTime, flightTime):
        """Returns the transfer with the specified start and flight times.
        
        Arguments:
            startTime (float): time in seconds since epoch of transfer start
            flightTIme (float): time in seconds of transfer duration
            
        Returns:
            The transfer at with the specified start and flight times
        """

        if self.transferType == 'ballistic':
            trs = Transfer(self.startOrbit, self.endOrbit, startTime,       \
                            flightTime, False, self.ignoreInsertion,        \
                            self.cheapStartOrb, self.cheapEndOrb)

        elif self.transferType == 'plane change':
            trs = Transfer(self.startOrbit, self.endOrbit, startTime,       \
                            flightTime, True, self.ignoreInsertion,         \
                            self.cheapStartOrb, self.cheapEndOrb)

        elif self.transferType == 'optimal':
            btr = Transfer(self.startOrbit, self.endOrbit,                  \
                                   startTime, flightTime,                   \
                                   False, self.ignoreInsertion,             \
                                   self.cheapStartOrb, self.cheapEndOrb)
            ptr = Transfer(self.startOrbit, self.endOrbit,                  \
                                   startTime, flightTime,                   \
                                   True, self.ignoreInsertion,              \
                                   self.cheapStartOrb, self.cheapEndOrb)
            bdv = btr.get_total_delta_v()
            pdv = ptr.get_total_delta_v()

            if bdv <= pdv:
                trs = btr
            else:
                trs = ptr

        else:
            raise Exception('uncrecognized transfer type')

        # gen = trs.genetic_refine()
        return trs  # , gen
Ejemplo n.º 36
0
def updateTransfer(userId, transferId):
    """update transfer in db with matching Entity id

    Returns:
        [Response] -- [updated transfer,response code]
    """
    try:
        if validateTransferBody(connexion.request.json) is True:
            transfer = dsHelper.updateEntity(
                transferId, Transfer(userId=userId, **connexion.request.json))
            transfer.get_dict().pop("deleted", None)
            return transfer.get_dict(), 200
        return validateTransferBody(connexion.request.json), 400
    except Exception as e:
        print(e)
        traceback.print_tb(e.__traceback__)
        return False, 400
    return None
Ejemplo n.º 37
0
    def add_transfer(self, outgoing_id, receiving_id, transfer_amount):
        """ Convenient transfer wrapper """

        transfer = Transfer(user_id=self.user_id,
                            outgoing_program=outgoing_id,
                            receiving_program=receiving_id,
                            outgoing_amount=transfer_amount)
        db.session.add(transfer)

        outgoing_program = self.get_balance(outgoing_id)
        receiving_program = self.get_balance(receiving_id)
        transfer_ratio = ratio_instance(outgoing_id, receiving_id).ratio_to()

        # Update balance for outgoing & receiving program in balances table
        outgoing_program.transferred_from(transfer_amount)
        receiving_program.transferred_to(transfer_amount, transfer_ratio)

        return transfer
Ejemplo n.º 38
0
def list_syncs():
    now = datetime.utcnow()
    items = []
    for res in Sync.find(sort=[('processed', DESCENDING)]):
        res['name'] = '%s to %s' % (_get_params_str(res['src']), _get_params_str(res['dst']))
        if res.get('transfer_id'):
            transfer = Transfer.find_one({'_id': ObjectId(res['transfer_id'])})
        else:
            transfer = None
        res['transfer'] = transfer or {}
        if not transfer:
            res['status'] = 'pending'
        elif transfer['finished'] and transfer['finished'] > now - timedelta(hours=res['recurrence'] + 24):
            res['status'] = 'ok'
        else:
            res['status'] = 'queued'

        items.append(res)

    return serialize({'result': items})
Ejemplo n.º 39
0
 def show_menu(self):
     print()
     print('  MAIN MENU')
     print('  ---------')
     print('  1: View Pokemon List')
     print('  2: Transfer Menu')
     print('  3: Evolve Menu')
     print('  4: Rename Menu')
     print('  0: Exit')
     choice = int(input("\nEnter choice: "))
     if choice == 1:
         self.print_pokemons(self.pokemons)
     elif choice == 2:
         Transfer(self).run()
     elif choice == 3:
         Evolve(self).run()
     elif choice == 4:
         Renamer(self).run()
     elif choice == 0:
         quit()
     else:
         quit()
Ejemplo n.º 40
0
 def update(self):
     if self.status != _TRANSFER:
         self.catcher = self.current.update(result=self.catcher)
         if self.status == _MINI and self.catcher != None:
             print "switching to tbg"
             self.current.reset()
             self.current = self.board
             self.status = _TBG
             self.currWait = self.wait
         elif self.status == _TBG and self.catcher != None:
             print "switching to transfer"
             self.easywin = EasyWin(self.screen)
             self.status = _TRANSFER
             self.counter += 1
             if self.counter == len(self.minigames):
                 self.counter = 0
             select = self.minigames[self.counter]
             # select = self.minigames[random.randint(0,len(self.minigames)-1)]
             self.current = Transfer(self.screen, select[1][random.randint(0, len(select[1]) - 1)], select[0])
             # self.current = self.minigames[random.randint(0,len(self.minigames)-1)]
             # self.status = _MINI
             self.currWait = self.wait
         elif self.status == _INTRO and self.catcher != None:
             self.catcher = None
             self.current.reset()
             self.current = self.board
             self.status = _TBG
             self.currWait = self.wait
     else:
         # clear queue
         # self.currWait -= 1
         result = self.current.update()
         pygame.event.get()
         if result != None:
             print "switching to mini"
             self.current = result
             self.status = _MINI
Ejemplo n.º 41
0
async def upload_file(
        *,
        path,
        force: bool = Query(
            False,
            title='Force create even existed, old file will be deleted',
        ),
        transfer_id: str = Query(
            None,
            alias='transfer',
            title='Transfer UUIDv4',
        ),
        request: Request,
):
    ensure_me(request)
    f = File(path)
    ensure_not_type(f, NodeType.Dir)
    if f and not force:
        raise HTTPException(409, 'Existed')
    if transfer_id:
        transfer = Transfer(transfer_id, int(request.headers['content-length']))
    else:
        transfer = None
    await f.create(request.stream(), transfer=transfer)
Ejemplo n.º 42
0
def postTransfer(userId):
    """add a new transfer to db

    Returns:
        [Response] -- [id of the newly inserted datastore entity,status code]
    """
    try:
        if validateTransferBody(connexion.request.json) is True:
            if (transactionCheck :=
                    makeTransaction(  # noqa: E231,E203,E999,E251,E261
                        userId, connexion.request.json["amount"])) is not True:
                print(transactionCheck)
                return (
                    {
                        "error":
                        "error creating transaction: {}".format(
                            transactionCheck["error"])
                    },
                    400,
                )
            transferId = dsHelper.putEntity(
                Transfer(userId=userId, **connexion.request.json))
            return {"transferId": transferId}, 201
        return validateTransferBody(connexion.request.json), 400
Ejemplo n.º 43
0
    def rekordbox_sync(self, drive):
        username = os.getlogin()
        files = Transfer(
            f"{drive[:3]}Users\\{username}\\AppData\\Roaming\\Pioneer\\rekordbox\\"
        )
        # rb_files = ["automixPlaylist6.xml"]
        rb_files = [
            "master.db", "master.backup.db", "networkAnalyze6.db",
            "masterPlaylists6.xml", "automixPlaylist6.xml"
        ]

        for rb_file in rb_files:
            files.prepare_to_send_file(self.s, files.path, rb_file)

        for root, d, file in os.walk(f"{files.path}share\\PIONEER"):
            if files.send_root(self.s, root):
                for f in file:
                    files.prepare_to_send_file(self.s, root, f)

        message_box = Popups(
            "Finished",
            "Finished sending database, artwork and waveform files")
        message_box.message_box()
Ejemplo n.º 44
0
def manage_torrents():
    client = get_torrent_client()

    for transfer in Transfer.find({
            'started': {'$ne': None},
            'finished': None,
            'type': 'torrent',
            }):
        hash = transfer['info'].get('hash')
        if not hash:
            continue
        try:
            torrent = client.get_torrent(hash=hash)
        except TorrentError:
            continue
        if not torrent:
            transfer['finished'] = datetime.utcnow()
            logger.debug('torrent %s is not queued', hash)
        else:
            transfer['info'] = torrent
            transfer['transferred'] = torrent.transferred
            transfer['total'] = torrent.size
            transfer['progress'] = torrent.progress
        Transfer.save(transfer, safe=True)

    default_dir = str(Settings.get_settings('paths')['default'])
    for torrent in client.iter_torrents():
        transfer = Transfer.find_one({'info.hash': torrent.hash},
                sort=[('created', DESCENDING)])
        if not transfer or (transfer['finished'] \
                and torrent.date_added > transfer['finished']):
            now = datetime.utcnow()
            Transfer.add(torrent.magnet_url, default_dir,
                    type='torrent', added=now, started=now,
                    info={'hash': torrent.hash})
            logger.info('started torrent transfer %s to %s', torrent.magnet_url, default_dir)
        elif transfer['finished']:
            client.remove_torrent(hash=torrent.hash, delete_data=True)
            logger.debug('removed finished torrent "%s" (%s)', torrent.name, torrent.hash)
        else:
            manage_torrent(client, hash=torrent.hash, dst=transfer['dst'])
Ejemplo n.º 45
0
        print("\nAs",country,"we give you 2 options: ")
        print("\t1. Accept", imm_level, "new people into your country")
        print("\t2. Give us", req, "units of", most_needed)

        choice = input("Put your choice here: ")
        while choice not in ["1", "2"]:
            choice = input("Please enter 1 or 2: ")
        if choice == "1":
            state = imm.execute()
            print("Immigration has been executed.")
        elif choice == "2":
            t = Thresholds(state, myCountry)

            # check if our country has enough resources to make this transaction
            if t.is_valid_transfer(most_needed, req):
                transfer = Transfer(state, myCountry, country, (most_needed,req))
                state = transfer.execute()
                print("Transfer has been executed.")
            
            # if our country lacks the resource, force the immigration
            else:
                print("You don't have enough of this resource to complete the transfer.")
                state = imm.execute()
                print("Immigration has been executed.")
    
    t = Thresholds(state, myCountry)

    # all resources at comfortable level
    if t.is_perfect():
        print("YOU WIN!")
        finished = True
Ejemplo n.º 46
0
class Account:
    def __init__(self):
        self.user_orders = list()

        x = ComplexBuy(0, 0, 0)
        self.user_orders.append(x)
        self.user_orders.remove(x)

        self.transfer = None

    def new_order(self, type, amount, opening_price, limit_price,
                  today_tomorrow):
        if today_tomorrow == 'today':
            if type == 'buy':
                self.user_orders.append(
                    ComplexBuy(opening_price, limit_price, amount))
            elif type == 'sell':
                self.user_orders.append(
                    ComplexSell(opening_price, limit_price, amount))

        elif today_tomorrow == 'tomorrow':
            if type == 'buy':
                self.user_orders.append(
                    ComplexTomorrowBuy(opening_price, limit_price, amount))
            elif type == 'sell':
                self.user_orders.append(
                    ComplexTomorrowSell(opening_price, limit_price, amount))

    def cancel_order(self):
        # TODO cancel_order should send NUN if needed!
        """""
            Cancels the last order of the user_orders, if possible
            if the last order is None or is finished or is already cancelled, nothing will happen and
             -1 will be returned

            :returns 0 if Cancellation is successful
            :returns -1 if Cancellation is not successful!
            """
        if self.get_last_order() is not None and \
                not self.get_last_order().is_done() and \
                not self.get_last_order().is_cancelled:
            self.get_last_order().is_cancelled = True
            return 0
        else:
            return -1

    def get_last_order(self):
        if self.user_orders is not None and \
                len(self.user_orders) > 0:
            return self.user_orders[-1]
        else:
            return None

    def last_order_status(self):
        order_status = 'وضعیت سفارش: '
        if self.has_active_order():
            order_status += '🔵\n' + self.get_last_order().raw_report()
        else:
            order_status += 'سفارش فعالی موجود نیست 🔴'
        order_status += '\n'
        return order_status

    def has_active_order(self):
        # TODO if Buy or sell, in addition to is_done, has active or inactive boolean,
        #  it too should be checked here!
        return self.get_last_order() is not None and \
               not self.get_last_order().is_done() and \
               not self.get_last_order().is_cancelled

    def orders_report_by_num(self, num_go_back):
        return_value = ''
        for order in self.user_orders[-num_go_back:]:
            return_value += order.short_report()

        if not return_value:
            return_value = 'سفارشی یافت نشد! ❗️'

        return return_value

    def orders_report_by_name(self, searched_order):
        searched = Translate.order(searched_order)
        searched_price = searched['price']
        searched_volume = searched['amount']
        return_value = ''
        for order in self.user_orders:
            if order.is_equal(searched_price, searched_volume):
                return_value += order.complete_report()

        if not return_value:
            return_value = 'سفارشی یافت نشد! ❗️'

        return return_value

    def new_transfer(self):
        if self.transfer is None:
            self.transfer = Transfer()
        else:
            self.transfer.turn_on()

    def change_transfer_rule(self, transfer_rule_text):
        translated_transfer_text = Translate.transfer_rule(transfer_rule_text)
        if translated_transfer_text is not None:
            if translated_transfer_text[
                    'buy_price_limit'] > translated_transfer_text[
                        'sell_price_limit']:
                return -1
            else:
                self.transfer.change_rule(
                    translated_transfer_text['amount_limit'],
                    translated_transfer_text['buy_price_limit'],
                    translated_transfer_text['sell_price_limit'])
                return 0
        else:
            return -2

    def is_transfer_active(self):
        return self.transfer is not None and \
               self.transfer.is_active

    def transfer_status(self):
        transfer_status = 'وضعیت جابه‌جایی: \n'
        if self.transfer is None:
            transfer_status += 'قانونی برای جابه‌جایی تعریف نشده است! 🔴'
        else:
            transfer_status += 'قانون جابه‌جایی: \n' + self.transfer.raw_report(
            ) + '\n'
            if self.transfer.is_active:
                transfer_status += 'جابه‌جایی فعال است! 🔵'
            else:
                transfer_status += 'جابه‌جایی غیر فعال است! 🔴'
        transfer_status += '\n'

        return transfer_status

    def transfer_report(self):
        if self.transfer is not None:
            return self.transfer.report()

    def account_status(self):
        return self.last_order_status() + '\n' + self.transfer_status()
Ejemplo n.º 47
0
        return k_I(self.nu,
                   nf,
                   np.linalg.norm(B),
                   sin_theta,
                   s=2.5,
                   q=q_e,
                   m=m_e)


if __name__ == '__main__':
    jet = Jet()
    from transfer import Transfer
    transf = Transfer(jet, math.pi / 4, (
        10,
        10,
    ), (
        0.1,
        0.1,
    ))
    origin = np.array([0., 0.1, 9.])
    direction = transf.los_direction
    from geometry import Ray
    ray = Ray(origin, direction)
    n = -np.array(direction)

    jet.create_ray(ray, n=10, max_dtau=1. * 10**(-16))
    # t1, t2 = jet.geometry.hit(ray)
    # k = 10
    # dt = abs(t2 - t1) / k
    # t_edges = [t2 + i * dt for i in xrange(k)]
    # t_cells = [min(t1, t2) + (i + 0.5) * dt for i in xrange(k - 1)]
Ejemplo n.º 48
0
def get_nzb_transfer(id):
    return Transfer.find_one({'info.nzo_id': id},
            sort=[('created', DESCENDING)])
Ejemplo n.º 49
0
 def test_merge_save_model(self):
     c = Params()
     c.load_config_file(JSON_PATH)
     cnn = Transfer(c, MODEL_NAME, PATCH_TYPE)
     cnn.merge_save_model()
Ejemplo n.º 50
0
def remove_transfer():
    data = request.json
    if not data.get('id'):
        return jsonify(error='missing id')
    Transfer.cancel(data['id'])
    return jsonify(result=True)
Ejemplo n.º 51
0
class CatTar(CommandOp):
    ##################################################################
    # This function is called when an object of the class is created.
    # 
    # We basically define required parameters and extract relevant 
    # command line information.
    # 
    ##################################################################
    def __init__(self, conf_obj):
        """This class returns an object which has the required 
        configuration values. Please add/delete data from __init__ 
        to suit requirements."""
        self.test = 0
        self.conf_obj = conf_obj
        self.transSftp = Transfer('sftp')
        #for next version this should be changed to 
        #ftp_trans.FtpTrans(self.conf_obj)
        self.ftp_obj = ftp_trans.FtpTrans()
        self.runid = self.conf_obj.runid
        self.backup_dir = self.conf_obj.backup_path
        #Change below code to modify logging format
        if(not os.path.isdir(self.backup_dir)):
            raise error.FnfError(self.backup_dir, 
                                 "Error: backup dir not found"+\
                                  self.backup_dir+"; Exiting")
        self.root_dir = self.conf_obj.root_path
        self.runid_dir = os.getcwd()
        self.transCopy = Transfer('copy')
        self.summaryName = ""

    ##################################################################
    # Creates directories of users under backup directory
    # Calls calcpath for further processing (for every lane and for 
    # every user
    ##################################################################
    def create_dir(self):
        logging.debug("dir list: "+str(self.conf_obj.lane_user_dict))
        for lane, name in self.conf_obj.lane_user_dict.iteritems():
            if not self.conf_obj.perm_dict[lane][0]:
                continue
            os.chdir(self.runid_dir)
            self.s_what = "s_"+lane+"_"
            if not os.path.isdir(name):
                os.mkdir(name)
                #Recreating dir tree on the ftp server
            os.chdir(name)
            self.cur_name_dir = os.getcwd()
            self.calcpath()
        self.add_htm()
    
    ##################################################################
    #This function populates different path variables and calls concat
    #as many times as required. Concat handles further processing
    ##################################################################
    def calcpath(self):
        self.Bustard_path = self.conf_obj.Bustard_path
        self.Gerald_path = self.conf_obj.Gerald_path
        os.chdir(self.Bustard_path)
        reqd_list = commands.getoutput("ls "+self.s_what+\
                                       "[1-9]_*_qseq.txt").split("\n")
        ind = []
        try:
            ind.append(reqd_list[0].split('_')[2])
        except IndexError:
            logging.debug("here is the problem causing ind "+str(ind))
        #The below for loop takes care of the case where there could 
        #be 2 kinds of files for same lane. 
        #Example:s_2_1_... s_2_2_...
        for file in reqd_list:
            if file.split('_')[2] not in ind:
                ind.append(file.split('_')[2])
        #Files which need to be included in the tar file. You could 
        #directly add to this list to make things easier
        for i in ind:
            #if there is only one set of files for every lane
            if len(ind) == 1:
                self.files = {self.s_what+'eland_extended.txt':'', 
                              self.s_what+'sequence.txt':'', 
                              self.s_what+'sorted.txt':'', 
                              self.s_what+'export.txt':''}
            #otherwise more than one set of files
            else:
                self.files = {self.s_what+i+'_'+'eland_extended.txt'\
                              :'', 
                              self.s_what+i+'_'+'sequence.txt':'', 
                              self.s_what+i+'_'+'sorted.txt':'', 
                              self.s_what+i+'_'+'export.txt':''}
            #Add in paths where corresponding files can be found
            for file in self.files.iterkeys():
                if not file.endswith("qseq.txt"):
                    self.files[file] = self.Gerald_path
                else:
                    self.files[file] = self.Bustard_path
            self.concat(i)


    ##################################################################
    # Concatenates all txt files under Bustard
    ##################################################################
    def concat(self, i_str):
        #Separate out the name of the current user. 
        #Used only for logging purposes
        cur_name = self.cur_name_dir.split("/")[-1]
        os.chdir(self.Bustard_path)
        concat_files = ''
        reqd_list = commands.getoutput("ls "+self.s_what+i_str+\
                                       "_*_qseq.txt").split("\n")
        logging.info("Starting loop for user "+cur_name)
        for file in reqd_list:
            concat_files += file +' '
        logging.info("Starting concatenation of files")
        logging.debug("Starting concatenation of files under "+\
                      self.Bustard_path)
        logging.debug('cat '+self.s_what+i_str+'_qseq.txt ')
        if(self.test):
            print 'cat '+concat_files+'> '+\
                  os.path.join(self.cur_name_dir,\
                  self.s_what+i_str)+'_qseq.txt \n'
        else:
            os.system('cat '+concat_files+'> '+\
                      os.path.join(self.cur_name_dir,\
                      self.s_what+i_str)+'_qseq.txt ')
        logging.info("Finished concatenation of files")
        logging.debug("Finished concatenation of files under "+\
                      self.Bustard_path)
        self.copy_tar()

    ##################################################################
    # Copies all files under Bustard and Gerald to the path for the 
    # user. Then tars the files.
    ##################################################################
    def copy_tar(self):
        all_paths = ''
        #Put all files to be copied in a single string
        #Will it be better to do it one by one?
        for file, path in self.files.iteritems():
            all_paths += os.path.join(path, file) + ' '
        logging.info("Starting copying of files")
        logging.debug("Starting copying of files under "+\
                      self.Bustard_path+" and "+ self.Gerald_path+\
                      " to "+self.cur_name_dir)
        logging.debug("cp "+all_paths+" "+self.cur_name_dir)
        if(self.test):
            print "cp "+all_paths+" "+self.cur_name_dir+"\n"
        else:
            for f in all_paths.split():
                self.transCopy.copy(f.strip(), self.cur_name_dir)
        logging.info("Finished copying of files")
        os.chdir(self.cur_name_dir)
        logging.info("Renaming of files")
        for file in os.listdir(os.getcwd()):
            if file.endswith(".txt"):
                if(self.test):
                    print "mv "+file+" "+self.runid+'_'+file+"\n"
                else:
                    #TODO: raghu. check error
                    self.move(file, self.runid+'_'+file)
        for file in os.listdir(os.getcwd()):
            if file.endswith(".txt"):
                logging.info("Starting tarring of file "+file)
                logging.debug("gzip "+file)
                if(self.test):
                    print "gzip "+file
                else:
                    os.system("gzip "+file)
                logging.info("Finished tarring of file "+file)
    
    ##################################################################
    #Copies summary, plots and htm from Bustard and Gerald into a temp
    #directory. Tars the temp directory and deletes the intermediate 
    #directory.
    ##################################################################
    def add_htm(self):
        self.Bustard_path = self.conf_obj.Bustard_path
        self.Gerald_path = self.conf_obj.Gerald_path
        logging.info("Starting summary tar creation")
        os.chdir(self.runid_dir)
        os.mkdir(self.runid+"summary")
        os.chdir(self.runid+"summary")
        logging.info("Copying htm files from Bustard")
        for file in glob.glob(os.path.join(self.Bustard_path, 
                  "*.[hH][Tt][Mm]")):
            self.transCopy.copy(file, ".")
        logging.info("Copying Plots folder from Bustard")
        self.transCopy.copy(os.path.join(self.Bustard_path, "Plots"),
                        os.path.join(os.getcwd(),"Plots"))
        os.mkdir(self.Gerald_path.split('/')[-1])
        os.chdir(self.Gerald_path.split('/')[-1])
        logging.info("Copying htm files from Gerald ")
        for file in glob.glob(os.path.join(self.Gerald_path, 
                                     "*.[hH][Tt][Mm]")):
            self.transCopy.copy(file, ".")
        logging.info("Copying Plots folder from Gerald") 
        self.transCopy.copy(os.path.join(self.Gerald_path, "Plots"),
                      os.path.join(os.getcwd(), "Plots"))
        logging.info("Starting tarring of Summary")
        os.chdir(self.runid_dir)
        os.system("tar -czf "+self.runid+"_summary.tar.gz "+\
                  self.runid+"summary")
        logging.info("Finished tarring of Summary")
        self.copy_file()
    
    ##################################################################
    # Copies the required files for different users.
    ##################################################################
    def copy_file(self):
        logging.info("Making copies of summary tar")
        os.chdir(self.runid_dir)
        logging.debug(self.conf_obj.copy_to_runid)
        logging.debug(self.conf_obj.copy_to_users)
        logging.debug(str(self.conf_obj.lane_user_dict)+" HI")
        for file in self.conf_obj.copy_to_runid:
            self.transCopy.copy(file, os.getcwd())
        for file in self.conf_obj.copy_to_users:
            for name in self.conf_obj.lane_user_dict.itervalues():
                self.transCopy.copy(file, name)
                if file.split('/')[-1] == 'Summary.htm':
                    os.chdir(name)
                    logging.debug("Renaming Summary.htm "+os.getcwd())
                    self.move('Summary.htm', self.runid+'_Summary.htm')
                    os.chdir('..')
        os.chdir(self.runid_dir)
        self.removeDir(self.runid+"summary")

    def ftp_files(self):
        i = 0
        #############################################################
        #name_folder format
        #
        #1. For Non-quest
        #   Length of list = 2
        #   [<name of user>, <folder to cd to on quest>]
        #2. For Quest
        #   Length of list = 4
        #   [<name of user>, <folder to cd to on quest>, <sample id>,
        #    <experiment id>]
        #############################################################
        for lane, name_folder in self.conf_obj.ftp_user_dict.\
                iteritems():
            if not self.conf_obj.perm_dict[lane][1]:
                continue
            os.chdir(self.runid_dir)
            os.chdir(name_folder[0])
            file_list = commands.getoutput("ls *s_"+lane+"*").\
                                           split("\n")
            file_list.append(self.runid+"_summary.tar.gz")
            file_list.append(self.runid+"_Summary.htm")
            if len(name_folder) < 4:
                self.ftp_obj.connect()
                self.ftp_obj.chdir(name_folder[1])
                try:
                    self.ftp_obj.mkdir(self.runid)
                except ftplib.error_perm:
                    self.ftp_obj.connect()
                    self.ftp_obj.chdir(name_folder[1])
                    logging.debug("runid dir already there")
                self.ftp_obj.chdir(self.runid)
                try:
                    self.ftp_obj.mkdir(name_folder[0])
                except ftplib.error_perm:
                    self.ftp_obj.connect()
                    self.ftp_obj.chdir(name_folder[1])
                    self.ftp_obj.chdir(self.runid)
                    logging.debug("user dir also already there")
                self.ftp_obj.chdir(name_folder[0])
            else:
                if self.conf_obj.ftp_quest:
                    self.ftp_obj.connect()
                    self.ftp_obj.chdir(name_folder[1])
            for file in file_list:
                logging.info("Transferring file %s of user %s" \
                        %(file, name_folder[0]))
                if self.conf_obj.ftp_quest:
                    self.ftp_obj.copy_file(file)
                else:
                    self.transCopy.copy(file, '/cccstor/'+\
                              name_folder[1]+'/'+file)
                    logging.debug('cp '+file+' /cccstor/'+\
                              name_folder[1]+'/'+file)
            if self.conf_obj.ftp_quest:
                self.ftp_obj.close()
            logging.info("Finished transfer !!!")
            if len(name_folder) == 4:
                sid = name_folder[2]
                eid = name_folder[3]
                self.conf_obj.server.UpdateQuestSampleResults(sid,\
                                     eid, file_list, self.runid, \
                                     "IlluminaGAII", "heWex45--")
    def sftp_files(self):
        #name_folder = [uname, passwd, email id]
        for lane, name_folder in self.conf_obj.sftp_user_dict.\
                iteritems():
            if not self.conf_obj.perm_dict[lane][1]:
                continue
            os.chdir(self.runid_dir)
            os.chdir(name_folder[0])
            file_list = commands.getoutput("ls *s_"+lane+"*").\
                                           split("\n")
            file_list.append(self.runid+"_summary.tar.gz")
            file_list.append(self.runid+"_Summary.htm")
            self.transSftp.setServerParams(self.conf_obj.sftp_server,
                                           name_folder[0],
                                           name_folder[1])
            for file in file_list:
                logging.info("Transferring file %s of user %s" \
                        %(file, name_folder[0]))
                self.transSftp.copy(file, "/data/repos/solexa/"+\
                        name_folder[0]+"/"+file)
            self.mail_obj = mail.Mail()
            if(name_folder[2] != ""):
                email_text = "Results for Illumina run "+self.runid+\
                             " has been uploaded to our SFTP server.\n"+\
                             "SFTP server: "+self.conf_obj.sftp_server+\
                             "\nUsername: "******"\nPassword: "******"\n"
                email_text += "Use winscp to download the files\n"+\
                              "http://winscp.net/eng/download.php\n"
                email_text += "\nRegards,\n\n"+\
                "Biomedical Informatics Shared Resource\n"+ \
                "The Ohio State University Comprehensive Cancer"+\
                "Center\n"+\
                "210 Biomedical Research Tower\n 460 W 12th Ave\n"+\
                "Columbus, OH 43210\n (614) 366-1538\n"
                self.mail_obj.mail_diff(name_folder[2], \
                                        "Illumina run "+self.runid, \
                                        email_text)
Ejemplo n.º 52
0
def list_transfers():
    items = [_get_transfer(t) for t in Transfer.find({'finished': None})]
    return serialize({'result': items})
Ejemplo n.º 53
0
    def test_evaluate_cnn_svm_rf_model(self):
        c = Params()
        c.load_config_file(JSON_PATH)
        cnn = Transfer(c, MODEL_NAME, PATCH_TYPE)

        cnn.evaluate_cnn_svm_rf_model(SAMPLE_FIlENAME, 100)
Ejemplo n.º 54
0
    def test_extract_features_for_train(self):
        c = Params()
        c.load_config_file(JSON_PATH)
        cnn = Transfer(c, MODEL_NAME, PATCH_TYPE)

        cnn.extract_features_for_train(SAMPLE_FIlENAME, 100)
Ejemplo n.º 55
0
class pyUpdater:
    def __init__(self, verbose=False):
        self.classname = "pyUpdater"
        self.version = "0.1"
        self.configdir = "/opt/pyUpdater"
        self.configfile = "conf/pyUpdater.json"
        self.configCustomFile = ""
        self.verbose = verbose
        self.loadConfig()
        logging.basicConfig(filename=self.config['logfile'],
                            level=self.config['loglevel'],
                            format=self.config['logformat'])
        self.preservefile = self.config['whitelist'].split(';')
        self.createTmpDir()

        # INIT transfer module
        self.tr = Tr()
        self.init_transfer()

    def loadConfig(self):
        print(self.configCustomFile)
        if (self.configCustomFile == ""):
            with open(self.configdir + '/' + self.configfile, 'r') as f:
                self.config = json.load(f)
        else:
            with open(self.configCustomFile, 'r') as f:
                self.config = json.load(f)

    def setConfigFile(self, path):
        self.configCustomFile = path
        self.loadConfig()

    def init_transfer(self):
        self.tr.server = self.config['transfer']['server']
        self.tr.username = self.config['transfer']['username']
        self.tr.password = self.config['transfer']['password']

    def createTmpDir(self):
        self.tmpdir = td.mkdtemp()
        self.log("Create temporary directory: " + self.tmpdir)

    def extractUpdate(self):
        self.log("Extract update!")
        archive_name = os.path.join(self.tmpdir, self.config['updatefile'])
        shutil.unpack_archive(archive_name, self.config['dstdir'])

    def backup(self):
        archive_name = os.path.join(self.config['backupdir'], 'backup')
        shutil.make_archive(archive_name, 'gztar', self.config['dstdir'])
        self.log("Create backup file: " + archive_name + ".tar.gz")

    def restore(self):
        archive_name = os.path.join(self.config['backupdir'],
                                    'backup') + '.tar.gz'
        shutil.unpack_archive(archive_name, self.config['dstdir'])
        self.log("Restored file in " + archive_name + ".tar.gz to " +
                 self.config['dstdir'])

    def update(self):
        if self.checkupdate():
            self.log("New update present. Start updating ...")
            self.backup_whitelist()
            self.remove_old_file()
            self.extractUpdate()
            self.restore_whitelist()
        else:
            self.log("New version not found!")
            quit(0)

    def dump(self):
        self.log("Dump database")
        self.dump_database()
        self.tr.send(self.config['backupdir'] + self.dumpname,
                     "/tmp" + self.dumpname)

    def restoredump(self):
        self.log("Restore database from dump")
        self.restore_database()
        # todo: remote copy

    def checkupdate(self):
        self.log("Start update procedure.")
        url = self.config['updateserver'] + "/" + self.config['updatefile']
        r = requests.get(url)
        if r.status_code != 200:
            self.log("Update not present. Error code: " +
                     r.status_code.__str__())
            return False
        else:
            with open(os.path.join(self.tmpdir, self.config['updatefile']),
                      'wb') as f:
                f.write(r.content)
        return True

    def backup_whitelist(self):
        for f in self.preservefile:
            self.log("Save file: " + os.path.join(self.config['dstdir'], f))
            shutil.copy2(os.path.join(self.config['dstdir'], f),
                         self.tmpdir + '/')

    def restore_whitelist(self):
        for f in self.preservefile:
            self.log("Restore file: " + os.path.join(self.config['dstdir'], f))
            filename = os.path.basename(self.tmpdir + '/' + f)
            shutil.copy2(self.tmpdir + '/' + filename,
                         os.path.join(self.config['dstdir'], f))

    def remove_old_file(self):
        for r, d, f in os.walk(self.config['dstdir']):
            for file in f:
                os.remove(os.path.join(r, file))

    def dump_database(self):
        try:
            timestamp = str(int(time.time()))
            self.dumpname = "/dump_" + self.config[
                'db_name'] + "_" + timestamp + ".sql.gz"
            p = subprocess.Popen("mysqldump -h " + self.config['db_host'] +
                                 " -u " + self.config['db_user'] + " -p" +
                                 self.config['db_pass'] + " " +
                                 self.config['db_name'] + " | gzip > " +
                                 self.config['backupdir'] + self.dumpname,
                                 shell=True)
            # Wait for completion
            p.communicate()
            # Check for errors
            if (p.returncode != 0):
                raise "Backup return code: " + p.returncode
            self.log("Backup done for " + self.config['db_name'])
        except:
            self.log("Backup failed for " + self.config['db_name'])

    def restore_database(self):
        try:
            # trovare ultimo dump sotto la directory dei backup
            list_of_files = glob.glob(self.config['backupdir'] + '/*.sql.gz')
            latest_file = max(list_of_files, key=os.path.getctime)
            if latest_file != '':
                p = subprocess.Popen("zcat  " + latest_file + " | mysql -h " +
                                     self.config['db_host'] + " -u " +
                                     self.config['db_user'] + " -p" +
                                     self.config['db_pass'] + " " +
                                     self.config['db_name'],
                                     shell=True)
                # Wait for completion
                p.communicate()
                # Check for errors
                if p.returncode != 0:
                    raise "Restore return code: " + p.returncode
                self.log("Restore done for " + latest_file)
            # no dump found
            else:
                self.log("No dumps found in " + self.config['backupdir'] +
                         ". Cannot restore anything, sorry.")
        except:
            self.log("Restore failed for " + latest_file)

    def log(self, msg, level='debug'):
        if self.verbose:
            print(msg)
        logging.debug(msg)

    def test(self):
        # check if destination directory exist and is writeable
        print("")
        print("-------------------# CHECK FILESYSTEM #-------------------")
        if os.path.exists(self.config['dstdir']):
            print(bcolors.OKBLUE + '[ok]' + bcolors.ENDC,
                  self.config['dstdir'] + ' exist.')
            if os.access(self.config['dstdir'], os.W_OK):
                print(bcolors.OKBLUE + '[ok]' + bcolors.ENDC,
                      self.config['dstdir'] + ' is writeable.')
            else:
                print(bcolors.FAIL + '[fail]' + bcolors.ENDC,
                      self.config['dstdir'] + ' is not writeable.')
        else:
            print(bcolors.FAIL + '[fail]' + bcolors.ENDC,
                  self.config['dstdir'] + ' not exist. Create this folder.')

        # check if backup directory exist and is writeable
        if os.path.exists(self.config['backupdir']):
            print(bcolors.OKBLUE + '[ok]' + bcolors.ENDC,
                  self.config['backupdir'] + ' exist.')
            if os.access(self.config['backupdir'], os.W_OK):
                print(bcolors.OKBLUE + '[ok]' + bcolors.ENDC,
                      self.config['backupdir'] + ' is writeable.')
            else:
                print(bcolors.FAIL + '[fail]' + bcolors.ENDC,
                      self.config['backupdir'] + ' is not writeable.')
        else:
            print(bcolors.FAIL + '[fail]' + bcolors.ENDC,
                  self.config['backupdir'] + ' not exist. Create this folder.')

        # check if config file exists and readable
        if os.path.exists(self.configdir + '/' + self.configfile):
            print(bcolors.OKBLUE + '[ok]' + bcolors.ENDC,
                  self.configdir + '/' + self.configfile + ' exist.')
            if os.access(self.configdir + '/' + self.configfile, os.R_OK):
                print(bcolors.OKBLUE + '[ok]' + bcolors.ENDC,
                      self.configdir + '/' + self.configfile + ' is readable.')
            else:
                print(
                    bcolors.FAIL + '[fail]' + bcolors.ENDC, self.configdir +
                    '/' + self.configfile + ' is not readable.')
        else:
            print(
                bcolors.FAIL + '[fail]' + bcolors.ENDC, self.configdir + '/' +
                self.configfile + ' not exist. Create this folder.')

        print("----------------------# END #-----------------------------")
        print("")
        print("")
        print("---------------------# DB CHECK #-------------------------")
        # check database connection
        try:
            connection = mysql.connector.connect(
                host=self.config['db_host'],
                database=self.config['db_name'],
                user=self.config['db_user'],
                password=self.config['db_pass'])
            if connection.is_connected():
                db_info = connection.get_server_info()
                print(
                    bcolors.OKBLUE + '[ok]' + bcolors.ENDC +
                    " connected to MySQL database... MySQL Server version on ",
                    db_info)
                cursor = connection.cursor()
                cursor.execute("select database();")
                record = cursor.fetchone()
                print("Your connected to - ", record)
        except Error as e:
            print(bcolors.FAIL + '[fail]' + bcolors.ENDC +
                  " error while connecting to MySQL")
            pass

        print("----------------------# END #-----------------------------")
        print("")
        print("")
        print("---------------------# SSH CHECK #-------------------------")
        if self.tr.checkconnection():
            print(
                bcolors.OKBLUE + '[ok]' + bcolors.ENDC,
                'server: ' + self.tr.server + ' and user: '******' - connection succesfuly.')
        else:
            print(
                bcolors.FAIL + '[fail]' + bcolors.ENDC,
                'server: ' + self.tr.server + ' and user: '******' - connection failed.')
        print("----------------------# END #-----------------------------")
        print("")

    def __del__(self):
        shutil.rmtree(self.tmpdir)
Ejemplo n.º 56
0
Archivo: hmf.py Proyecto: tbs1980/hmf
class MassFunction(object):
    """
    An object containing all relevant quantities for the mass function.
    
    The purpose of this class is to calculate many quantities associated with 
    the dark matter halo mass function (HMF). The class is initialized to form a 
    cosmology and takes in various options as to how to calculate all
    further quantities. 
    
    All required outputs are provided as ``@property`` attributes for ease of 
    access.
    
    Contains an update() method which can be passed arguments to update, in the
    most optimal manner. All output quantities are calculated only when needed 
    (but stored after first calculation for quick access).
    
    Quantities related to the transfer function can be accessed through the 
    ``transfer`` property of this object.
    
    Parameters
    ----------   
    M : array_like, optional, default ``np.linspace(10,15,501)``
        The masses at which to perform analysis [units :math:`\log_{10}M_\odot h^{-1}`]. 
                     
    mf_fit : str or callable, optional, default ``"SMT"``
        A string indicating which fitting function to use for :math:`f(\sigma)`
                       
        Available options:
                                           
        1. ``'PS'``: Press-Schechter form from 1974
        #. ``'ST'``: Sheth-Mo-Tormen empirical fit 2001 (deprecated!)
        #. ``'SMT'``: Sheth-Mo-Tormen empirical fit from 2001
        #. ``'Jenkins'``: Jenkins empirical fit from 2001
        #. ``'Warren'``: Warren empirical fit from 2006
        #. ``'Reed03'``: Reed empirical from 2003
        #. ``'Reed07'``: Reed empirical from 2007
        #. ``'Tinker'``: Tinker empirical from 2008
        #. ``'Watson'``: Watson empirical 2012
        #. ``'Watson_FoF'``: Watson Friend-of-friend fit 2012
        #. ``'Crocce'``: Crocce 2010
        #. ``'Courtin'``: Courtin 2011
        #. ``'Angulo'``: Angulo 2012
        #. ``'Angulo_Bound'``: Angulo sub-halo function 2012
        #. ``'Bhattacharya'``: Bhattacharya empirical fit 2011
        #. ``'Behroozi'``: Behroozi extension to Tinker for high-z 2013

        Alternatively, one may define a callable function, with the signature
        ``func(self)``, where ``self`` is a :class:`MassFunction` object (and
        has access to all its attributes). This may be passed here. 
        
    delta_wrt : str, {``"mean"``, ``"crit"``}
        Defines what the overdensity of a halo is with respect to, mean density
        of the universe, or critical density.
                       
    delta_h : float, optional, default ``200.0``
        The overdensity for the halo definition, with respect to ``delta_wrt``
                       
    user_fit : str, optional, default ``""``
        A string defining a mathematical function in terms of `x`, used as
        the fitting function, where `x` is taken as :math:`\( \sigma \)`. Will only
        be applicable if ``mf_fit == "user_model"``.
                                       
    cut_fit : bool, optional, default ``True``
        Whether to forcibly cut :math:`f(\sigma)` at bounds in literature.
        If false, will use whole range of `M`.
           
    delta_c : float, default ``1.686``
        The critical overdensity for collapse, :math:`\delta_c`
        
    kwargs : keywords
        These keyword arguments are sent to the `hmf.transfer.Transfer` class.
        
        Included are all the cosmological parameters (see the docs for details).
        
    """


    def __init__(self, M=None, mf_fit="ST", delta_h=200.0,
                 delta_wrt='mean', cut_fit=True, z2=None, nz=None,
                 delta_c=1.686, mv_scheme="trapz", **kwargs):
        """
        Initializes some parameters      
        """
        if M is None:
            M = np.linspace(10, 15, 501)

        # A list of all available kwargs (sent to Cosmology via Transfer)
        self._cp = ["sigma_8", "n", "w", "cs2_lam", "t_cmb", "y_he", "N_nu",
                    "omegan", "H0", "h", "omegab",
                    "omegac", "omegav", "omegab_h2", "omegac_h2",
                    "force_flat", "default"]

        # Set up a simple dictionary of kwargs which can be later updated
        self._cpdict = {k:v for k, v in kwargs.iteritems() if k in self._cp}

        # Set all given parameters.
        self.mf_fit = mf_fit
        self.M = M
        self.delta_h = delta_h
        self.delta_wrt = delta_wrt
        self.cut_fit = cut_fit
        self.z2 = z2
        self.nz = nz
        self.delta_c = delta_c
        self.transfer = Transfer(**kwargs)
        self.mv_scheme = mv_scheme

        tools.check_kr(self.M[0], self.M[-1], self.cosmo.mean_dens,
                       self.transfer.lnk[0], self.transfer.lnk[-1])
    def update(self, **kwargs):
        """
        Update the class with the given arguments in an optimal manner.
        
        Accepts any argument that the constructor takes.
        """
        for key, val in kwargs.iteritems():

#             The following takes care of everything specifically in this class
            if "_MassFunction__" + key in self.__dict__:
                try: doset = np.any(getattr(self, key) != val)
                except ValueError: doset = not np.array_equal(getattr(self, key), val)
                if doset:
                    setattr(self, key, val)

            # We need to handle deletes in this class by parameters in Transfer here
            if key is 'z':
                if val != self.transfer.z:
                    del self.sigma

        # All parameters being sent to Transfer:
        the_rest = {k:v for k, v in kwargs.iteritems() if "_MassFunction__" + k not in self.__dict__}

        # Some things are basically deleted when anything in Transfer is updated
        if len(the_rest) > 0:
            del self.delta_halo
        if len(the_rest) > 1 or (len(the_rest) == 1 and 'z' not in the_rest):
            del self._sigma_0

        # The rest are sent to the Transfer class (stupid values weeded out there)
        self.transfer.update(**the_rest)

        tools.check_kr(self.M[0], self.M[-1], self.cosmo.mean_dens,
                       self.transfer.lnk[0], self.transfer.lnk[-1])
    # --- SET PROPERTIES -------------------------------------------------------
    @property
    def M(self):
        return self.__M

    @M.setter
    def M(self, val):
        try:
            if len(val) == 1:
                raise ValueError("M must be a sequence of length > 1")
        except TypeError:
            raise TypeError("M must be a sequence of length > 1")

        if np.any(np.abs(np.diff(val, 2)) > 1e-5) or val[1] < val[0]:
            raise ValueError("M must be a linearly increasing vector! " + str(val[0]) + " " + str(val[1]))

        # Delete stuff dependent on it
        del self._sigma_0
        self.__M = 10 ** val

    @property
    def delta_c(self):
        return self.__delta_c

    @delta_c.setter
    def delta_c(self, val):
        try:
            val = float(val)
        except ValueError:
            raise ValueError("delta_c must be a number: ", val)

        if val <= 0:
            raise ValueError("delta_c must be > 0 (", val, ")")
        if val > 10.0:
            raise ValueError("delta_c must be < 10.0 (", val, ")")

        self.__delta_c = val

        del self.fsigma

    @property
    def mv_scheme(self):
        return self.__mv_scheme

    @mv_scheme.setter
    def mv_scheme(self, val):
        if val not in ['trapz', 'simps', 'romb']:
            raise ValueError("mv_scheme wrong")
        else:
            self.__mv_scheme = val
            del self._sigma_0

    @property
    def mf_fit(self):
        return self.__mf_fit

    @mf_fit.setter
    def mf_fit(self, val):
        # mf_fit may be a callable or a string. Try callable first.
        try:
            val(self)
        except:
            try:
                val = str(val)
            except:
                raise ValueError("mf_fit must be a string or callable, got ", val)

            if val not in Fits.mf_fits + ["Behroozi"]:
                raise ValueError("mf_fit is not in the list of available fitting functions: ", val)

        # Also delete stuff dependent on it
        del self.fsigma

        self.__mf_fit = val

    @property
    def delta_h(self):
        return self.__delta_h

    @delta_h.setter
    def delta_h(self, val):
        try:
            val = float(val)
        except ValueError:
            raise ValueError("delta_halo must be a number: ", val)

        if val <= 0:
            raise ValueError("delta_halo must be > 0 (", val, ")")
        if val > 10000:
            raise ValueError("delta_halo must be < 10,000 (", val, ")")

        self.__delta_h = val

        # Delete stuff dependent on it
        del self.delta_halo

    @property
    def delta_wrt(self):
        return self.__delta_wrt

    @delta_wrt.setter
    def delta_wrt(self, val):
        if val not in ['mean', 'crit']:
            raise ValueError("delta_wrt must be either 'mean' or 'crit' (", val, ")")

        self.__delta_wrt = val
        del self.delta_halo


    @property
    def z2(self):
        return self.__z2

    @z2.setter
    def z2(self, val):
        if val is None:
            self.__z2 = val
            return

        try:
            val = float(val)
        except ValueError:
            raise ValueError("z must be a number (", val, ")")

        if val <= self.transfer.z:
            raise ValueError("z2 must be larger than z")
        else:
            self.__z2 = val

        del self.dndm

    @property
    def nz(self):
        return self.__nz

    @nz.setter
    def nz(self, val):
        if val is None:
            self.__nz = val
            return

        try:
            val = int(val)
        except ValueError:
            raise ValueError("nz must be an integer")

        if val < 1:
            raise ValueError("nz must be >= 1")
        else:
            self.__nz = val

        del self.dndm

    @property
    def cut_fit(self):
        return self.__cut_fit

    @cut_fit.setter
    def cut_fit(self, val):
        if not isinstance(val, bool):
            raise ValueError("cut_fit must be a bool, " + str(val))

        del self.fsigma
        self.__cut_fit = val


    #--------------------------------  START NON-SET PROPERTIES ----------------------------------------------
    @property
    def cosmo(self):
        """ :class:`hmf.cosmo.Cosmology` object aliased from `self.transfer.cosmo`"""
        return self.transfer.cosmo

    @property
    def delta_halo(self):
        """ Overdensity of a halo w.r.t mean density"""
        try:
            return self.__delta_halo
        except:
            if self.delta_wrt == 'mean':
                self.__delta_halo = self.delta_h

            elif self.delta_wrt == 'crit':
                self.__delta_halo = self.delta_h / cp.density.omega_M_z(self.transfer.z, **self.cosmo.cosmolopy_dict())
            return self.__delta_halo

    @delta_halo.deleter
    def delta_halo(self):
        try:
            del self.__delta_halo
            del self.fsigma
        except:
            pass

    @property
    def _sigma_0(self):
        """
        The normalised mass variance at z=0 :math:`\sigma`
        
        Notes
        -----
        
        .. math:: \sigma^2(R) = \frac{1}{2\pi^2}\int_0^\infty{k^2P(k)W^2(kR)dk}
        
        """

        try:
            return self.__sigma_0
        except:
            self.__sigma_0 = tools.mass_variance(self.M, self.transfer._lnP_0,
                                                 self.transfer.lnk,
                                                 self.cosmo.mean_dens,
                                                 self.mv_scheme)
            return self.__sigma_0

    @_sigma_0.deleter
    def _sigma_0(self):
        try:
            del self.__sigma_0
            del self._dlnsdlnm
            del self.sigma
        except:
            pass

    @property
    def _dlnsdlnm(self):
        """
        The value of :math:`\left|\frac{\d \ln \sigma}{\d \ln M}\right|`, ``len=len(M)``
        
        Notes
        -----
        
        .. math:: frac{d\ln\sigma}{d\ln M} = \frac{3}{2\sigma^2\pi^2R^4}\int_0^\infty \frac{dW^2(kR)}{dM}\frac{P(k)}{k^2}dk
        
        """
        try:
            return self.__dlnsdlnm
        except:
            self.__dlnsdlnm = tools.dlnsdlnm(self.M, self._sigma_0, self.transfer._lnP_0,
                                             self.transfer.lnk,
                                             self.cosmo.mean_dens)
            return self.__dlnsdlnm

    @_dlnsdlnm.deleter
    def _dlnsdlnm(self):
        try:
            del self.__dlnsdlnm
            del self.dndm
            del self.n_eff
        except:
            pass

    @property
    def sigma(self):
        """
        The mass variance at `z`, ``len=len(M)``
        """
        try:
            return self.__sigma
        except:
            self.__sigma = self._sigma_0 * self.transfer.growth
            return self.__sigma

    @sigma.deleter
    def sigma(self):
        try:
            del self.__sigma
            del self.fsigma
            del self.lnsigma
        except:
            pass

    @property
    def lnsigma(self):
        """
        Natural log of inverse mass variance, ``len=len(M)``
        """
        try:
            return self.__lnsigma
        except:
            self.__lnsigma = np.log(1 / self.sigma)
            return self.__lnsigma

    @lnsigma.deleter
    def lnsigma(self):
        try:
            del self.__lnsigma
            del self.fsigma
        except:
            pass

    @property
    def n_eff(self):
        """
        Effective spectral index at scale of halo radius, ``len=len(M)``
        """
        try:
            return self.__n_eff
        except:
            self.__n_eff = tools.n_eff(self._dlnsdlnm)
            return self.__n_eff

    @n_eff.deleter
    def n_eff(self):
        try:
            del self.__n_eff
        except:
            pass

    @property
    def fsigma(self):
        """
        The multiplicity function, :math:`f(\sigma)`, for `mf_fit`. ``len=len(M)``
        """
        try:
            return self.__fsigma
        except:
            try:
                self.__fsigma = self.mf_fit(self)
            except:
                fits_class = Fits(self)
                self.__fsigma = fits_class.nufnu()

            if np.sum(np.isnan(self.__fsigma)) > 0.8 * len(self.__fsigma):
                # the input mass range is almost completely outside the cut
                logger.warning("The specified mass-range was almost entirely \
                                outside of the limits from the fit. Ignored fit range...")
                self.cut_fit = False
                try:
                    self.__fsigma = self.mf_fit(self)
                except:
                    self.__fsigma = fits_class.nufnu()

            return self.__fsigma

    @fsigma.deleter
    def fsigma(self):
        try:
            del self.__fsigma
            del self.dndm
        except:
            pass

    @property
    def dndm(self):
        """
        The number density of haloes, ``len=len(M)`` [units :math:`h^4 M_\odot^{-1} Mpc^{-3}`]
        """
        try:
            return self.__dndm
        except:
            if self.z2 is None:  # #This is normally the case
                self.__dndm = self.fsigma * self.cosmo.mean_dens * np.abs(self._dlnsdlnm) / self.M ** 2
                if self.mf_fit == 'Behroozi':
                    a = 1 / (1 + self.transfer.z)
                    theta = 0.144 / (1 + np.exp(14.79 * (a - 0.213))) * (self.M / 10 ** 11.5) ** (0.5 / (1 + np.exp(6.5 * a)))
                    ngtm_tinker = self._ngtm()
                    ngtm_behroozi = 10 ** (theta + np.log10(ngtm_tinker))
                    dthetadM = 0.144 / (1 + np.exp(14.79 * (a - 0.213))) * \
                        (0.5 / (1 + np.exp(6.5 * a))) * (self.M / 10 ** 11.5) ** \
                        (0.5 / (1 + np.exp(6.5 * a)) - 1) / (10 ** 11.5)
                    self.__dndm = self.__dndm * 10 ** theta - ngtm_behroozi * np.log(10) * dthetadM
            else:  # #This is for a survey-volume weighted calculation
                if self.nz is None:
                    self.nz = 10
                zedges = np.linspace(self.transfer.z, self.z2, self.nz)
                zcentres = (zedges[:-1] + zedges[1:]) / 2
                dndm = np.zeros_like(zcentres)
                vol = np.zeros_like(zedges)
                vol[0] = cp.distance.comoving_volume(self.transfer.z,
                                            **self.cosmo.cosmolopy_dict())
                for i, zz in enumerate(zcentres):
                    self.update(z=zz)
                    dndm[i] = self.fsigma * self.cosmo.mean_dens * np.abs(self._dlnsdlnm) / self.M ** 2
                    if self.mf_fit == 'Behroozi':
                        a = 1 / (1 + self.transfer.z)
                        theta = 0.144 / (1 + np.exp(14.79 * (a - 0.213))) * (self.M / 10 ** 11.5) ** (0.5 / (1 + np.exp(6.5 * a)))
                        ngtm_tinker = self._ngtm()
                        ngtm_behroozi = 10 ** (theta + np.log10(ngtm_tinker))
                        dthetadM = 0.144 / (1 + np.exp(14.79 * (a - 0.213))) * (0.5 / (1 + np.exp(6.5 * a))) * (self.M / 10 ** 11.5) ** (0.5 / (1 + np.exp(6.5 * a)) - 1) / (10 ** 11.5)
                        dndm[i] = dndm[i] * 10 ** theta - ngtm_behroozi * np.log(10) * dthetadM

                    vol[i + 1] = cp.distance.comoving_volume(z=zedges[i + 1],
                                                    **self.cosmo.cosmolopy_dict())

                vol = vol[1:] - vol[:-1]  # Volume in shells
                integrand = vol * dndm
                numerator = intg.simps(integrand, x=zcentres)
                denom = intg.simps(vol, zcentres)
                self.__dndm = numerator / denom
            return self.__dndm

    @dndm.deleter
    def dndm(self):
        try:
            del self.__dndm
            del self.dndlnm
            del self.dndlog10m
        except:
            pass


    @property
    def dndlnm(self):
        """
        The differential mass function in terms of natural log of `M`, ``len=len(M)`` [units :math:`h^3 Mpc^{-3}`]
        """
        try:
            return self.__dndlnm
        except:
            self.__dndlnm = self.M * self.dndm
            return self.__dndlnm

    @dndlnm.deleter
    def dndlnm(self):
        try:
            del self.__dndlnm
            del self.ngtm
            del self.nltm
            del self.mgtm
            del self.mltm
            del self.how_big
        except:
            pass

    @property
    def dndlog10m(self):
        """
        The differential mass function in terms of log of `M`, ``len=len(M)`` [units :math:`h^3 Mpc^{-3}`]
        """
        try:
            return self.__dndlog10m
        except:
            self.__dndlog10m = self.M * self.dndm * np.log(10)
            return self.__dndlog10m

    @dndlog10m.deleter
    def dndlog10m(self):
        try:
            del self.__dndlog10m
        except:
            pass

    def _upper_ngtm(self, M, mass_function, cut):
        """Calculate the mass function above given range of `M` in order to integrate"""
        ### WE CALCULATE THE MASS FUNCTION ABOVE THE COMPUTED RANGE ###
        # mass_function is logged already (not log10 though)
        m_upper = np.linspace(np.log(M[-1]), np.log(10 ** 18), 500)
        if cut:  # since its been cut, the best we can do is a power law
            mf_func = spline(np.log(M), mass_function, k=1)
            mf = mf_func(m_upper)
        else:
            # We try to calculate the hmf as far as we can normally
            new_pert = copy.deepcopy(self)
            new_pert.update(M=np.log10(np.exp(m_upper)))
            mf = np.log(np.exp(m_upper) * new_pert.dndm)

            if np.isnan(mf[-1]):  # Then we couldn't get up all the way, so have to do linear ext.
                if np.isnan(mf[1]):  # Then the whole extension is nan and we have to use the original (start at 1 because 1 val won't work either)
                    mf_func = spline(np.log(M), mass_function, k=1)
                    mf = mf_func(m_upper)
                else:
                    mfslice = mf[np.logical_not(np.isnan(mf))]
                    m_nan = m_upper[np.isnan(mf)]
                    m_true = m_upper[np.logical_not(np.isnan(mf))]
                    mf_func = spline(m_true, mfslice, k=1)
                    mf[len(mfslice):] = mf_func(m_nan)
        return m_upper, mf

    def _lower_ngtm(self, M, mass_function, cut):
        ### WE CALCULATE THE MASS FUNCTION BELOW THE COMPUTED RANGE ###
        # mass_function is logged already (not log10 though)
        m_lower = np.linspace(np.log(10 ** 3), np.log(M[0]), 500)
        if cut:  # since its been cut, the best we can do is a power law
            mf_func = spline(np.log(M), mass_function, k=1)
            mf = mf_func(m_lower)
        else:
            # We try to calculate the hmf as far as we can normally
            new_pert = copy.deepcopy(self)
            new_pert.update(M=np.log10(np.exp(m_lower)))
            mf = np.log(np.exp(m_lower) * new_pert.dndm)

            if np.isnan(mf[0]):  # Then we couldn't go down all the way, so have to do linear ext.
                mfslice = mf[np.logical_not(np.isnan(mf))]
                m_nan = m_lower[np.isnan(mf)]
                m_true = m_lower[np.logical_not(np.isnan(mf))]
                mf_func = spline(m_true, mfslice, k=1)
                mf[:len(mfslice)] = mf_func(m_nan)
        return m_lower, mf

    def _ngtm(self):
        """
        Calculate n(>m).
        
        This function is separated from the property because of the Behroozi fit
        """
        # set M and mass_function within computed range
        M = self.M[np.logical_not(np.isnan(self.dndlnm))]
        mass_function = self.dndlnm[np.logical_not(np.isnan(self.dndlnm))]

        # Calculate the mass function (and its integral) from the highest M up to 10**18
        if M[-1] < 10 ** 18:
            m_upper, mf = self._upper_ngtm(M, np.log(mass_function), M[-1] < self.M[-1])

            int_upper = intg.simps(np.exp(mf), dx=m_upper[2] - m_upper[1], even='first')
        else:
            int_upper = 0

        # Calculate the cumulative integral (backwards) of mass_function (Adding on the upper integral)
        ngtm = np.concatenate((intg.cumtrapz(mass_function[::-1], dx=np.log(M[1]) - np.log(M[0]))[::-1], np.zeros(1))) + int_upper

        # We need to set ngtm back in the original length vector with nans where they were originally
        if len(ngtm) < len(self.M):
            ngtm_temp = np.zeros_like(self.dndlnm)
            ngtm_temp[:] = np.nan
            ngtm_temp[np.logical_not(np.isnan(self.dndlnm))] = ngtm
            ngtm = ngtm_temp

        return ngtm

    @property
    def ngtm(self):
        """
        The cumulative mass function above `M`, ``len=len(M)`` [units :math:`h^3 Mpc^{-3}`]
        """
        try:
            return self.__ngtm
        except:
            self.__ngtm = self._ngtm()
            return self.__ngtm

    @ngtm.deleter
    def ngtm(self):
        try:
            del self.__ngtm
            del self.how_big
        except:
            pass

    @property
    def mgtm(self):
        """
        Mass in haloes `>M`, ``len=len(M)`` [units :math:`M_\odot h^2 Mpc^{-3}`]
        """
        try:
            return self.__mgtm
        except:
            M = self.M[np.logical_not(np.isnan(self.dndlnm))]
            mass_function = self.dndlnm[np.logical_not(np.isnan(self.dndlnm))]

            # Calculate the mass function (and its integral) from the highest M up to 10**18
            if M[-1] < 10 ** 18:
                m_upper, mf = self._upper_ngtm(M, np.log(mass_function), M[-1] < self.M[-1])
                int_upper = intg.simps(np.exp(mf + m_upper) , dx=m_upper[2] - m_upper[1], even='first')
            else:
                int_upper = 0

            # Calculate the cumulative integral (backwards) of mass_function (Adding on the upper integral)
            self.__mgtm = np.concatenate((intg.cumtrapz(mass_function[::-1] * M[::-1], dx=np.log(M[1]) - np.log(M[0]))[::-1], np.zeros(1))) + int_upper

            # We need to set ngtm back in the original length vector with nans where they were originally
            if len(self.__mgtm) < len(self.M):
                mgtm_temp = np.zeros_like(self.dndlnm)
                mgtm_temp[:] = np.nan
                mgtm_temp[np.logical_not(np.isnan(self.dndlnm))] = self.__mgtm
                self.__mgtm = mgtm_temp
            return self.__mgtm
    @mgtm.deleter
    def mgtm(self):
        try:
            del self.__mgtm
        except:
            pass

    @property
    def nltm(self):
        """
        Inverse cumulative mass function, ``len=len(M)`` [units :math:`h^3 Mpc^{-3}`]
        """
        try:
            return self.__nltm
        except:
            # set M and mass_function within computed range
            M = self.M[np.logical_not(np.isnan(self.dndlnm))]
            mass_function = self.dndlnm[np.logical_not(np.isnan(self.dndlnm))]

            # Calculate the mass function (and its integral) from 10**3 up to lowest M
            if M[0] > 10 ** 3:
                m_lower, mf = self._lower_ngtm(M, np.log(mass_function), M[0] > self.M[0])

                int_lower = intg.simps(np.exp(mf), dx=m_lower[2] - m_lower[1], even='first')
            else:
                int_lower = 0

            # Calculate the cumulative integral of mass_function (Adding on the lower integral)
            self.__nltm = np.concatenate((np.zeros(1), intg.cumtrapz(mass_function, dx=np.log(M[1]) - np.log(M[0])))) + int_lower

            # We need to set ngtm back in the original length vector with nans where they were originally
            if len(self.__nltm) < len(self.M):
                nltm_temp = np.zeros_like(self.dndlnm)
                nltm_temp[:] = np.nan
                nltm_temp[np.logical_not(np.isnan(self.dndlnm))] = self.__nltm
                self.__nltm = nltm_temp

            return self.__nltm
    @nltm.deleter
    def nltm(self):
        try:
            del self.__nltm
        except:
            pass

    @property
    def mltm(self):
        """
        Total mass in haloes `<M`, ``len=len(M)`` [units :math:`M_\odot h^2 Mpc^{-3}`]
        """
        try:
            return self.__mltm
        except:
            # Set M within calculated range
            M = self.M[np.logical_not(np.isnan(self.dndlnm))]
            mass_function = self.dndlnm[np.logical_not(np.isnan(self.dndlnm))]

            # Calculate the mass function (and its integral) from 10**3 up to lowest M
            if M[0] > 10 ** 3:
                m_lower, mf = self._lower_ngtm(M, np.log(mass_function), M[0] > self.M[0])

                int_lower = intg.simps(np.exp(mf + m_lower), dx=m_lower[2] - m_lower[1], even='first')
            else:
                int_lower = 0

            # Calculate the cumulative integral of mass_function (Adding on the lower integral)
            self.__mltm = np.concatenate((np.zeros(1), intg.cumtrapz(mass_function * M, dx=np.log(M[1]) - np.log(M[0])))) + int_lower

            # We need to set ngtm back in the original length vector with nans where they were originally
            if len(self.__mltm) < len(self.M):
                nltm_temp = np.zeros_like(self.dndlnm)
                nltm_temp[:] = np.nan
                nltm_temp[np.logical_not(np.isnan(self.dndlnm))] = self.__mltm
                self.__mltm = nltm_temp

            return self.__mltm

    @property
    def how_big(self):
        """ 
        Size of simulation volume in which to expect one halo of mass M, ``len=len(M)`` [units :math:`Mpch^{-1}`]
        """

        return self.ngtm ** (-1. / 3.)

    @how_big.deleter
    def how_big(self):
        try:
            del self.how_big
        except:
            pass
Ejemplo n.º 57
0
print("Precione 6 para realizar um estorno com split")
print("Precione 7 para verificar o status de um postback")

esco = raw_input("")

if (esco == "1"):

    print("A transação numero " + str(Split()) + " foi criada!")

elif (esco == "2"):

    print("A assinatura numero " + str(Recorrencia()) + " foi criada!")

elif (esco == "3"):

    retorno = Transfer()
    print("A transferencia: " + str(retorno[0]) + ", de " + str(retorno[1]) +
          " centavos foi realizada para a conta bancária do recebedor " +
          str(retorno[2]) + "!")

elif (esco == "4"):

    retorno = Antecipacao()
    print("A antecipação: " + str(retorno[0]) + ", de " + str(retorno[1]) +
          " centavos foi criada para o recebedor " + str(retorno[2]) + "!")

elif (esco == "5"):

    print("A assinatura com split numero " + str(RecorrenciaSplit()) +
          " foi criada!")
Ejemplo n.º 58
0
import time
from operator import itemgetter
from transfer import Transfer, Mail
from config import *


print("\n# {:*^90} #".format(""))
print("# {:^80} #".format(" 欢迎使用12306余票查询服务 "))
print("# {:^90} #".format(" Author: yuanyuanzijin "))
print("# {:^90} #".format(f" Version: {Transfer.__tf_version__} - {Transfer.__update_time__} "))
print("# {:*^90} #\n".format(""))

tf = Transfer(date=DATE, transfer_cities=TRANSFER_CITIES, max_transfer_times=MAX_TRANSFER,
              expect_total_time=EXPECT_TOTAL_TIME, retry_times=RETRY_TIMES, expire_time=EXPIRE_TIME,
              min_expire_time=MIN_EXPIRE_TIME, auto_expire=AUTO_EXPIRE)

print('\n')
print("{:*^50}".format(" 任务信息如下 "))
print(f"出发日期:{DATE}")
print(f"始发城市:{TRANSFER_CITIES[0]}")
print(f"终到城市:{TRANSFER_CITIES[-1]}")
print(f"可选换乘城市:{', '.join(TRANSFER_CITIES[1:-1])}")
print(f"最大换乘次数:{MAX_TRANSFER}")
print(f"期望总时间:<{EXPECT_TOTAL_TIME}h")
print("{:*^56}".format(""))
print(f"理论换乘路线:{len(tf.tasks)}个")
print(f"初始缓存时间:{EXPIRE_TIME}s")
print(f"最小缓存时间:{MIN_EXPIRE_TIME}s")
print(f"邮件最小间隔:{MAIL_MIN_INTERVAL}s")
print(f"邮件通知:{'开启' if EMAIL_ENABLE else '关闭'}")
print(f"通知邮箱:{RECEIVE_EMAIL}")
Ejemplo n.º 59
0
 def new_transfer(self):
     if self.transfer is None:
         self.transfer = Transfer()
     else:
         self.transfer.turn_on()
Ejemplo n.º 60
0
        return self.DAILY_REPORT

    def daily_activity_report_output(self):
        self.file_name = 'SuburbanDigitalAdReport_' + self.date_strftime(
        ) + '.txt'
        output = open(self.file_name, 'w')
        output.write('Date: ' + str(self.todays_date()) + '\n' +
                     str(self.daily_activity_report()))
        output.close()

    def monthly_activity_report(self):
        self.MONTHLY_REPORT = str(self.analytics.getMonthlyReport())
        return self.MONTHLY_REPORT

    def monthly_activity_report_output(self):
        self.file_name = 'MonthlyDealerBudgetTemplate_' + self.date_strftime(
        ) + '.txt'
        output = open(self.file_name, 'w')
        output.write('Date: ' + str(self.todays_date()) + '\n' +
                     str(self.monthly_activity_report()))
        output.close()


if __name__ == "__main__":
    suburban = SuburbanCollection()
    print(suburban.todays_date())
    suburban.daily_activity_report_output()
    suburban.monthly_activity_report_output()
    send = Transfer()
    send.ftp_daily_report()