Ejemplo n.º 1
0
    def sar_top_mp3(self):
        from NXSpider.bin.models import artist_mo

        if self.param_check(['artist'],
                            sys._getframe().f_code.co_name) is False:
            return

        download_type = self.parse_download()
        artists = self.app.pargs.artist.split(',')  # type: list

        for arid in artists:
            detail = api.get_artists_songs(arid)
            if detail is None:
                continue

            artist_detail = detail['artist']
            artist_detail['mp3'] = detail['hotSongs']

            log.print_info(u"<{}>".format(artist_detail['name']))
            artist_mo.parse_model(
                artist_detail,
                download_type=download_type,
                file_check=Config().get_file_check(),
                shortcuts_stack=[] if Config().get_shortcut() else None)

        log.print_info("spider complete!~")
        pass
Ejemplo n.º 2
0
    def sur_pls(self):
        from NXSpider.bin.models import playlist_mo

        if self.param_check(['user'], sys._getframe().f_code.co_name) is False:
            return

        download_type = self.parse_download()
        user_id = self.app.pargs.user
        playlists = api.user_playlist(user_id,
                                      offset=self.app.pargs.offset or 0,
                                      limit=self.app.pargs.limit or 50)

        log.print_info("playlists bellow will be crawled")
        print_playlist(playlists)

        for pl_obj in playlists:
            playlist_detail = api.get_playlist_detail(pl_obj['id'])
            if playlist_detail:
                log.print_info(u"<{}> author:{}".format(
                    playlist_detail['name'],
                    playlist_detail['creator']['nickname'],
                ))
                playlist_mo.parse_model(
                    playlist_detail,
                    download_type=download_type,
                    file_check=Config().get_file_check(),
                    shortcuts_stack=[] if Config().get_shortcut() else None)
        log.print_info("spider complete!~")
        pass
Ejemplo n.º 3
0
    def sab(self):
        from NXSpider.bin.models import album_mo

        if self.param_check(['album'],
                            sys._getframe().f_code.co_name) is False:
            return

        download_type = self.parse_download()
        albums = self.app.pargs.album.split(',')  # type: list

        for pid in albums:
            album_detail = api.get_album_detail(pid)
            if album_detail is None:
                continue

            log.print_info(u"{} artist:{}".format(
                "<" + album_detail['name'] + ">",
                album_detail['artist']['name'],
            ))
            album_mo.parse_model(
                album_detail,
                download_type=download_type,
                file_check=Config().get_file_check(),
                shortcuts_stack=[] if Config().get_shortcut() else None)

        log.print_info("spider complete!~")
        pass
Ejemplo n.º 4
0
    def login_smv(self):
        from NXSpider.bin.models import no_rec_mv_mo

        if self.param_check(['lu'], sys._getframe().f_code.co_name) is False:
            return

        plaintext_pwd = self.app.pargs.lp or None
        if plaintext_pwd is None:
            import getpass
            plaintext_pwd = getpass.getpass("Please input your password:"******"none")))
            exit()

        mvs = api.my_mvs(session)
        mvs = [api.get_mv_detail(d['id']) for d in mvs]
        mvs = [d for d in mvs if d]

        for mv in mvs:
            no_rec_mv_mo.parse_model(
                mv,
                download_type=['mv'],
                file_check=Config().get_file_check(),
                shortcuts_stack=[] if Config().get_shortcut() else None)

        log.print_info("spider complete!~")

        pass
Ejemplo n.º 5
0
    def spls(self):
        from NXSpider.bin.models import playlist_mo

        if self.param_check(['playlist'],
                            sys._getframe().f_code.co_name) is False:
            return

        download_type = self.parse_download()
        playlists = self.app.pargs.playlist.split(',')  # type: list

        for pid in playlists:
            playlist_detail = api.get_playlist_detail(pid)
            if playlist_detail:
                log.print_info(u"<{}> author:{}".format(
                    playlist_detail['name'],
                    playlist_detail['creator']['nickname'],
                ))
                playlist_mo.parse_model(
                    playlist_detail,
                    download_type=download_type,
                    file_check=Config().get_file_check(),
                    shortcuts_stack=[] if Config().get_shortcut() else None)

        log.print_info("spider complete!~")
        pass
Ejemplo n.º 6
0
 def config_check(self):
     self.config_show()
     try:
         config = Config()
         if config.config_test():
             log.print_info('config check complete, all is well done!')
     except:
         log.print_err('config check failed, pls re config')
Ejemplo n.º 7
0
    def config_spider(self):
        config = Config()
        config_dict = config.config  # type: dict
        is_config = False
        try:
            if self.app.pargs.path_download is not None:
                paths = self.app.pargs.path_download.split(',')  # type: list
                if default_path_key in paths:
                    index = paths.index(default_path_key)
                    paths.remove(default_path_key)
                    paths.insert(index, default_download_dir)

                final_paths = []
                for p in paths:
                    try:
                        # some error need pass
                        if os.path.isdir(p) is False:
                            os.mkdir(p)
                        final_paths.append(p)
                    except:
                        log.print_warn("path may be wrong and be deleted: {}".format(p))
                        pass

                if not final_paths:
                    final_paths.append(default_download_dir)

                log.print_info('path will be set as: ' + ','.join(final_paths))

                config_dict['download_path'] = final_paths
                is_config = True

            if self.app.pargs.mv_resolution is not None:
                r = int(self.app.pargs.mv_resolution)
                if r not in mv_resolutions:
                    log.print_warn("-mvr resolution config skip, value must be 240,480,720,1080")
                config_dict['mv_def_resolution'] = r
                is_config = True

            if self.app.pargs.media_tag is not None:
                config_dict['media_tag'] = True if self.app.pargs.media_tag.lower() == 'true'\
                                                   or self.app.pargs.media_tag == '1' else False
                is_config = True

            if self.app.pargs.media_tag_163 is not None:
                config_dict['media_tag_163'] = True if self.app.pargs.media_tag_163.lower() == 'true' \
                                                       or self.app.pargs.media_tag_163 == '1' else False
                is_config = True

        except:
            log.print_err("input error, pls check")
            raise
        if is_config:
            config.save_config_file()
        log.print_info("config success")
        self.config_show()
Ejemplo n.º 8
0
    def download_file(self, doc):
        """
        download file from music 163
        :param doc:
        :return:
        """
        file_relative_path = self.download_relative_path(doc)
        path = Config().get_path()
        content = self.request_file(doc)

        if content is None:
            log.print_err(u"file download failed : %s" % file_relative_path)
            return False

        try:
            file_name = os.path.join(path, file_relative_path)

            # dir make
            dir_name = os.path.dirname(file_name)
            if not os.path.exists(dir_name):
                os.makedirs(dir_name)

            # file write
            with open(file_name, "wb") as code:
                code.write(content)

            self.download_file_tag(file_name, doc)

            log.print_info(u"file download complete: %s" % file_relative_path)
            self.download_log(doc)
            return True
        except Exception as e:
            log.print_err("file save failed : %s, err: %s" %
                          (file_relative_path, e))
            return False
Ejemplo n.º 9
0
def attach_mp4_tag(doc, file):
    """
    :type doc: Mp4Model
    :param doc:
    :param file:
    :return:
    """
    authors = u",".join([x['name'] for x in doc.artists])
    data = {
        'title': doc['name'],
        'artist': authors,
    }

    if Config().get_media_tag_163():
        comment_plaintext = u'mv:{"title":"%s","mvId":%d,"artistId":%d,' \
                            u'"artistName":"%s","pubTime":"%s","bitrate":%d}' \
                            % (doc['name'], doc.id, doc.artists[0].id,
                               doc.artists[0]['name'], doc['publishTime'],
                               doc['download_video_r'])
        comment = "163 key(Don't modify):" + aes_ecb(comment_plaintext, aes_code).decode()
        data['comment'] = comment

    try:
        mp4 = MP4(file)
        for k, v in data.items():
            if k not in mugagen_mp4_key_map:
                continue
            mp4[mugagen_mp4_key_map[k]] = v

        mp4.save()
    except Exception as e:
        return False
    return True
Ejemplo n.º 10
0
    def sur_pls(self):
        from NXSpider.bin.models import playlist_mo

        if self.param_check(['user'], sys._getframe().f_code.co_name) is False:
            return

        download_type = self.parse_download()
        user_id = self.app.pargs.user
        playlists = api.user_playlist(user_id)

        from terminaltables import AsciiTable
        table = AsciiTable([["ID", "Name", "User", "PlayCount"]])
        table_data = [[
            str(item['id']),
            item['name'],
            item['creator']['nickname'],
            str(item['playCount']),
        ] for item in playlists]
        table.table_data.extend(table_data)
        log.print_info("playlists bellow will be crawled")
        print(table.table)

        for pl_obj in playlists:
            playlist_detail = api.get_playlist_detail(pl_obj['id'])
            if playlist_detail:
                log.print_info(u"<{}> author:{}".format(
                    playlist_detail['name'],
                    playlist_detail['creator']['nickname'],
                ))
                playlist_mo.parse_model(playlist_detail,
                                        download_type=download_type,
                                        file_check=Config().get_file_check())
        log.print_info("spider complete!~")
        pass
Ejemplo n.º 11
0
    def create_shortcut(self, doc, shortcuts_stack):
        """
        :param doc:
        :param shortcuts_stack:
        :type shortcuts_stack: list[str]
        :return:
        """
        from NXSpider.utility.shortcut import symlink
        if not (hasattr(doc, model_is_download) and doc[model_is_download]
                and hasattr(doc, model_download_path) and os.path.exists(
                    doc[model_download_path]) and self.download_filename(doc)):
            return

        file_path = doc[model_download_path]
        shortcut_root = os.path.join(Config().get_path(), 'shortcuts')
        file_name = self.download_filename(doc)
        if file_path and file_path is not True:
            if shortcuts_stack:
                path = os.path.join(shortcut_root, *shortcuts_stack)
                os.makedirs(path, exist_ok=True)
                target = os.path.join(path, file_name)
                symlink(file_path, os.path.join(shortcut_root, target))

            for k in self.shortcut_self_path(doc):
                path = os.path.join(shortcut_root, k)
                os.makedirs(path, exist_ok=True)
                target = os.path.join(path, file_name)
                symlink(file_path, os.path.join(shortcut_root, target))
        pass
Ejemplo n.º 12
0
    def scls_pls(self):
        from NXSpider.bin.models import playlist_mo

        if self.param_check(['cls'], sys._getframe().f_code.co_name) is False:
            return

        download_type = self.parse_download()
        class_name = self.app.pargs.cls
        class_name = py2_decoding(class_name)

        if class_name != u"全部" and py2_encoding(
                class_name) not in api.ALL_CLASSES:
            log.print_err(
                "class name is wrong, pls check by run : nxspider sw-pl-classes"
            )
            return

        playlists = api.get_top_playlists(category=class_name,
                                          offset=self.app.pargs.offset or 0,
                                          limit=self.app.pargs.limit
                                          or 50)  # type: list

        for pl_obj in playlists:
            playlist_detail = api.get_playlist_detail(pl_obj['id'])
            if playlist_detail:
                log.print_info(u"<{}> author:{}".format(
                    playlist_detail['name'],
                    playlist_detail['creator']['nickname'],
                ))
                playlist_mo.parse_model(playlist_detail,
                                        download_type=download_type,
                                        file_check=Config().get_file_check())
        log.print_info("spider complete!~")
        pass
Ejemplo n.º 13
0
    def stop_mvs(self):
        from NXSpider.bin.models import no_rec_mv_mo

        mvs = api.top_mvs(offset=self.app.pargs.offset or 0,
                          limit=self.app.pargs.limit or 50)
        mvs = [api.get_mv_detail(d['id']) for d in mvs]
        mvs = [d for d in mvs if d]

        for mv in mvs:
            no_rec_mv_mo.parse_model(
                mv,
                download_type=['mv'],
                file_check=Config().get_file_check(),
                shortcuts_stack=[] if Config().get_shortcut() else None)

        log.print_info("spider complete!~")

        pass
Ejemplo n.º 14
0
 def debug_save_json(self, obj):
     with tools.ignored(Exception):
         relative_path = os.path.join(
             self.__file_type__ + '.debug',
             self.__file_type__ + "_" + str(obj['id']) + '.json')
         file_name = os.path.join(Config().get_path(), relative_path)
         with codecs.open(file_name, "wb", encoding='utf8') as code:
             test = json.dumps(obj, ensure_ascii=False)
             code.write(test)
Ejemplo n.º 15
0
    def login_spls(self):
        if self.param_check(['lu', 'lp'],
                            sys._getframe().f_code.co_name) is False:
            return

        from NXSpider.bin.models import playlist_mo

        plaintext_pwd = self.app.pargs.lp or None
        if plaintext_pwd is None:
            import getpass
            plaintext_pwd = getpass.getpass("Please input your password:"******"none")))
            exit()

        user_id = res['account']['id']
        download_type = self.parse_download()
        playlists = api.user_playlist(user_id,
                                      offset=self.app.pargs.offset or 0,
                                      limit=self.app.pargs.limit or 1000)

        log.print_info("playlists bellow will be crawled")
        print_playlist(playlists)

        for pl_obj in playlists:
            playlist_detail = api.get_playlist_detail(pl_obj['id'])
            if playlist_detail:
                log.print_info(u"<{}> author:{}".format(
                    playlist_detail['name'],
                    playlist_detail['creator']['nickname'],
                ))
                playlist_mo.parse_model(
                    playlist_detail,
                    download_type=download_type,
                    file_check=Config().get_file_check(),
                    shortcuts_stack=[] if Config().get_shortcut() else None)
        log.print_info("spider complete!~")
        pass
Ejemplo n.º 16
0
    def smp3s(self):
        from NXSpider.bin.models import dw_mp3_mo

        if self.param_check(['mp3'], sys._getframe().f_code.co_name) is False:
            return

        download_type = self.parse_download()
        mp3s = self.app.pargs.mp3.split(',')  # type: list
        details = api.get_mp3_details(mp3s)

        for mid, detail in details.items():
            log.print_info(u"<{}>".format(detail['name']))
            dw_mp3_mo.parse_model(
                detail,
                download_type=download_type,
                file_check=Config().get_file_check(),
                shortcuts_stack=[] if Config().get_shortcut() else None)

        log.print_info("spider complete!~")
        pass
Ejemplo n.º 17
0
 def url_load(self, doc):
     """
     implement pls
     :param doc:
     :return:
     :rtype: str
     """
     try:
         target_r = get_target_r(doc, Config().get_mv_resolution())
         doc['download_video_r'] = target_r
         return get_video_link(doc['id'], target_r)
     except:
         return None
Ejemplo n.º 18
0
    def config_mongo(self):
        config = Config()
        config_dict = config.config  # type: dict
        mongo_key = 'mongo'
        is_config = False
        try:
            if self.app.pargs.mhost is not None:
                config_dict[mongo_key]['host'] = self.app.pargs.mhost
                config_dict['no_mongo'] = False
                is_config = True

            if self.app.pargs.mport is not None:
                config_dict[mongo_key]['port'] = int(self.app.pargs.mport)
                is_config = True

            if self.app.pargs.muser is not None:
                config_dict[mongo_key]['username'] = self.app.pargs.muser
                is_config = True

            if self.app.pargs.mpassword is not None:
                config_dict[mongo_key]['password'] = self.app.pargs.mpassword
                is_config = True

            if self.app.pargs.mdbname is not None:
                config_dict[mongo_key]['name'] = self.app.pargs.mdbname
                is_config = True

            if self.app.pargs.nomongo is not None:
                config_dict['no_mongo'] = True if self.app.pargs.nomongo.lower() == 'true'\
                                                   or self.app.pargs.nomongo == '1' else False
                is_config = True
        except:
            log.print_err("input error, pls check")
            raise
        if is_config:
            config.save_config_file()
        log.print_info("config success")
        self.config_show()
Ejemplo n.º 19
0
    def sar_albums(self):
        from NXSpider.bin.models import artist_album_mo

        if self.param_check(['artist'],
                            sys._getframe().f_code.co_name) is False:
            return

        download_type = self.parse_download()
        artists = self.app.pargs.artist.split(',')  # type: list

        for arid in artists:
            detail = api.get_artist_album(arid,
                                          offset=self.app.pargs.offset or 0,
                                          limit=self.app.pargs.limit or 50)
            if detail is None:
                continue

            artist_detail = detail['artist']
            album_details = [
                api.get_album_detail(d['id']) for d in detail['hotAlbums']
            ]
            album_details = [d for d in album_details if d]
            artist_detail['albums'] = album_details

            log.print_info(u"<{}>".format(artist_detail['name']))
            log.print_info("albums bellow will be crawled")
            print_albums(artist_detail['albums'])

            artist_album_mo.parse_model(
                artist_detail,
                download_type=download_type,
                file_check=Config().get_file_check(),
                shortcuts_stack=[] if Config().get_shortcut() else None)

        log.print_info("spider complete!~")
        pass
Ejemplo n.º 20
0
def attach_mp3_idv3(doc, file):
    """
    :type doc: Mp3Model
    :param doc:
    :param file:
    :return:
    """
    artists = [('["%s",%d]' % (x['name'], x.id)) for x in doc.artists]
    artists_str = reduce(lambda x, y: x + "," + y, artists)

    authors = reduce(lambda x, y: x + ',' + y,
                     [x['name'] for x in doc.artists])
    data = {
        'title': doc['name'],
        'artist': authors,
        'album': doc['album']['name'],
        'album_artist': authors,
        'track_num': str(doc['no']),
    }

    if Config().get_media_tag_163():
        comment_plaintext = u'music:{"musicId":%d,"musicName":"%s","bitrate":320000,' \
                            u'"albumId":%d,"album":"%s", "artist":[%s]}' \
                            % (doc.id, doc['name'], doc.album.id, doc.album['name'],
                               artists_str)
        comment = "163 key(Don't modify):" + aes_ecb(comment_plaintext,
                                                     aes_code).decode()
        data['comment'] = comment

    try:
        mp3 = ID3(file, v2_version=3)
        for k, v in data.items():
            if k not in mutagen_idv3_key_map:
                continue

            if k == 'comment':
                mp3.add(id3.COMM(lang='XXX', text=v))
                continue

            attr_type = getattr(id3, mutagen_idv3_key_map[k], None)
            if attr_type:
                mp3.add(attr_type(text=v))

        mp3.save(v2_version=3)
    except Exception as e:
        return False
    return True
Ejemplo n.º 21
0
    def download_check(self, doc, check_file=False):
        """
        :param doc:
        :param check_file: check dist file to download
        :type doc: DynamicDocument
        :return:
        """
        if check_file:
            file_relative_path = self.download_relative_path(doc)
            for path in Config().get_paths():
                file_path = os.path.join(path, file_relative_path)
                if os.path.exists(file_path):
                    self.download_log(doc, download_path=file_path)
                    return file_path
            self.download_log(doc, downloaded=False)
            return False

        return hasattr(doc, model_is_download) and doc[model_is_download]
Ejemplo n.º 22
0
    def sar_albums(self):
        from NXSpider.bin.models import artist_album_mo

        if self.param_check(['artist'],
                            sys._getframe().f_code.co_name) is False:
            return

        download_type = self.parse_download()
        artists = self.app.pargs.artist.split(',')  # type: list

        for arid in artists:
            detail = api.get_artist_album(arid,
                                          offset=self.app.pargs.offset or 0,
                                          limit=self.app.pargs.limit or 50)
            if detail is None:
                continue

            artist_detail = detail['artist']
            album_details = [
                api.get_album_detail(d['id']) for d in detail['hotAlbums']
            ]
            album_details = [d for d in album_details if d]
            artist_detail['albums'] = album_details

            from terminaltables import AsciiTable
            table = AsciiTable([["ID", "Album", "Artist", "ArtistID"]])
            table_data = [[
                str(item['id']),
                item['name'],
                ','.join([ar['name'] for ar in item['artists']]),
                ','.join([str(ar['id']) for ar in item['artists']]),
            ] for item in artist_detail['albums']]
            table.table_data.extend(table_data)

            log.print_info(u"<{}>".format(artist_detail['name']))
            log.print_info("albums bellow will be crawled")
            print(table.table)

            artist_album_mo.parse_model(artist_detail,
                                        download_type=download_type,
                                        file_check=Config().get_file_check())

        log.print_info("spider complete!~")
        pass
Ejemplo n.º 23
0
def get_target_r(obj, limit_r=Config().get_mv_resolution()):
    max_valid = max([x['resolution'] for x in obj['resolutions']])
    return limit_r if max_valid > limit_r else max_valid
Ejemplo n.º 24
0
 def download_file_tag(self, filename, doc):
     if self.__file_type__ in ['mp3', 'mp4', 'mv'] \
             and Config().get_media_tag():
         attach_media_tag(doc, filename)
Ejemplo n.º 25
0
 def config_clear(self):
     Config().config_reset()
     log.print_info("config has been reset, u need re-config from beginning pls")
     self.config_show()
Ejemplo n.º 26
0
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# created by Lipson on 2018/6/20.
# email to [email protected]
#

from NXSpider.common.config import Config
if Config().get_no_mongo():
    from NXSpider.model.dict_model import *
else:
    from NXSpider.model.mongo_model import *

model_download_url = 'download_url'
model_is_download = 'downloaded'
model_download_path = 'download_path'

__all__ = [
    'ConfigModel',
    'UserModel',
    'AlbumModel',
    'PlaylistModel',
    'Mp4Model',
    'VideoModel',
    'ArtistModel',
    'Mp3Model',
    'AuthorModel',
    'update_dynamic_doc',
    'model_download_url',
    'model_is_download',
    'get_one_model_by_key',
Ejemplo n.º 27
0
import os
import shutil

from NXSpider.common.config import Config
from NXSpider.model.mongo_model import Mp3Model, Mp4Model
from NXSpider.spider.mp3 import Mp3
from NXSpider.spider.mv import MV
from NXSpider.common import tools, log, constant
from NXSpider.spider.base_driver import Music163Obj
from NXSpider.utility.media_tag import attach_media_tag

media_types = {
    'mp3': [Mp3Model, Mp3],
    'mp4': [Mp4Model, MV],
}
paths = Config().get_paths()


def attach_media_idv3_by_db():
    """
    attach media idv3 by paths in config
    :return:
    """
    for suffix, type_setting in media_types:
        model, driver = type_setting  # type: Mp3Model or Mp4Model, Music163Obj
        objs = model.objects(downloaded=True)
        for obj in objs:
            file_path = driver.download_check(obj, check_file=True)
            if not file_path:
                continue
Ejemplo n.º 28
0
#
# created by Lipson on 2018/4/19.
# email to [email protected]
#
from datetime import datetime

import pymongo
from mongoengine import connect, DynamicDocument, Document, signals
from mongoengine.fields import *
from pymongo.errors import ServerSelectionTimeoutError

from NXSpider.common import log
from NXSpider.common.config import Config
from NXSpider.model.export import *

mongodb_conf = Config().get_mongo()

try:

    client = pymongo.MongoClient(host=mongodb_conf['host'],
                                 port=mongodb_conf['port'],
                                 connectTimeoutMS=3000,
                                 serverSelectionTimeoutMS=3000)
    test_connect = client.database.test.count()
    del client
except ServerSelectionTimeoutError as e:
    log.print_err("mongodb server config error")
    exit()

model_download_url = 'download_url'
model_is_download = 'downloaded'
Ejemplo n.º 29
0
 def config_show(self):
     config_dict = Config().config
     log.print_info("config will be show fellow:")
     print(json.dumps(config_dict, ensure_ascii=False, indent=1))