from opac_proc.datastore.mongodb_connector import get_opac_webapp_db_name from opac_proc.loaders.base import BaseLoader from opac_proc.datastore.models import ( LoadNews, TransformNews) from opac_proc.datastore.identifiers_models import NewsIdModel from opac_schema.v1.models import News as OpacNews from opac_proc.web import config from opac_proc.logger_setup import getMongoLogger if config.DEBUG: logger = getMongoLogger(__name__, "DEBUG", "load") else: logger = getMongoLogger(__name__, "INFO", "load") OPAC_WEBAPP_DB_NAME = get_opac_webapp_db_name() class NewsLoader(BaseLoader): transform_model_class = TransformNews transform_model_instance = None opac_model_class = OpacNews opac_model_instance = None load_model_class = LoadNews load_model_instance = None
from xylose.scielodocument import Article from opac_proc.datastore.models import (ExtractArticle, TransformArticle, TransformIssue, TransformJournal) from opac_proc.datastore.identifiers_models import ArticleIdModel from opac_proc.transformers.base import BaseTransformer from opac_proc.extractors.decorators import update_metadata from opac_proc.web import config from opac_proc.logger_setup import getMongoLogger from opac_proc.core.assets import AssetPDF, AssetXML, AssetHTMLS if config.DEBUG: logger = getMongoLogger(__name__, "DEBUG", "transform") else: logger = getMongoLogger(__name__, "INFO", "transform") class ArticleTransformer(BaseTransformer): extract_model_class = ExtractArticle extract_model_instance = None transform_model_class = TransformArticle transform_model_instance = None ids_model_class = ArticleIdModel ids_model_instance = None def get_extract_model_instance(self, key):
# coding: utf-8 from datetime import datetime from opac_proc.datastore.models import ExtractIssue from opac_proc.extractors.base import BaseExtractor from opac_proc.extractors.decorators import update_metadata from opac_proc.web import config from opac_proc.logger_setup import getMongoLogger if config.DEBUG: logger = getMongoLogger(__name__, "DEBUG", "extract") else: logger = getMongoLogger(__name__, "INFO", "extract") class IssueExtractor(BaseExtractor): acronym = None issue_id = None extract_model_class = ExtractIssue def __init__(self, acronym, issue_id): super(IssueExtractor, self).__init__() self.acronym = acronym self.issue_id = issue_id self.get_instance_query = { 'collection': self.acronym, 'code': self.issue_id, } @update_metadata
# coding: utf-8 from opac_proc.datastore.models import (TransformCollection, LoadCollection) from opac_proc.loaders.base import BaseLoader from opac_proc.datastore.identifiers_models import CollectionIdModel from opac_schema.v1.models import Collection as OpacCollection from opac_schema.v1.models import CollectionMetrics from opac_proc.web import config from opac_proc.logger_setup import getMongoLogger if config.DEBUG: logger = getMongoLogger(__name__, "DEBUG", "load") else: logger = getMongoLogger(__name__, "INFO", "load") class CollectionLoader(BaseLoader): transform_model_class = TransformCollection transform_model_instance = None opac_model_class = OpacCollection opac_model_instance = None load_model_class = LoadCollection load_model_instance = None ids_model_class = CollectionIdModel ids_model_name = 'CollectionIdModel' ids_model_instance = None
# coding: utf-8 import requests from datetime import datetime from opac_proc.datastore.models import ExtractCollection from opac_proc.datastore.identifiers_models import CollectionIdModel from opac_proc.extractors.base import BaseExtractor from opac_proc.extractors.decorators import update_metadata from opac_proc.web import config from opac_proc.logger_setup import getMongoLogger if config.DEBUG: logger = getMongoLogger(__name__, "DEBUG", "extract") else: logger = getMongoLogger(__name__, "INFO", "extract") PUBLICATION_SIZE_ENDPOINT = 'ajx/publication/size' ARTICLE_META_COLLECTION_ENDPOINT = 'api/v1/collection/' class CollectionExtractor(BaseExtractor): acronym = None extract_model_class = ExtractCollection ids_model_class = CollectionIdModel ids_model_name = 'CollectionIdModel' def __init__(self): super(CollectionExtractor, self).__init__() self.acronym = config.OPAC_PROC_COLLECTION self.get_instance_query = {'acronym': self.acronym}
# coding: utf-8 import datetime from opac_proc.datastore.models import ( ExtractNews, TransformNews) from opac_proc.transformers.base import BaseTransformer from opac_proc.extractors.decorators import update_metadata from opac_proc.datastore.identifiers_models import NewsIdModel from opac_proc.web import config from opac_proc.logger_setup import getMongoLogger if config.DEBUG: logger = getMongoLogger(__name__, "DEBUG", "transform") else: logger = getMongoLogger(__name__, "INFO", "transform") class NewsTransformer(BaseTransformer): extract_model_class = ExtractNews extract_model_instance = None transform_model_class = TransformNews transform_model_instance = None ids_model_class = NewsIdModel ids_model_instance = None def get_extract_model_instance(self, key): return self.extract_model_class.objects.get(uuid=key)