Esempi in Python per make_invenio_opener

Esempio n. 1

File: crossrefutils.py Progetto: aw-bib/tind-invenio

import sys
import urllib
import urllib2
import datetime
from xml.dom.minidom import parse
from time import sleep

from invenio.config import CFG_ETCDIR, CFG_CROSSREF_USERNAME, \
 CFG_CROSSREF_PASSWORD, CFG_CROSSREF_EMAIL
from invenio.bibconvert_xslt_engine import convert
from invenio.bibrecord import record_get_field_value
from invenio.urlutils import make_invenio_opener
from invenio.jsonutils import json, json_unicode_to_utf8

CROSSREF_OPENER = make_invenio_opener('crossrefutils')

FIELDS_JOURNAL = 'issn,title,author,volume,issue,page,year,type,doi'.split(',')
FIELDS_BOOK = ('isbn,ser_title,vol_title,author,volume,edition_number,'
               + 'page,year,component_number,type,doi').split(',')

# Exceptions classes
class CrossrefError(Exception):
    """Crossref errors"""
    def __init__(self, code):
        """Initialisation"""
        self.code = code

    def __str__(self):
        """Returns error code"""
        return repr(self.code)

Esempio n. 2

File: bfe_issn.py Progetto: aw-bib/tind-invenio

# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""BibFormat element - Print ISSN corresponding to given journal name
"""

__revision__ = "$Id$"

import pprint
import sys
import re
import getopt
from invenio.search_engine import perform_request_search
from invenio.search_engine_utils import get_fieldvalues
from invenio.config import CFG_CERN_SITE
from invenio.urlutils import make_invenio_opener

BIBFORMAT_OPENER = make_invenio_opener('BibFormat')

if CFG_CERN_SITE:
    journal_name_tag = '773__p'
else:
    journal_name_tag = '909C4p'

issns = {   'aapps bull.': '0218-2203',
            'account. manag. inf. technol.': '0959-8022',
            'acm comput. surv.': '0360-0300',
            'acm sigplan not.': '0362-1340',
            'acm trans. comput. syst.': '0734-2071',
            'acm trans. comput.-hum. interact.': '1073-0516',
            'acm trans. database syst.': '0362-5915',
            'acm trans. graph.': '0730-0301',
            'acm trans. inf. syst. secur.': '1094-9224',

Esempio n. 3

File: bibcirculation_utils.py Progetto: bopopescu/invenio

                                CFG_BIBCIRCULATION_AMAZON_ACCESS_KEY, \
                                CFG_BIBCIRCULATION_WORKING_DAYS, \
                                CFG_BIBCIRCULATION_HOLIDAYS, \
                                CFG_CERN_SITE, \
                                CFG_BIBCIRCULATION_ITEM_STATUS_ON_LOAN, \
                                CFG_BIBCIRCULATION_ITEM_STATUS_ON_SHELF, \
                                CFG_BIBCIRCULATION_ITEM_STATUS_IN_PROCESS, \
                                CFG_BIBCIRCULATION_REQUEST_STATUS_PENDING, \
                                CFG_BIBCIRCULATION_REQUEST_STATUS_WAITING, \
                                CFG_BIBCIRCULATION_LOAN_STATUS_ON_LOAN, \
                                CFG_BIBCIRCULATION_LOAN_STATUS_EXPIRED, \
                                CFG_BIBCIRCULATION_LOAN_STATUS_RETURNED
from invenio.urlutils import create_html_link, make_invenio_opener
from invenio.messages import gettext_set_language

BIBCIRCULATION_OPENER = make_invenio_opener('BibCirculation')

DICC_REGEXP = re.compile("^\{('[^']*': ?('[^']*'|\"[^\"]+\"|[0-9]*|None)(, ?'[^']*': ?('[^']*'|\"[^\"]+\"|[0-9]*|None))*)?\}$")


def search_user(column, string):
    if string is not None:
        string = string.strip()

    if CFG_CERN_SITE == 1:
        if column == 'name':
            result = db.search_borrower_by_name(string)
        else:
            if column == 'email':
                try:
                    result = db.search_borrower_by_email(string)

Esempio n. 4

File: bibencode_tester.py Progetto: AlbertoPeon/invenio

from invenio import bibencode_encode
from invenio import bibencode_metadata
import invenio.config
from invenio.bibencode_encode import encode_video
from invenio.bibencode_extract import extract_frames
from invenio.textutils import wait_for_user
from invenio.urlutils import make_invenio_opener
from os.path import basename
import os
from urlparse import urlsplit
import shutil
import urllib2

from invenio.testutils import make_test_suite, run_test_suite

BIBENCODE_OPENER = make_invenio_opener('BibEncode')

## original URL
video_url = "http://media.xiph.org/video/derf/y4m/blue_sky_1080p25.y4m"
video01 = invenio.config.CFG_TMPDIR + "/blue_sky_1080p25.y4m"
video01_out01 = invenio.config.CFG_TMPDIR + "/blue_sky_1080p.mp4"
video01_out02 = invenio.config.CFG_TMPDIR + "/blue_sky_1080p.ogg"
video01_out03 = invenio.config.CFG_TMPDIR + "/blue_sky_1080p.webm"
video01_out04 = invenio.config.CFG_TMPDIR + "/blue_sky_720p.mp4"
video01_out05 = invenio.config.CFG_TMPDIR + "/blue_sky_720p.ogg"
video01_out06 = invenio.config.CFG_TMPDIR + "/blue_sky_720p.webm"
video01_out07 = invenio.config.CFG_TMPDIR + "/blue_sky_480p.mp4"
video01_out08 = invenio.config.CFG_TMPDIR + "/blue_sky_480p.ogg"
video01_out09 = invenio.config.CFG_TMPDIR + "/blue_sky_480p.webm"

movie_no_aspect = invenio.config.CFG_TMPDIR + "/blue_sky_1080p_anamorphic.webm"

Esempio n. 5

File: bibcirculation_utils.py Progetto: AlbertoPeon/invenio

import invenio.bibcirculation_dblayer as db
from invenio.bibcirculation_config import \
                                CFG_BIBCIRCULATION_WORKING_DAYS, \
                                CFG_BIBCIRCULATION_HOLIDAYS, \
                                CFG_CERN_SITE, \
                                CFG_BIBCIRCULATION_ITEM_STATUS_ON_LOAN, \
                                CFG_BIBCIRCULATION_ITEM_STATUS_ON_SHELF, \
                                CFG_BIBCIRCULATION_ITEM_STATUS_IN_PROCESS, \
                                CFG_BIBCIRCULATION_REQUEST_STATUS_PENDING, \
                                CFG_BIBCIRCULATION_REQUEST_STATUS_WAITING, \
                                CFG_BIBCIRCULATION_LOAN_STATUS_ON_LOAN, \
                                CFG_BIBCIRCULATION_LOAN_STATUS_EXPIRED, \
                                CFG_BIBCIRCULATION_LOAN_STATUS_RETURNED

DICC_REGEXP = re.compile("^\{('[^']*': ?('[^']*'|\"[^\"]+\"|[0-9]*|None)(, ?'[^']*': ?('[^']*'|\"[^\"]+\"|[0-9]*|None))*)?\}$")
BIBCIRCULATION_OPENER = make_invenio_opener('BibCirculation')

def search_user(column, string):
    if string is not None:
        string = string.strip()

    if CFG_CERN_SITE == 1:
        if column == 'name':
            result = db.search_borrower_by_name(string)
        else:
            if column == 'email':
                try:
                    result = db.search_borrower_by_email(string)
                except:
                    result = ()
            else:

Esempio n. 6

import urllib2, time, os, sys, re
from invenio.config import CFG_TMPDIR, \
                           CFG_PLOTEXTRACTOR_SOURCE_BASE_URL, \
                           CFG_PLOTEXTRACTOR_SOURCE_TARBALL_FOLDER, \
                           CFG_PLOTEXTRACTOR_SOURCE_PDF_FOLDER, \
                           CFG_PLOTEXTRACTOR_DOWNLOAD_TIMEOUT
from invenio.plotextractor_config import CFG_PLOTEXTRACTOR_DESY_BASE, \
                                         CFG_PLOTEXTRACTOR_DESY_PIECE
from invenio.search_engine import get_record
from invenio.bibrecord import record_get_field_instances, \
                              field_get_subfield_values
from invenio.shellutils import run_shell_command
from invenio.plotextractor_output_utils import write_message
from invenio.urlutils import make_invenio_opener

PLOTEXTRACTOR_OPENER = make_invenio_opener('plotextractor')

PDF_EXTENSION = '.pdf'

ARXIV_HEADER = 'arXiv:'
HEP_EX = ['hep-ex/', 9405, ARXIV_HEADER + 'hep-ex_']  # experimental
# a note about hep-ex: the hep-ex papers from 9403 nad 9404 are stored
# in arXiv's servers as hep-ph
HEP_LAT = ['hep-lat/', 9107, ARXIV_HEADER + 'hep-lat_']  # lattice
HEP_PH = ['hep-ph/', 9203, ARXIV_HEADER + 'hep-ph_']  # phenomenology
HEP_TH = ['hep-th/', 9108, ARXIV_HEADER + 'hep-th_']  # theory

HEP_AREAS = [HEP_EX, HEP_LAT, HEP_PH, HEP_TH]

URL = 0
BEGIN_YEAR_MONTH_INDEX = 1

Esempio n. 7

import urllib2
import urllib
import mimetools
from invenio import intbitset
from invenio.urlutils import make_invenio_opener
from invenio.jsonutils import json
from invenio.config import CFG_SOLR_URL, \
                           CFG_WEBSEARCH_FULLTEXT_SNIPPETS, \
                           CFG_WEBSEARCH_FULLTEXT_SNIPPETS_CHARS

if CFG_SOLR_URL:
    import solr
    SOLR_CONNECTION = solr.SolrConnection(CFG_SOLR_URL)  # pylint: disable=E1101

SOLRUTILS_OPENER = make_invenio_opener('solrutils')


def solr_get_facets(bitset, solr_url):
    facet_query_url = "%s/invenio_facets" % solr_url
    # now use the bitset to fetch the facet data
    r = urllib2.Request(facet_query_url)
    data = bitset.fastdump()
    boundary = mimetools.choose_boundary()

    # fool solr into thinking we're uploading a file so it will read our data as a stream
    contents = '--%s\r\n' % boundary
    contents += 'Content-Disposition: form-data; name="bitset"; filename="bitset"\r\n'
    contents += 'Content-Type: application/octet-stream\r\n'
    contents += '\r\n' + data + '\r\n'
    contents += '--%s--\r\n\r\n' % boundary

Esempio n. 8

File: solrutils_bibindex_searcher.py Progetto: chezjohnny/invenio

import urllib2
import urllib
import mimetools
from invenio import intbitset
from invenio.urlutils import make_invenio_opener
from invenio.jsonutils import json
from invenio.config import CFG_SOLR_URL


if CFG_SOLR_URL:
    import solr
    SOLR_CONNECTION = solr.SolrConnection(CFG_SOLR_URL) # pylint: disable=E1101


SOLRUTILS_OPENER = make_invenio_opener('solrutils')

def solr_get_facets(bitset, solr_url):
    facet_query_url = "%s/invenio_facets" % solr_url
    # now use the bitset to fetch the facet data
    r = urllib2.Request(facet_query_url)
    data = bitset.fastdump()
    boundary = mimetools.choose_boundary()

    # fool solr into thinking we're uploading a file so it will read our data as a stream
    contents = '--%s\r\n' % boundary
    contents += 'Content-Disposition: form-data; name="bitset"; filename="bitset"\r\n'
    contents += 'Content-Type: application/octet-stream\r\n'
    contents += '\r\n' + data + '\r\n'
    contents += '--%s--\r\n\r\n' % boundary
    r.add_data(contents)

Esempio n. 9

File: crossrefutils.py Progetto: tsgit/invenio

import sys
import urllib
import urllib2
import datetime
from xml.dom.minidom import parse
from time import sleep

from invenio.config import (CFG_ETCDIR, CFG_CROSSREF_USERNAME,
                            CFG_CROSSREF_PASSWORD, CFG_CROSSREF_EMAIL)
from invenio.bibconvert_xslt_engine import convert
from invenio.bibrecord import record_get_field_value
from invenio.urlutils import make_invenio_opener
from invenio.jsonutils import json, json_unicode_to_utf8

CROSSREF_OPENER = make_invenio_opener('crossrefutils')

FIELDS_JOURNAL = 'issn,title,author,volume,issue,page,year,type,doi'.split(',')
FIELDS_BOOK = ('isbn,ser_title,vol_title,author,volume,edition_number,' +
               'page,year,component_number,type,doi').split(',')


# Exceptions classes
class CrossrefError(Exception):
    """Crossref errors"""
    def __init__(self, code):
        """Initialisation"""
        self.code = code

    def __str__(self):
        """Returns error code"""

Esempio n. 10

File: filedownloadutils.py Progetto: aw-bib/tind-invenio

Raises InvenioFileDownloadError exception.
"""

import urllib2
import time
import os
import socket
import urllib
import tempfile
import shutil
import sys

from invenio.urlutils import make_invenio_opener

URL_OPENER = make_invenio_opener('filedownloadutils')

from invenio.config import (CFG_TMPSHAREDDIR,
                            CFG_BIBUPLOAD_FFT_ALLOWED_LOCAL_PATHS,
                            CFG_WEBSUBMIT_STORAGEDIR)

#: block size when performing I/O.
CFG_FILEUTILS_BLOCK_SIZE = 1024 * 8


class InvenioFileDownloadError(Exception):
    """A generic download exception."""
    def __init__(self, msg, code=None):
        Exception.__init__(self, msg)
        self.code = code

Esempio n. 11

from invenio import bibencode_encode
from invenio import bibencode_metadata
import invenio.config
from invenio.bibencode_encode import encode_video
from invenio.bibencode_extract import extract_frames
from invenio.textutils import wait_for_user
from invenio.urlutils import make_invenio_opener
from os.path import basename
import os
from urlparse import urlsplit
import shutil
import urllib2

from invenio.testutils import make_test_suite, run_test_suite

BIBENCODE_OPENER = make_invenio_opener('BibEncode')

# original URL
video_url = "http://media.xiph.org/video/derf/y4m/blue_sky_1080p25.y4m"
video01 = invenio.config.CFG_TMPDIR + "/blue_sky_1080p25.y4m"
video01_out01 = invenio.config.CFG_TMPDIR + "/blue_sky_1080p.mp4"
video01_out02 = invenio.config.CFG_TMPDIR + "/blue_sky_1080p.ogg"
video01_out03 = invenio.config.CFG_TMPDIR + "/blue_sky_1080p.webm"
video01_out04 = invenio.config.CFG_TMPDIR + "/blue_sky_720p.mp4"
video01_out05 = invenio.config.CFG_TMPDIR + "/blue_sky_720p.ogg"
video01_out06 = invenio.config.CFG_TMPDIR + "/blue_sky_720p.webm"
video01_out07 = invenio.config.CFG_TMPDIR + "/blue_sky_480p.mp4"
video01_out08 = invenio.config.CFG_TMPDIR + "/blue_sky_480p.ogg"
video01_out09 = invenio.config.CFG_TMPDIR + "/blue_sky_480p.webm"

movie_no_aspect = invenio.config.CFG_TMPDIR + "/blue_sky_1080p_anamorphic.webm"

Esempio n. 12

File: filedownloadutils.py Progetto: michamos/invenio

Raises InvenioFileDownloadError exception.
"""

import urllib2
import time
import os
import socket
import urllib
import tempfile
import shutil
import sys

from invenio.urlutils import make_invenio_opener

URL_OPENER = make_invenio_opener('filedownloadutils')

from invenio.config import (CFG_TMPSHAREDDIR,
                            CFG_BIBUPLOAD_FFT_ALLOWED_LOCAL_PATHS,
                            CFG_WEBSUBMIT_STORAGEDIR)

#: block size when performing I/O.
CFG_FILEUTILS_BLOCK_SIZE = 1024 * 8


class InvenioFileDownloadError(Exception):
    """A generic download exception."""
    def __init__(self, msg, code=None):
        Exception.__init__(self, msg)
        self.code = code

Esempio n. 13

File: bibclassify_ontology_reader.py Progetto: aw-bib/tind-invenio

    rdflib_exceptions_Error = None

from invenio import bibclassify_config as bconfig

log = bconfig.get_logger("bibclassify.ontology_reader")
from invenio import config

# only if not running in a stanalone mode
if bconfig.STANDALONE:
    dbquery = None
    from urllib2 import urlopen
else:
    from invenio import dbquery
    from invenio.urlutils import make_invenio_opener

    urlopen = make_invenio_opener("BibClassify").open

_contains_digit = re.compile("\d")
_starts_with_non = re.compile("(?i)^non[a-z]")
_starts_with_anti = re.compile("(?i)^anti[a-z]")
_split_by_punctuation = re.compile("(\W+)")


_CACHE = {}


def get_cache(taxonomy_id):
    """Returns thread-safe cache for the given taxonomy id,
    @var taxonomy_id: str, identifier of the taxonomy
    @return: dictionary object (empty if no taxonomy_id
        is found), you must not change anything inside it.

Esempio n. 14

File: plotextractor_getter.py Progetto: AlbertoPeon/invenio

import urllib2, time, os, sys, re
from invenio.config import CFG_TMPDIR, \
                           CFG_PLOTEXTRACTOR_SOURCE_BASE_URL, \
                           CFG_PLOTEXTRACTOR_SOURCE_TARBALL_FOLDER, \
                           CFG_PLOTEXTRACTOR_SOURCE_PDF_FOLDER, \
                           CFG_PLOTEXTRACTOR_DOWNLOAD_TIMEOUT
from invenio.plotextractor_config import CFG_PLOTEXTRACTOR_DESY_BASE, \
                                         CFG_PLOTEXTRACTOR_DESY_PIECE
from invenio.search_engine import get_record
from invenio.bibrecord import record_get_field_instances, \
                              field_get_subfield_values
from invenio.shellutils import run_shell_command
from invenio.plotextractor_output_utils import write_message
from invenio.urlutils import make_invenio_opener

PLOTEXTRACTOR_OPENER = make_invenio_opener('plotextractor')

PDF_EXTENSION = '.pdf'

ARXIV_HEADER = 'arXiv:'
HEP_EX = ['hep-ex/', 9405, ARXIV_HEADER + 'hep-ex_'] # experimental
# a note about hep-ex: the hep-ex papers from 9403 nad 9404 are stored
# in arXiv's servers as hep-ph
HEP_LAT = ['hep-lat/', 9107, ARXIV_HEADER + 'hep-lat_'] # lattice
HEP_PH = ['hep-ph/', 9203, ARXIV_HEADER + 'hep-ph_'] # phenomenology
HEP_TH = ['hep-th/', 9108, ARXIV_HEADER + 'hep-th_'] # theory

HEP_AREAS = [HEP_EX, HEP_LAT, HEP_PH, HEP_TH]

URL = 0
BEGIN_YEAR_MONTH_INDEX = 1

Esempio n. 15

File: bibrankgkb.py Progetto: aw-bib/tind-invenio

 -v,  --verbose=LEVEL       verbose level (from 0 to 9, default 1)
"""

__revision__ = "$Id$"

import getopt
import sys
import time
import re
import ConfigParser

from invenio.urlutils import make_invenio_opener
from invenio.config import CFG_ETCDIR
from invenio.dbquery import run_sql

BIBRANK_OPENER = make_invenio_opener('BibRank')

opts_dict = {}
task_id = -1

def bibrankgkb(config):
    """Generates a .kb file based on input from the configuration file"""

    if opts_dict["verbose"] >= 1:
        write_message("Running: Generate Knowledgebase.")
    journals = {}
    journal_src = {}
    i = 0

    #Reading the configuration file
    while config.has_option("bibrankgkb","create_%s" % i):

Esempio n. 16

This module is STANDALONE safe
"""

import os
import re
import sys
import tempfile
import urllib2
from invenio import bibclassify_config as bconfig

if bconfig.STANDALONE:
    from urllib2 import urlopen
else:
    from invenio.urlutils import make_invenio_opener
    urlopen = make_invenio_opener('BibClassify').open

log = bconfig.get_logger("bibclassify.text_extractor")

_ONE_WORD = re.compile("[A-Za-z]{2,}")


def text_lines_from_local_file(document, remote=False):
    """Returns the fulltext of the local file.
    @var document: fullpath to the file that should be read
    @var remote: boolean, if True does not count lines (gosh!)
    @return: list of lines if st was read or an empty list"""

    # FIXME - this does not care if we open anything, including binary files

    try:

Esempio n. 17

File: bibclassify_text_extractor.py Progetto: AlbertoPeon/invenio

This module is STANDALONE safe
"""

import os
import re
import sys
import tempfile
import urllib2
from invenio import bibclassify_config as bconfig

if bconfig.STANDALONE:
    from urllib2 import urlopen
else:
    from invenio.urlutils import make_invenio_opener
    urlopen = make_invenio_opener('BibClassify').open

log = bconfig.get_logger("bibclassify.text_extractor")


_ONE_WORD = re.compile("[A-Za-z]{2,}")

def text_lines_from_local_file(document, remote=False):
    """Returns the fulltext of the local file.
    @var document: fullpath to the file that should be read
    @var remote: boolean, if True does not count lines (gosh!)
    @return: list of lines if st was read or an empty list"""

    # FIXME - this does not care if we open anything, including binary files

    try:

Esempio n. 18

 -v,  --verbose=LEVEL       verbose level (from 0 to 9, default 1)
"""

__revision__ = "$Id$"

import getopt
import sys
import time
import re
import ConfigParser

from invenio.urlutils import make_invenio_opener
from invenio.config import CFG_ETCDIR
from invenio.dbquery import run_sql

BIBRANK_OPENER = make_invenio_opener('BibRank')

opts_dict = {}
task_id = -1


def bibrankgkb(config):
    """Generates a .kb file based on input from the configuration file"""

    if opts_dict["verbose"] >= 1:
        write_message("Running: Generate Knowledgebase.")
    journals = {}
    journal_src = {}
    i = 0

    #Reading the configuration file