import csv import datetime import json import logging import os import sys import urllib import urllib2 import urlparse from biryani1 import baseconv, custom_conv, states, strings from ckantoolbox import ckanconv app_name = os.path.splitext(os.path.basename(__file__))[0] conv = custom_conv(baseconv, ckanconv, states) log = logging.getLogger(app_name) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('config', help='path of configuration file') parser.add_argument( 'csv_file_path', help='path of CSV file containing the groups to use by organization') parser.add_argument('-v', '--verbose', action='store_true', help='increase output verbosity') global args
import itertools import json import logging import os import sys import urllib2 import urlparse from biryani1 import baseconv, custom_conv, datetimeconv, states, strings from ckantoolbox import ckanconv from . import helpers app_name = os.path.splitext(os.path.basename(__file__))[0] conv = custom_conv(baseconv, ckanconv, datetimeconv, states) format_by_mime_type = { u'application/api': u'api', u'application/visualisation': u'viz', u'application/vnd.ms-excel': u'xls', u'application/x-gzip': u'gzip', u'application/xhtml+xml': u'html', u'application/xml': u'xml', u'text/tsv': u'tsv', } groups_title_translations = { u'Agriculture & Pêche': u'Économie & Travail', u'Arts & Culture': u'Culture', u'Autres': None, u'Banque & Assurance': u'Économie & Travail', u'Climat & Météorologie': u'Habitat & Écologie',
import logging import os import shutil import sys import thread import time import urllib2 import urlparse from biryani1 import baseconv, custom_conv, datetimeconv from lxml import etree app_name = os.path.splitext(os.path.basename(__file__))[0] args = None conv = custom_conv(baseconv, datetimeconv) existing_files_path = set() html_parser = etree.HTMLParser() log = logging.getLogger(app_name) processing_html = False pool = set() remaining_html_pages = set() remaining_links = None rejected_urls = set() visited_data_indexes = set() visited_search_indexes = set() def main(): parser = argparse.ArgumentParser(description = __doc__) parser.add_argument('download_dir', help = 'directory where to store downloaded HTML pages')
# GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. import collections import json import logging import urllib2 import urlparse from biryani1 import baseconv, custom_conv, jsonconv, states conv = custom_conv(baseconv, jsonconv, states) headers = { 'User-Agent': 'OpenFisca-France-Tests/0.2dev (https://www.github.com/openfisca/openfisca-france [email protected])', } log = logging.getLogger(__name__) web_api_url = 'http://localhost:2014/' api_response_to_value = conv.pipe( conv.make_input_to_json(object_pairs_hook = collections.OrderedDict), conv.not_none, conv.test_isinstance(dict), conv.struct( dict( apiVersion = conv.pipe( conv.test_equals('1.0'),
# but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. import collections import json import logging import urllib2 import urlparse from biryani1 import baseconv, custom_conv, jsonconv, states conv = custom_conv(baseconv, jsonconv, states) headers = { 'User-Agent': 'OpenFisca-France-Tests/0.2dev (https://www.github.com/openfisca/openfisca-france [email protected])', } log = logging.getLogger(__name__) web_api_url = 'http://localhost:2014/' api_response_to_value = conv.pipe( conv.make_input_to_json(object_pairs_hook=collections.OrderedDict), conv.not_none, conv.test_isinstance(dict), conv.struct( dict( apiVersion=conv.pipe( conv.test_equals('1.0'),