import csv
import datetime
import json
import logging
import os
import sys
import urllib
import urllib2
import urlparse
from biryani1 import baseconv, custom_conv, states, strings
from ckantoolbox import ckanconv
app_name = os.path.splitext(os.path.basename(__file__))[0]
conv = custom_conv(baseconv, ckanconv, states)
log = logging.getLogger(app_name)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('config', help='path of configuration file')
parser.add_argument(
'csv_file_path',
help='path of CSV file containing the groups to use by organization')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='increase output verbosity')
global args
import itertools
import json
import logging
import os
import sys
import urllib2
import urlparse
from biryani1 import baseconv, custom_conv, datetimeconv, states, strings
from ckantoolbox import ckanconv
from . import helpers
app_name = os.path.splitext(os.path.basename(__file__))[0]
conv = custom_conv(baseconv, ckanconv, datetimeconv, states)
format_by_mime_type = {
u'application/api': u'api',
u'application/visualisation': u'viz',
u'application/vnd.ms-excel': u'xls',
u'application/x-gzip': u'gzip',
u'application/xhtml+xml': u'html',
u'application/xml': u'xml',
u'text/tsv': u'tsv',
}
groups_title_translations = {
u'Agriculture & Pêche': u'Économie & Travail',
u'Arts & Culture': u'Culture',
u'Autres': None,
u'Banque & Assurance': u'Économie & Travail',
u'Climat & Météorologie': u'Habitat & Écologie',
import logging
import os
import shutil
import sys
import thread
import time
import urllib2
import urlparse
from biryani1 import baseconv, custom_conv, datetimeconv
from lxml import etree
app_name = os.path.splitext(os.path.basename(__file__))[0]
args = None
conv = custom_conv(baseconv, datetimeconv)
existing_files_path = set()
html_parser = etree.HTMLParser()
log = logging.getLogger(app_name)
processing_html = False
pool = set()
remaining_html_pages = set()
remaining_links = None
rejected_urls = set()
visited_data_indexes = set()
visited_search_indexes = set()
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('download_dir', help = 'directory where to store downloaded HTML pages')
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import collections
import json
import logging
import urllib2
import urlparse
from biryani1 import baseconv, custom_conv, jsonconv, states
conv = custom_conv(baseconv, jsonconv, states)
headers = {
'User-Agent': 'OpenFisca-France-Tests/0.2dev (https://www.github.com/openfisca/openfisca-france [email protected])',
}
log = logging.getLogger(__name__)
web_api_url = 'http://localhost:2014/'
api_response_to_value = conv.pipe(
conv.make_input_to_json(object_pairs_hook = collections.OrderedDict),
conv.not_none,
conv.test_isinstance(dict),
conv.struct(
dict(
apiVersion = conv.pipe(
conv.test_equals('1.0'),
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import collections
import json
import logging
import urllib2
import urlparse
from biryani1 import baseconv, custom_conv, jsonconv, states
conv = custom_conv(baseconv, jsonconv, states)
headers = {
'User-Agent':
'OpenFisca-France-Tests/0.2dev (https://www.github.com/openfisca/openfisca-france [email protected])',
}
log = logging.getLogger(__name__)
web_api_url = 'http://localhost:2014/'
api_response_to_value = conv.pipe(
conv.make_input_to_json(object_pairs_hook=collections.OrderedDict),
conv.not_none,
conv.test_isinstance(dict),
conv.struct(
dict(
apiVersion=conv.pipe(
conv.test_equals('1.0'),