def all_plugins():
"""Return a dict of plugin name -> Plugin for all plugins, including core.
Plugins are registered via the ``dxr.plugins`` setuptools entry point,
which may point to either a module (in which case a Plugin will be
constructed based on the contents of the module namespace) or a Plugin
object (which will be returned directly). The entry point name is what the
user types into the config file under ``enabled_plugins``.
The core plugin, which provides many of DXR's cross-language, built-in
features, is always the first plugin when iterating over the returned
dict. This lets other plugins override bits of its elasticsearch mappings
and analyzers when we're building up the schema.
"""
global _plugin_cache
def name_and_plugin(entry_point):
"""Return the name of an entry point and the Plugin it points to."""
object = entry_point.load()
plugin = (object if isinstance(object, Plugin) else
Plugin.from_namespace(object.__dict__))
plugin.name = entry_point.name
return entry_point.name, plugin
if _plugin_cache is None:
# Iterating over entrypoints could be kind of expensive, with the FS
# reads and all.
_plugin_cache = OrderedDict([('core', core_plugin())])
_plugin_cache.update(
name_and_plugin(point)
for point in iter_entry_points('dxr.plugins'))
return _plugin_cache
def read_all(self):
"""read all available sheets"""
result = OrderedDict()
for sheet in self.native_book.sheets:
data_dict = self.read_sheet(sheet)
result.update(data_dict)
return result
def read_all(self):
result = OrderedDict()
self.native_book = self._get_book()
for sheet in self.native_book.sheets():
data_dict = self.read_sheet(sheet)
result.update(data_dict)
return result
def gen_mapping(infile):
"""Generate a mapping file based on infile csv
"""
fh = open(infile)
cont = fh.read()
fh.close()
row = cont.split(LINE_END)[0]
from ordereddict import OrderedDict
out = OrderedDict()
for x in row.split(","):
#y = space_word(x)
out.update({ x : x})
ret = """# coding: utf-8
'''
mapping from Google Contact standard field name to
our customized fields name.
Edit it to make it fit your fields.
About Mapping:
-----------------
MAPPING = {
"Goolge_Field" : "Our Field",
...
}
So we need to check and edit the left side of the dict(i.e Key)
to make the right side(i.e Value) match the left as possible.
'''
"""
from django.utils import simplejson
ret += "MAPPING = " + simplejson.dumps(out, indent=4)
outfile = open(OUT_FILE_NAME, "w")
outfile.write(ret)
def read_all(self):
result = OrderedDict()
self.native_book = self._get_book()
for sheet in self.native_book.sheets():
data_dict = self.read_sheet(sheet)
result.update(data_dict)
return result
def test_update(self):
self.assertRaises(TypeError,
OrderedDict().update, [('a', 1), ('b', 2)],
None) # too many args
pairs = [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5)]
od = OrderedDict()
od.update(dict(pairs))
self.assertEqual(sorted(od.items()), pairs) # dict input
od = OrderedDict()
od.update(**dict(pairs))
self.assertEqual(sorted(od.items()), pairs) # kwds input
od = OrderedDict()
od.update(pairs)
self.assertEqual(list(od.items()), pairs) # pairs input
od = OrderedDict()
od.update([('a', 1), ('b', 2), ('c', 9), ('d', 4)], c=3, e=5)
self.assertEqual(list(od.items()), pairs) # mixed input
# Make sure that direct calls to update do not clear previous contents
# add that updates items are not moved to the end
d = OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 44), ('e', 55)])
d.update([('e', 5), ('f', 6)], g=7, d=4)
self.assertEqual(list(d.items()), [('a', 1), ('b', 2), ('c', 3),
('d', 4), ('e', 5), ('f', 6),
('g', 7)])
def begetDataBody(self, *pa, **kwa):
""" Returns duple (data,body) for request from arguments
positional arguments may be either strings, sequences of key,value
duples, or dictionaries.
Body is string concated from all string positional arguments.
Non string positional arguments and keyword arguments are ignored.
Data is an ordereddict where the items are extracted from
self.preloads, any sequenced duples and/or dicts in the positonal
arguments, and from the keyword arguments.
String positional arguments are ignored.
"""
body = ''
data = ODict(self.preloads) #load defaults
for a in pa: #extract key/value pairs from positional parameters
if isinstance(a, basestring): # string
body = '%s%s' % (body, a) #concat
else: # dict or sequence of duples
data.update(a)
data.update(kwa) #so insert keyword arguments
if isinstance(body, unicode):
body = body.encode('utf-8')
if 'utf-8' not in self.headers['content-type']:
self.headers['content-type'] += '; charset=utf-8'
return (data, body)
def read_all(self):
"""read all available sheets"""
result = OrderedDict()
for sheet in self.native_book.sheets:
data_dict = self.read_sheet(sheet)
result.update(data_dict)
return result
def _get_csv_data(user_profile):
data = OrderedDict()
data['application_status_ID'] = user_profile.applicationstatus.pk
data['profile'] = _get_profile_data(user_profile)
data.update(_get_csv_form_data(user_profile))
# data ['optional_subjects'] = _get_other_subjects_data( user_profile )
return data
def _produce_ordered_dict():
data_dict = OrderedDict()
data_dict.update({"Sheet1": [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]]})
data_dict.update({"Sheet2": [[4, 4, 4, 4], [5, 5, 5, 5], [6, 6, 6, 6]]})
data_dict.update(
{"Sheet3": [[u"X", u"Y", u"Z"], [1, 4, 7], [2, 5, 8], [3, 6, 9]]})
return data_dict
def all_plugins():
"""Return a dict of plugin name -> Plugin for all plugins, including core.
Plugins are registered via the ``dxr.plugins`` setuptools entry point,
which may point to either a module (in which case a Plugin will be
constructed based on the contents of the module namespace) or a Plugin
object (which will be returned directly). The entry point name is what the
user types into the config file under ``enabled_plugins``.
The core plugin, which provides many of DXR's cross-language, built-in
features, is always the first plugin when iterating over the returned
dict. This lets other plugins override bits of its elasticsearch mappings
and analyzers when we're building up the schema.
"""
global _plugin_cache
def name_and_plugin(entry_point):
"""Return the name of an entry point and the Plugin it points to."""
object = entry_point.load()
plugin = (object if isinstance(object, Plugin) else
Plugin.from_namespace(object.__dict__))
plugin.name = entry_point.name
return entry_point.name, plugin
if _plugin_cache is None:
# Iterating over entrypoints could be kind of expensive, with the FS
# reads and all.
_plugin_cache = OrderedDict([('core', core_plugin())])
_plugin_cache.update(name_and_plugin(point) for point in
iter_entry_points('dxr.plugins'))
return _plugin_cache
def _parse(self, menu):
di = OrderedDict()
for value in menu:
id = value.get('id', value.get('title'))
extra = value.get('extra', None)
separator = False
if extra:
separator = extra.get('separator', None)
title = value.get('title')
if isinstance(title, str):
translated_title = translate(_p(title), context=self.request)
else:
translated_title = translate(value.get('title'), domain=_p, context=self.request)
su = OrderedDict(label=translated_title,
action=self._action(value.get('action', '')),
icon=value.get('icon', ''),
_class=value.get('class', ''),
separator_before=separator,
_disabled=(not value.get('action', False) and separator) and True or False,
)
if value.get('submenu', None):
su.update(OrderedDict(submenu=self._parse(value.get('submenu'))))
di[id] = su
return di
def find_crashed_impalads(self, start_time):
"""If any impalads are found not running, they will assumed to have crashed. A crash
info message will be return for each stopped impalad. The return value is a dict
keyed by impalad. See Impalad.find_last_crash_message() for info about the returned
messages. 'start_time' is used to filter log messages and core dumps, it should
be set to the time when the Impala service was started. Impalads that have
non-generic crash info will be sorted last in the returned dict.
"""
stopped_impalads = self.find_stopped_impalads()
if not stopped_impalads:
return dict.fromkeys(stopped_impalads)
messages = OrderedDict()
impalads_with_message = dict()
for i, message in izip(
stopped_impalads,
self.for_each_impalad(
lambda i: i.find_last_crash_message(start_time),
impalads=stopped_impalads)):
if message:
impalads_with_message[i] = "%s crashed:\n%s" % (i.host_name,
message)
else:
messages[
i] = "%s crashed but no info could be found" % i.host_name
messages.update(impalads_with_message)
return messages
def flatten_dict(self, d):
flat_dict = OrderedDict()
for key, item in d.iteritems():
key = str(key)
flat_item = self.flatten_item(item)
nested_item = self.nest_flat_item(flat_item, key)
flat_dict.update(nested_item)
return flat_dict
def flatten_dict(self, d):
flat_dict = OrderedDict()
for key, item in d.iteritems():
key = str(key)
flat_item = self.flatten_item(item)
nested_item = self.nest_flat_item(flat_item, key)
flat_dict.update(nested_item)
return flat_dict
def read_all(self):
result = OrderedDict()
for sheet in self.native_book:
if self.skip_hidden_sheets and sheet.sheet_state == 'hidden':
continue
data_dict = self.read_sheet(sheet)
result.update(data_dict)
return result
def _produce_ordered_dict():
data_dict = OrderedDict()
data_dict.update({"Sheet1": [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]]})
data_dict.update({"Sheet2": [[4, 4, 4, 4], [5, 5, 5, 5], [6, 6, 6, 6]]})
data_dict.update(
{"Sheet3": [[u"X", u"Y", u"Z"], [1, 4, 7], [2, 5, 8], [3, 6, 9]]}
)
return data_dict
def read_all(self):
result = OrderedDict()
self.native_book = self._get_book()
for sheet in self.native_book.sheets():
if self.skip_hidden_sheets and sheet.visibility != 0:
continue
data_dict = self.read_sheet(sheet)
result.update(data_dict)
return result
def read_all(self):
result = OrderedDict()
self._native_book = self._get_book()
for sheet in self._native_book.sheets():
if self.skip_hidden_sheets and sheet.visibility != 0:
continue
data_dict = self.read_sheet(sheet)
result.update(data_dict)
return result
class TestXlsNXlsmMultipleSheets(PyexcelMultipleSheetBase):
def setUp(self):
self.testfile = "multiple1.xls"
self.testfile2 = "multiple1.xlsm"
self.content = OrderedDict()
self.content.update({"Sheet1": [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]]})
self.content.update({"Sheet2": [[4, 4, 4, 4], [5, 5, 5, 5], [6, 6, 6, 6]]})
self.content.update({"Sheet3": [[u'X', u'Y', u'Z'], [1, 4, 7], [2, 5, 8], [3, 6, 9]]})
self._write_test_file(self.testfile)
def tearDown(self):
self._clean_up()
class TestXlsNXlsxMultipleSheets(PyexcelMultipleSheetBase):
def setUp(self):
self.testfile = "multiple1.xlsm"
self.testfile2 = "multiple1.xlsx"
self.content = OrderedDict()
self.content.update({"Sheet1": [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]]})
self.content.update({"Sheet2": [[4, 4, 4, 4], [5, 5, 5, 5], [6, 6, 6, 6]]})
self.content.update({"Sheet3": [[u'X', u'Y', u'Z'], [1, 4, 7], [2, 5, 8], [3, 6, 9]]})
self._write_test_file(self.testfile)
def tearDown(self):
self._clean_up()
def _parse_(self,rootgrp):
## get global metadata
dataset = OrderedDict()
for attr in rootgrp.ncattrs():
dataset.update({attr:getattr(rootgrp,attr)})
self.update({'dataset':dataset})
## get variables
variables = OrderedDict()
for key,value in rootgrp.variables.iteritems():
subvar = OrderedDict()
for attr in value.ncattrs():
if attr.startswith('_'): continue
subvar.update({attr:getattr(value,attr)})
variables.update({key:{'dimensions':value.dimensions,
'attrs':subvar,
'dtype':str(value.dtype)}})
self.update({'variables':variables})
## get dimensions
dimensions = OrderedDict()
for key,value in rootgrp.dimensions.iteritems():
subdim = {key:{'len':len(value),'isunlimited':value.isunlimited()}}
dimensions.update(subdim)
self.update({'dimensions':dimensions})
def _parse_(self, rootgrp):
## get global metadata
dataset = OrderedDict()
for attr in rootgrp.ncattrs():
dataset.update({attr: getattr(rootgrp, attr)})
self.update({'dataset': dataset})
## get variables
variables = OrderedDict()
for key, value in rootgrp.variables.iteritems():
subvar = OrderedDict()
for attr in value.ncattrs():
if attr.startswith('_'): continue
subvar.update({attr: getattr(value, attr)})
variables.update({
key: {
'dimensions': value.dimensions,
'attrs': subvar,
'dtype': str(value.dtype)
}
})
self.update({'variables': variables})
## get dimensions
dimensions = OrderedDict()
for key, value in rootgrp.dimensions.iteritems():
subdim = {
key: {
'len': len(value),
'isunlimited': value.isunlimited()
}
}
dimensions.update(subdim)
self.update({'dimensions': dimensions})
def getAttributes(self, section, stripNamespaces=False):
"""
Returns given section attributes.
Usage::
>>> content = ["[Section A]\\n", "; Comment.\\n", "Attribute 1 = \\"Value A\\"\\n", "\\n", \
"[Section B]\\n", "Attribute 2 = \\"Value B\\"\\n"]
>>> sectionsFileParser = SectionsFileParser()
>>> sectionsFileParser.content = content
>>> sectionsFileParser.parse()
<foundations.parsers.SectionsFileParser object at 0x125698322>
>>> sectionsFileParser.getAttributes("Section A")
OrderedDict([(u'Section A|Attribute 1', u'Value A')])
>>> sectionsFileParser.preserveOrder=False
>>> sectionsFileParser.getAttributes("Section A")
{u'Section A|Attribute 1': u'Value A'}
>>> sectionsFileParser.preserveOrder=True
>>> sectionsFileParser.getAttributes("Section A", stripNamespaces=True)
OrderedDict([(u'Attribute 1', u'Value A')])
:param section: Section containing the requested attributes.
:type section: unicode
:param stripNamespaces: Strip namespaces while retrieving attributes.
:type stripNamespaces: bool
:return: Attributes.
:rtype: OrderedDict or dict
"""
LOGGER.debug("> Getting section '{0}' attributes.".format(section))
attributes = OrderedDict() if self.__preserveOrder else dict()
if not self.sectionExists(section):
return attributes
if stripNamespaces:
for attribute, value in self.__sections[section].iteritems():
attributes[foundations.namespace.removeNamespace(attribute, rootOnly=True)] = value
else:
attributes.update(self.__sections[section])
LOGGER.debug("> Attributes: '{0}'.".format(attributes))
return attributes
def find_crashed_impalads(self, start_time):
"""If any impalads are found not running, they will assumed to have crashed. A crash
info message will be return for each stopped impalad. The return value is a dict
keyed by impalad. See Impalad.find_last_crash_message() for info about the returned
messages. 'start_time' is used to filter log messages and core dumps, it should
be set to the time when the Impala service was started. Impalads that have
non-generic crash info will be sorted last in the returned dict.
"""
stopped_impalads = self.find_stopped_impalads()
if not stopped_impalads:
return dict.fromkeys(stopped_impalads)
messages = OrderedDict()
impalads_with_message = dict()
for i, message in izip(stopped_impalads, self.for_each_impalad(
lambda i: i.find_last_crash_message(start_time), impalads=stopped_impalads)):
if message:
impalads_with_message[i] = "%s crashed:\n%s" % (i.host_name, message)
else:
messages[i] = "%s crashed but no info could be found" % i.host_name
messages.update(impalads_with_message)
return messages
def test_to_dict(self):
"""
Note: data file with column headers are tested
in test_filters.py
"""
r = pe.Reader(self.testfile)
result = OrderedDict()
result.update({"Series_1": [1, 2, 3, 4]})
result.update({"Series_2": [5, 6, 7, 8, ]})
result.update({"Series_3": [9, 10, 11, 12]})
actual = pe.utils.to_dict(r.rows())
assert actual.keys() == result.keys()
assert result == actual
result = {
"Series_1": 1,
"Series_2": 2,
"Series_3": 3,
"Series_4": 4,
"Series_5": 5,
"Series_6": 6,
"Series_7": 7,
"Series_8": 8,
"Series_9": 9,
"Series_10": 10,
"Series_11": 11,
"Series_12": 12
}
actual = pe.utils.to_dict(r.enumerate())
assert result == actual
def package_merge(packagelist):
''' merge package strings together, and sort out duplicates'''
try:
from ordereddict import OrderedDict
except ImportError:
from deployment.ordereddict import OrderedDict
x= "".join(packagelist)
y= OrderedDict()
line = 0
for a in x.splitlines():
if a.strip():
if a[0] == "#":
line+=1
y.update({a: line})
else:
if a not in y:
line+=1
y.update({a:line})
else:
pass # warn("removing duplicate: "+ str(a))
else:
line+=1
y.update({"": line})
z = "\n".join(y)
return z
class TestSingleSheetReaderForMulitpleSheetBook:
def setUp(self):
self.testfile = "multiple1.xls"
self.content = OrderedDict()
self.content.update({"Sheet1": [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]]})
self.content.update({"Sheet2": [[4, 4, 4, 4], [5, 5, 5, 5], [6, 6, 6, 6]]})
self.content.update({"Sheet3": [[u'X', u'Y', u'Z'], [1, 4, 7], [2, 5, 8], [3, 6, 9]]})
w = pe.BookWriter(self.testfile)
w.write_book_from_dict(self.content)
w.close()
def test_non_default_sheet_as_single_sheet_reader(self):
r = pe.Reader(self.testfile, "Sheet1")
data = pe.utils.to_array(r)
assert data == self.content["Sheet1"]
r2 = pe.Reader(self.testfile, "Sheet2")
data = pe.utils.to_array(r2)
assert data == self.content["Sheet2"]
r3 = pe.Reader(self.testfile, "Sheet3")
data = pe.utils.to_array(r3)
assert data == self.content["Sheet3"]
def test_non_default_sheet_as_single_sheet_reader_series(self):
r = pe.SeriesReader(self.testfile, "Sheet3")
data = pe.utils.to_array(r.rows())
assert data == self.content["Sheet3"][1:]
def test_non_default_sheet_as_single_sheet_plain_reader(self):
r = pe.PlainReader(self.testfile, "Sheet2")
data = pe.utils.to_array(r.rows())
assert data == self.content["Sheet2"]
def test_non_default_sheet_as_single_sheet_filterable_reader(self):
r = pe.FilterableReader(self.testfile, "Sheet2")
data = pe.utils.to_array(r.rows())
assert data == self.content["Sheet2"]
def tearDown(self):
if os.path.exists(self.testfile):
os.unlink(self.testfile)
def format(self, record):
# Get message and time
record.message = record.getMessage()
record.asctime = self.formatTime(record, self.datefmt)
record.utctime = "{0}.{1:03.0f}Z".format(record.asctime, record.msecs)
# Serialize record keys to json (just the configured ones and extra)
record_dict = OrderedDict([(key, getattr(record, log_key, None))
for key, log_key in six.iteritems(self._keys_fmt)])
if hasattr(record, 'additional') and record.additional:
record_dict.update(record.additional.items())
# Add exception info (if available)
if record.exc_info:
record.exc_text = self.formatException(record.exc_info)
record_dict.update({'exc_text': record.exc_text})
# Remove blanks items
if self.remove_blanks:
empty_keys = [k for k, v in six.iteritems(record_dict) if not v]
for key in empty_keys:
del record_dict[key]
return self.encode(record_dict)
def test_update(self):
self.assertRaises(TypeError, OrderedDict().update, [('a', 1), ('b',
2)], None) # too many args
pairs = [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5)]
od = OrderedDict()
od.update(dict(pairs))
self.assertEqual(sorted(od.items()), pairs) # dict input
od = OrderedDict()
od.update(**dict(pairs))
self.assertEqual(sorted(od.items()), pairs) # kwds input
od = OrderedDict()
od.update(pairs)
self.assertEqual(list(od.items()), pairs) # pairs input
od = OrderedDict()
od.update([('a', 1), ('b', 2), ('c', 9), ('d', 4)], c=3, e=5)
self.assertEqual(list(od.items()), pairs) # mixed input
# Make sure that direct calls to update do not clear previous contents
# add that updates items are not moved to the end
d = OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 44), ('e', 55)])
d.update([('e', 5), ('f', 6)], g=7, d=4)
self.assertEqual(list(d.items()),
[('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5), ('f', 6), ('g', 7)])
def test_copying(self):
# Check that ordered dicts are copyable, deepcopyable, picklable,
# and have a repr/eval round-trip
pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
od = OrderedDict(pairs)
update_test = OrderedDict()
update_test.update(od)
for i, dup in enumerate([
od.copy(),
copy.copy(od),
copy.deepcopy(od),
pickle.loads(pickle.dumps(od, 0)),
pickle.loads(pickle.dumps(od, 1)),
pickle.loads(pickle.dumps(od, 2)),
pickle.loads(pickle.dumps(od, -1)),
eval(repr(od)),
update_test,
OrderedDict(od),
]):
self.assert_(dup is not od)
self.assertEquals(dup, od)
self.assertEquals(list(dup.items()), list(od.items()))
self.assertEquals(len(dup), len(od))
self.assertEquals(type(dup), type(od))
def test_copying(self):
# Check that ordered dicts are copyable, deepcopyable, picklable,
# and have a repr/eval round-trip
pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
od = OrderedDict(pairs)
update_test = OrderedDict()
update_test.update(od)
for i, dup in enumerate([
od.copy(),
copy.copy(od),
copy.deepcopy(od),
pickle.loads(pickle.dumps(od, 0)),
pickle.loads(pickle.dumps(od, 1)),
pickle.loads(pickle.dumps(od, 2)),
pickle.loads(pickle.dumps(od, -1)),
eval(repr(od)),
update_test,
OrderedDict(od),
]):
self.assert_(dup is not od)
self.assertEquals(dup, od)
self.assertEquals(list(dup.items()), list(od.items()))
self.assertEquals(len(dup), len(od))
self.assertEquals(type(dup), type(od))
def to_dict(o):
"""convert a reader iterator to a dictionary"""
the_dict = OrderedDict()
series = "Series_%d"
count = 1
for c in o:
if type(c) == dict:
the_dict.update(c)
elif isinstance(c, Sheet):
the_dict.update({c.name: to_array(c)})
else:
key = series % count
the_dict.update({key: c})
count += 1
return the_dict
class ExcelResponseTestCase(TestCase):
def setUp(self):
self.client = Client()
self.data = [[1, 2, 3], [4, 5, 6]]
self.single_sheet = [['X', 'Y', 'Z'], [1, 2, 3], [4, 5, 6]]
self.book_content = OrderedDict()
self.book_content.update(
{"Sheet1": [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]]})
self.book_content.update(
{"Sheet2": [[4, 4, 4, 4], [5, 5, 5, 5], [6, 6, 6, 6]]})
self.book_content.update(
{"Sheet3": [[u'X', u'Y', u'Z'], [1, 4, 7], [2, 5, 8], [3, 6, 9]]})
def test_download(self):
for file_type in FILE_TYPE_MIME_TABLE.keys():
print(file_type)
response = self.client.get("/polls/download/" + file_type)
assert response['Content-Type'] == FILE_TYPE_MIME_TABLE[file_type]
sheet = pe.get_sheet(file_type=file_type,
file_content=response.content)
sheet.format(int)
array = sheet.to_array()
assert array == self.data
def test_download_attachment(self):
test_file_name = "test"
for file_type in FILE_TYPE_MIME_TABLE.keys():
print(file_type)
response = self.client.get("/polls/download_attachment/" +
file_type + "/" + test_file_name)
assert response['Content-Type'] == FILE_TYPE_MIME_TABLE[file_type]
assert response[
'Content-Disposition'] == "attachment; filename=%s.%s" % (
test_file_name, file_type)
sheet = pe.get_sheet(file_type=file_type,
file_content=response.content)
sheet.format(int)
array = sheet.to_array()
assert array == self.data
def test_parse_single_sheet(self):
test_sample = {
"array": {
u'result': [[u'X', u'Y', u'Z'], [1.0, 2.0, 3.0],
[4.0, 5.0, 6.0]]
},
"dict": {
u'Y': [2.0, 5.0],
u'X': [1.0, 4.0],
u'Z': [3.0, 6.0]
},
"records": {
u'result': [{
u'Y': 2.0,
u'X': 1.0,
u'Z': 3.0
}, {
u'Y': 5.0,
u'X': 4.0,
u'Z': 6.0
}]
}
}
for data_struct_type in test_sample.keys():
sheet = pe.Sheet(self.single_sheet)
tmp_filename = "test.file.xls"
sheet.save_as(tmp_filename)
with open(tmp_filename, "rb") as fp:
response = self.client.post('/polls/parse/' + data_struct_type,
data={"file": fp})
assert json.loads(response.content.decode(
'utf-8')) == test_sample[data_struct_type]
os.unlink(tmp_filename)
def test_parse_book(self):
test_sample = ["book", "book_dict"]
expected_dict = {
u'Sheet1': [[1.0, 1.0, 1.0, 1.0], [2.0, 2.0, 2.0, 2.0],
[3.0, 3.0, 3.0, 3.0]],
u'Sheet3': [[u'X', u'Y', u'Z'], [1.0, 4.0, 7.0], [2.0, 5.0, 8.0],
[3.0, 6.0, 9.0]],
u'Sheet2': [[4.0, 4.0, 4.0, 4.0], [5.0, 5.0, 5.0, 5.0],
[6.0, 6.0, 6.0, 6.0]]
}
for data_struct_type in test_sample:
sheet = pe.Book(self.book_content)
tmp_filename = "test.xls"
sheet.save_as(tmp_filename)
with open(tmp_filename, "rb") as fp:
response = self.client.post('/polls/parse/' + data_struct_type,
data={"file": fp})
assert json.loads(
response.content.decode('utf-8')) == expected_dict
os.unlink(tmp_filename)
def test_exchange(self):
for in_file_type in FILE_TYPE_MIME_TABLE.keys():
sheet = pe.Sheet(self.data)
tmp_filename = "test.%s" % in_file_type
sheet.save_as(tmp_filename)
for file_type in FILE_TYPE_MIME_TABLE.keys():
print("Post %s -> Respond %s" % (in_file_type, file_type))
with open(tmp_filename, "rb") as fp:
response = self.client.post('/polls/exchange/' + file_type,
data={"file": fp})
assert response['Content-Type'] == FILE_TYPE_MIME_TABLE[
file_type]
sheet = pe.get_sheet(file_type=file_type,
file_content=response.content)
sheet.format(int)
array = sheet.to_array()
assert array == self.data
os.unlink(tmp_filename)
class ExcelResponseTestCase(TestCase):
def setUp(self):
self.client = Client()
self.data = [[1, 2, 3], [4, 5, 6]]
self.single_sheet = [["X", "Y", "Z"], [1, 2, 3], [4, 5, 6]]
self.book_content = OrderedDict()
self.book_content.update(
{"Sheet1": [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]]})
self.book_content.update(
{"Sheet2": [[4, 4, 4, 4], [5, 5, 5, 5], [6, 6, 6, 6]]})
self.book_content.update(
{"Sheet3": [[u"X", u"Y", u"Z"], [1, 4, 7], [2, 5, 8], [3, 6, 9]]})
def test_download(self):
for file_type in FILE_TYPE_MIME_TABLE.keys():
response = self.client.get("/polls/download/" + file_type)
assert response["Content-Type"] == FILE_TYPE_MIME_TABLE[file_type]
sheet = pe.get_sheet(file_type=file_type,
file_content=response.content)
sheet.format(int)
array = sheet.to_array()
assert array == self.data
def test_download_attachment_with_ascii_name(self):
test_file_name = "test"
self._download_and_verify_file_name(test_file_name)
def test_download_attachment_with_unicode_name(self):
test_file_name = u"中文文件名"
self._download_and_verify_file_name(test_file_name.encode("utf-8"))
def test_download_attachment_with_unicode_name_as_string(self):
test_file_name = "中文文件名"
self._download_and_verify_file_name(test_file_name)
def _download_and_verify_file_name(self, file_name):
for file_type in FILE_TYPE_MIME_TABLE.keys():
url_encoded_file_name = urllib_quote(file_name)
response = self.client.get(("/polls/download_attachment/%s/%s" %
(file_type, url_encoded_file_name)))
assert response["Content-Type"] == FILE_TYPE_MIME_TABLE[file_type]
assert response["Content-Disposition"] == (
"attachment; filename=%s.%s;filename*=utf-8''%s.%s" %
(url_encoded_file_name, file_type, url_encoded_file_name,
file_type))
sheet = pe.get_sheet(file_type=file_type,
file_content=response.content)
sheet.format(int)
array = sheet.to_array()
assert array == self.data
def test_parse_single_sheet(self):
test_sample = {
"array": {
u"result": [[u"X", u"Y", u"Z"], [1.0, 2.0, 3.0],
[4.0, 5.0, 6.0]]
},
"dict": {
u"Y": [2.0, 5.0],
u"X": [1.0, 4.0],
u"Z": [3.0, 6.0]
},
"records": {
u"result": [
{
u"Y": 2.0,
u"X": 1.0,
u"Z": 3.0
},
{
u"Y": 5.0,
u"X": 4.0,
u"Z": 6.0
},
]
},
}
for data_struct_type in test_sample.keys():
sheet = pe.Sheet(self.single_sheet)
tmp_filename = "test.file.xls"
sheet.save_as(tmp_filename)
with open(tmp_filename, "rb") as fp:
response = self.client.post("/polls/parse/" + data_struct_type,
data={"file": fp})
self.assertEqual(
json.loads(response.content.decode("utf-8")),
test_sample[data_struct_type],
)
os.unlink(tmp_filename)
def test_parse_book(self):
test_sample = ["book", "book_dict"]
expected_dict = {
u"Sheet1": [
[1.0, 1.0, 1.0, 1.0],
[2.0, 2.0, 2.0, 2.0],
[3.0, 3.0, 3.0, 3.0],
],
u"Sheet3": [
[u"X", u"Y", u"Z"],
[1.0, 4.0, 7.0],
[2.0, 5.0, 8.0],
[3.0, 6.0, 9.0],
],
u"Sheet2": [
[4.0, 4.0, 4.0, 4.0],
[5.0, 5.0, 5.0, 5.0],
[6.0, 6.0, 6.0, 6.0],
],
}
for data_struct_type in test_sample:
sheet = pe.Book(self.book_content)
tmp_filename = "test.xls"
sheet.save_as(tmp_filename)
with open(tmp_filename, "rb") as fp:
response = self.client.post("/polls/parse/" + data_struct_type,
data={"file": fp})
self.assertEqual(json.loads(response.content.decode("utf-8")),
expected_dict)
os.unlink(tmp_filename)
def test_exchange(self):
for in_file_type in FILE_TYPE_MIME_TABLE.keys():
sheet = pe.Sheet(self.data)
tmp_filename = "test.%s" % in_file_type
sheet.save_as(tmp_filename)
for file_type in FILE_TYPE_MIME_TABLE.keys():
with open(tmp_filename, "rb") as fp:
response = self.client.post("/polls/exchange/" + file_type,
data={"file": fp})
self.assertEqual(response["Content-Type"],
FILE_TYPE_MIME_TABLE[file_type])
sheet = pe.get_sheet(file_type=file_type,
file_content=response.content)
sheet.format(int)
array = sheet.to_array()
assert array == self.data
os.unlink(tmp_filename)
def include_schema():
for schema, value in globals.edg_schema.items()[:]:
schema_data = OrderedDict()
schema_files = [
globals.edg_conf['conf']['metadata_info'][schema]['schema_path']
]
# Continue to next schema if this schema has no other include files
if not value.get('include_schema'):
continue
# Check for self inclusion
if list(set(schema_files) & set(value.get('include_schema'))):
print "Error: Cannot self include schema in %s : schema_include" % schema
return False
# Check for duplicate inclusion
schema_files = value.get('include_schema')
if len(list(set(schema_files))) != len(schema_files):
print "Error: File duplication in %s : schema_include list " % schema
return False
for schema_file in schema_files:
# Check whether file exist or not
try:
with open(schema_file) as f:
pass
except IOError as e:
print 'Error : Included schema file: "%s", do not exist' % schema_file
return False
# Parse JSON data in schema file
schema_data_json_file = open(schema_file)
str_data = bytearray(schema_data_json_file.read()).decode("utf-8")
schema_data_json = re.sub(r'<Application>',
globals.edg_schema[schema]['name'],
str_data)
try:
schema_data[schema_file] = json.loads(
schema_data_json, object_pairs_hook=OrderedDict)
schema_data_json_file.close()
except ValueError as e:
print "Error parsing JSON in included schema file : ", schema_file
print e
return False
print "Include Schema File " + schema_file + " Parsed"
# Check for more than one level of include
if schema_data[schema_file].get('include_schema'):
print "Error: more than one level of schema nesting in", schema, ": include_schema :", schema_file
return False
# Schema structure validation and expansion
if 'name' not in schema_data[schema_file]:
print schema_file, "schema does not contain mandatory 'name' tag"
return False
if 'fields' not in schema_data[schema_file]:
print schema_file, "schema does not contain mandatory 'fields' tag"
return False
if not validate_fields(schema_file,
schema_data[schema_file]['fields']):
return False
# Tweak for maintaining order of included fields
ordered_fields = OrderedDict()
for schema_file in schema_files:
key_intersect = list(
set(ordered_fields.keys())
& set(schema_data[schema_file]['fields'].keys()))
if (key_intersect):
print "Following duplicate keys found in nested schema file: ", schema_file, "in", schema
print key_intersect
return False
ordered_fields.update(schema_data[schema_file]['fields'])
# Now add fields of the Top schema
key_intersect = list(
set(ordered_fields.keys()) & set(value['fields'].keys()))
if (key_intersect):
print "Following duplicate keys found in top level schema file of", schema
print key_intersect
return False
ordered_fields.update(value['fields'])
globals.edg_schema[schema]['fields'] = ordered_fields
#value['fields'].update(schema_data[schema_file]['fields'])
return True
def construct_yaml_map(self, node):
data = OrderedDict()
yield data
value = self.construct_mapping(node)
data.update(value)
data = OrderedDict()
string = 'system/breakwindows@XE'
conn = cx.connect(string)
c = conn.cursor()
query = 'SELECT STATE, PERCENTAGE FROM LITERACY WHERE YEAR = 2001'
c.execute(query)
ra = []
# Fetching data from oracle
for n in c:
ra.append(n)
#Converting tuple to list
for i in range(len(ra)):
ra[i] = list(ra[i])
query = 'SELECT PERCENTAGE FROM LITERACY WHERE YEAR = 2011'
c.execute(query)
# Fetchin data from oracle
for key, n in enumerate(c):
ra[key].append(n[0])
ra.sort()
data.update({"Sheet 1": ra})
save_data("/home/negi/Documents/Topological_Crime_Analysis/Literacy/testing.xls", data)
def SecondButtonPress(url,
HostPage,
page=None,
elm="",
elm2="",
wform=0,
addkey=None,
removekey=None,
cookies={},
wait=0,
captchakey=None,
captchaimg=None,
captchacookies={},
split=None,
GetUserAgent=None):
domain = HostPage.split('/')[2]
payload = OrderedDict()
headers = OrderedDict()
headers[
'Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
headers['Accept-Charset'] = 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
headers['Accept-Encoding'] = 'gzip,deflate,sdch'
headers['Accept-Language'] = 'en-US,en;q=0.8'
headers['Cache-Control'] = 'max-age=0'
headers['Connection'] = 'keep-alive'
headers['Referer'] = url
if GetUserAgent == None:
headers['User-Agent'] = UserAgent
else:
headers['User-Agent'] = GetUserAgent
session = requests.session()
requests.utils.add_dict_to_cookiejar(session.cookies, cookies)
if page != None:
s = page
else:
s = session.get(HostPage, headers=headers)
try:
form = HTML.ElementFromString(s.content)
except:
form = HTML.ElementFromString(s)
try:
whichform = form.xpath('//' + elm + 'form')[wform]
if len(whichform.xpath('./' + elm2 + 'input')) != 0:
for input in whichform.xpath('./' + elm2 + 'input'):
if input.get('name') != None:
key = input.get('name')
value = input.get('value')
if key != 'method_premium':
if not payload.has_key(key):
payload[key] = [value]
else:
payload[key].append(value)
else:
for input in form.xpath('//input'):
if input.get('name') != None:
key = input.get('name')
value = input.get('value')
if key != 'method_premium':
if not payload.has_key(key):
payload[key] = [value]
else:
payload[key].append(value)
if captchakey != None:
try:
payload[captchakey] = GetImgValue(url=captchaimg,
HostPage=HostPage,
UserAgent=UserAgent,
cookies=captchacookies,
split=split)
except:
payload[captchakey] = "Processing Issue"
if addkey != None:
payload.update(addkey)
if removekey != None:
for key in removekey:
try:
del payload[key]
except KeyError:
pass
Log(payload)
if wait != 0:
#wait required
time.sleep(wait)
headers['Content-Type'] = 'application/x-www-form-urlencoded'
headers['Origin'] = 'http://' + domain
headers['Referer'] = HostPage
formaction = form.xpath('//' + elm + 'form')[wform].get('action')
if formaction != None and formaction != "":
if formaction.split('/')[0] == 'http:':
HostPage = formaction
elif len(formaction.split('/')) == 2:
HostPage = 'http://' + HostPage.split('/')[2] + formaction
elif len(formaction.split('/')) == 1:
HostPage = HostPage.rpartition('/')[0] + '/' + formaction
r = session.post(HostPage,
data=payload,
headers=headers,
allow_redirects=True)
r.raise_for_status()
r.cookies = session.cookies
return r
except:
s.cookies = session.cookies
return s
class TestAddBooks:
def _write_test_file(self, file):
"""
Make a test file as:
1,1,1,1
2,2,2,2
3,3,3,3
"""
self.rows = 3
pyexcel.save_book_as(bookdict=self.content, dest_file_name=file)
def setUp(self):
self.testfile = "multiple1.xls"
self.testfile2 = "multiple2.xls"
self.testfile3 = "multiple3.xls"
self.content = OrderedDict()
self.content.update(
{"Sheet1": [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]]})
self.content.update(
{"Sheet2": [[4, 4, 4, 4], [5, 5, 5, 5], [6, 6, 6, 6]]})
self.content.update(
{"Sheet3": [[u'X', u'Y', u'Z'], [1, 4, 7], [2, 5, 8], [3, 6, 9]]})
self._write_test_file(self.testfile)
self._write_test_file(self.testfile2)
def test_load_a_single_sheet(self):
b1 = pyexcel.get_book(file_name=self.testfile, sheet_name="Sheet1")
assert len(b1.sheet_names()) == 1
assert b1['Sheet1'].to_array() == self.content['Sheet1']
def test_load_a_single_sheet2(self):
b1 = pyexcel.load_book(self.testfile, sheet_index=2)
assert len(b1.sheet_names()) == 1
assert b1['Sheet3'].to_array() == self.content['Sheet3']
@raises(IndexError)
def test_load_a_single_sheet3(self):
pyexcel.get_book(file_name=self.testfile, sheet_index=10000)
@raises(ValueError)
def test_load_a_single_sheet4(self):
pyexcel.get_book(file_name=self.testfile, sheet_name="Not exist")
def test_delete_sheets(self):
b1 = pyexcel.load_book(self.testfile)
assert len(b1.sheet_names()) == 3
del b1["Sheet1"]
assert len(b1.sheet_names()) == 2
try:
del b1["Sheet1"]
assert 1 == 2
except KeyError:
assert 1 == 1
del b1[1]
assert len(b1.sheet_names()) == 1
try:
del b1[1]
assert 1 == 2
except IndexError:
assert 1 == 1
def test_delete_sheets2(self):
"""repetitively delete first sheet"""
b1 = pyexcel.load_book(self.testfile)
del b1[0]
assert len(b1.sheet_names()) == 2
del b1[0]
assert len(b1.sheet_names()) == 1
del b1[0]
assert len(b1.sheet_names()) == 0
def test_add_book1(self):
"""
test this scenario: book3 = book1 + book2
"""
b1 = pyexcel.BookReader(self.testfile)
b2 = pyexcel.BookReader(self.testfile2)
b3 = b1 + b2
content = pyexcel.utils.to_dict(b3)
sheet_names = content.keys()
assert len(sheet_names) == 6
for name in sheet_names:
if "Sheet3" in name:
assert content[name] == self.content["Sheet3"]
elif "Sheet2" in name:
assert content[name] == self.content["Sheet2"]
elif "Sheet1" in name:
assert content[name] == self.content["Sheet1"]
def test_add_book1_in_place(self):
"""
test this scenario book1 += book2
"""
b1 = pyexcel.BookReader(self.testfile)
b2 = pyexcel.BookReader(self.testfile2)
b1 += b2
content = pyexcel.utils.to_dict(b1)
sheet_names = content.keys()
assert len(sheet_names) == 6
for name in sheet_names:
if "Sheet3" in name:
assert content[name] == self.content["Sheet3"]
elif "Sheet2" in name:
assert content[name] == self.content["Sheet2"]
elif "Sheet1" in name:
assert content[name] == self.content["Sheet1"]
def test_add_book2(self):
"""
test this scenario book3 = book1 + sheet3
"""
b1 = pyexcel.BookReader(self.testfile)
b2 = pyexcel.BookReader(self.testfile2)
b3 = b1 + b2["Sheet3"]
content = pyexcel.utils.to_dict(b3)
sheet_names = content.keys()
assert len(sheet_names) == 4
for name in sheet_names:
if "Sheet3" in name:
assert content[name] == self.content["Sheet3"]
elif "Sheet2" in name:
assert content[name] == self.content["Sheet2"]
elif "Sheet1" in name:
assert content[name] == self.content["Sheet1"]
def test_add_book2_in_place(self):
"""
test this scenario book3 = book1 + sheet3
"""
b1 = pyexcel.BookReader(self.testfile)
b2 = pyexcel.BookReader(self.testfile2)
b1 += b2["Sheet3"]
content = pyexcel.utils.to_dict(b1)
sheet_names = content.keys()
assert len(sheet_names) == 4
for name in sheet_names:
if "Sheet3" in name:
assert content[name] == self.content["Sheet3"]
elif "Sheet2" in name:
assert content[name] == self.content["Sheet2"]
elif "Sheet1" in name:
assert content[name] == self.content["Sheet1"]
def test_add_book3(self):
"""
test this scenario book3 = sheet1 + sheet2
"""
b1 = pyexcel.BookReader(self.testfile)
b2 = pyexcel.BookReader(self.testfile2)
b3 = b1["Sheet1"] + b2["Sheet3"]
content = pyexcel.utils.to_dict(b3)
sheet_names = content.keys()
assert len(sheet_names) == 2
assert content["Sheet3"] == self.content["Sheet3"]
assert content["Sheet1"] == self.content["Sheet1"]
def test_add_book4(self):
"""
test this scenario book3 = sheet1 + book
"""
b1 = pyexcel.BookReader(self.testfile)
b2 = pyexcel.BookReader(self.testfile2)
b3 = b1["Sheet1"] + b2
content = pyexcel.utils.to_dict(b3)
sheet_names = content.keys()
assert len(sheet_names) == 4
for name in sheet_names:
if "Sheet3" in name:
assert content[name] == self.content["Sheet3"]
elif "Sheet2" in name:
assert content[name] == self.content["Sheet2"]
elif "Sheet1" in name:
assert content[name] == self.content["Sheet1"]
def test_add_book_error(self):
"""
test this scenario: book3 = sheet1 + book
"""
b1 = pyexcel.BookReader(self.testfile)
try:
b1 + 12
assert 1 == 2
except TypeError:
assert 1 == 1
try:
b1 += 12
assert 1 == 2
except TypeError:
assert 1 == 1
def tearDown(self):
if os.path.exists(self.testfile):
os.unlink(self.testfile)
if os.path.exists(self.testfile2):
os.unlink(self.testfile2)
class ExcelResponseTestCase(TestCase):
def setUp(self):
self.client = Client()
self.data = [
[1, 2, 3],
[4, 5, 6]
]
self.single_sheet = [
['X', 'Y', 'Z'],
[1, 2, 3],
[4, 5, 6]
]
self.book_content = OrderedDict()
self.book_content.update({"Sheet1": [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]]})
self.book_content.update({"Sheet2": [[4, 4, 4, 4], [5, 5, 5, 5], [6, 6, 6, 6]]})
self.book_content.update({"Sheet3": [[u'X', u'Y', u'Z'], [1, 4, 7], [2, 5, 8], [3, 6, 9]]})
def test_download(self):
for file_type in FILE_TYPE_MIME_TABLE.keys():
print(file_type)
response = self.client.get("/polls/download/"+file_type)
assert response['Content-Type'] == FILE_TYPE_MIME_TABLE[file_type]
sheet = pe.get_sheet(file_type=file_type, file_content=response.content)
sheet.format(int)
array = sheet.to_array()
assert array == self.data
def test_parse_single_sheet(self):
test_sample = {
"array": {u'result': [[u'X', u'Y', u'Z'], [1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]},
"dict": {u'Y': [2.0, 5.0], u'X': [1.0, 4.0], u'Z': [3.0, 6.0]},
"records": {u'result': [{u'Y': 2.0, u'X': 1.0, u'Z': 3.0}, {u'Y': 5.0, u'X': 4.0, u'Z': 6.0}]}
}
for data_struct_type in test_sample.keys():
sheet = pe.Sheet(self.single_sheet)
tmp_filename = "test.xls"
sheet.save_as(tmp_filename)
with open(tmp_filename, "rb") as fp:
response = self.client.post('/polls/parse/'+data_struct_type,
data={"file": fp})
assert json.loads(response.content.decode('utf-8')) == test_sample[data_struct_type]
os.unlink(tmp_filename)
def test_parse_book(self):
test_sample = [
"book",
"book_dict"
]
expected_dict = {
u'Sheet1': [[1.0, 1.0, 1.0, 1.0], [2.0, 2.0, 2.0, 2.0], [3.0, 3.0, 3.0, 3.0]],
u'Sheet3': [[u'X', u'Y', u'Z'], [1.0, 4.0, 7.0], [2.0, 5.0, 8.0], [3.0, 6.0, 9.0]],
u'Sheet2': [[4.0, 4.0, 4.0, 4.0], [5.0, 5.0, 5.0, 5.0], [6.0, 6.0, 6.0, 6.0]]}
for data_struct_type in test_sample:
sheet = pe.Book(self.book_content)
tmp_filename = "test.xls"
sheet.save_as(tmp_filename)
with open(tmp_filename, "rb") as fp:
response = self.client.post('/polls/parse/'+data_struct_type,
data={"file": fp})
assert json.loads(response.content.decode('utf-8')) == expected_dict
os.unlink(tmp_filename)
def test_exchange(self):
for in_file_type in FILE_TYPE_MIME_TABLE.keys():
sheet = pe.Sheet(self.data)
tmp_filename = "test.%s" % in_file_type
sheet.save_as(tmp_filename)
for file_type in FILE_TYPE_MIME_TABLE.keys():
print("Post %s -> Respond %s" % (in_file_type, file_type))
with open(tmp_filename, "rb") as fp:
response = self.client.post('/polls/exchange/'+file_type,
data={"file": fp})
assert response['Content-Type'] == FILE_TYPE_MIME_TABLE[file_type]
sheet = pe.get_sheet(file_type=file_type, file_content=response.content)
sheet.format(int)
array = sheet.to_array()
assert array == self.data
os.unlink(tmp_filename)
from pyexcel_xls import save_data
from ordereddict import OrderedDict
data = OrderedDict()
f = open('census01', 'r')
k = []
for (i, line) in enumerate(f.readlines()):
line = line.split('|')
line[2] = line[2]
line[3] = line[3][:len(line[3]) - 2]
line[1] = line[1].replace(',', '')
k.append(line)
data.update({"Sheet 1": k})
data.update({"Sheet 2": [["row 1", "row 2", "row 3", "row 4"]]})
save_data("your_file.xls", data)
f.close()
from pyexcel_xls import save_data
from ordereddict import OrderedDict
data = OrderedDict()
f = open('census01', 'r')
k = []
for (i, line) in enumerate(f.readlines()):
line = line.split('|')
line[2] = line[2]
line[3] = line[3][:len(line[3]) - 2]
line[1] = line[1].replace(',', '')
k.append(line)
data.update({"Sheet 1": k})
data.update({"Sheet 2": [["row 1", "row 2", "row 3","row 4"]]})
save_data("your_file.xls", data)
f.close()
def SecondButtonPress(
url,
HostPage,
page=None,
elm="",
elm2="",
wform=0,
addkey=None,
removekey=None,
cookies={},
wait=0,
captchakey=None,
captchaimg=None,
captchacookies={},
split=None,
GetUserAgent=None,
):
domain = HostPage.split("/")[2]
payload = OrderedDict()
headers = OrderedDict()
headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
headers["Accept-Charset"] = "ISO-8859-1,utf-8;q=0.7,*;q=0.3"
headers["Accept-Encoding"] = "gzip,deflate,sdch"
headers["Accept-Language"] = "en-US,en;q=0.8"
headers["Cache-Control"] = "max-age=0"
headers["Connection"] = "keep-alive"
headers["Referer"] = url
if GetUserAgent == None:
headers["User-Agent"] = UserAgent
else:
headers["User-Agent"] = GetUserAgent
session = requests.session()
requests.utils.add_dict_to_cookiejar(session.cookies, cookies)
if page != None:
s = page
else:
s = session.get(HostPage, headers=headers)
try:
form = HTML.ElementFromString(s.content)
except:
form = HTML.ElementFromString(s)
try:
whichform = form.xpath("//" + elm + "form")[wform]
if len(whichform.xpath("./" + elm2 + "input")) != 0:
for input in whichform.xpath("./" + elm2 + "input"):
if input.get("name") != None:
key = input.get("name")
value = input.get("value")
if key != "method_premium":
if not payload.has_key(key):
payload[key] = [value]
else:
payload[key].append(value)
else:
for input in form.xpath("//input"):
if input.get("name") != None:
key = input.get("name")
value = input.get("value")
if key != "method_premium":
if not payload.has_key(key):
payload[key] = [value]
else:
payload[key].append(value)
if captchakey != None:
try:
payload[captchakey] = GetImgValue(
url=captchaimg, HostPage=HostPage, UserAgent=UserAgent, cookies=captchacookies, split=split
)
except:
payload[captchakey] = "Processing Issue"
if addkey != None:
payload.update(addkey)
if removekey != None:
for key in removekey:
try:
del payload[key]
except KeyError:
pass
Log(payload)
if wait != 0:
# wait required
time.sleep(wait)
headers["Content-Type"] = "application/x-www-form-urlencoded"
headers["Origin"] = "http://" + domain
headers["Referer"] = HostPage
formaction = form.xpath("//" + elm + "form")[wform].get("action")
if formaction != None and formaction != "":
if formaction.split("/")[0] == "http:":
HostPage = formaction
elif len(formaction.split("/")) == 2:
HostPage = "http://" + HostPage.split("/")[2] + formaction
elif len(formaction.split("/")) == 1:
HostPage = HostPage.rpartition("/")[0] + "/" + formaction
r = session.post(HostPage, data=payload, headers=headers, allow_redirects=True)
r.raise_for_status()
r.cookies = session.cookies
return r
except:
s.cookies = session.cookies
return s
def build_settings_form(user, settings):
"""
Create a set of fields and builds a form class
returns SettingForm class
"""
fields = OrderedDict()
for setting in settings:
# Do not display standard regform settings
if setting.scope_category == "events" and setting.name.startswith("regform_"):
continue
try:
setting_value = force_unicode(setting.get_value())
except DjangoUnicodeDecodeError:
setting_value = ""
if setting.input_type in ["text", "textarea"]:
options = {
"label": setting.label,
"help_text": setting.description,
"initial": setting_value,
"required": False,
}
if setting.input_type == "textarea":
options["widget"] = forms.Textarea(attrs={"rows": 5, "cols": 30})
if setting.client_editable:
fields.update({"%s" % setting.name: forms.CharField(**options)})
else:
if user.is_superuser:
fields.update({"%s" % setting.name: forms.CharField(**options)})
elif setting.input_type == "select":
if setting.input_value == "<form_list>":
choices = get_form_list(user)
required = False
elif setting.input_value == "<box_list>":
choices = get_box_list(user)
required = False
elif setting.input_value == "<group_list>":
choices, initial = get_group_list(user)
required = True
if not setting_value:
setting_value = initial
elif setting.input_value == "<timezone_list>":
choices = zones.PRETTY_TIMEZONE_CHOICES
required = True
elif setting.input_value == "<language_list>":
choices = get_languages_with_local_name()
required = True
else:
# Allow literal_eval in settings in order to pass a list from the setting
# This is useful if you want different values and labels for the select options
try:
choices = tuple([(k, v) for k, v in literal_eval(setting.input_value)])
required = False
# By adding #<box_list> on to the end of a setting, this will append the boxes
# as select items in the list as well.
if "<box_list>" in setting.input_value:
box_choices = get_box_list(user)[1:]
choices = (("Content", choices), ("Boxes", box_choices))
except:
choices = tuple([(s.strip(), s.strip()) for s in setting.input_value.split(",")])
required = True
options = {
"label": setting.label,
"help_text": setting.description,
"initial": setting_value,
"choices": choices,
"required": required,
}
if setting.client_editable:
fields.update({"%s" % setting.name: forms.ChoiceField(**options)})
else:
if user.is_superuser:
fields.update({"%s" % setting.name: forms.ChoiceField(**options)})
elif setting.input_type == "file":
from tendenci.core.files.models import File as TendenciFile
file_display = ""
try:
try:
val = int(setting_value)
except ValueError:
val = 0
try:
tfile = TendenciFile.objects.get(pk=val)
except Exception:
tfile = None
if tfile:
if tfile.file.name.lower().endswith((".jpg", ".jpe", ".png", ".gif", ".svg")):
tfile_alt = tfile.file.name.lower()[:-4]
file_display = '<img src="/files/%s/" alt="%s" title="%s">' % (tfile.pk, tfile_alt, tfile_alt)
else:
file_display = tfile.file.name
except TendenciFile.DoesNotExist:
file_display = "No file"
options = {
"label": setting.label,
"help_text": "%s<br> Current File: %s" % (setting.description, file_display),
#'initial': tfile and tfile.file, # Removed this so the file doesn't save over and over
"required": False,
}
if setting.client_editable:
fields.update({"%s" % setting.name: forms.FileField(**options)})
else:
if user.is_superuser:
fields.update({"%s" % setting.name: forms.FileField(**options)})
attributes = {
"settings": settings,
"base_fields": fields,
"clean": clean_settings_form,
"save": save_settings_form,
"user": user,
}
return type("SettingForm", (forms.BaseForm,), attributes)
data = OrderedDict()
string = 'system/breakwindows@XE'
conn = cx.connect(string)
c = conn.cursor()
query = 'SELECT STATE, COUNT FROM POPULATION WHERE YEAR = 2001'
c.execute(query)
ra = []
# Fetching data from oracle
for n in c:
ra.append(n)
#Converting tuple to list
for i in range(len(ra)):
ra[i] = list(ra[i])
query = 'SELECT COUNT FROM POPULATION WHERE YEAR = 2011'
c.execute(query)
# Fetchin data from oracle
for key, n in enumerate(c):
ra[key].append(n[0])
data.update({"Sheet 1": ra})
save_data(
"/home/negi/Documents/Topological_Crime_Analysis/Population/testing.xls",
data)
if word.isalpha() and not (word == 'State' or word == 'UT'):
if flag:
state = state + ' ' + word
else:
state = word
flag = True
# print state
# removing sl no
line.remove(line[0])
# removing name of state
for i in range(state.count(' ')):
line.remove(line[0])
line.remove(line[1])
line.remove(line[3])
line.remove(line[3])
line.remove(line[0])
line.insert(0, state)
line[1] = line[1].replace(',', '')
line = line[:4]
print line
k.append(line)
f.close()
data.update({"Sheet 1": k})
save_data("population_census2011.xls", data)
def build_settings_form(user, settings):
"""
Create a set of fields and builds a form class
returns SettingForm class
"""
fields = OrderedDict()
for setting in settings:
# Do not display standard regform settings
if setting.scope_category == 'events' and setting.name.startswith(
'regform_'):
continue
try:
setting_value = force_unicode(setting.get_value())
except DjangoUnicodeDecodeError:
setting_value = ''
if setting.input_type in ['text', 'textarea']:
options = {
'label': setting.label,
'help_text': setting.description,
'initial': setting_value,
'required': False
}
if setting.input_type == 'textarea':
options['widget'] = forms.Textarea(attrs={
'rows': 5,
'cols': 30
})
if setting.client_editable:
fields.update(
{"%s" % setting.name: forms.CharField(**options)})
else:
if user.is_superuser:
fields.update(
{"%s" % setting.name: forms.CharField(**options)})
elif setting.input_type == 'select':
if setting.input_value == '<form_list>':
choices = get_form_list(user)
required = False
elif setting.input_value == '<box_list>':
choices = get_box_list(user)
required = False
elif setting.input_value == '<group_list>':
choices, initial = get_group_list(user)
required = True
if not setting_value:
setting_value = initial
elif setting.input_value == '<timezone_list>':
choices = zones.PRETTY_TIMEZONE_CHOICES
required = True
elif setting.input_value == '<language_list>':
choices = get_languages_with_local_name()
required = True
else:
# Allow literal_eval in settings in order to pass a list from the setting
# This is useful if you want different values and labels for the select options
try:
choices = tuple([
(k, v) for k, v in literal_eval(setting.input_value)
])
required = False
# By adding #<box_list> on to the end of a setting, this will append the boxes
# as select items in the list as well.
if '<box_list>' in setting.input_value:
box_choices = get_box_list(user)[1:]
choices = (('Content', choices), ('Boxes',
box_choices))
except:
choices = tuple([(s.strip(), s.strip())
for s in setting.input_value.split(',')])
required = True
options = {
'label': setting.label,
'help_text': setting.description,
'initial': setting_value,
'choices': choices,
'required': required,
}
if setting.client_editable:
fields.update(
{"%s" % setting.name: forms.ChoiceField(**options)})
else:
if user.is_superuser:
fields.update(
{"%s" % setting.name: forms.ChoiceField(**options)})
elif setting.input_type == 'file':
from tendenci.core.files.models import File as TendenciFile
file_display = ''
try:
try:
val = int(setting_value)
except ValueError:
val = 0
try:
tfile = TendenciFile.objects.get(pk=val)
except Exception:
tfile = None
if tfile:
if tfile.file.name.lower().endswith(
('.jpg', '.jpe', '.png', '.gif', '.svg')):
tfile_alt = tfile.file.name.lower()[:-4]
file_display = '<img src="/files/%s/" alt="%s" title="%s">' % (
tfile.pk, tfile_alt, tfile_alt)
else:
file_display = tfile.file.name
except TendenciFile.DoesNotExist:
file_display = "No file"
options = {
'label':
setting.label,
'help_text':
"%s<br> Current File: %s" %
(setting.description, file_display),
#'initial': tfile and tfile.file, # Removed this so the file doesn't save over and over
'required':
False
}
if setting.client_editable:
fields.update(
{"%s" % setting.name: forms.FileField(**options)})
else:
if user.is_superuser:
fields.update(
{"%s" % setting.name: forms.FileField(**options)})
attributes = {
'settings': settings,
'base_fields': fields,
'clean': clean_settings_form,
'save': save_settings_form,
'user': user,
}
return type('SettingForm', (forms.BaseForm, ), attributes)
class Config(object):
""" KGEN configuration parameter holder """
def __init__(self):
# setup config parameters
self._attrs = OrderedDict()
self.opt_handlers = OrderedDict()
# KGEN operation mode
self._attrs['check_mode'] = False
# Fortran parameters
self._attrs['fort'] = OrderedDict()
self._attrs['fort']['maxlinelen'] = 132
# logging parameters
self._attrs['logging'] = OrderedDict()
self._attrs['logging']['select'] = OrderedDict()
# callsite parameters
self._attrs['callsite'] = OrderedDict()
self._attrs['callsite']['filepath'] = ''
self._attrs['callsite']['span'] = (-1, -1)
self._attrs['callsite']['namepath'] = ''
# self._attrs['callsite']['lineafter'] = -1
# external tool parameters
self._attrs['bin'] = OrderedDict()
self._attrs['bin']['pp'] = 'cpp'
self._attrs['bin']['cpp_flags'] = '-w -traditional'
self._attrs['bin']['fpp_flags'] = '-w'
# search parameters
self._attrs['search'] = OrderedDict()
self._attrs['search']['skip_intrinsic'] = True
self._attrs['search']['except'] = []
self._attrs['search']['promote_exception'] = False
# path parameters
self._attrs['path'] = OrderedDict()
self._attrs['path']['outdir'] = '.'
self._attrs['path']['state'] = 'state'
self._attrs['path']['kernel'] = 'kernel'
# source file parameters
self._attrs['source'] = OrderedDict()
self._attrs['source']['isfree'] = None
self._attrs['source']['isstrict'] = None
self._attrs['source']['alias'] = OrderedDict()
self._attrs['source']['file'] = OrderedDict()
# include parameters
self._attrs['include'] = OrderedDict()
self._attrs['include']['macro'] = OrderedDict()
self._attrs['include']['path'] = ['.']
self._attrs['include']['type'] = OrderedDict()
self._attrs['include']['import'] = OrderedDict()
self._attrs['include']['file'] = OrderedDict()
# add mpi frame code in kernel driver
self._attrs['add_mpi_frame'] = OrderedDict()
self._attrs['add_mpi_frame']['enabled'] = False
self._attrs['add_mpi_frame']['np'] = '2'
self._attrs['add_mpi_frame']['mpiexec'] = 'mpiexec'
# exclude parameters
self._attrs['exclude'] = OrderedDict()
# debugging parameters
self._attrs['debug'] = OrderedDict()
self._attrs['debug']['printvar'] = []
# plugin parameters
self._attrs['plugin'] = OrderedDict()
self._attrs['plugin']['priority'] = OrderedDict()
def apply(self, cfg):
import optparse
if not cfg:
raise ProgramException('Custom configuration is not provided.')
if hasattr(cfg, 'attrs'):
self._attrs.update(cfg.attrs)
# parsing arguments
parser = optparse.OptionParser(usage=cfg.usage, version=cfg.version)
# add default options
parser.add_option("-s",
"--syntax-check",
dest="syntax_check",
action='store_true',
default=False,
help="KGEN Syntax Check Mode")
parser.add_option("-i",
"--include-ini",
dest="include_ini",
action='store',
type='string',
default=None,
help="information used for analysis")
parser.add_option("-e",
"--exclude-ini",
dest="exclude_ini",
action='store',
type='string',
default=None,
help="information excluded for analysis")
parser.add_option("-I",
dest="include",
action='append',
type='string',
default=None,
help="include path information used for analysis")
parser.add_option("-D",
dest="macro",
action='append',
type='string',
default=None,
help="macro information used for analysis")
parser.add_option("--outdir",
dest="outdir",
action='store',
type='string',
default=None,
help="path to create outputs")
parser.add_option("--source",
dest="source",
action='append',
type='string',
default=None,
help="Setting source file related properties")
parser.add_option("--skip-intrinsic",
dest="skip_intrinsic",
action='store_true',
default=False,
help=optparse.SUPPRESS_HELP)
parser.add_option("--noskip-intrinsic",
dest="noskip_intrinsic",
action='store_true',
default=False,
help=optparse.SUPPRESS_HELP)
parser.add_option(
"--intrinsic",
dest="intrinsic",
action='append',
type='string',
default=None,
help=
"Specifying resolution for intrinsic procedures during searching")
parser.add_option("--debug",
dest="debug",
action='append',
type='string',
help=optparse.SUPPRESS_HELP)
parser.add_option("--logging",
dest="logging",
action='append',
type='string',
help=optparse.SUPPRESS_HELP)
parser.add_option("--add-mpi-frame",
dest="add_mpi_frame",
type='string',
default=None,
help='Add MPI frame codes in kernel_driver.')
# add custom options
if hasattr(cfg, 'options'):
for opt_handler, args, kwargs in cfg.options:
parser.add_option(*args, **kwargs)
self.opt_handlers[kwargs['dest']] = opt_handler
opts, args = parser.parse_args()
if len(args) < 1:
print 'ERROR: No call-site information is provided in command line.'
sys.exit(-1)
if opts.syntax_check:
self._process_default_flags(opts)
self._attrs['check_mode'] = args
return
# old options
if opts.skip_intrinsic:
print "skip-intrinsic flag is discarded. Please use --intrinsic skip instead"
sys.exit(-1)
if opts.noskip_intrinsic:
print "noskip-intrinsic flag is discarded. Please use --intrinsic noskip instead"
sys.exit(-1)
callsite = args[0].split(':', 1)
if not os.path.isfile(callsite[0]):
print 'ERROR: %s can not be found.' % callsite[0]
sys.exit(-1)
# set callsite filepath
self.callsite['filepath'] = callsite[0]
# set namepath if exists in command line argument
if len(callsite) == 2:
self.callsite['namepath'] = callsite[1].lower()
elif len(callsite) > 2:
print 'ERROR: Unrecognized call-site information(Syntax -> filepath[:subprogramname]): %s' % str(
callsite)
sys.exit(-1)
# process default flags
self._process_default_flags(opts)
# process custom flags
for optname, opt_handler in self.opt_handlers.items():
opt = getattr(opts, optname, None)
if opt and optname in self.opt_handlers:
self.opt_handlers[optname](opt)
def _process_default_flags(self, opts):
# check if exists fpp or cpp
output = ''
try:
output = exec_cmd('which cpp', show_error_msg=False).strip()
except Exception as e:
pass
if output.endswith('cpp'):
self.bin['pp'] = output
else:
output = ''
try:
output = exec_cmd('which fpp', show_error_msg=False).strip()
except Exception as e:
pass
if output.endswith('fpp'):
self.bin['pp'] = output
else:
print 'ERROR: neither cpp or fpp is found'
sys.exit(-1)
# parsing intrinsic skip option
if opts.intrinsic:
subflags = []
for line in opts.intrinsic:
subflags.extend(line.split(','))
for subf in subflags:
if subf and subf.find('=') > 0:
key, value = subf.split('=')
if key == 'except':
self._attrs['search']['except'].extend(
value.split(';'))
elif key == 'add_intrinsic':
Intrinsic_Procedures.extend(
[name.lower() for name in value.split(';')])
else:
raise UserException(
'Unknown intrinsic sub option: %s' % subf)
else:
if subf == 'skip':
self._attrs['search']['skip_intrinsic'] = True
elif subf == 'noskip':
self._attrs['search']['skip_intrinsic'] = False
else:
raise UserException(
'Unknown intrinsic option(s) in %s' % subf)
# parsing include parameters
if opts.include:
for inc in opts.include:
inc_eq = inc.split('=')
if len(inc_eq) == 1:
for inc_colon in inc_eq[0].split(':'):
self._attrs['include']['path'].append(inc_colon)
if len(inc_eq) == 1:
for inc_colon in inc_eq[0].split(':'):
self._attrs['include']['path'].append(inc_colon)
elif len(inc_eq) == 2:
# TODO: support path for each file
pass
else:
raise UserException('Wrong format include: %s' % inc)
if opts.include_ini:
process_include_option(opts.include_ini, self._attrs['include'])
if opts.exclude_ini:
process_exclude_option(opts.exclude_ini, self._attrs['exclude'])
# parsing macro parameters
if opts.macro:
for line in opts.macro:
for macro in line.split(','):
macro_eq = macro.split('=')
if len(macro_eq) == 1:
self._attrs['include']['macro'][macro_eq[0]] = '1'
elif len(macro_eq) == 2:
self._attrs['include']['macro'][
macro_eq[0]] = macro_eq[1]
else:
raise UserException('Wrong format include: %s' % inc)
files = None
if opts.source:
for line in opts.source:
flags = OrderedDict()
for subflag in line.lower().split(','):
if subflag.find('=') > 0:
key, value = subflag.split('=')
if key == 'file':
flags[key] = value.split(':')
elif key == 'alias':
p1, p2 = value.split(':')
if p1.endswith('/'): p1 = p1[:-1]
if p2.endswith('/'): p2 = p2[:-1]
self._attrs['source']['alias'][p1] = p2
else:
flags[key] = value
else:
flags[subflag] = None
isfree = None
isstrict = None
if flags.has_key('format'):
if flags['format'] == 'free': isfree = True
elif flags['format'] == 'fixed': isfree = False
else:
raise UserException(
'format subflag of source flag should be either free or fixed.'
)
if flags.has_key('strict'):
if flags['strict'] == 'yes': isstrict = True
elif flags['strict'] == 'no': isstrict = False
else:
raise UserException(
'strict subflag of source flag should be either yes or no.'
)
if flags.has_key('file'):
subflags = OrderedDict()
if isfree: subflags['isfree'] = isfree
if isstrict: subflags['isstrict'] = isstrict
for file in flags['file']:
abspath = os.path.abspath(file)
if files is None: files = []
files.append(abspath)
self._attrs['source']['file'][abspath] = subflags
else:
if isfree: self._attrs['source']['isfree'] = isfree
if isstrict: self._attrs['source']['isstrict'] = isstrict
# dupulicate paths per each alias
if files is None:
newpath = set()
for path in self._attrs['include']['path']:
newpath.add(path)
for p1, p2 in self._attrs['source']['alias'].iteritems():
if path.startswith(p1):
newpath.add(p2 + path[len(p1):])
elif path.startswith(p2):
newpath.add(p1 + path[len(p2):])
self._attrs['include']['path'] = list(newpath)
newfile = OrderedDict()
for path, value in self._attrs['include']['file'].iteritems():
newfile[path] = value
for p1, p2 in self._attrs['source']['alias'].iteritems():
if path.startswith(p1):
newpath = p2 + path[len(p1):]
newfile[newpath] = deepcopy(value)
elif path.startswith(p2):
newpath = p1 + path[len(p2):]
newfile[newpath] = deepcopy(value)
self._attrs['include']['file'] = newfile
for path, value in self._attrs['include']['file'].iteritems():
if value.has_key('path'):
newpath = set()
for path in value['path']:
newpath.add(path)
for p1, p2 in self._attrs['source']['alias'].iteritems():
if path.startswith(p1):
newpath.add(p2 + path[len(p1):])
elif path.startswith(p2):
newpath.add(p1 + path[len(p2):])
value['path'] = list(newpath)
# parsing debugging options
if opts.debug:
for dbg in opts.debug:
param_path, value = dbg.split('=')
param_split = param_path.lower().split('.')
value_split = value.lower().split(',')
curdict = self._attrs['debug']
for param in param_split[:-1]:
curdict = curdict[param]
exec('curdict[param_split[-1]] = value_split')
# parsing logging options
if opts.logging:
for log in opts.logging:
param_path, value = log.split('=')
param_split = param_path.lower().split('.')
value_split = value.lower().split(',')
curdict = self._attrs['logging']
for param in param_split[:-1]:
curdict = curdict[param]
exec('curdict[param_split[-1]] = value_split')
# mpi frame code in kernel driver
if opts.add_mpi_frame:
self._attrs['add_mpi_frame']['enabled'] = True
for checkparams in opts.add_mpi_frame.split(','):
key, value = checkparams.split('=')
key = key.lower()
if key in ['np', 'mpiexec']:
self._attrs['add_mpi_frame'][key] = value
else:
print 'WARNING: %s is not supported add_mpi_frame parameter' % key
if opts.outdir:
self._attrs['path']['outdir'] = opts.outdir
# create state directories and change working directory
if not os.path.exists(self._attrs['path']['outdir']):
os.makedirs(self._attrs['path']['outdir'])
os.chdir(self._attrs['path']['outdir'])
def __getattr__(self, name):
return self._attrs[name]
def build_settings_form(user, settings):
"""
Create a set of fields and builds a form class
returns SettingForm class
"""
fields = OrderedDict()
for setting in settings:
# Do not display standard regform settings
if setting.scope_category == 'events' and setting.name.startswith('regform_'):
continue
try:
setting_value = force_unicode(setting.get_value())
except DjangoUnicodeDecodeError:
setting_value = ''
if setting.input_type in ['text', 'textarea']:
options = {
'label': setting.label,
'help_text': setting.description,
'initial': setting_value,
'required': False
}
if setting.input_type == 'textarea':
options['widget'] = forms.Textarea(attrs={'rows': 5, 'cols': 30});
if setting.client_editable:
fields.update({"%s" % setting.name: forms.CharField(**options)})
else:
if user.is_superuser:
fields.update({"%s" % setting.name: forms.CharField(**options)})
elif setting.input_type in ['select', 'selectmultiple']:
if setting.input_value == '<form_list>':
choices = get_form_list(user)
required = False
elif setting.input_value == '<box_list>':
choices = get_box_list(user)
required = False
elif setting.input_value == '<group_list>':
choices, initial = get_group_list(user)
required = True
if not setting_value:
setting_value = initial
elif setting.input_value == '<timezone_list>':
choices = zones.PRETTY_TIMEZONE_CHOICES
required = True
elif setting.input_value == '<language_list>':
choices = get_languages_with_local_name()
required = True
elif setting.input_value == '<country_list>':
choices = (('', '-----------'),) + tuple(COUNTRIES)
required = False
else:
# Allow literal_eval in settings in order to pass a list from the setting
# This is useful if you want different values and labels for the select options
try:
choices = tuple([(k, v)for k, v in literal_eval(setting.input_value)])
required = False
# By adding #<box_list> on to the end of a setting, this will append the boxes
# as select items in the list as well.
if '<box_list>' in setting.input_value:
box_choices = get_box_list(user)[1:]
choices = (('Content', choices), ('Boxes', box_choices))
except:
choices = tuple([(s.strip(), s.strip())for s in setting.input_value.split(',')])
required = True
options = {
'label': setting.label,
'help_text': setting.description,
'initial': setting_value,
'choices': choices,
'required': required,
}
if setting.client_editable or user.is_superuser:
if setting.input_type == 'selectmultiple':
fields.update({"%s" % setting.name: forms.MultipleChoiceField(**options)})
else:
fields.update({"%s" % setting.name: forms.ChoiceField(**options)})
elif setting.input_type == 'file':
from tendenci.apps.files.models import File as TendenciFile
file_display = ''
try:
try:
val = int(setting_value)
except ValueError:
val = 0
try:
tfile = TendenciFile.objects.get(pk=val)
except Exception:
tfile = None
if tfile:
if tfile.file.name.lower().endswith(('.jpg', '.jpe', '.png', '.gif', '.svg')):
tfile_alt = tfile.file.name.lower()[:-4]
file_display = '<img src="/files/%s/" alt="%s" title="%s">' % (tfile.pk, tfile_alt, tfile_alt)
else:
file_display = tfile.file.name
except TendenciFile.DoesNotExist:
file_display = "No file"
options = {
'label': setting.label,
'help_text': "%s<br> Current File: %s" % (setting.description, file_display),
#'initial': tfile and tfile.file, # Removed this so the file doesn't save over and over
'required': False
}
if setting.client_editable:
fields.update({"%s" % setting.name: forms.FileField(**options)})
else:
if user.is_superuser:
fields.update({"%s" % setting.name: forms.FileField(**options)})
attributes = {
'settings': settings,
'base_fields': fields,
'clean': clean_settings_form,
'save': save_settings_form,
'user': user,
}
return type('SettingForm', (forms.BaseForm,), attributes)
def build_settings_form(user, settings):
"""
Create a set of fields and builds a form class
returns SettingForm class
"""
fields = OrderedDict()
for setting in settings:
try:
setting_value = force_unicode(setting.get_value())
except DjangoUnicodeDecodeError:
setting_value = ''
if setting.input_type == 'text':
options = {
'label': setting.label,
'help_text': setting.description,
'initial': setting_value,
'required': False
}
if setting.client_editable:
fields.update({"%s" % setting.name: forms.CharField(**options)})
else:
if user.is_superuser:
fields.update({"%s" % setting.name: forms.CharField(**options)})
elif setting.input_type == 'select':
if setting.input_value == '<form_list>':
choices = get_form_list(user)
required = False
elif setting.input_value == '<box_list>':
choices = get_box_list(user)
required = False
elif setting.input_value == '<group_list>':
choices, initial = get_group_list(user)
required = True
if not setting_value:
setting_value = initial
else:
# Allow literal_eval in settings in order to pass a list from the setting
# This is useful if you want different values and labels for the select options
try:
choices = tuple([(k, v)for k, v in literal_eval(setting.input_value)])
required = False
# By adding #<box_list> on to the end of a setting, this will append the boxes
# as select items in the list as well.
if '<box_list>' in setting.input_value:
box_choices = get_box_list(user)[1:]
choices = (('Content', choices), ('Boxes', box_choices))
except:
choices = tuple([(s.strip(), s.strip())for s in setting.input_value.split(',')])
required = True
options = {
'label': setting.label,
'help_text': setting.description,
'initial': setting_value,
'choices': choices,
'required': required,
}
if setting.client_editable:
fields.update({"%s" % setting.name: forms.ChoiceField(**options)})
else:
if user.is_superuser:
fields.update({"%s" % setting.name: forms.ChoiceField(**options)})
elif setting.input_type == 'file':
from tendenci.core.files.models import File as TendenciFile
file_display = ''
try:
try:
val = int(setting_value)
except ValueError:
val = 0
try:
tfile = TendenciFile.objects.get(pk=val)
except Exception:
tfile = None
if tfile:
if tfile.file.name.lower().endswith(('.jpg', '.jpe', '.png', '.gif', '.svg')):
file_display = '<img src="/files/%s/">' % tfile.pk
else:
file_display = tfile.file.name
except TendenciFile.DoesNotExist:
file_display = "No file"
options = {
'label': setting.label,
'help_text': "%s<br> Current File: %s" % (setting.description, file_display),
#'initial': tfile and tfile.file, # Removed this so the file doesn't save over and over
'required': False
}
if setting.client_editable:
fields.update({"%s" % setting.name: forms.FileField(**options)})
else:
if user.is_superuser:
fields.update({"%s" % setting.name: forms.FileField(**options)})
attributes = {
'settings': settings,
'base_fields': fields,
'clean': clean_settings_form,
'save': save_settings_form,
'user': user,
}
return type('SettingForm', (forms.BaseForm,), attributes)
def construct_yaml_map(self, node):
data = OrderedDict()
yield data
value = self.construct_mapping(node)
data.update(value)
class Element(object):
def __init__(self,
elem_name,
primaries,
parameter_dict,
slurp_flags,
fn_list=None,
del_fn=None):
self.elem_name = elem_name
self.parameter = parameter_dict
self.primaries = primaries
self.writables = OrderedDict()
for fn in fn_list:
self.writables.update(fn.unique_writables)
self.write_fns = dict([(fn.name, fn) for fn in fn_list])
self.del_fn = del_fn
self.slurp_flags = slurp_flags
def __repr__(self):
return '%s, %s, %s, %s' % (self.elem_name, str(
self.parameter.keys()), str(
self.primaries), str(self.writables.keys()))
def required_writables(self):
"""Return a list of the required writable options"""
return self.writables.keys()
def required_readables(self):
"""Return a list of the required readable options.
This includes the primaries at the start of the returned list, in the
order they are required in and the readable names of the writable
options at the end."""
readables = list(self.primaries)
for option in self.required_writables():
if option not in readables:
readables.append(self.writables[option].read_param)
return readables
def _slurp(self, req_read_names, sid=-1, flags=None):
"""Return a iterator of the slurped data"""
if not flags:
flags = self.slurp_flags
if app_settings.USE_CASPY:
return caspy_info(self.elem_name,
req_read_names,
sid=sid,
flags=flags)
else:
return subsystem_info(self.elem_name,
req_read_names,
sid=sid,
flags=flags)
slurp = _slurp
def needs_transform(self, options):
for opt in options:
if self.writables[opt].trns_fn:
return True
def map_func(self, read_opts, options):
if self.needs_transform(options):
trns = [self.writables[opt_name].trns_fn for opt_name in options]
def transform(vals):
return [
trns_fn(val) if trns_fn else val
for trns_fn, val in zip(trns, vals)
]
if len(read_opts) == len(options):
# just transform the data:
def zipper(options, vals):
return zip(options, transform(vals))
else:
# Some read data needs to be added.
val_ind = [
read_opts.index(self.writables[opt_name].read_param)
for opt_name in options
]
def zipper(options, vals):
return zip(options,
transform([vals[ind] for ind in val_ind]))
return zipper
elif len(read_opts) != len(options):
# Some read data needs to be added.
val_ind = [
read_opts.index(self.writables[opt_name].read_param)
for opt_name in options
]
def zipper(options, vals):
return zip(options, [vals[ind] for ind in val_ind])
return zipper
else:
# just a regular zipping of files.
return zip
class TestAddBooks:
def _write_test_file(self, file):
"""
Make a test file as:
1,1,1,1
2,2,2,2
3,3,3,3
"""
self.rows = 3
pyexcel.save_book_as(bookdict=self.content,dest_file_name=file)
def setUp(self):
self.testfile = "multiple1.xls"
self.testfile2 = "multiple2.xls"
self.testfile3 = "multiple3.xls"
self.content = OrderedDict()
self.content.update({"Sheet1": [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]]})
self.content.update({"Sheet2": [[4, 4, 4, 4], [5, 5, 5, 5], [6, 6, 6, 6]]})
self.content.update({"Sheet3": [[u'X', u'Y', u'Z'], [1, 4, 7], [2, 5, 8], [3, 6, 9]]})
self._write_test_file(self.testfile)
self._write_test_file(self.testfile2)
def test_load_a_single_sheet(self):
b1 = pyexcel.load_book(self.testfile, sheet_name="Sheet1")
assert len(b1.sheet_names()) == 1
assert b1['Sheet1'].to_array() == self.content['Sheet1']
def test_load_a_single_sheet2(self):
b1 = pyexcel.load_book(self.testfile, sheet_index=2)
assert len(b1.sheet_names()) == 1
assert b1['Sheet3'].to_array() == self.content['Sheet3']
@raises(IndexError)
def test_load_a_single_sheet3(self):
pyexcel.load_book(self.testfile, sheet_index=10000)
@raises(ValueError)
def test_load_a_single_sheet4(self):
pyexcel.load_book(self.testfile, sheet_name="Not exist")
def test_delete_sheets(self):
b1 = pyexcel.load_book(self.testfile)
assert len(b1.sheet_names()) == 3
del b1["Sheet1"]
assert len(b1.sheet_names()) == 2
try:
del b1["Sheet1"]
assert 1==2
except KeyError:
assert 1==1
del b1[1]
assert len(b1.sheet_names()) == 1
try:
del b1[1]
assert 1==2
except IndexError:
assert 1==1
def test_delete_sheets2(self):
"""repetitively delete first sheet"""
b1 = pyexcel.load_book(self.testfile)
del b1[0]
assert len(b1.sheet_names()) == 2
del b1[0]
assert len(b1.sheet_names()) == 1
del b1[0]
assert len(b1.sheet_names()) == 0
def test_add_book1(self):
"""
test this scenario: book3 = book1 + book2
"""
b1 = pyexcel.BookReader(self.testfile)
b2 = pyexcel.BookReader(self.testfile2)
b3 = b1 + b2
content = pyexcel.utils.to_dict(b3)
sheet_names = content.keys()
assert len(sheet_names) == 6
for name in sheet_names:
if "Sheet3" in name:
assert content[name] == self.content["Sheet3"]
elif "Sheet2" in name:
assert content[name] == self.content["Sheet2"]
elif "Sheet1" in name:
assert content[name] == self.content["Sheet1"]
def test_add_book1_in_place(self):
"""
test this scenario book1 += book2
"""
b1 = pyexcel.BookReader(self.testfile)
b2 = pyexcel.BookReader(self.testfile2)
b1 += b2
content = pyexcel.utils.to_dict(b1)
sheet_names = content.keys()
assert len(sheet_names) == 6
for name in sheet_names:
if "Sheet3" in name:
assert content[name] == self.content["Sheet3"]
elif "Sheet2" in name:
assert content[name] == self.content["Sheet2"]
elif "Sheet1" in name:
assert content[name] == self.content["Sheet1"]
def test_add_book2(self):
"""
test this scenario book3 = book1 + sheet3
"""
b1 = pyexcel.BookReader(self.testfile)
b2 = pyexcel.BookReader(self.testfile2)
b3 = b1 + b2["Sheet3"]
content = pyexcel.utils.to_dict(b3)
sheet_names = content.keys()
assert len(sheet_names) == 4
for name in sheet_names:
if "Sheet3" in name:
assert content[name] == self.content["Sheet3"]
elif "Sheet2" in name:
assert content[name] == self.content["Sheet2"]
elif "Sheet1" in name:
assert content[name] == self.content["Sheet1"]
def test_add_book2_in_place(self):
"""
test this scenario book3 = book1 + sheet3
"""
b1 = pyexcel.BookReader(self.testfile)
b2 = pyexcel.BookReader(self.testfile2)
b1 += b2["Sheet3"]
content = pyexcel.utils.to_dict(b1)
sheet_names = content.keys()
assert len(sheet_names) == 4
for name in sheet_names:
if "Sheet3" in name:
assert content[name] == self.content["Sheet3"]
elif "Sheet2" in name:
assert content[name] == self.content["Sheet2"]
elif "Sheet1" in name:
assert content[name] == self.content["Sheet1"]
def test_add_book3(self):
"""
test this scenario book3 = sheet1 + sheet2
"""
b1 = pyexcel.BookReader(self.testfile)
b2 = pyexcel.BookReader(self.testfile2)
b3 = b1["Sheet1"] + b2["Sheet3"]
content = pyexcel.utils.to_dict(b3)
sheet_names = content.keys()
assert len(sheet_names) == 2
assert content["Sheet3"] == self.content["Sheet3"]
assert content["Sheet1"] == self.content["Sheet1"]
def test_add_book4(self):
"""
test this scenario book3 = sheet1 + book
"""
b1 = pyexcel.BookReader(self.testfile)
b2 = pyexcel.BookReader(self.testfile2)
b3 = b1["Sheet1"] + b2
content = pyexcel.utils.to_dict(b3)
sheet_names = content.keys()
assert len(sheet_names) == 4
for name in sheet_names:
if "Sheet3" in name:
assert content[name] == self.content["Sheet3"]
elif "Sheet2" in name:
assert content[name] == self.content["Sheet2"]
elif "Sheet1" in name:
assert content[name] == self.content["Sheet1"]
def test_add_book_error(self):
"""
test this scenario: book3 = sheet1 + book
"""
b1 = pyexcel.BookReader(self.testfile)
try:
b1 + 12
assert 1==2
except TypeError:
assert 1==1
try:
b1 += 12
assert 1==2
except TypeError:
assert 1==1
def tearDown(self):
if os.path.exists(self.testfile):
os.unlink(self.testfile)
if os.path.exists(self.testfile2):
os.unlink(self.testfile2)
def build_settings_form(user, settings):
"""
Create a set of fields and builds a form class
returns SettingForm class
"""
fields = OrderedDict()
for setting in settings:
if setting.input_type == 'text':
options = {
'label': setting.label,
'help_text': setting.description,
'initial': setting.get_value(),
'required': False
}
if setting.client_editable:
fields.update(
{"%s" % setting.name: forms.CharField(**options)})
else:
if user.is_superuser:
fields.update(
{"%s" % setting.name: forms.CharField(**options)})
elif setting.input_type == 'select':
if setting.input_value == '<form_list>':
choices = get_form_list(user)
required = False
elif setting.input_value == '<box_list>':
choices = get_box_list(user)
required = False
else:
# Allow literal_eval in settings in order to pass a list from the setting
# This is useful if you want different values and labels for the select options
try:
choices = tuple([
(k, v) for k, v in literal_eval(setting.input_value)
])
required = False
# By adding #<box_list> on to the end of a setting, this will append the boxes
# as select items in the list as well.
if '<box_list>' in setting.input_value:
box_choices = get_box_list(user)[1:]
choices = (('Content', choices), ('Boxes',
box_choices))
except:
choices = tuple([(s.strip(), s.strip())
for s in setting.input_value.split(',')])
required = True
options = {
'label': setting.label,
'help_text': setting.description,
'initial': setting.get_value(),
'choices': choices,
'required': required,
}
if setting.client_editable:
fields.update(
{"%s" % setting.name: forms.ChoiceField(**options)})
else:
if user.is_superuser:
fields.update(
{"%s" % setting.name: forms.ChoiceField(**options)})
elif setting.input_type == 'file':
from tendenci.core.files.models import File as TendenciFile
file_display = ''
try:
try:
val = int(setting.get_value())
except:
val = 0
try:
tfile = TendenciFile.objects.get(pk=val)
except:
tfile = None
if tfile:
if tfile.file.name.lower().endswith(
('.jpg', '.jpe', '.png', '.gif', '.svg')):
file_display = '<img src="/files/%s/">' % tfile.pk
else:
file_display = tfile.file.name
except TendenciFile.DoesNotExist:
file_display = "No file"
options = {
'label':
setting.label,
'help_text':
"%s<br> Current File: %s" %
(setting.description, file_display),
#'initial': tfile and tfile.file, # Removed this so the file doesn't save over and over
'required':
False
}
if setting.client_editable:
fields.update(
{"%s" % setting.name: forms.FileField(**options)})
else:
if user.is_superuser:
fields.update(
{"%s" % setting.name: forms.FileField(**options)})
attributes = {
'settings': settings,
'base_fields': fields,
'clean': clean_settings_form,
'save': save_settings_form,
'user': user,
}
return type('SettingForm', (forms.BaseForm, ), attributes)
