def run():
datasheetPath = os.path.join(os.path.dirname(__file__),
'data/Data_Shanghai-Movement-Check_2015.csv')
dataset = tablib.import_set(open(datasheetPath, encoding='utf-8').read())
StudentResource().import_data(dataset)
StudentEvaluationResource().import_data(dataset)
talentDatasheetPath = os.path.join(os.path.dirname(__file__),
'data/三区细筛名单.csv')
talentDataset = tablib.import_set(
open(datasheetPath, encoding='utf-8').read())
StudentEvaluationTalentResource().import_data(talentDataset)
print('Done!')
def reload_database_tag(filename):
tag_resource = TagResource()
#curve_resource = CurveResource(None)
with open(filename, 'r') as f:
string = f.read()
dataset_new = tablib.import_set(string)
result = tag_resource.import_data(dataset_new, dry_run=False, raise_errors=True)
def import_files(self, request, query_set):
for q in query_set:
data = tablib.import_set(open(q.file.file.name, "rb").read(),
format="xlsx")
changed_headers = []
for header in data.headers:
changed_headers.append(header.lower().replace(":", "").replace(
".", "").strip().replace(" ", "_"))
data.headers = changed_headers
building_res = models.BuildingResource()
res = building_res.import_data(data,
dry_run=False,
raise_errors=True)
csv_file = models.CsvFile.objects.get(name=q.__str__())
if res.has_errors() or res.has_validation_errors():
csv_file.status = models.CsvFile.UNSUCCESS
message_str = "File with name {} wasn't imported.".format(
q.__str__())
message_level = messages.WARNING
else:
csv_file.status = models.CsvFile.SUCCESS
message_str = "File with name {} was imported.".format(
q.__str__())
message_level = messages.SUCCESS
self.message_user(request, message_str, message_level)
csv_file.save()
def convert_file(fname):
data = tablib.import_set(open(fname, 'rb'))
data.headers = ('round', 'subj', 'csv')
for (test_round, subj, ext) in data:
if ext is None:
continue
yield flatten_row(int(test_round), int(subj), fix_csv_line(read_csv_line(ext)))
def handle(self, *args, **kwargs):
file_path = kwargs["file_path"]
year = kwargs["year"]
with open(file_path, "rb") as f:
file_bin_input = f.read()
dataset = tablib.import_set(file_bin_input)
for raw in dataset.dict:
# load to ProcesType
processtype_data = prepare_procestype(raw, year)
# if current nummer already exists - update it
p, created = ProcesType.objects.update_or_create(
nummer=processtype_data["nummer"],
defaults=processtype_data,
jaar=processtype_data["jaar"],
)
# load to Resultaat
resultaat_data = prepare_resultaat(raw, year)
# if current resultaat already exists - update it
r, created = Resultaat.objects.update_or_create(
proces_type=resultaat_data["proces_type"],
generiek_resultaat=resultaat_data["generiek_resultaat"],
nummer=resultaat_data["nummer"],
defaults=resultaat_data,
)
def importAccountBalance():
account_csv = os.path.join(settings.FTP_IMPORT_DIR, 'CompSaldos.CSV')
try:
# Get modified date/time
mtime = os.path.getmtime(account_csv)
last_modified_date = datetime.fromtimestamp(mtime)
except OSError:
mtime = 0
file_data = CSVFilesData.objects.get(file='CompSaldos')
# Update if modified date/time has changed
if file_data.modified_date != str(last_modified_date):
print('ACTUALIZO SALDOS DE CUENTA CORRIENTE')
f = open(account_csv, 'r')
dataset = tablib.import_set(f.read(),
format='csv',
delimiter=';',
headers=False)
dataset.headers = ('customer_id', 'date', 'voucher', 'balance')
account_resource = AccountBalanceResource()
# Test import
result = account_resource.import_data(dataset, dry_run=True)
# If result has no errors then import (create or update) the data
if not result.has_errors():
account_resource.import_data(dataset, dry_run=False)
file_data.modified_date = last_modified_date
file_data.save()
else:
print('NO ACTUALIZO SALDOS DE CUENTA CORRIENTE')
def import_data(zf, model_dataset_map, dry_run=False):
error = False
for model, dataset in model_dataset_map.items():
dataset = tablib.import_set(zf.read(dataset))
data = model().import_data(dataset, dry_run=dry_run)
error = error or data.has_errors()
return error
def import_data(zf, model_dataset_map, dry_run=False):
error = False
for model, dataset in model_dataset_map.items():
dataset = tablib.import_set(zf.read(dataset))
data = model().import_data(dataset, dry_run=dry_run)
error = error or data.has_errors()
return error
def read(self, **kwargs):
"""Read xlsx format
:return: Dataset object
"""
with open(self.file, 'rb') as file:
return tablib.import_set(file, **kwargs)
def dataset(self):
"""Create a valid dataset from data in the sourcefile.
We create a tablib.Dataset object.
We clean up the headers.
We email ourselves if the source file cannot be converted \
to a datset, and we save that sourcefile in a tempoary \
location, as well as emailing it, for analysis.
"""
value = None
datastream = self.sourcefile.read()
try:
raw_dataset = tablib.import_set(datastream)
except AttributeError, e:
# TODO: need to get more specific exception
dt = datetime.datetime.now().isoformat()
this_file = TEMP_FILES_DIR + \
'/failed_import_{timestamp}_{filename}'.format(
timestamp=dt,
filename=unicode(self.sourcefile)
)
with open(this_file, 'wb+') as tmp_file:
for chunk in self.sourcefile.chunks():
tmp_file.write(chunk)
# email ourselves that we have a file to check
subject = 'Open Budget: Failed File Import'
message = 'The file is attached.'
sender = EMAIL_HOST_USER
recipients = [ADMINS]
send_mail(subject, message, sender, recipients)
raise e
def test_csv_import_set_skip_lines(self):
csv_text = ('garbage,line\n'
'\n'
'id,name,description\n'
'12,Smith,rounded\n')
dataset = tablib.import_set(csv_text, format="csv", skip_lines=2)
self.assertEqual(dataset.headers, ['id', 'name', 'description'])
def _extract_data(self, data_source):
"""Create a Dataset object from the data source."""
with open(data_source) as f:
stream = f.read()
raw_dataset = tablib.import_set(stream)
return raw_dataset
def _extract_data(self, data_source):
"""Create a Dataset object from the data source."""
with open(data_source) as f:
stream = f.read()
raw_dataset = tablib.import_set(stream)
return raw_dataset
def _extract_data(self, data_source):
"""Create a Dataset object from the data source."""
with open(data_source) as f:
stream = f.read()
extracted = tablib.import_set(stream)
# TODO: Notify which sources could not be serialized by tablib
return extracted
def test_csv_formatter_support_kwargs(self):
"""Test CSV import and export with formatter configuration."""
data.append(self.john)
data.append(self.george)
data.headers = self.headers
expected = 'first_name;last_name;gpa\nJohn;Adams;90\nGeorge;Washington;67\n'
kwargs = dict(delimiter=';', lineterminator='\n')
_csv = data.export('csv', **kwargs)
self.assertEqual(expected, _csv)
# the import works but consider default delimiter=','
d1 = tablib.import_set(_csv, format="csv")
self.assertEqual(1, len(d1.headers))
d2 = tablib.import_set(_csv, format="csv", **kwargs)
self.assertEqual(3, len(d2.headers))
def test_csv_formatter_support_kwargs(self):
"""Test CSV import and export with formatter configuration."""
data.append(self.john)
data.append(self.george)
data.headers = self.headers
expected = 'first_name;last_name;gpa\nJohn;Adams;90\nGeorge;Washington;67\n'
kwargs = dict(delimiter=';', lineterminator='\n')
_csv = data.export('csv', **kwargs)
self.assertEqual(expected, _csv)
# the import works but consider default delimiter=','
d1 = tablib.import_set(_csv, format="csv")
self.assertEqual(1, len(d1.headers))
d2 = tablib.import_set(_csv, format="csv", **kwargs)
self.assertEqual(3, len(d2.headers))
def _extract_data(self, data_source):
"""Create a Dataset object from the data source."""
with open(data_source) as f:
stream = f.read()
extracted = tablib.import_set(stream)
# TODO: Notify which sources could not be serialized by tablib
return extracted
def test_csv_import_set_ragged(self):
"""Import CSV set when not all rows have the same length."""
csv_text = ("H1,H2,H3\n" "A,B\n" "C,D,E\n" "\n" "F\n")
dataset = tablib.import_set(csv_text, format="csv")
self.assertEqual(
str(dataset), 'H1|H2|H3\n'
'--|--|--\n'
'A |B | \n'
'C |D |E \n'
'F | | ')
def test_export(self):
"""If export is in query string, response should be a csv."""
response = self.client.get('%s?export' % reverse('users_report'))
self.assertEqual(response['Content-Disposition'],
'attachment; filename=users.csv')
self.assertEqual(response['Content-Type'], 'text/csv')
data = import_set(response.content)
# There should be at least the header row and one user row
self.assertGreater(data.height, 2)
self.assertEqual(data.width, 14)
def get_data(self, stream):
try:
raw_dataset = tablib.import_set(stream)
except AttributeError as e:
# TODO: need to get more specific exception
self._import_error()
raise e
# `_normalize_headers` may transform the headers and returns the same data object
# we then return the dict view of the data
return self._normalize_headers(raw_dataset).dict
def get_data(self, stream):
try:
raw_dataset = tablib.import_set(stream)
except AttributeError as e:
# TODO: need to get more specific exception
self._import_error()
raise e
# `_normalize_headers` may transform the headers and returns the same data object
# we then return the dict view of the data
return self._normalize_headers(raw_dataset).dict
def read(in_file):
"""
Returns a tablib.dataset object if it can be created from the data
Otherwise returns None
"""
data = read_bytes(in_file)
for reader in READER_TYPES:
# try each reader until we find a way to import data
data_set = tablib.import_set(reader(data))
if data_set:
return data_set
def read(in_file):
"""
Returns a tablib.dataset object if it can be created from the data
Otherwise returns None
"""
data = read_bytes(in_file)
for reader in READER_TYPES:
# try each reader until we find a way to import data
data_set = tablib.import_set(reader(data))
if data_set:
return data_set
def import_files(self, request, query_set):
for q in query_set:
try:
data = tablib.import_set(open(q.file.file.name, "rb").read(),
format="xlsx")
changed_headers = []
for header in data.headers:
changed_headers.append(header.lower().replace(
":", "").replace(".", "").strip().replace(" ", "_"))
data.headers = changed_headers
building_res = models.BuildingResource()
res = building_res.import_data(data,
dry_run=False,
raise_errors=True)
csv_file = models.CsvFile.objects.get(name=q.__str__())
if res.has_errors() or res.has_validation_errors():
csv_file.status = models.CsvFile.FAILURE
message_str = _(
"File with name '{file_name}' wasn't imported.".format(
file_name=q.__str__()))
message_level = messages.WARNING
else:
csv_file.status = models.CsvFile.SUCCESS
message_str = _(
"File with name '{file_name}' was imported.".format(
file_name=q.__str__()))
message_level = messages.SUCCESS
csv_file.save()
except BadZipFile:
self.save_file_as_failed(q)
message_str = _(
"File with name '{file_name}' wasn't an XLSX.".format(
file_name=q.__str__()))
message_level = messages.ERROR
except ValueError as e:
self.save_file_as_failed(q)
message_str = _(
"File with name '{file_name}' couldn't be imported. The error received was: `{error_args}`"
.format(
file_name=q.__str__(),
error_args=e.args[0],
))
message_level = messages.ERROR
except Exception as e:
self.save_file_as_failed(q)
raise e
self.message_user(request, message_str, message_level)
def reload_database(filename):
print "loading tags... ",
reload_database_tag(filename + ".tag")
print 'done...'
print 'parsing text file for curves...'
curve_resource = CurveResourceWithId()
with open(filename, 'r') as f:
string = f.read()
print "Loading curves into database"
dataset_new = tablib.import_set(string)
print "succesfully parsed the text file... importing curves"
result = curve_resource.import_data(dataset_new, dry_run=False, raise_errors=True)
print 'curves imported succesfully... reconstructing hierarchy'
reload_hierarchy(dataset_new)
print 'hierarchy reconstructed! --> import successfull'
def test_export(self):
"""If export is in query string, response should be a csv."""
response = self.client.get('%s?export' % reverse('users_report'))
self.assertEqual(
response['Content-Disposition'],
'attachment; filename=users.csv'
)
self.assertEqual(
response['Content-Type'],
'text/csv'
)
data = import_set(response.content)
# There should be at least the header row and one user row
self.assertGreater(data.height, 2)
self.assertEqual(data.width, 14)
def handle(self, *args, **options):
# Following file is mandatory to exist on S3 bucket
file = get_s3_file_stream(
file_name='countries-and-territories-trading-blocs-25.0.csv',
bucket_name=settings.AWS_STORAGE_BUCKET_NAME_DATA_SCIENCE,
)
data = tablib.import_set(file, format='csv', headers=True)
for item in data:
if not PersonalisationTradingBlocTag.objects.filter(
slug=slugify(item[4])).exists():
PersonalisationTradingBlocTag.objects.create(name=item[4],
slug=slugify(
item[4]))
self.stdout.write(self.style.SUCCESS('All done, bye!'))
def post_parking():
valids = []
with open("Parking_2015.csv", "rb") as parking_file:
dataset = tablib.import_set(parking_file.read())
for data in dataset.dict:
parking_record = create_parking_record(data)
print parking_record
r = perform_post('parkings', json.dumps(parking_record))
print "'parking' posted", r.status_code
if r.status_code == 201:
response = r.json()
if response['_status'] == 'OK':
valids.append(response['_id'])
return valids
def handle(self, *args, **options):
# Following file is mandatory to exist on S3 bucket
file = get_s3_file_stream(
file_name='countries-territories-and-regions-1.0.csv',
bucket_name=settings.AWS_STORAGE_BUCKET_NAME_DATA_SCIENCE,
)
data = tablib.import_set(file, format='csv', headers=True)
# add only countries and iso2 as slug
for item in data:
if item[2] == 'Country':
if not PersonalisationCountryTag.objects.filter(
name=item[1]).exists():
PersonalisationCountryTag.objects.create(name=item[1],
slug=item[4])
self.stdout.write(self.style.SUCCESS('All done, bye!'))
def get_data(self, stream):
logging.info('Executing TablibImporter._get_parser_from_post')
try:
raw_dataset = tablib.import_set(stream)
except AttributeError as e:
self.import_error()
raise e
# `normalize_headers`, and return the updated dataset object.
raw_dataset.headers = self.normalize_headers(raw_dataset.headers)
# return the dict view of the data
return raw_dataset.dict
def load_dataset(self, stream, input_format=None, from_encoding=None):
"""
Loads data from ``stream`` given valid tablib ``input_format``
and returns tablib dataset
If ``from_encoding`` is specified, data will be converted to `utf-8`
characterset.
"""
if from_encoding:
text = unicode(stream.read(), from_encoding).encode('utf-8')
else:
text = stream.read()
if not input_format:
data = tablib.import_set(text)
else:
data = tablib.Dataset()
input_format.import_set(data, text)
return data
def import_files(self, request, query_set):
for q in query_set:
data = tablib.import_set(open(q.file.file.name, 'rb').read())
changed_headers = []
for header in data.headers:
changed_headers.append(header.lower().replace(':', '').replace('.', '').strip().replace(' ', '_'))
data.headers = changed_headers
building_res = models.BuildingResource()
res = building_res.import_data(data, False, False)
csv_file = models.CsvFile.objects.get(name=q.__str__())
if res.has_errors() or res.has_validation_errors():
csv_file.status = models.CsvFile.UNSUCCESS
else:
csv_file.status = models.CsvFile.SUCCESS
self.message_user(request, "File with name {} {} imported.".format(q.__str__(),
"was" if csv_file.status == models.CsvFile.SUCCESS
else "wasn't"),
messages.WARNING if csv_file.status == models.CsvFile.UNSUCCESS else messages.SUCCESS)
csv_file.save()
def import_csv(request):
if 'file' not in request.data:
raise ParseError("Empty content")
# Save temp file
f = request.data['file']
file_name = secure_filename(request.data['fileName'])
file_id = file_name
tmp_file_dir = 'static/upload/'
tmp_file_path = '{tmp_file_dir}/{file_name}'.format(
tmp_file_dir=tmp_file_dir, file_name=file_name)
stored_path = default_storage.save(tmp_file_path, ContentFile(f.read()))
# Read csv file from the temp file
new_http_header_value_descriptions = open(stored_path).read()
# Read dataset with custom delimiter
dataset = tablib.import_set(new_http_header_value_descriptions,
format='csv',
delimiter=settings.IMPORT_EXPORT_CSV_DELIMITER,
headers=False)
# Check header was appeared
first_row = dataset[0]
if first_row and first_row[0] == 'name':
# Remove first row
del dataset[0]
dataset.headers = ['name', 'value', 'description']
http_header_value_description_resource = HttpHeaderValueDescriptionResource(
)
# Test import now
result = http_header_value_description_resource.import_data(dataset,
dry_run=True)
# Remove temp file
os.remove(stored_path)
if not result.has_errors():
# Actually import now
http_header_value_description_resource.import_data(dataset,
dry_run=False)
return Response(status=status.HTTP_200_OK)
return Response(status=status.HTTP_403_FORBIDDEN)
def readForumsXLSX(self, inFile, loadDocshares=True):
'''
Read an existing Excel file having OWNERs marked, and update my forums tree.
:param inFile: Excel file having the Forums structure and OWNERs marked
must have same columns and order as is generated by writeForumsXLSX.
:param loadDocshares: if True, proceed to load the selected docshares.
'''
# column indexes in sheet:
OWNER = 0
FORUMNAME = 2
with open(inFile, 'rb') as file:
sheet = tablib.import_set(file, format='xlsx', headers=True)
for row in sheet:
if row[OWNER]:
forum = self.forums.find(row[FORUMNAME])
if forum:
forum['attrs']['OWNER'] = row[OWNER]
else:
print('Forum not found: ' + row[FORUMNAME])
if loadDocshares:
self.loadDocshares()
def importCustomer():
customer_csv = os.path.join(settings.FTP_IMPORT_DIR, 'Clientes.CSV')
try:
# Get modified date/time
mtime = os.path.getmtime(customer_csv)
last_modified_date = datetime.fromtimestamp(mtime)
except OSError:
mtime = 0
file_data = CSVFilesData.objects.get(file='Clientes')
# Update if modified date/time has changed
if file_data.modified_date != str(last_modified_date):
print('ACTUALIZO BASE DE CLIENTES')
f = open(customer_csv, 'r')
dataset = tablib.import_set(f.read(),
format='csv',
delimiter=';',
headers=False)
dataset.headers = ('customer_id', 'doc_type', 'cuit', 'name',
'address', 'city', 'telephone', 'discount')
customer_resource = CustomerResource()
# Test import
result = customer_resource.import_data(dataset, dry_run=True)
# If result has no errors then import (create or update) the data
if not result.has_errors():
customer_resource.import_data(dataset, dry_run=False)
file_data.modified_date = last_modified_date
file_data.save()
PublishUpdate()
else:
print('NO ACTUALIZO BASE DE CLIENTES')
def setUp(self):
self.dataset = tablib.import_set(open(TESTDATA_FILENAME, "rb").read())
self.raw = {
"Procestypenummer": 1,
"Procestypenaam": "Instellen en inrichten organisatie",
"Procestypeomschrijving": "Instellen en inrichten organisatie",
"Procestypetoelichting": "Dit procestype betreft het instellen van een nieuw organisatieonderdeel of een nieuw orgaan waar het orgaan in deelneemt. Dit procestype betreft eveneens het inrichten van het eigen orgaan. Dit kan kleinschalig plaatsvinden bijvoorbeeld het wijzigen van de uitvoering van een wettelijke taak of grootschalig wanneer er een organisatiewijziging wordt doorgevoerd.",
"procesobject": "De vastgestelde organisatie inrichting",
"Nr.": "1.1",
"Generiek / specifiek": "Generiek",
"Resultaat": "Ingericht",
"Omschrijving": "",
"Herkomst": "Risicoanalyse",
"Waardering": "Vernietigen",
"Procestermijn": "Nihil",
"Bewaartermijn": "10 jaar",
"Toelichting": "Invoering nieuwe werkwijze",
"Algemeen bestuur en inrichting organisatie": "",
"Bedrijfsvoering en personeel": "Bedrijfsvoering en personeel",
"Publieke informatie en registratie": "",
"Burgerzaken": "",
"Veiligheid": "",
"Verkeer en vervoer": "",
"Economie": "",
"Onderwijs": "",
"Sport, cultuur en recreatie": "",
"Sociaal domein": "",
"Volksgezondheid en milieu": "",
"VHROSV": "",
"Heffen belastingen etc": "",
"Alle taakgebieden": "",
}
self.specifiek = self.raw.copy()
self.specifiek.update({"Nr.": "1.1.2", "Generiek / specifiek": "Specifiek"})
def importProducts():
products_csv = os.path.join(settings.FTP_IMPORT_DIR, 'Productos.CSV')
try:
# Get modified date/time
mtime = os.path.getmtime(products_csv)
last_modified_date = datetime.fromtimestamp(mtime)
except OSError:
mtime = 0
file_data = CSVFilesData.objects.get(file='Productos')
# Update if modified date/time has changed
if file_data.modified_date != str(last_modified_date):
print('ACTUALIZO BASE DE PRODUCTOS')
f = open(products_csv, 'r')
dataset = tablib.import_set(f.read(),
format='csv',
delimiter=';',
headers=False)
dataset.headers = ('product_id', 'name', 'brand', 'product_line',
'unit', 'price', 'offer', 'offer_price', 'package')
products_resource = ProductsResource()
# Test import
result = products_resource.import_data(dataset, dry_run=True)
# If result has no errors then import (create or update) the data
if not result.has_errors():
products_resource.import_data(dataset, dry_run=False)
file_data.modified_date = last_modified_date
file_data.save()
PublishUpdate()
else:
print('NO ACTUALIZO BASE DE PRODUCTOS')
def importInvoices():
invoice_header_csv = os.path.join(settings.FTP_IMPORT_DIR,
'Comprobantes.CSV')
invoice_items_csv = os.path.join(settings.FTP_IMPORT_DIR, 'Comp_Items.CSV')
try:
# Get modified date/time
header_mtime = os.path.getmtime(invoice_header_csv)
items_mtime = os.path.getmtime(invoice_items_csv)
header_last_modified_date = datetime.fromtimestamp(header_mtime)
items_last_modified_date = datetime.fromtimestamp(items_mtime)
except OSError:
header_mtime = 0
items_mtime = 0
file_header_data = CSVFilesData.objects.get(file='Comprobantes')
file_items_data = CSVFilesData.objects.get(file='Comp_Items')
# Update if modified date/time has changed
if file_header_data.modified_date != str(header_last_modified_date):
print('ACTUALIZO CABECERA DE FACTURAS')
f = open(invoice_header_csv, 'r')
dataset = tablib.import_set(f.read(),
format='csv',
delimiter=';',
headers=False)
dataset.headers = ('customer_id', 'invoice_type', 'invoice_branch',
'invoice_number', 'date', 'iva', 'taxes')
invoice_header_resource = InvoicesResource()
# Test import
result = invoice_header_resource.import_data(dataset, dry_run=True)
# If result has no errors then import (create or update) the data
if not result.has_errors():
invoice_header_resource.import_data(dataset, dry_run=False)
file_header_data.modified_date = header_last_modified_date
file_header_data.save()
else:
print('NO ACTUALIZO CABECERA DE FACTURAS')
if file_items_data.modified_date != str(items_last_modified_date):
print('ACTUALIZO ITEMS DE FACTURAS')
f = open(invoice_items_csv, 'r')
dataset = tablib.import_set(f.read(),
format='csv',
delimiter=';',
headers=False)
dataset.headers = ('invoice_type', 'invoice_branch', 'invoice_number',
'product_id', 'price', 'quantity', 'amount',
'product_description')
invoice_items_resource = InvoiceItemsResource()
# Test import
result = invoice_items_resource.import_data(dataset, dry_run=True)
# If result has no errors then import (create or update) the data
if not result.has_errors():
invoice_items_resource.import_data(dataset, dry_run=False)
file_items_data.modified_date = items_last_modified_date
file_items_data.save()
else:
print('NO ACTUALIZO ITEMS DE FACTURAS')
def create_dataset(self, in_stream, **kwargs):
return tablib.import_set(in_stream, format=self.get_title())
def convert(input, output):
""" Converts the existing translators excel to be compatible with the
seantis.people excel import.
"""
print_error = lambda msg: click.secho(textwrap.dedent(msg), fg='red')
with open(input, 'rb') as excel:
ds = tablib.import_set(excel)
if not all(ds.headers):
print_error(u"""
The headers could not be detected. This means that you are trying
to convert the original xls(x) by the client, which doesn't work
with this script because it never meant to leave the Windows world.
To fix, open the excel file in Excel, select all, copy all, create
a new excel file and paste it all there.
Store the new excel file as xlsx and call the script with the
path to that file again.
""")
sys.exit(1)
if not output.endswith('.csv'):
print_error(u"The export file is required to end with .csv")
sys.exit(1)
# we don't need those columns anymore
for header in (u"AVANTI", u"Verzeichnis Übersetzende", u"Mutation"):
del ds[header]
# split zipcode and town
zipcodes = [s.split(' ')[0].strip() for s in ds[u"Plz Ort"]]
towns = [s.split(' ')[1].strip() for s in ds[u"Plz Ort"]]
del ds["Plz Ort"]
ds.append_col(zipcodes, u"PLZ")
ds.append_col(towns, u"Ort")
# rename certain headers
header_map = {
u"Bank Kto ltd auf": u"Bank Konto lautet auf",
u"eMail": u"E-Mail",
u"Telefon P": u"Telefon Privat",
u"Telefon G": u"Telefon Geschäft",
u"Andere": u"Andere Zertifikate",
u"Geb Datum": u"Geburtsdatum"
}
for old_name, new_name in header_map.items():
ds.append_col(ds[old_name], new_name)
del ds[old_name]
# certificates are done differently
certificates = []
for zhaw, ogzh in zip(ds[u"ZHAW Zertifikat"], ds[u"OGZH Zertifikat"]):
if zhaw == 1 and ogzh == 1:
certificates.append(u"ZHAW, OGZH")
if zhaw == 1 and ogzh == 0:
certificates.append(u"ZHAW")
if zhaw == 0 and ogzh == 1:
certificates.append(u"OGZH")
if zhaw == 0 and ogzh == 0:
certificates.append(u"")
del ds[u"ZHAW Zertifikat"]
del ds[u"OGZH Zertifikat"]
ds.append_col(certificates, u"Zertifikate")
# cleanup a few things
new_ds = tablib.Dataset()
new_ds.headers = ds.headers
for r, row in enumerate(ds):
new_row = [c for c in row]
for c, col in enumerate(row):
if col is None:
new_row[c] = ''
if isinstance(col, basestring):
if col == '-':
new_row[c] = ''
if '@' in col and '#mailto:' in col:
new_row[c] = col[:col.find('#mailto')]
if isinstance(col, int):
if col <= 1:
new_row[c] = int == 1 and True or False
else:
new_row[c] = str(col)
new_ds.append(new_row)
ds = new_ds
# write the result
with open(output, 'w') as csv:
csv.write(ds.csv)
def main(input_xls, dest, accept=False):
with open(input_xls, 'rb') as f:
sheet = tablib.import_set(f.read())
for row in sheet.dict:
process_entry(row, dest, accept=accept)
from pybrain.structure.modules import SoftmaxLayer
close_min_Value = 7.2386
close_max_Value = 133
volume_min_Value = 13046.45
volume_max_Value = 841930.153
opens_min_Value = 7.3879
opens_max_Value = 134.455
high_min_Value = 7.5557
high_max_Value = 134.54
low_min_Value = 7.1657
low_max_Value = 131.4
#importing data
data = open("HistoricalQuotes.csv").read()
dataset = tablib.import_set(data)
header = dataset.headers
rows = dataset._data
ds = SupervisedDataSet(4, 1)
n = buildNetwork(4, 20, 20, 1, bias=True, hiddenclass=TanhLayer)
print("Network Built")
#inputs setting up datasets
for row in rows[:2000]:
close = float(row[1])
volume = float(row[2])
opens = float(row[3])
high = float(row[4])
def import_zip(zipname):
curve_resource = CurveResourceNoId(zipname)
with zipfile.PyZipFile(zipname, 'r') as f:
string = f.read('dataset.csv')
dataset_new = tablib.import_set(string)
result = curve_resource.import_data(dataset_new, dry_run=False, raise_errors=True)
import tablib
from pybrain.structure import LinearLayer, SigmoidLayer
from pybrain.structure import FeedForwardNetwork
from pybrain.structure import FullConnection
from pybrain.supervised.trainers import BackpropTrainer
from pybrain import *
from pybrain.datasets import SupervisedDataSet
from pybrain.tools.shortcuts import buildNetwork
from pybrain.supervised.trainers import BackpropTrainer
from pybrain.datasets import SupervisedDataSet
from pybrain.structure import TanhLayer
#importing data
data = open("HistoricalQuotes.csv").read()
dataset = tablib.import_set(data)
header = dataset.headers
rows = dataset._data
ds = SupervisedDataSet(1, 1)
ds.addSample((0), (0,))
ds.addSample((0), (1,))
ds.addSample((1), (0,))
ds.addSample((1), (1,))
ds.addSample((1), (0,))
for inpt, target in ds:
print (inpt, target)
__author__ = 'zh'
__date__ = '6/17/15'
from pymongo import MongoClient
from bson import ObjectId
import pickle
import datetime
import time
import sys
import logging
import tablib
import xlrd
import decimal
client = MongoClient('mongodb://*****:*****@123.56.131.68:7900/admin')
db_wukong = client['wukong-release']
c_courier = db_wukong['courier']
my_input_stream = open('debt.xls', 'rb').read()
my_dataset = tablib.import_set(my_input_stream)
for debt_record in my_dataset:
try:
if debt_record[0] == '552d1fbc778d17053aba7057':
courier_id = ObjectId(debt_record[0])
debt_amount = int(decimal.Decimal(debt_record[5])*100)
print debt_amount
except:
continue
def run():
datasheetPath = os.path.join(os.path.dirname(__file__), 'data/certificate-data-04.02.2016.csv')
dataset = tablib.import_set(open(datasheetPath, encoding='utf-8').read())
StudentEvaluationResource().import_data(dataset)
print('Done!')
import tablib
import openpyxl
import xlrd
f = open('C:/Users/Administrator/Desktop/data.xlsx', mode='rb')
data = tablib.import_set(f.read(), 'xlsx')
f.close()
print(data[0:3])
print(data["url"])
title = item[title_column] if title_column in item else "Untitled"
if identifier_column in item and original_id in mapping.id_to_omeka_id:
previous_id = mapping.id_to_omeka_id[original_id]
upload(previous_id, original_id, jsonstr, title, URLs, files, iterations)
mapdata = []
id_sheet = tablib.import_set(mapping.id_to_omeka_id)
mapdata.append({'title': 'Omeka Mapping', 'data': mapping.supplied_element_names})
mapdata.append({'title': 'ID Mapping', 'data': id_mapping})
mapdata.append({'title': 'Downloads', 'data': mapping.downloads})
mapdata.append({'title': 'Multiple Uploads', 'data': mapping.multiples})
new_book = tablib.Databook()
new_book.yaml = yaml.dump(mapdata)
with open(mapfile,"wb") as f:
f.write(new_book.xlsx)
logger.info("Finished")
