def __init__(self, argv):
self.gus = GUS(api_key=environ('GUSREGON_API_KEY'),
sandbox=environ('GUSREGON_SANDBOX', True))
self.s = requests.Session()
self.argv = argv
self.args = self.get_build_args(argv[1:])
self.s.auth = (self.args.user, self.args.password)
def handle(self, no_progress, verbose, comment, *args, **options):
gus = GUS(api_key=settings.GUSREGON_API_KEY,
sandbox=settings.GUSREGON_SANDBOX)
if settings.GUSREGON_SANDBOX is True:
self.stderr.write(
"You are using sandbox mode for the REGON database. Data may be incorrect. "
"Set the environemnt variable GUSREGON_SANDBOX and GUSREGON_API_KEY correctly."
)
self.inserted, self.updated, self.errored, self.skipped = 0, 0, 0, 0
qs = self.get_queryset()
for regon in self.get_iter(qs, no_progress):
with transaction.atomic() and reversion.create_revision():
data = gus.search(regon=regon.data['regon14'],
report_type='PublDaneRaportLokalnePrawnej',
list=True)
for item in data:
# item_data = gus.search(regon=item['regon14'])
item_data = item
try:
obj = REGON.objects.get(regon=item['regon14'])
if obj.data != item_data:
obj.data = item_data
obj.save()
self.updated += 1
else:
self.skipped += 1
except REGON.DoesNotExist:
REGON.objects.create(regon=item['regon14'],
data=item_data)
self.inserted += 1
self.stdout.write(
"There is {} courts, which {} skipped, {} updated and {} inserted."
.format(self.updated + self.inserted + self.skipped, self.skipped,
self.updated, self.inserted))
def __init__(self, argv):
self.gus = GUS(
api_key=environ("GUSREGON_API_KEY"),
sandbox=environ("GUSREGON_SANDBOX", True),
)
self.s = requests.Session()
self.argv = argv
self.args = self.get_build_args(argv[1:])
self.s.auth = (self.args.user, self.args.password)
self.done_count = 0
self.total_count = None
def handle(self, krs, regon, nip, comment, google, no_progress, no_regon,
no_nip, *args, **options):
gus = GUS(api_key=settings.GUSREGON_API_KEY,
sandbox=settings.GUSREGON_SANDBOX)
if settings.GUSREGON_SANDBOX is True:
self.stderr.write(
"You are using sandbox mode for the REGON database. Data may be incorrect. "
"Set the environemnt variable GUSREGON_SANDBOX and GUSREGON_API_KEY correctly."
)
self.inserted, self.updated, self.errored, self.skipped = 0, 0, 0, 0
if not any([krs, regon, nip, google]):
raise CommandError(
"Provide at least one '--krs' or '--regon' or '--nip' or '--google' is required. "
)
queries = self.get_queryset(krs, nip, regon, google, no_regon, no_nip)
with transaction.atomic() and reversion.create_revision():
reversion.set_comment(comment)
for query in self.get_iter(queries, no_progress):
data = gus.search(**query)
if data:
regon_id = query.get(
'regon', data.get('regon14', data.get('regon9')))
try:
regon = REGON.objects.regon(regon_id)
if regon.data != data:
regon.data = data
regon.save()
self.updated += 1
else:
self.skipped += 1
except REGON.DoesNotExist:
regon = REGON(regon=regon_id, data=data)
self.stdout.write("Added {}".format(
regon.data.get('nazwa', '')))
regon.save()
self.inserted += 1
else:
self.stderr.write("Unable to find {}".format(query))
self.errored += 1
total = self.inserted + self.updated + self.errored
self.stdout.write(
("There is {} REGON changed, which "
"{} updated, "
"{} skipped, "
"{} inserted and "
"{} errored.").format(total, self.updated, self.skipped,
self.inserted, self.errored))
def handle(self, comment, institutions_id, update, no_progress, *args, **options):
gus = GUS(api_key=settings.GUSREGON_API_KEY, sandbox=settings.GUSREGON_SANDBOX)
if settings.GUSREGON_SANDBOX is True:
self.stderr.write("You are using sandbox mode for the REGON database. Data may be incorrect. "
"Set the environemnt variable GUSREGON_SANDBOX and GUSREGON_API_KEY correctly.")
inserted, updated, errored, skipped = 0, 0, 0, 0
qs = self.get_queryset(update, institutions_id)
for institution in self.get_iter(qs, no_progress):
with transaction.atomic() and reversion.create_revision():
try:
data = gus.search(regon=normalize_regon(institution.regon))
except TypeError as e:
regon_obj = REGON.objects.create(institution=institution, regon=institution.regon)
REGONError.objects.create(regon=regon_obj, exception=repr(e))
errored += 1
self.stderr.write("Errored for {} in {} as {}".format(institution.regon, institution.pk, repr(e)))
continue
if hasattr(institution, 'regon_data'):
if institution.regon_data.data != data:
institution.regon_data.regon = institution.regon
institution.regon_data.data = data
institution.regon_data.save()
updated += 1
else:
skipped += 1
else:
REGON.objects.create(institution=institution,
regon=institution.regon,
data=data)
inserted += 1
reversion.set_comment(comment)
time.sleep(2)
total = updated + inserted + errored + skipped
self.stdout.write(("There is {} REGON changed, which "
"{} updated, "
"{} skipped, "
"{} inserted and "
"{} errored.").format(total, updated, skipped, inserted, errored))
class Command(object):
REQUIRED_FIELDS = [
'name', 'email', 'regon', 'terc', 'regon_parent', 'tags'
]
def __init__(self, argv):
self.gus = GUS(api_key=environ('GUSREGON_API_KEY'),
sandbox=environ('GUSREGON_SANDBOX', True))
self.s = requests.Session()
self.argv = argv
self.args = self.get_build_args(argv[1:])
self.s.auth = (self.args.user, self.args.password)
def get_build_args(self, argv):
parser = argparse.ArgumentParser()
parser.add_argument('--input',
required=True,
type=argparse.FileType('r'),
help="Input CSV-file")
parser.add_argument('--host', required=True, help="Host of instance")
parser.add_argument('--user',
required=True,
help="User to authentication")
parser.add_argument('--password',
required=True,
help="Password to authentication")
return parser.parse_args(argv)
def _match(self, host, **query):
response = self.s.get(url=urljoin(host, "/api/institutions/"),
params=query)
data = response.json()
return data['results'][0]['pk'] if data['results'] else None
def _find(self, host, **query):
response = self.s.get(url=urljoin(host, "/api/institutions/"),
params=query)
data = response.json()
if not data['results'] or len(data['results']) != 1:
import ipdb
ipdb.set_trace()
return data['results'][0]['pk']
def insert_row(self, host, name, email, terc, regon, tags, regon_parent,
**extra):
data = {
"name": name,
"tags": tags.split(','),
"jst": normalize_jst(terc),
"email": email,
"regon": regon,
"extra": {
'regon': self.gus.search(regon=regon)
}
}
if regon_parent:
data.update(
{"parents_ids": [self._find(host, regon=regon_parent)]})
pk = self._match(host, regon=regon)
if pk:
data.update({"id": pk})
response = self.s.patch(url=urljoin(
urljoin(host, "/api/institutions/"),
str(pk) + "/"),
json=data)
else:
response = self.s.post(url=urljoin(host, "/api/institutions/"),
json=data)
if response.status_code == 500:
print(name.encode('utf-8'),
" response 500",
response.status_code,
":",
file=sys.stderr)
print(response.text, file=sys.stderr)
return
json = response.json()
if response.status_code == 201:
print(name.encode('utf-8'), " created as PK", json['pk'])
elif response.status_code == 200:
print(name.encode('utf-8'), " updated as PK", json['pk'])
else:
print(name.encode('utf-8'),
"response ",
response.status_code,
":",
file=sys.stderr)
print(json, file=sys.stderr)
def fields_validation(self, fields):
result = True
for field_name in set(self.REQUIRED_FIELDS) - set(fields):
print("There is missing %s field. " % (field_name, ) +
"Required fields name is {}".format(self.REQUIRED_FIELDS))
result = False
return result
def run(self):
reader = csv.DictReader(self.args.input)
if not self.fields_validation(reader.fieldnames):
print("Script stop, due previos erros.")
return 5
data = list(reader)
with trange(len(data), leave=True) as t:
for row in data:
self.insert_row(host=self.args.host, **row)
t.set_description(row['name'])
t.update(1)
def test_different_endpoints():
gus = GUS(sandbox=True)
assert gus.service._binding_options.get('address') == ENDPOINT_SANDBOX
gus = GUS(api_key='my_api_key')
assert gus.service._binding_options.get('address') == ENDPOINT
def test_required_api_key():
with pytest.raises(AttributeError):
GUS()
def client():
return GUS(sandbox=True)
class Command:
REQUIRED_FIELDS = ["name", "email", "tags"]
def __init__(self, argv):
self.gus = GUS(
api_key=environ("GUSREGON_API_KEY"),
sandbox=environ("GUSREGON_SANDBOX", True),
)
self.s = requests.Session()
self.argv = argv
self.args = self.get_build_args(argv[1:])
self.s.auth = (self.args.user, self.args.password)
self.done_count = 0
self.total_count = None
def get_build_args(self, argv):
parser = argparse.ArgumentParser()
parser.add_argument(
"--input", required=True, type=argparse.FileType("r"), help="Input CSV-file"
)
parser.add_argument("--host", required=True, help="Host of instance")
parser.add_argument("--user", required=True, help="User to authentication")
parser.add_argument(
"--password", required=True, help="Password to authentication"
)
return parser.parse_args(argv)
def _match(self, host, **query):
response = self.s.get(url=urljoin(host, "/api/institutions/"), params=query)
data = response.json()
return data["results"][0]["pk"] if data["results"] else None
def _find(self, host, **query):
response = self.s.get(url=urljoin(host, "/api/institutions/"), params=query)
data = response.json()
if not data.get("results") or len(data["results"]) != 1:
import ipdb
ipdb.set_trace()
return data["results"][0]["pk"]
def insert_row( # noqa: C901
self, host, name, email, tags, regon=None, regon_parent=None, **extra
):
regon_data = None
if regon:
regon = regon.strip(">").strip("*").strip()
regon_data = self.gus.search(regon=regon)
if not regon_data:
raise Exception("Invalid regon data", regon, name)
data = {"name": name, "tags": tags.split(","), "email": email}
if regon:
data.update({"regon": regon})
terc = None
if "terc" in extra:
terc = extra.get("terc")
data.update({"jst": normalize_jst(terc), "extra": {"regon": regon_data}})
if not terc and regon_data:
terc = "".join(
[
regon_data["adsiedzwojewodztwo_symbol"],
regon_data["adsiedzpowiat_symbol"],
regon_data["adsiedzgmina_symbol"],
]
)
if not terc:
raise Exception("Missing terc and unable to found", name)
data.update({"jst": normalize_jst(terc)})
if regon_data:
data.update({"extra": {"regon": regon_data}})
if regon_parent:
data.update({"parents_ids": [self._find(host, regon=regon_parent)]})
if regon:
pk = self._match(host, regon=regon)
if pk:
data.update({"id": pk})
response = self.s.patch(
url=urljoin(urljoin(host, "/api/institutions/"), str(pk) + "/"),
json=data,
)
else:
response = self.s.post(
url=urljoin(host, "/api/institutions/"), json=data
)
else:
response = self.s.post(url=urljoin(host, "/api/institutions/"), json=data)
if response.status_code >= 300:
print(
name.encode("utf-8"),
" response 500",
response.status_code,
":",
file=sys.stderr,
)
print(response.text, file=sys.stderr)
return
json = response.json()
self.done_count += 1
progress = (self.done_count / self.total_count) * 100
if response.status_code == 201:
print(progress, name.encode("utf-8"), " created as PK", json["pk"])
elif response.status_code == 200:
print(progress, name.encode("utf-8"), " updated as PK", json["pk"])
else:
print(
progress,
name.encode("utf-8"),
"response ",
response.status_code,
":",
file=sys.stderr,
)
print(json, file=sys.stderr)
def fields_validation(self, fields):
result = True
for field_name in set(self.REQUIRED_FIELDS) - set(fields):
print(
"There is missing {} field. ".format(field_name)
+ "Required fields name is {}".format(self.REQUIRED_FIELDS)
)
result = False
return result
def run(self):
reader = csv.DictReader(self.args.input)
if not self.fields_validation(reader.fieldnames):
print("Script stop, due previos erros.")
return 5
data = list(reader)
self.total_count = len(data)
for row in data:
self.insert_row(host=self.args.host, **row)
class Command(object):
JST_VOIVODESHIP_KEYS = [
'adsiedzwojewodztwo_symbol', 'adkorwojewodztwo_symbol'
]
JST_COUNTY_KEYS = [
'adsiedzpowiat_symbol',
'adkorpowiat_symbol',
]
JST_COMMUNITY_KEYS = [
"adsiedzgmina_symbol",
'adkorgmina_symbol',
]
REQUIRED_FIELDS = [
'parent__name', 'name', 'district', 'appeal', 'street', 'postcode',
'phone', 'fax', 'email', 'active', 'regon'
]
def __init__(self, argv):
self.gus = GUS(api_key=os.environ('GUSREGON_API_KEY'),
sandbox=os.environ('GUSREGON_SANDBOX', True))
self.s = requests.Session()
self.argv = argv
self.args = self.get_build_args(argv[1:])
self.s.auth = (self.args.user, self.args.password)
def get_build_args(self, argv):
parser = argparse.ArgumentParser()
parser.add_argument('--input',
required=True,
type=argparse.FileType('r'),
help="Input CSV-file")
parser.add_argument('--host', required=True, help="Host of instance")
parser.add_argument('--user',
required=True,
help="User to authentication")
parser.add_argument('--password',
required=True,
help="Password to authentication")
return parser.parse_args(argv)
def _get_jst_id(self, data):
for a, b, c in zip(self.JST_VOIVODESHIP_KEYS, self.JST_COUNTY_KEYS,
self.JST_COMMUNITY_KEYS):
if a in data and b in data and c in data:
value = data[a] + data[b] + data[c]
if value:
return value
def _get_court(self, host, name):
response = self.s.get(url=urljoin(host, "/api/institutions/"),
params={
'name': name
}).json()
if not response['results']:
import ipdb
ipdb.set_trace()
return response['results'][0]['pk']
def insert_row(self, host, name, email, regon, **extra):
data = self.gus.search(regon=regon)
if not data:
print("Unable find REGON {} for {}".format(regon, name),
file=sys.stderr)
return
tags = []
for keyword in ['Sąd Apelacyjny', 'Sąd Okręgowy', 'Sąd Rejonowy']:
if keyword in name:
tags = ['Sąd', keyword]
data = {
"name": name,
"tags": tags,
"jst": normalize_jst(self._get_jst_id(data)),
"email": email,
"extra": {
'db': extra,
'regon': data
},
"regon": regon
}
response = self.s.post(url=urljoin(host, "/api/institutions/"),
json=data)
if response.status_code == 500:
print(name.encode('utf-8'),
" response 500",
response.status_code,
":",
file=sys.stderr)
print(response.text, file=sys.stderr)
return
json = response.json()
if response.status_code == 400 and json.keys() == ['regon']:
court = self._get_court(host, name)
parent_court = self._get_court(host, extra['parent__name'])
url = urljoin(host, "/api/institutions/{}/".format(court))
data['parents_ids'] = [parent_court]
self.s.patch(url, json=data)
if response.status_code == 500:
print(name.encode('utf-8'),
" response 500",
response.status_code,
":",
file=sys.stderr)
print(response.text, file=sys.stderr)
return
if response.status_code == 201:
print(name.encode('utf-8'), " created as PK", json['pk'])
else:
print(name.encode('utf-8'),
"response ",
response.status_code,
":",
file=sys.stderr)
print(json, file=sys.stderr)
def update_parent(self, host, parent__name, name, **extra):
court = self._get_court(host, name)
parent_court = self._get_court(host, parent__name)
if not parent_court:
print(u"Unable to find parent court for {}".format(name))
return
url = urljoin(host, "/api/institutions/{}/".format(court))
params = {'parents_ids': [parent_court]}
def fields_validation(self, fields):
result = True
for field_name in set(self.REQUIRED_FIELDS) - set(fields):
print("There is missing %s field. " % (field_name, ) +
"Required fields name is {}".format(self.REQUIRED_FIELDS))
result = False
return result
def run(self):
reader = csv.DictReader(self.args.input)
if not self.fields_validation(reader.fieldnames):
print("Script stop, due previos erros.")
return
data = list(reader)
with trange(len(data) * 2, leave=True) as t:
for row in data:
self.insert_row(host=self.args.host, **row)
t.set_description(row['name'])
t.update(1)
for row in data:
t.set_description(row['name'])
self.update_parent(host=self.args.host, **row)
t.update(1)