def handle(self, *args, **options): peklun = User.objects.get(username="******") for root, dirs, files in os.walk(options["input_dir"]): for f in files: if not os.path.isfile(os.path.join(root, f)): continue fname, ext = os.path.splitext(f) human_name = re.sub(r"[\s_]+", " ", fname.decode("utf-8")) doc_san_name = f.decode("utf-8") with open(os.path.join(root, f), "rb") as fp: doc_content = fp.read() doc_hash = sha1(f).hexdigest() try: doc_instance = Document.objects.get(hash=doc_hash) self.stderr.write('Skipping file {}'.format(doc_san_name)) except Document.DoesNotExist: self.stdout.write('Adding file {}'.format(doc_san_name)) if doc_san_name: doc_instance = Document(name_uk=human_name, uploader=peklun, hash=doc_hash) doc_instance.doc.save(doc_san_name, ContentFile(doc_content)) doc_instance.save()
def form_valid(self, form): ''' :returns: *"Media"* is assigned to an external URL if it's filled, otherwise it points to the uploaded file. ''' try: instance = Document(file_upload=self.request.FILES['upload_video']) instance.save() if not form.instance.media: form.instance.media = instance.accessURL except: pass return super(OverdubUpdateView, self).form_valid(form)
def form_valid(self, form): # Auto set the following fields: form.instance.collection = get_object_or_404(ActivityCollection, pk=self.kwargs['pk']) form.instance.activity_type = self.activity_type try: instance = Document(file_upload=self.request.FILES['upload_video']) instance.save() if not form.instance.media: form.instance.media = instance.accessURL except: pass return super(OverdubCreateView, self).form_valid(form)
def createDocument(self, request): name = request.POST.get('name', None) if name: s = Document(name=name) s.save() result = { 'status': 'success', 'data': 'Document added successfully' } else: result = { 'status': 'failure', 'error': "Please enter the document name" } return result
def handle(self, *args, **opt): if opt['verbosity'] == 0: logging.basicConfig(level=logging.CRITICAL) elif opt['verbosity'] == 1: logging.basicConfig(level=logging.ERROR) elif opt['verbosity'] == 2: logging.basicConfig(level=logging.INFO) elif opt['verbosity'] == 3: logging.basicConfig(level=logging.DEBUG) if opt['poName'] == None: BaseName = os.path.splitext(os.path.basename(opt['file']))[0] else: BaseName = opt['poName'] PoFile = polib.pofile(opt['file']) docObj = None if Document.objects.filter(Name__exact=BaseName).count( ) > 0: # Check whether this doc has been inserted or not docObj = Document.objects.get(Name__exact=BaseName) if not opt['isUpdate']: logging.warning("PO文件已存在,跳过添加") sys.exit(0) FileObj = File(open(opt['file'], 'r'), name=BaseName) if docObj == None: # Construct a new doc obj docObj = Document() docObj.Name = BaseName docObj.PoFile = FileObj docObj.save() # Update or Insert if opt['isUpdate']: queryset = POEntry.objects.filter(Doc__Name__exact=docObj.Name) logging.info("删除原有条目(total:%d)" % (len(queryset))) queryset.delete() total = len(PoFile) idx = 0 with transaction.atomic(): for entry in PoFile: idx += 1 logging.debug("添加翻译条目(%d/%d)" % (idx, total)) POEntry(Raw=pickle.dumps(entry), Doc=docObj, Translated=entry.translated(), Msgid=entry.msgid, Msgstr=entry.msgstr).save()
def setUpDatebase(): Document = cm.Document if Document.objects.all().count() == 0: samplePo = tempfile.NamedTemporaryFile() with open("./core/testUtils/sample.po") as fp: samplePo.write(bytes(fp.read(), 'utf-8')) samplePo.seek(0) FileObj = File(samplePo, name='cvpcb') docobj = Document(Name='cvpcb', PoFile=FileObj) docobj.save() PoFile: File = docobj.PoFile fileName = samplePo.name PoFile: polib.POFile = polib.pofile(fileName) for entry in PoFile: POEntry(Raw=pickle.dumps(entry), Doc=docobj, Translated=entry.translated(), Msgid=entry.msgid, Msgstr=entry.msgstr).save() return PoFile else: Document = Document.objects.all()[0] return polib.pofile(Document.PoFile.file.name)
def handle(self, *args, **options): Document.objects.all().delete() documents = [] for file in glob.glob(options["files"]): json_dos = json.load(open(file))['document'] doc = Document( xsi_type=json_dos['@xsi:type'], uid=json_dos['uid'], legislature=json_dos.get('legislature'), cycleDeVie_chrono_dateCreation=json_dos['cycleDeVie']['chrono'] ['dateCreation'], cycleDeVie_chrono_dateDepot=json_dos['cycleDeVie']['chrono'] ['dateDepot'], cycleDeVie_chrono_datePublication=json_dos['cycleDeVie'] ['chrono']['datePublication'], cycleDeVie_chrono_datePublicationWeb=json_dos['cycleDeVie'] ['chrono']['datePublicationWeb'], denominationStructurelle=json_dos['denominationStructurelle'], provenance=json_dos.get('provenance'), titres_titrePrincipal=json_dos['titres']['titrePrincipal'], titres_titrePrincipalCourt=json_dos['titres'] ['titrePrincipalCourt'], dossierRef=json_dos['dossierRef'], classification_famille_depot_code=json_dos['classification'] ['famille']['depot']['code'], classification_famille_depot_libelle=json_dos['classification'] ['famille']['depot']['libelle'], classification_famille_classe_code=json_dos['classification'] ['famille']['classe']['code'], classification_famille_classe_libelle=json_dos[ 'classification']['famille']['classe']['libelle'], classification_famille_espece_code=json_dos['classification'] ['famille'].get('espece', {}).get('code'), classification_famille_espece_libelle=json_dos[ 'classification']['famille'].get('espece', {}).get('libelle'), classification_type_code=json_dos['classification']['type'] ['code'], classification_type_libelle=json_dos['classification']['type'] ['libelle'], classification_sousType_code=json_dos['classification'] ['sousType']['code'] if json_dos['classification']['sousType'] else None, classification_sousType_libelle=json_dos['classification'] ['sousType'].get('libelle') if json_dos['classification']['sousType'] else None, classification_sousType_libelleEdition=json_dos[ 'classification']['sousType'].get('libelleEdition') if json_dos['classification']['sousType'] else None, classification_statutAdoption=json_dos['classification'] ['statutAdoption'], correction=json_dos['correction'], notice_numNotice=json_dos['notice'].get('numNotice'), notice_formule=json_dos['notice'].get('formule'), notice_adoptionConforme=json_dos['notice']['adoptionConforme'], indexation=json_dos['indexation'], imprimerie_ISSN=json_dos['imprimerie'].get('ISSN') if json_dos['imprimerie'] else None, imprimerie_ISBN=json_dos['imprimerie'].get('ISBN') if json_dos['imprimerie'] else None, imprimerie_DIAN=json_dos['imprimerie'].get('DIAN') if json_dos['imprimerie'] else None, imprimerie_nbPage=json_dos['imprimerie'].get('nbPage') if json_dos['imprimerie'] else None, imprimerie_prix=json_dos['imprimerie']['prix'] if json_dos['imprimerie'] else None, ) documents.append(doc) print("creating", len(documents), "documents") Document.objects.bulk_create(documents)
def handle(self, *args, **options): peklun = User.objects.get(username="******") wks = get_spreadsheet().sheet1 for i, l in enumerate(wks.get_all_records()): # reopen it time from time to avoid disconnect by timeout if i % 2000 == 0 and i: wks = get_spreadsheet().sheet1 self.stdout.write('Processing line #{}'.format(i)) company_ipn = l.get("ІПН", "") company_name = l.get("Назва", "") person_id = l.get("id персони", "") company_id = l.get("id компанії", "") photo_url = l.get("Фото", "") person = None # First let's search for appropriate company company = self.process_company(company_id, company_ipn, company_name) # No company — no go if company is None: continue # Let's backwrite company id to the spreadsheet for further use if company.pk != company_id: company_id = company.pk wks.update_cell(i + 2, len(l.keys()), company.pk) person_name = l.get("ПІБ", "").strip() position = l.get("Посада", "").strip() person_dob = unicode(l.get("Дата народження", "")).strip() person_from = parse_date(l.get("Дата призначення", "")) person_to = parse_date(l.get("Дата звільнення", "")) doc_received = parse_date(l.get("Дата відповіді", "")) docs = l.get("Лінк на відповідь", "").strip() website = l.get("лінк на сайт", "").strip() # Now let's search for the person if person_name: last_name, first_name, patronymic, _ = parse_fullname( person_name) if not last_name: continue # First we search by person_id (if it's present) if person_id: try: person = Person.objects.get(pk=person_id) except Person.DoesNotExist: pass # If nothing is found we search for name (for now) if not person: try: person = Person.objects.get( first_name_uk__iexact=first_name, last_name_uk__iexact=last_name, patronymic_uk__iexact=patronymic) except Person.MultipleObjectsReturned: self.stderr.write( "Double person {}!".format(person_name)) except Person.DoesNotExist: pass # If nothing is found, let's create a record for that person if not person: person = Person() self.stderr.write( "Created new person {}".format(person_name)) person.first_name_uk = first_name person.last_name_uk = last_name person.patronymic_uk = patronymic Ua2RuDictionary.objects.get_or_create(term=first_name) Ua2RuDictionary.objects.get_or_create(term=last_name) Ua2RuDictionary.objects.get_or_create(term=patronymic) person.first_name_en = translitua(first_name) person.last_name_en = translitua(last_name) person.patronymic_en = translitua(patronymic) person.is_pep = True person.imported = True person.type_of_official = 1 # Parsing date (can be a full date or just a year or # year/month) if person_dob: person.dob = parse_date(person_dob) if len(person_dob) == 4: person.dob_details = 2 # Only year if len(person_dob) > 4 and len(person_dob) < 7: person.dob_details = 1 # month and year # Let's download the photo (if any) if not person.photo and photo_url: photo_name, photo_san_name, photo_content = download( photo_url, translitua(person_name)) if photo_name: person.photo.save(photo_san_name, ContentFile(photo_content)) else: self.stdout.write("Cannot download image %s for %s" % (photo_url, person_name)) person.save() # Let's write the person id back to the table. if person.pk != person_id: person_id = person.pk wks.update_cell(i + 2, len(l.keys()) - 1, person.pk) # Now let's download all supporting docs docs_downloaded = [] first_doc_name = False # There might be many of them for doc in docs.split(", "): doc_instance = None # we cannot download folders from google docs, so let's # skip them if doc and "folderview" not in doc \ and "drive/#folders" not in doc: doc = expand_gdrive_download_url(doc) doc_hash = sha1(doc).hexdigest() # Check, if docs try: doc_instance = Document.objects.get(hash=doc_hash) except Document.DoesNotExist: self.stdout.write( 'Downloading file {}'.format(doc)) doc_name, doc_san_name, doc_content = download(doc) doc_san_name = translitua(doc_san_name) if doc_name: doc_instance = Document(name_uk=doc_name, uploader=peklun, hash=doc_hash) doc_instance.doc.save(doc_san_name, ContentFile(doc_content)) doc_instance.save() else: self.stdout.write( 'Cannot download file {}'.format(doc)) if doc_instance: first_doc_name = doc_instance.name_uk docs_downloaded.append(doc_instance.doc.url) # Now let's setup links between person and companies links = Person2Company.objects.filter( (Q(date_established=person_from) | Q(date_established=mangle_date(person_from)) | Q(date_established__isnull=True)), (Q(date_finished=person_to) | Q(date_finished=mangle_date(person_to)) | Q(date_finished__isnull=True)), from_person=person, to_company=company) # Delete if there are doubling links # including those cases when dates were imported incorrectly # because of parse_date if len(links) > 1: links.delete() link, _ = Person2Company.objects.update_or_create( from_person=person, to_company=company, date_established=person_from, date_established_details=0, date_finished=person_to, date_finished_details=0) if not link.relationship_type: link.relationship_type = position # And translate them Ua2EnDictionary.objects.get_or_create( term=lookup_term(position)) # oh, and add links to supporting docs all_docs = docs_downloaded + website.split(", ") if all_docs: link.proof = ", ".join(filter(None, all_docs)) if first_doc_name: link.proof_title = first_doc_name link.date_confirmed = doc_received link.is_employee = True link.save()
def zip_import(self, request): if request.method == "GET": return render( request, "admin/core/document/zip_import.html", {"form": ZIPImportForm()}, ) if request.method == "POST": form = ZIPImportForm(request.POST, request.FILES) if not form.is_valid(): return render(request, "admin/core/document/zip_import.html", {"form": form}) with ZipFile(request.FILES["zipfile"]) as zip_arch: for finfo in zip_arch.infolist(): fullname = finfo.filename if fullname.endswith("/"): continue _, fname = os.path.split(fullname) if fname.startswith("._"): continue if finfo.file_size < 2048: continue try: dec_fname = unicode(fname) except UnicodeDecodeError: dec_fname = fname.decode("cp866") fname, ext = os.path.splitext(dec_fname) doc_san_name = fname.decode("utf-8") human_name = re.sub(r"[\s_]+", " ", doc_san_name) doc_san_name = "{}.{}".format(doc_san_name, ext) with zip_arch.open(fullname, "r") as fp: doc_content = fp.read() doc_hash = sha1(doc_content).hexdigest() try: doc_instance = Document.objects.get(hash=doc_hash) self.message_user( request, "Skipping file {} as it already exists".format( doc_san_name), level=messages.WARNING, ) except Document.DoesNotExist: self.message_user( request, "Adding file {}".format(doc_san_name)) if doc_san_name: doc_instance = Document(name_uk=human_name, uploader=request.user, hash=doc_hash) doc_instance.doc.save(doc_san_name, ContentFile(doc_content)) doc_instance.guess_doc_type() return redirect(reverse("admin:core_document_changelist"))
def savedocument(request): print(request) print("POST DATA") print(request.POST) print("FILES DATA") print(request.FILES) post_data = request.POST request.FILES.get('filepath') # doc = request.FILES['docs'] # thumbnail = request.FILES['thumb'] doc = request.FILES.get('docs') thumbnail = request.FILES.get('thumb') if doc and thumbnail: document = Document( title = post_data['title'], summary = post_data['summary'], description = post_data['description'], category = post_data['category'], doctype = post_data['type'], author = post_data['author'], document = request.FILES['docs'], thumbnail = request.FILES['thumb'], ) document.save() elif doc: document = Document( title = post_data['title'], summary = post_data['summary'], description = post_data['description'], category = post_data['category'], doctype = post_data['type'], author = post_data['author'], document = request.FILES['docs'], ) document.save() elif thumbnail: document = Document( title = post_data['title'], summary = post_data['summary'], description = post_data['description'], category = post_data['category'], doctype = post_data['type'], author = post_data['author'], thumbnail = request.FILES['thumb'], ) document.save() else: document = Document( title = post_data['title'], summary = post_data['summary'], description = post_data['description'], category = post_data['category'], doctype = post_data['type'], author = post_data['author'], ) document.save() return redirect('dashboard')