def upload_file(request): if request.method == 'POST': form = UploadFileForm(request.POST, request.FILES) if form.is_valid(): if validate_upload_file(request.FILES['file']) and request.FILES['file'].size <= settings.MAX_ARCHIVE_FILE_SIZE: link = Link(submitted_url=form.cleaned_data['url'], submitted_title=form.cleaned_data['title']) if request.user.is_authenticated(): link.created_by = request.user link.save() now = dt = datetime.now() time_tuple = now.timetuple() path_elements = [str(time_tuple.tm_year), str(time_tuple.tm_mon), str(time_tuple.tm_mday), str(time_tuple.tm_hour), str(time_tuple.tm_min), link.guid] linky_home_disk_path = settings.GENERATED_ASSETS_STORAGE + '/' + os.path.sep.join(path_elements) if not os.path.exists(linky_home_disk_path): os.makedirs(linky_home_disk_path) asset, created = Asset.objects.get_or_create(link=link) asset.base_storage_path = os.path.sep.join(path_elements) asset.save() file_name = '/cap.' + request.FILES['file'].name.split('.')[-1] if request.FILES['file'].name.split('.')[-1] == 'pdf': asset.pdf_capture = file_name else: asset.image_capture = file_name #print linky_home_disk_path + file_name #png = PythonMagick.Image(linky_home_disk_path + file_name) #png.write("file_out.png") #params = ['convert', linky_home_disk_path + file_name, 'out.png'] #subprocess.check_call(params) asset.save() request.FILES['file'].file.seek(0) f = open(linky_home_disk_path + file_name, 'w') f.write(request.FILES['file'].file.read()) os.fsync(f) f.close() response_object = {'status':'success', 'linky_id':link.guid, 'linky_hash':link.guid} """try: get_source.delay(link.guid, target_url, os.path.sep.join(path_elements), request.META['HTTP_USER_AGENT']) store_text_cap.delay(target_url, target_title, asset) except Exception, e: # TODO: Log the failed url asset.pdf_capture = 'failed' asset.save()""" return HttpResponse(json.dumps(response_object), 'application/json') else: return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'Invalid file.'}), 'application/json') else: return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'Missing file.'}), 'application/json')
def update_perma(link_guid): """ Update the vested/darchived status of a perma link, and download the assets if necessary """ # N.B. This function has two instances of downloading stuff from # the root server using a scheme that looks something like # settings.SERVER + reverse("url_pattern") # This is nice because it means we don't have to repeat our URL # patterns from urls.py, but it hardcodes the fact that the root # server is another Perma instance. It's unclear to me which is a # better fact to abstract, but this is easier for now. ## First, let's get the metadata for this link. The metadata ## contains information about where we should place the assets (if ## we decide that we need them). This is also a fast check to make ## sure the link GUID is actually real. metadata_server = settings.UPSTREAM_SERVER['address'] metadata_url = metadata_server + reverse("service_link_status", args=(link_guid,)) metadata = requests.get( metadata_url, headers=settings.UPSTREAM_SERVER.get('headers', {}) ).json() ## Next, let's see if we need to get the assets. If we have the ## Link object for this GUID, we're going to assume we already ## have what we need. It would make a little more sense to use the ## Asset object here instead, but we're definitely going to need ## to do stuff to the Link object so we might as well get that ## instead. In practice they should be ~one to one. try: link = Link.objects.get(guid=link_guid) except Link.DoesNotExist: ## We need to download the assets. We can download an archive ## from the assets server. assets_server = settings.UPSTREAM_SERVER['address'] assets_url = assets_server + reverse("mirroring:link_assets", args=(link_guid,)) # Temp paths can be relative because we're in run_in_tempdir() temp_zip_path = 'temp.zip' # Save remote zip file to disk, using streaming to avoid keeping large files in RAM. request = requests.get( assets_url, headers=settings.UPSTREAM_SERVER.get('headers', {}), stream=True) with open(temp_zip_path, 'wb') as f: for chunk in request.iter_content(1024): f.write(chunk) ## Extract the archive and change into the extracted folder. with zipfile.ZipFile(temp_zip_path, "r") as zipfh: #assets_path = os.path.dirname(os.path.join(settings.MEDIA_ROOT, metadata["path"])) zipfh.extractall() # creates folder named [guid] in current temp dir temp_extracted_path = os.path.basename(metadata['path']) # e.g. "1234-ABCD" # Save all extracted files to default_storage, using the path in metadata. for root, dirs, files in os.walk(temp_extracted_path): for file in files: source_file_path = os.path.join(root, file) # e.g. "1234-ABCD/cap.png" dest_file_path = os.path.join(os.path.dirname(metadata['path']), source_file_path) # e.g. 2014/6/10/18/37/1234-ABCD/cap.png with open(source_file_path, 'rb') as source_file: default_storage.store_file(source_file, dest_file_path) ## We can now get some additional metadata that we'll need to ## create the Link object. with open(os.path.join(temp_extracted_path, "metadata.json"), "r") as fh: link_metadata = json.load(fh) ## We now have everything we need to initialize the Link object. link = Link(guid=link_guid) link.submitted_url = link_metadata["submitted_url"] link.submitted_title = link_metadata["submitted_title"] link.created_by = None # XXX maybe we should do something with FakeUser here link.save(pregenerated_guid=True) # We need to save this so that we can create an Asset object # This is a stupid hack to overcome the fact that the Link has # auto_now_add=True, so it's always going to be saved to the # current time on first creation. link.creation_timestamp = unserialize_datetime(link_metadata["creation_timestamp"]) link.save() ## Lastly, let's create an Asset object for this Link. asset = Asset(link=link) asset.base_storage_path = metadata["path"] asset.image_capture = metadata["image_capture"] asset.warc_capture = metadata["source_capture"] asset.pdf_capture = metadata["pdf_capture"] asset.text_capture = metadata["text_capture"] asset.save() ## We can now add some of the data we got from the metadata to the Link object link.dark_archived = metadata["dark_archived"] link.vested = metadata["vested"] link.save() # If we have sub-mirrors, poke them to get a copy from us. if settings.DOWNSTREAM_SERVERS: run_task(poke_mirrors, link_guid=link_guid)
# Get the markup. We get the mime-type and the title from this. try: r = requests.get(target_url) parsed_html = lxml.html.fromstring(r.content) except IOError: logger.debug("Title capture from markup failed for %s, using the hostname" % target_url) if len(parsed_html): if parsed_html.find(".//title") is not None and parsed_html.find(".//title").text: target_title = parsed_html.find(".//title").text.strip() # We have some markup and a title. Let's create a linky from it link = Link(submitted_url=target_url, submitted_title=target_title) if request.user.is_authenticated(): link.created_by = request.user link.save() # Assets get stored in /storage/path/year/month/day/hour/unique-id/* # Get that path that we'll pass off to our workers to do the indexing. They'll store their results here now = dt = datetime.now() time_tuple = now.timetuple() path_elements = [str(time_tuple.tm_year), str(time_tuple.tm_mon), str(time_tuple.tm_mday), str(time_tuple.tm_hour), str(time_tuple.tm_min), link.guid] # Create a stub for our assets asset, created = Asset.objects.get_or_create(link=link) asset.base_storage_path = os.path.sep.join(path_elements) asset.save()
def upload_file(request): if request.method == 'POST': form = UploadFileForm(request.POST, request.FILES) if form.is_valid(): mime = MimeTypes() mime_type = mime.guess_type(request.FILES['file'].name) # Get mime type string from tuple if mime_type[0]: mime_type = mime_type[0] else: return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'Invalid file.'}), 'application/json') if validate_upload_file(request.FILES['file'], mime_type) and request.FILES['file'].size <= settings.MAX_ARCHIVE_FILE_SIZE: link = Link(submitted_url=form.cleaned_data['url'], submitted_title=form.cleaned_data['title']) if request.user.is_authenticated(): link.created_by = request.user link.save() now = dt = datetime.now() time_tuple = now.timetuple() path_elements = [str(time_tuple.tm_year), str(time_tuple.tm_mon), str(time_tuple.tm_mday), str(time_tuple.tm_hour), str(time_tuple.tm_min), link.guid] linky_home_disk_path = settings.GENERATED_ASSETS_STORAGE + '/' + os.path.sep.join(path_elements) if not os.path.exists(linky_home_disk_path): os.makedirs(linky_home_disk_path) asset, created = Asset.objects.get_or_create(link=link) asset.base_storage_path = os.path.sep.join(path_elements) asset.save() file_name = '/cap' + mime.guess_extension(mime_type) if mime_type == 'application/pdf': asset.pdf_capture = file_name else: asset.image_capture = file_name #print linky_home_disk_path + file_name #png = PythonMagick.Image(linky_home_disk_path + file_name) #png.write("file_out.png") #params = ['convert', linky_home_disk_path + file_name, 'out.png'] #subprocess.check_call(params) asset.save() request.FILES['file'].file.seek(0) f = open(linky_home_disk_path + file_name, 'w') f.write(request.FILES['file'].file.read()) os.fsync(f) f.close() response_object = {'status':'success', 'linky_id':link.guid, 'linky_hash':link.guid} return HttpResponse(json.dumps(response_object), 'application/json') else: return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'Invalid file.'}), 'application/json') else: return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'Missing file.'}), 'application/json')