Ejemplo n.º 1
0
    def _upload_video(self, video_file, callback, session, proxy):

        session = session or self.http_settings.session
        proxy = proxy or self.http_settings.proxy

        filename = path.Path(video_file).name
        filehash = self._generate_file_hash(filename)
        fields = []
        fields.append(('filename', filehash))
        fields.append(
            ('content', (path.Path(video_file).name, open(video_file, 'rb'))))

        encoder = MultipartEncoder(fields)

        if callback:
            monitor = MultipartEncoderMonitor(encoder, callback)
        else:
            monitor = MultipartEncoderMonitor(encoder)

        url = '{nginx_uploader_url}?filename={filehash}' \
            '&X-Progress-ID={filehash}'.format(nginx_uploader_url=self.nginx_uploader_url,
                                               filehash=filehash)

        self._get_upload_progress2(filehash)
        upload_video = session.post(
            url,
            data=monitor,
            proxies=proxy,
            headers={'Content-Type': monitor.content_type})

        return filehash
Ejemplo n.º 2
0
 def create_multipart_monitor(encoder, callback=None):
     if callback:
         if not hasattr(callback, '__call__'):
             raise InvalidUploadCallback(
                 'Callback {c} needs to be callable'.format(c=callback))
         return MultipartEncoderMonitor(encoder, callback)
     else:
         return MultipartEncoderMonitor(encoder)
Ejemplo n.º 3
0
    def upContent(self, request):
        """ Upload content to datastore server """
        oid = request['_id']['$oid']
        thumbkey = request['thumbnail_key']
        contentkey = request['content_key']

        thumbpath = self.dataDIR + oid + '/' + thumbkey
        contentpath = self.dataDIR + oid + '/' + contentkey

        thumburl = self.localServerURLformat % ('image/new')
        contenturl = self.localServerURLformat % ('content/new')

        thumbfid = 'thumb_' + oid
        contentfid = 'content_' + oid

        thumbfile = open(thumbpath, 'rb')
        enc = MultipartEncoder({thumbfid: (thumbkey, thumbfile)})
        mon = MultipartEncoderMonitor(enc)
        r = requests.post(thumburl,
                          data=mon,
                          headers={'Content-Type': mon.content_type})
        thumbfile.close()

        # Upload content
        contentsize = int(request['size'])

        def contentmonitor(monitor):
            progress = float(monitor.bytes_read) / contentsize
            mdisp.updateMsg(progress, self.getStorage(), self.getNetwork(),
                            self.getPower())
            return

        contentfile = open(contentpath, 'rb')
        enc = MultipartEncoder({contentfid: (contentkey, contentfile)})
        mon = MultipartEncoderMonitor(enc, contentmonitor)
        r = requests.post(contenturl,
                          data=mon,
                          headers={'Content-Type': mon.content_type})
        contentfile.close()

        # Trash content once uploaded
        os.remove(thumbpath)
        os.remove(contentpath)
        try:
            os.rmdir(self.dataDIR + oid)
        except:
            mdisp.errMsg(1234, 'rmdir fail')

        # Final status update
        mdisp.updateMsg(1.0, self.getStorage(), self.getNetwork(),
                        self.getPower())

        return
Ejemplo n.º 4
0
    def upload(self,
               video_file,
               title,
               tags,
               description,
               tag_ids,
               upload_id=None,
               callback=None,
               session=None,
               proxy=None):

        if isinstance(tag_ids, (list, tuple)):
            tag_ids = [("listch", str(t)) for t in tag_ids]
        else:
            tag_ids = [("listch", str(tag_ids))]

        upload_id = upload_id or self._initiate_new_upload()

        session = session or self.http_settings.session
        proxy = proxy or self.http_settings.proxy

        main_url = self._get_path_to_cgi()
        now = arrow.utcnow().timestamp
        q = "upload_id={upload_id}".format(upload_id=upload_id)
        url = "{main_url}ubr_upload.pl?{q}".format(main_url=main_url, q=q)

        fields = []
        fields.append(('MAX_FILE_SIZE', str(os.path.getsize(video_file))))
        fields.append(('upload_range', str(1)))
        fields.append(('adult', ''))
        fields.append(('field_myvideo_keywords', " ".join([t for t in tags])))
        fields.append(('field_myvideo_title', title))
        fields.append(('field_myvideo_descr', description))
        fields.append(
            ('upfile_0', (path.Path(video_file).name, open(video_file, 'rb'))))
        fields.extend(tag_ids)

        multipart_encoder = MultipartEncoder(fields)

        if callback:
            monitor = MultipartEncoderMonitor(multipart_encoder, callback)
        else:
            monitor = MultipartEncoderMonitor(multipart_encoder)

        upload = session.post(url,
                              data=monitor,
                              headers={'Content-Type': monitor.content_type},
                              proxies=proxy)
        self._start_progress_tracker(upload_id=upload_id)

        return upload_id
Ejemplo n.º 5
0
    def provider_upload(self, name, version, provider, file):
        r = self.api_get(self.api['provider']['upload'], args={
            'username': self.username,
            'boxname': name,
            'version': version,
            'provider': provider
        })

        data = r.json()

        encoder = MultipartEncoder({
            'file': (file, open(file, 'rb'), 'application/octet-stream')
        })
        bar = ProgressBar(expected_size=encoder.len, filled_char='=')

        def callback(monitor):
            bar.show(monitor.bytes_read)

        monitor = MultipartEncoderMonitor(encoder, callback)

        self.api_put(
            data['upload_path'], monitor, isjson=False, anonymous=True,
            headers={
                'Content-Type': monitor.content_type
            }
        )
        print('')
Ejemplo n.º 6
0
def upload(filepath, comment, url, headers):
    """Upload a file with progress bar.

    Parameters
    ----------
    filepath : string
        The local filepath of the file to upload.
    comment : string
        The comment for the file to upload.
    url : string
        The url of the file to upload.
    headers : dictionary
        The headers of the upload's request.

    Returns
    -------
    headers : dictionary | json
        The response of the upload's request.
    """

    filename = (os.sep).join(filepath.split(os.sep)[-2:]) 
    encoder = MultipartEncoder({'file': (filename, open(filepath, 'rb'), 'text/plain'), "comment":comment})

    with tqdm(desc=f"Submit {filename}",
              total=encoder.len, ncols=100,
              unit='o', unit_scale=True,
              unit_divisor=1024             ) as progress_bar :

        multipart_monitor = MultipartEncoderMonitor(encoder, lambda monitor: progress_bar.update(monitor.bytes_read - progress_bar.n))
        headers = {**headers, 'Content-Type': multipart_monitor.content_type,}

        response = requests.post(url, data=multipart_monitor, headers=headers, )
    return response
Ejemplo n.º 7
0
    def push(self, filepath):
        """Push the bytes from filepath to the Storage."""
        logger.debug("Starting to push %r", str(filepath))

        def _progress(monitor):
            # XXX Facundo 2020-07-01: use a real progress bar
            if monitor.bytes_read <= monitor.len:
                progress = 100 * monitor.bytes_read / monitor.len
                print("Uploading... {:.2f}%\r".format(progress), end="", flush=True)

        with filepath.open("rb") as fh:
            encoder = MultipartEncoder(
                fields={"binary": (filepath.name, fh, "application/octet-stream")}
            )

            # create a monitor (so that progress can be displayed) as call the real pusher
            monitor = MultipartEncoderMonitor(encoder, _progress)
            response = _storage_push(monitor, self.storage_base_url)

        if not response.ok:
            raise CommandError(
                "Failure while pushing file: [{}] {!r}".format(
                    response.status_code, response.content
                )
            )

        result = response.json()
        if not result["successful"]:
            raise CommandError("Server error while pushing file: {}".format(result))

        upload_id = result["upload_id"]
        logger.debug("Uploading bytes ended, id %s", upload_id)
        return upload_id
Ejemplo n.º 8
0
def upload(path, fname):
    server = get_server()
    total = check_size(path)
    url = "https://{}.zippyshare.com/upload".format(server)
    pb = tqdm(total=total, unit='B', unit_scale=True)
    data = {
        'name': fname,
        'file': (fname, open(path, 'rb'), get_mime_type(fname))
    }
    if cfg.private == True:
        data['private'] = "true"
    else:
        data['notprivate'] = "true"
    multi = MultipartEncoder(fields=data)
    try:
        monitor = MultipartEncoderMonitor(
            multi, lambda multi: pb.update(monitor.bytes_read - pb.n))
        s.headers.update({'Content-Type': monitor.content_type})
        r = s.post(url, data=monitor)
        r.raise_for_status()
    finally:
        if s.headers.get('Content-Type'):
            del s.headers['Content-Type']
        pb.close()
    return r.text
Ejemplo n.º 9
0
def upload_file(upload_url, upload_fields, filepath, callback=None):
    """Upload a pre-signed file to Cloudsmith."""
    upload_fields = upload_fields.items()
    upload_fields.append(
        ('file', (os.path.basename(filepath), click.open_file(filepath,
                                                              'rb'))))
    encoder = MultipartEncoder(upload_fields)
    monitor = MultipartEncoderMonitor(encoder, callback=callback)

    config = cloudsmith_api.Configuration()
    if config.proxy:
        proxies = {'http': config.proxy, 'https': config.proxy}
    else:
        proxies = None

    headers = {'content-type': monitor.content_type}

    client = get_files_api()
    headers['user-agent'] = client.api_client.user_agent

    resp = requests.post(upload_url,
                         data=monitor,
                         headers=headers,
                         proxies=proxies)

    try:
        resp.raise_for_status()
    except requests.RequestException as exc:
        raise ApiException(resp.status_code,
                           headers=exc.response.headers,
                           body=exc.response.content)
Ejemplo n.º 10
0
    def silence_upload(endpoint, data, callback=None, return_value=None):

        if Api.bearer is None:
            return False

        m = MultipartEncoder(fields=data)
        if callback is not None:
            m = MultipartEncoderMonitor(m, callback)

        try:
            bearer = Config.load_token()
            resp = requests.post(Api.apiUrl + endpoint,
                                 data=m,
                                 headers={
                                     "Content-Type": m.content_type,
                                     "Accept": "application/json",
                                     "Authorization": "Bearer " + bearer
                                 })
            if return_value:
                return_value.value = resp

            return resp
        except:
            print("Failed", endpoint, data)
            return False
        pass
Ejemplo n.º 11
0
def post_file(api: Api, name, fp):
    fp.seek(0, 2)
    file_size = fp.tell()
    fp.seek(0, 0)

    upload = api.post_uploads(name, file_size)

    progress_bar = None

    def update_callback(monitor):
        progress_bar.n = monitor.bytes_read
        progress_bar.refresh()

    data = MultipartEncoder(fields={'file': (name, fp)})
    monitor = MultipartEncoderMonitor(data, update_callback)
    progress_bar = tqdm(desc=name, total=data.len, unit='iB', unit_scale=True)
    headers = {
        'Content-Type': monitor.content_type,
        'Content-Length': str(data.len)
    }
    res = api.session.put(upload['upload_url'], data=monitor, headers=headers)

    if progress_bar.n != data.len:
        raise IOError('file size mismatches')
    progress_bar.close()
    res.raise_for_status()
    return upload['id']
Ejemplo n.º 12
0
    def upload(self, url, files, files_size, params=None, json=None, timeout=3600):
        if files_size > self.MAX_UPLOAD_SIZE:
            raise PolyaxonShouldExitError(
                "Files too large to sync, please keep it under {}.\n"
                "If you have data files in the current directory, "
                "please add them directly to your data volume, or upload them "
                "separately using `polyxon data` command and remove them from here.\n".format(
                    self.format_sizeof(self.MAX_UPLOAD_SIZE)))

        files = to_list(files)
        if json:
            files.append(('json', json.dumps(json)))

        multipart_encoder = MultipartEncoder(
            fields=files
        )

        # Attach progress bar
        progress_callback, bar = self.create_progress_callback(multipart_encoder)
        multipart_encoder_monitor = MultipartEncoderMonitor(multipart_encoder, progress_callback)
        try:
            response = self.put(url=url,
                                params=params,
                                data=multipart_encoder_monitor,
                                headers={"Content-Type": multipart_encoder.content_type},
                                timeout=timeout)
        finally:
            # always make sure we clear the console
            bar.done()

        return response
Ejemplo n.º 13
0
def uploadFile(file):
    """
    Upload file to server
    :param file:
    :return: json
    """
    global config
    key = config['key']
    file = config['file']
    base = os.path.basename(file)
    url = config['server'] + '/localupload'
    json = {}

    session = requests.Session()
    with open(file, 'rb') as f:
        form = MultipartEncoder({
            "documents": (base, f, "application/octet-stream"),
            "composite": "NONE"
        })
        m = MultipartEncoderMonitor(form, upload_callback)

        headers = {"Prefer": "respond-async", "Content-Type": m.content_type}
        params = {"key": key}
        resp = session.post(url, params=params, headers=headers, data=m)
        json = resp.json()
    session.close()

    return json
Ejemplo n.º 14
0
        def _upload_file(file, targetpath, fullpath):
            try:
                param = {"path": f"{targetpath}/{file}"}
                print(param)
                print(
                    requests.get(self.URL + "/upload",
                                 headers=self.headers,
                                 params=param).json())
                upload_url = requests.get(self.URL + "/upload",
                                          headers=self.headers,
                                          params=param).json()["href"]

                file_size = os.path.getsize(fullpath)
                pbar = tqdm(total=file_size)
                callback = track_upload_progress(pbar)

                encoder = MultipartEncoder(fields={
                    'file': ('filename', open(fullpath, 'rb'), 'text/plain')
                })
                encoder_monitor = MultipartEncoderMonitor(encoder, callback)

                requests.put(
                    upload_url,
                    data=encoder_monitor,
                    headers={'Content-Type': encoder_monitor.content_type})
                pbar.close()
                print(f'Файл "{file}" успешно загружен на Яндекс.Диск\n')
            except KeyError:
                print(f'Файл "{file}" был ранее загружен на Яндекс.Диск\n')
Ejemplo n.º 15
0
    def files(self, payload):
        """Upload a file to a hydroshare resource.

        :param payload:
            file: File object to upload to server
            folder: folder path to upload the file to
        :return: json object
            resource_id: string resource id,
            file_name: string name of file
        """
        url = "{url_base}/resource/{pid}/files/".format(
            url_base=self.hs.url_base, pid=self.pid)

        encoder = MultipartEncoder({
            "file": (payload['file'], open(payload['file'], 'r')),
            "folder":
            payload['folder']
        })
        monitor = MultipartEncoderMonitor(encoder, default_progress_callback)

        r = self.hs._request('POST',
                             url,
                             None,
                             data=monitor,
                             headers={'Content-Type': monitor.content_type})
        return r.text
Ejemplo n.º 16
0
def upload_large_file(url, file_path, data, chunk_callback=None):
    if not path.isfile(file_path):
        print(f'File {file_path} not found')
        return
    file = open(file_path, 'rb')
    base_name = path.basename(file_path)
    data.update({'file': (base_name, file, get_mime_type(file_path))})
    encoder = MultipartEncoder(data)

    prev_bytes_read = 0

    def progress_callback(x):
        nonlocal prev_bytes_read
        if chunk_callback:
            chunk_callback(x.bytes_read - prev_bytes_read)
        prev_bytes_read = x.bytes_read

    monitor = MultipartEncoderMonitor(encoder, progress_callback)
    headers = {
        'Content-Type': monitor.content_type,
        'Authorization': session.token
    }
    with requests.post(url, data=monitor, headers=headers, stream=True) as res:
        if res.status_code == 401:
            login_and_retry()
        return res
Ejemplo n.º 17
0
  def submit(self):

    self.submit_url = self._get_submit_url()

    mkzip(os.path.dirname(sys.argv[0]), SUBMISSION_FILENAME, self.filenames, self.max_zip_size)

    fd = open(SUBMISSION_FILENAME, "rb")

    m = MultipartEncoder(fields={'zipfile': ('student.zip', fd, 'application/zip')})
    monitor = MultipartEncoderMonitor(m, self.upload_progress_callback)

    try:
      r = self.s.post(self.submit_url, 
                      data=monitor,
                      headers={'Content-Type': monitor.content_type})
      r.raise_for_status()
    except requests.exceptions.HTTPError as e:
      if r.status_code == 403:
        raise RuntimeError("You don't have access to this quiz.")
      elif r.status_code in [404,429,500]:
        try:
          response_json = r.json()
          message = response_json.get("message") or "An internal server error occurred."
        except:
          message = "An unknown error occurred"
        raise RuntimeError(message)
      else:
        raise

    fd.close()

    self.submission = r.json()
Ejemplo n.º 18
0
    def upload_file_to_device(self):
        if not self.parent.geoshark_widget.device_on_connect or self.server is None:
            return
        file = self.left_file_model.filePath(self.lefttableview.currentIndex())
        filename = file.split('/')[-1]
        url = 'http://{}/data/{}'.format(self.server, '/'.join(self.right_file_model_path))
        filesize = os.path.getsize(file)
        if filesize == 0:
            show_error(_('File error'), _('File size must be non zero.'))
            return
        progress = ProgressBar(text=_('Upload File Into GeoShark'), window_title=_('Upload file to GeoShark'))
        encoder = MultipartEncoder(
            fields={'upload_file': (filename, open(file, 'rb'))}  # added mime-type here
        )
        data = MultipartEncoderMonitor(encoder, lambda monitor: progress.update((monitor.bytes_read/filesize)*99))

        try:
            res = requests.post(url, data=data, headers={'Content-Type': encoder.content_type}, timeout=5)
        except requests.exceptions.RequestException:
            progress.close()
            show_error(_('GeoShark error'), _('GeoShark is not responding.'))
            return
        if res.ok:
            progress.update(100)
            self.right_file_model_update()
    def validate(self, project_id, fims_metadata, expedition_code, upload,
                 is_public):
        with open(fims_metadata, 'rb') as f:
            e = MultipartEncoder({
                'fimsMetadata': (fims_metadata, f, "application/octet-stream"),
                'upload':
                str(upload),
                'projectId':
                project_id,
                'expeditionCode':
                expedition_code,
                'public':
                str(is_public)
            })

            m = MultipartEncoderMonitor(e, self.upload_progress)

            headers = {'Content-Type': m.content_type}
            print("0% uploaded")
            r = self.session.post(self.rest_root + self.validate_url,
                                  allow_redirects=False,
                                  headers=headers,
                                  data=m)

        if r.status_code > 299:
            print('status code: %s' % r.status_code)
            print(r.json()['usrMessage'] or 'Server Error')
            sys.exit()

        return r.json()
Ejemplo n.º 20
0
    def _post_stream(self, path, files=None, form_data=None):
        from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor

        def my_callback(monitor):
            progress = (monitor.bytes_read / monitor.len) * 100
            print("\r uploading...:%d%%(%d/%d)" %
                  (progress, monitor.bytes_read, monitor.len),
                  end=" ")

        url = self._make_url(path)
        headers = self._get_headers()
        m = MultipartEncoderMonitor(
            MultipartEncoder(fields={
                **files,
                **form_data
            }), my_callback)
        res = requests.post(url,
                            data=m,
                            headers={
                                **{
                                    'Content-Type': m.content_type
                                },
                                **headers
                            })
        return self._parse_response(res)
Ejemplo n.º 21
0
 def _multipart_post(self, data):
     encoder = MultipartEncoder(fields=data)
     monitor = MultipartEncoderMonitor(encoder)
     r = requests.post(self.file_host_url,
                       data=monitor,
                       headers={'Content-Type': monitor.content_type})
     return r
Ejemplo n.º 22
0
 def _request_fn(self, url, data, headers, files, requests_kwargs,
                 progress):
     if files:  # use streaming
         newfiles = dict([(k, (getattr(files[k], 'name',
                                       'upload_filename'), files[k]))
                          for k in files])
         intersect = set(data.keys()) & set(newfiles.keys())
         if intersect:
             raise CKANAPIError('field-name for files ("{}")'.format(
                 ', '.join(list(intersect))) +
                                ' cannot also be field name in data_dict.')
         data.update(newfiles)
         m = MultipartEncoder(data)
         if progress:
             m = MultipartEncoderMonitor(m, progress(m))
         headers.update({'Content-Type': m.content_type})
         r = self.session.post(url,
                               data=m,
                               headers=headers,
                               allow_redirects=False,
                               **requests_kwargs)
     else:
         r = self.session.post(url,
                               data=data,
                               headers=headers,
                               files=files,
                               allow_redirects=False,
                               **requests_kwargs)
     # allow_redirects=False because: if a post is redirected (e.g. 301 due
     # to a http to https redirect), then the second request is made to the
     # new URL, but *without* the data. This gives a confusing "No request
     # body data" error. It is better to just return the 301 to the user, so
     # we disallow redirects.
     return r.status_code, r.text
Ejemplo n.º 23
0
def multipart_post(data):
    encoder = MultipartEncoder(fields=data)
    monitor = MultipartEncoderMonitor(encoder)
    r = requests.post("https://catbox.moe/user/api.php",
                      data=monitor,
                      headers={'Content-Type': monitor.content_type})
    return r
Ejemplo n.º 24
0
    def handle(self, *args, **options):
        self.stderr.write(
            "`syncdata` command is deprecated and will be removed in 0.13.0 in favor of `sync`. Type `kolibri manage sync --help` for available options."
        )

        self.stdout.write("Uploading database to central server...\n")

        buff = io.BytesIO()

        zip_archive = zipfile.ZipFile(buff,
                                      mode="w",
                                      compression=zipfile.ZIP_DEFLATED)

        zip_archive.write(DB_PATH, "db.sqlite3")

        zip_archive.close()

        encoder = MultipartEncoder({
            "project":
            options["project"],
            "file": ("db.sqlite3.zip", buff, "application/octet-stream"),
        })
        monitor = MultipartEncoderMonitor(encoder, create_callback(encoder))
        r = requests.post(
            CENTRAL_SERVER_DB_UPLOAD_URL,
            data=monitor,
            headers={"Content-Type": monitor.content_type},
        )
        print("\nUpload finished! (Returned status {0} {1})".format(
            r.status_code, r.reason))
Ejemplo n.º 25
0
def upload_with_progress_bar(data, url, kwargs, label=None, token=None):
    """
    Uses multipart data to show progress of upload to the user.
    Requires request.files['data'].read() instead of request.data on the backend site.
    :param data: path to file to upload
    :param url: target url
    :param kwargs: additional args for the post request
    :param label: label of progress bar
    :param token: token to authorize the request
    :return: response from server
    """
    encoder = MultipartEncoder({'data': ('data', data, 'text/plain')})

    with tqdm(desc=label,
              total=encoder.len,
              disable=not label,
              dynamic_ncols=True,
              unit='B',
              unit_scale=True,
              unit_divisor=1024) as bar:
        multipart_monitor = MultipartEncoderMonitor(
            encoder, lambda monitor: bar.update(monitor.bytes_read - bar.n))
        r = requests.post(url,
                          data=multipart_monitor,
                          headers={
                              'Content-Type': multipart_monitor.content_type,
                              'X-Token': token
                          },
                          params=kwargs)

    return r
Ejemplo n.º 26
0
    def upload(self):  #上传
        self.progress = 0
        self.fileName = urlPath + scheme_Name + '.ipa'  #'/Users/Jerry/Desktop/doctor-iphone/doctor-iphone.ipa'
        self.fileName = '/Users/Jerry/Desktop/Demo/test/test.ipa'
        self.file_size = os.path.getsize(self.fileName) / 1024 / 1024.0
        url = "http://www.pgyer.com/apiv2/app/upload?" \
              "_api_key=2e8571d626b9a8c8b752e59624481847&" \
              "userKey=e34bf7d5e70d78339d60ea7bb68867c8&" \
              "enctype=multipart/form-data"
        file = open(self.fileName)
        filejson = {'file': ('doctor-iphone', file)}  # 后边的是其他参数
        e = MultipartEncoder(fields=filejson)
        m = MultipartEncoderMonitor(e, callback=self.callBack)
        # r = requests.post(url, files=filejson)#没有进度展示
        r = requests.post(url,
                          data=m,
                          headers={'Content-Type': m.content_type})
        j = r.json()
        if j:
            jsReult = j
            print('\n')
            if j['code'] == '0':

                print('**upload success!\n ipa下载地址:' +
                      jsReult['buildShortcutUrl'] + '**')
            else:
                print('**fail:' + jsReult['message'] + '**')
Ejemplo n.º 27
0
    def upload_file(
        self,
        io: IO,
        remotepath: str,
        ondup="overwrite",
        callback: Callable[[MultipartEncoderMonitor], None] = None,
    ):
        """Upload the content of io to remotepath

        WARNING: This api can not set local_ctime and local_mtime
        """

        assert remotepath.startswith(
            "/"), "`remotepath` must be an absolute path"
        remotePath = Path(remotepath)

        url = PcsNode.File.url()
        params = {
            "method": "upload",
            "ondup": ondup,
            "dir": remotePath.parent.as_posix(),
            "filename": remotePath.name,
            "BDUSS": self._bduss,
        }

        m = MultipartEncoder(fields={"file": ("file", io, "")})
        monitor = MultipartEncoderMonitor(m, callback=callback)

        resp = self._request(Method.Post, url, params=params, data=monitor)
        return resp.json()
Ejemplo n.º 28
0
def upload(ymlfile):
    infourl = 'http://fir.im/api/v2/app/info/%s?token=%s&type=%s'
    uploadurl = 'http://up.qiniu.com/'
    updateinfourl = 'http://fir.im/api/v2/app/%s?token=%s'
    if len(ymlfile) == 0:
        f = open('config.yml')
    else:
        f = open(ymlfile[0])

    x = yaml.load(f)
    infourl = infourl % (x['appid'], x['token'], x['type'])
    print '===== do get info from ', infourl, ' ====='
    response = requests.get(infourl)
    print json.loads(response.text)
    pkgkey = json.loads(response.text)['bundle']['pkg']['key']
    pkgtoken = json.loads(response.text)['bundle']['pkg']['token']
    print pkgkey
    print pkgtoken

    print '===== do upload file',x['filepath'], 'to', uploadurl, ' ====='
    e = MultipartEncoder(
    fields={'key': pkgkey, 'token': pkgtoken,
            'file': ('filename', open(glob(x['filepath'])[0], 'rb'), 'text/plain')}
    )
    callback = my_callback(e)
    m = MultipartEncoderMonitor(e, callback)
    uploadresponse = requests.post(uploadurl, data=m, headers={'Content-Type': m.content_type})
    # print uploadresponse.text

    appOid = json.loads(uploadresponse.text)['appOid']
    updateinfourl = updateinfourl % (appOid, x['token'])
    print '===== do update app info ', updateinfourl, ' ====='
    updateinforesponse = requests.put(updateinfourl, x)
Ejemplo n.º 29
0
 def post_request(self, path, json=None, files=None, log=log_responses, bar=False):
     if files is not None and bar:
         _files_dict = {
                 key: (key, file_, 'text/plain') for key, file_ in files.items()}
         encoder = MultipartEncoder(_files_dict)
         with tqdm(total=encoder.len, 
                   unit_scale=True, desc='Uploading') as tqdm_bar:
             monitor = MultipartEncoderMonitor(encoder, 
                     lambda monitor: tqdm_bar.update(
                         monitor.bytes_read - tqdm_bar.n))
             response = requests.post(
                 urljoin(self.server_address, path),
                 json=json,
                 data=monitor,
                 headers={"Access-Token": self.access_token,
                          "Content-Type": monitor.content_type})
     else:
         response = requests.post(
             urljoin(self.server_address, path),
             json=json,
             files=files,
             headers={"Access-Token":self.access_token})
     if log:
         log_response(response)
     if response.status_code != 200:
         message = response.status_code
         try:
             json = response.json()
             if 'message' in json:
                 message = json['message']
         except JSONDecodeError:
             message = 'No json to retrieve message from'
         raise ServerError(message, response.content)
     return response
Ejemplo n.º 30
0
def test_storage_push_succesful():
    """Bytes are properly pushed to the Storage."""
    test_monitor = MultipartEncoderMonitor(
        MultipartEncoder(fields={"binary": ("filename", "somefile", "application/octet-stream")})
    )

    with patch("requests.Session") as mock:
        _storage_push(test_monitor, "http://test.url:0000")
    cm_session_mock = mock().__enter__()

    # check request was properly called
    url = "http://test.url:0000/unscanned-upload/"
    headers = {
        "Content-Type": test_monitor.content_type,
        "Accept": "application/json",
        "User-Agent": build_user_agent(),
    }
    cm_session_mock.post.assert_called_once_with(url, headers=headers, data=test_monitor)

    # check the retries were properly setup
    (protocol, adapter), _ = cm_session_mock.mount.call_args
    assert protocol == "https://"
    assert isinstance(adapter, HTTPAdapter)
    assert adapter.max_retries.backoff_factor == 2
    assert adapter.max_retries.total == 5
    assert adapter.max_retries.status_forcelist == [500, 502, 503, 504]