Beispiel #1
0
def _download_biogrid_data(url):
    """Downloads zipped, tab-separated Biogrid data in .tab2 format.

    Parameters:
    -----------
    url : str
        URL of the BioGrid zip file.

    Returns
    -------
    csv.reader
        A csv.reader object for iterating over the rows (header has already
        been skipped).
    """
    res = requests.get(biogrid_file_url)
    if res.status_code != 200:
        raise Exception('Unable to download Biogrid data: status code %s'
                        % res.status_code)
    zip_bytes = BytesIO(res.content)
    zip_file = ZipFile(zip_bytes)
    zip_info_list = zip_file.infolist()
    # There should be only one file in this zip archive
    if len(zip_info_list) != 1:
        raise Exception('There should be exactly zipfile in BioGrid zip '
                        'archive: %s' % str(zip_info_list))
    unzipped_bytes = zip_file.read(zip_info_list[0]) # Unzip the file
    biogrid_str = StringIO(unzipped_bytes.decode('utf8')) # Make file-like obj
    csv_reader = csv.reader(biogrid_str, delimiter='\t') # Get csv reader
    next(csv_reader) # Skip the header
    return csv_reader
Beispiel #2
0
def MyUnzipFile(in_file_path, target_root_path):
    """Unzip file.

>>> unzip_file("d:\\zout.zip", "d:\\")"""
    zin = ZipFile(in_file_path, "r")
    zin.extractall(target_root_path)
    zin.close()
Beispiel #3
0
    def get(self):
        lines = []
        jps = models.JourneyPattern.query().order(models.JourneyPattern.line)
        for jp in jps:
            if jp.line not in lines:
                logging.info(jp.line)
                lines.append(jp.line)
            logging.info([s.get().name for s in jp.stops])      
          
        return 

#         return taskqueue.add(url='/timetable', queue_name='default', params={'file_name': 'tfl_1-BAK_-390106-y05.xml'})

        if self.is_local():
            taskqueue.add(url='/timetable', queue_name='default')
        else:
            bucket_name = os.environ.get('BUCKET_NAME', app_identity.get_default_gcs_bucket_name())
#             bucket_name = 'jeg376-tm470.appspot.com'
            filename = '/%s/stream.zip' % bucket_name
            gcs_file = cloudstorage.open(filename)
    
            tfl_zip = ZipFile(gcs_file, 'r')
            tfl_data = tfl_zip.read('LULDLRRiverTramCable.zip')
            tfl_data = ZipFile(StringIO.StringIO(tfl_data), 'r')
            
            tube_file_matcher = re.compile('^tfl_\d-\w{3}_.*\.xml$')
            for file_name in tfl_data.namelist():
                if tube_file_matcher.match(file_name):
                    taskqueue.add(url='/timetable', queue_name='default', params={'file_name': file_name})
Beispiel #4
0
def get_info(in_stream):
    """ Return the version and submitter strings from zipfile byte stream. """
    arch = ZipFile(in_stream, 'r')
    try:
        return unpack_info(arch.read('__INFO__'))
    finally:
        arch.close()
Beispiel #5
0
class UnZip():
    """Unzip the given file into a temporary directory."""
    def __init__(self, input_file):
        self.input_file = input_file

    def unzip(self):
        self.zip_file = ZipFile(self.input_file, 'r')
        self.tempdir = tempfile.mkdtemp()
        self.zip_file.extractall(self.tempdir)
        return self.tempdir

    def cleanup(self):
        self.zip_file.close()
        for root, dirs, files in os.walk(self.tempdir, topdown=False):
            for name in files:
                os.remove(os.path.join(root, name))
            for name in dirs:
                os.rmdir(os.path.join(root, name))
        os.rmdir(self.tempdir)

    def __enter__(self):
        return self.unzip()

    def __exit__(self, *args):
        self.cleanup()
        return False
def temp_shapefile_from_zip(zip_path):
    """
    Given a path to a ZIP file, unpack it into a temp dir and return the path
    to the shapefile that was in there.  Doesn't clean up after itself unless
    there was an error.

    If you want to cleanup later, you can derive the temp dir from this path.
    """
    zf = ZipFile(zip_path)
    tempdir = mkdtemp()
    shape_path = None
    # Copy the zipped files to a temporary directory, preserving names.
    for name in zf.namelist():
        data = zf.read(name)
        outfile = os.path.join(tempdir, name)
        if name.endswith('.shp'):
            shape_path = outfile
        f = open(outfile, 'w')
        f.write(data)
        f.close()

    if shape_path is None:
        log.warn("No shapefile, cleaning up")
        # Clean up after ourselves.
        for file in os.listdir(tempdir):
            os.unlink(os.path.join(tempdir, file))
        os.rmdir(tempdir)
        raise ValueError("No shapefile found in zip")

    return shape_path
Beispiel #7
0
def extract(zip):
    """Extrait le fichier '/word/document.xml'
    d'une archive word dans le répertoire courant
    """

    myzip = ZipFile(zip, 'a')
    myzip.extract('word/document.xml', './../')
Beispiel #8
0
    def _render_zip(self, req, filename, repos, diff):
        """ZIP archive with all the added and/or modified files."""
        new_rev = diff.new_rev
        req.send_response(200)
        req.send_header('Content-Type', 'application/zip')
        req.send_header('Content-Disposition', 'attachment;'
                        'filename=%s.zip' % filename)

        from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED

        buf = StringIO()
        zipfile = ZipFile(buf, 'w', ZIP_DEFLATED)
        for old_node, new_node, kind, change in repos.get_changes(**diff):
            if kind == Node.FILE and change != Changeset.DELETE:
                assert new_node
                zipinfo = ZipInfo()
                zipinfo.filename = new_node.path.encode('utf-8')
                # Note: unicode filenames are not supported by zipfile.
                # UTF-8 is not supported by all Zip tools either,
                # but as some does, I think UTF-8 is the best option here.
                zipinfo.date_time = time.gmtime(new_node.last_modified)[:6]
                zipinfo.compress_type = ZIP_DEFLATED
                zipfile.writestr(zipinfo, new_node.get_content().read())
        zipfile.close()

        buf.seek(0, 2) # be sure to be at the end
        req.send_header("Content-Length", buf.tell())
        req.end_headers()

        req.write(buf.getvalue())
Beispiel #9
0
def import_view(request):
    """
    Gets the existing declared parsers for the current project.
    This view handles only the file based import parsers.
    """
    choices = []
    choices_url = []
    render_dict = {}

    choices, choices_url, classes = discover_available_parsers()

    form = ImportDatasetFormWithFile(choices, prefix="with-file")
    form_without_file = ImportDatasetForm(
        choices_url, prefix="without-file")

    if request.method == 'POST':
        if 'upload-file' in request.POST:
            form = ImportDatasetFormWithFile(
                choices, request.POST, request.FILES, prefix="with-file")

            if form.is_valid():
                print(request.FILES)
                uploaded = request.FILES['with-file-zipfile']

                destination_dir, destination_file = create_tmp_destination(
                    uploaded.name)

                with open(destination_file, 'w+') as f:
                    f.write(uploaded.file.read())
                    zfile = ZipFile(f)
                    for name in zfile.namelist():
                        try:
                            zfile.extract(
                                name, os.path.dirname(os.path.realpath(f.name)))
                            if name.endswith('shp'):
                                parser = classes[int(form['parser'].value())]
                                import_datas.delay(
                                    '/'.join((destination_dir, name)),
                                    parser.__name__, parser.__module__
                                )
                                continue
                        except Exception:
                            raise

        if 'import-web' in request.POST:
            form_without_file = ImportDatasetForm(
                choices_url, request.POST, prefix="without-file")

            if form_without_file.is_valid():
                parser = classes[int(form_without_file['parser'].value())]
                import_datas_from_web.delay(
                    parser.__name__, parser.__module__
                )

    # Hide second form if parser has no web based imports.
    render_dict['form'] = form
    if choices_url:
        render_dict['form_without_file'] = form_without_file

    return render(request, 'common/import_dataset.html', render_dict)
Beispiel #10
0
    def acquire_resource(self, target_path, format_dict):
        """
        Downloads the zip file and extracts the files listed in
        :meth:`zip_file_contents` to the target path.

        """
        import cStringIO as StringIO
        from zipfile import ZipFile

        target_dir = os.path.dirname(target_path)
        if not os.path.isdir(target_dir):
            os.makedirs(target_dir)

        url = self.url(format_dict)

        shapefile_online = self._urlopen(url)

        zfh = ZipFile(StringIO.StringIO(shapefile_online.read()), 'r')

        for member_path in self.zip_file_contents(format_dict):
            ext = os.path.splitext(member_path)[1]
            target = os.path.splitext(target_path)[0] + ext
            member = zfh.getinfo(member_path)
            with open(target, 'wb') as fh:
                fh.write(zfh.open(member).read())

        shapefile_online.close()
        zfh.close()

        return target_path
Beispiel #11
0
    def aqcuire_all_resources(self, format_dict):
        import cStringIO as StringIO
        from zipfile import ZipFile

        # Download archive.
        url = self.url(format_dict)
        shapefile_online = self._urlopen(url)
        zfh = ZipFile(StringIO.StringIO(shapefile_online.read()), 'r')
        shapefile_online.close()

        # Iterate through all scales and levels and extract relevant files.
        modified_format_dict = dict(format_dict)
        scales = ('c', 'l', 'i', 'h', 'f')
        levels = (1, 2, 3, 4)
        for scale, level in itertools.product(scales, levels):
            modified_format_dict.update({'scale': scale, 'level': level})
            target_path = self.target_path(modified_format_dict)
            target_dir = os.path.dirname(target_path)
            if not os.path.isdir(target_dir):
                os.makedirs(target_dir)

            for member_path in self.zip_file_contents(modified_format_dict):
                ext = os.path.splitext(member_path)[1]
                target = os.path.splitext(target_path)[0] + ext
                member = zfh.getinfo(member_path)
                with open(target, 'wb') as fh:
                    fh.write(zfh.open(member).read())

        zfh.close()
Beispiel #12
0
def get_data_famafrench(name, start=None, end=None):
    start, end = _sanitize_dates(start, end)

    # path of zip files
    zipFileURL = "http://mba.tuck.dartmouth.edu/pages/faculty/ken.french/ftp/"

    url = urllib.urlopen(zipFileURL + name + ".zip")
    zipfile = ZipFile(StringIO(url.read()))
    data = zipfile.open(name + ".txt").readlines()

    file_edges = np.where(np.array([len(d) for d in data]) == 2)[0]

    datasets = {}
    for i in range(len(file_edges) - 1):
        dataset = [d.split() for d in data[(file_edges[i] + 1):
                                           file_edges[i + 1]]]
        if(len(dataset) > 10):
            ncol = np.median(np.array([len(d) for d in dataset]))
            header_index = np.where(
                np.array([len(d) for d in dataset]) == (ncol - 1))[0][-1]
            header = dataset[header_index]
            # to ensure the header is unique
            header = [str(j + 1) + " " + header[j] for j in range(len(header))]
            index = np.array(
                [d[0] for d in dataset[(header_index + 1):]], dtype=int)
            dataset = np.array(
                [d[1:] for d in dataset[(header_index + 1):]], dtype=float)
            datasets[i] = DataFrame(dataset, index, columns=header)

    return datasets
Beispiel #13
0
    def save_pickle_in_cfile(self, local_fname, networkref):
        """ Creates a pickled version of the graph and stores it in the
        cfile
        
        Parameters
        ----------
        local_fname: string
            The filename used in the Pickle folder to store
        networkref: NetworkX Graph instance
            The NetworkX graph to pickle
        
        """

        logger.info('Write a generated graph pickle to the connectome file.')
        picklefilepath = os.path.join(tempfile.gettempdir(),local_fname)
        from networkx import write_gpickle
        # add nodekeys, edgekeys, graphid to helpernode 'n0' before storage
        helperdict = {'nodekeys': networkref.nodekeys.copy(), \
                      'edgekeys': networkref.edgekeys.copy(), \
                      'graphid' : networkref.networkid }
        networkref.graph.add_node('n0')
        networkref.graph.node['n0'] = helperdict
        write_gpickle(networkref.graph, picklefilepath)
        networkref.graph.remove_node('n0')
        
        from zipfile import ZipFile, ZIP_DEFLATED
        tmpzipfile = ZipFile(self.data.fullpathtofile, 'a', ZIP_DEFLATED)
        # store it in the zip file
        tmpzipfile.write(picklefilepath, 'Pickle/' + local_fname)
        tmpzipfile.close()
        
        # remove pickle file from system
        logger.debug('Unlink: %s' % picklefilepath)
        os.unlink(picklefilepath)
Beispiel #14
0
    def do_export(self, REQUEST=None):
        """ """
        if REQUEST and not self.getParentNode().checkPermissionView():
            raise Unauthorized

        errors = []

        my_container = self.getParentNode()

        objects_to_archive = self.gather_objects(my_container)

        file_like_object = StringIO()
        zip_file = ZipFile(file_like_object, 'w')
        archive_files = []
        try:
            for obj in objects_to_archive:
                added_path = None
                if self.is_exportable(obj):
                    added_path = self.add_object_to_zip(obj, zip_file)
                if added_path:
                    archive_files.append((obj.title_or_id(),
                                          getattr(obj, 'meta_label',
                                                  obj.meta_type),
                                          added_path))

            self.add_index(zip_file, archive_files)
            zip_file.close()
        except Exception, e:
            errors.append(e)
Beispiel #15
0
def getTranslations(type, localesDir, defaultLocale, projectName, key):
  result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/export?key=%s' % (projectName, key)).read()
  if result.find('<success') < 0:
    raise Exception('Server indicated that the operation was not successful\n' + result)

  result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/download/all.zip?key=%s' % (projectName, key)).read()
  zip = ZipFile(StringIO(result))
  dirs = {}
  for info in zip.infolist():
    if not info.filename.endswith('.json'):
      continue

    dir, file = os.path.split(info.filename)
    if not re.match(r'^[\w\-]+$', dir) or dir == defaultLocale:
      continue
    if type == 'chrome' and file.count('.') == 1:
      origFile = file
    else:
      origFile = re.sub(r'\.json$', '', file)
    if type == 'gecko' and not origFile.endswith('.dtd') and not origFile.endswith('.properties'):
      continue

    mapping = langMappingChrome if type == 'chrome' else langMappingGecko
    for key, value in mapping.iteritems():
      if value == dir:
        dir = key
    if type == 'chrome':
      dir = dir.replace('-', '_')

    data = zip.open(info.filename).read()
    if data == '[]':
      continue

    if not dir in dirs:
      dirs[dir] = set()
    dirs[dir].add(origFile)

    path = os.path.join(localesDir, dir, origFile)
    if not os.path.exists(os.path.dirname(path)):
      os.makedirs(os.path.dirname(path))
    if type == 'chrome' and origFile.endswith('.json'):
      postprocessChromeLocale(path, data)
    elif type == 'chrome':
      data = json.loads(data)
      if origFile in data:
        fileHandle = codecs.open(path, 'wb', encoding='utf-8')
        fileHandle.write(data[origFile]['message'])
        fileHandle.close()
    else:
      fromJSON(path, data)

  # Remove any extra files
  for dir, files in dirs.iteritems():
    baseDir = os.path.join(localesDir, dir)
    if not os.path.exists(baseDir):
      continue
    for file in os.listdir(baseDir):
      path = os.path.join(baseDir, file)
      if os.path.isfile(path) and (file.endswith('.json') or file.endswith('.properties') or file.endswith('.dtd')) and not file in files:
        os.remove(path)
Beispiel #16
0
def layer_type(filename):
    """Finds out if a filename is a Feature or a Vector
       returns a gsconfig resource_type string
       that can be either 'featureType' or 'coverage'
    """
    base_name, extension = os.path.splitext(filename)

    shp_exts = ['.shp',]
    cov_exts = ['.tif', '.tiff', '.geotiff', '.geotif']
    csv_exts = ['.csv']
    kml_exts = ['.kml']

    if extension.lower() == '.zip':
        zf = ZipFile(filename)
        # ZipFile doesn't support with statement in 2.6, so don't do it
        try:
            for n in zf.namelist():
                b, e = os.path.splitext(n.lower())
                if e in shp_exts or e in cov_exts or e in csv_exts:
                    base_name, extension = b,e
        finally:
            zf.close()

    if extension.lower() in shp_exts + csv_exts + kml_exts:
         return FeatureType.resource_type
    elif extension.lower() in cov_exts:
         return Coverage.resource_type
    else:
        msg = ('Saving of extension [%s] is not implemented' % extension)
        raise GeoNodeException(msg)
Beispiel #17
0
    def pack(self, output_dir, devel=False, force=False, keep_temp=False):
        self.output_dir = os.path.realpath(output_dir)
        if not os.path.exists(self.output_dir):
            os.makedirs(self.output_dir)
        self.force = force
        self.keep_temp = keep_temp

        paths = []
        self.merge_modules = {}

        # create runtime package
        p = self._create_msi_installer(PackageType.RUNTIME)
        paths.append(p)

        # create devel package
        if devel and not isinstance(self.package, App):
            p = self._create_msi_installer(PackageType.DEVEL)
            paths.append(p)

        # create zip with merge modules
        self.package.set_mode(PackageType.RUNTIME)
        zipf = ZipFile(os.path.join(self.output_dir, '%s-merge-modules.zip' %
                                    self._package_name()), 'w')
        for p in self.merge_modules[PackageType.RUNTIME]:
            zipf.write(p)
        zipf.close()

        if not keep_temp:
            for msms in self.merge_modules.values():
                for p in msms:
                    os.remove(p)

        return paths
Beispiel #18
0
    def get_results(self):
        """Get analysis results.
        @return: data.
        """
        root = self._get_root(container="cuckoo", create=False)

        if not os.path.exists(root):
            return False

        zip_data = StringIO()
        zip_file = ZipFile(zip_data, "w", ZIP_DEFLATED)

        root_len = len(os.path.abspath(root))
        
        for root, dirs, files in os.walk(root):
            archive_root = os.path.abspath(root)[root_len:]
            for name in files:
                path = os.path.join(root, name)
                archive_name = os.path.join(archive_root, name)
                zip_file.write(path, archive_name, ZIP_DEFLATED)
        
        zip_file.close()
        data = xmlrpclib.Binary(zip_data.getvalue())
        zip_data.close()

        return data
Beispiel #19
0
def zip_layer_folder(dir_path, layer_name):
    """
    Create a zip archive with the content of the folder located at `dir_path`
    and name it with `layer_name`.

    Parameters
    ----------
    dir_path: str
        The path to the temporary folder in which are located the files to
        be zipped.

    layer_name: str
        The name of the concerned layer (will be used as file name for the
        zip archive).

    Returns
    -------
    raw_content: str
        The zip archive
    archive_name: str
        The name of the archive (used later in the header of the response).
    """
    filenames = os.listdir(dir_path)
    zip_stream = BytesIO()
    myZip = ZipFile(zip_stream, "w", compression=ZIP_DEFLATED)
    for filename in filenames:
        if not filename.endswith('.geojson'):
            f_name = path_join(dir_path, filename)
            myZip.write(f_name, filename, ZIP_DEFLATED)
    myZip.close()
    zip_stream.seek(0)
    return zip_stream.read(), ''.join([layer_name, ".zip"])
Beispiel #20
0
def load_otto_group():
    """
    Loads and returns several variables for the data set from Kaggle's Otto Group Product Classification competition.
    Link: https://www.kaggle.com/c/otto-group-product-classification-challenge

    Returns
    ----------
    data : array-like
        Pandas data frame containing the entire data set.

    X : array-like
        Training input samples.

    y : array-like
        Target values.
    """
    file_location = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'otto_group.zip')
    z = ZipFile(file_location)
    data = pd.read_csv(z.open('train.csv'))
    data = data.set_index('id')

    # move the label to the first position
    cols = data.columns.tolist()
    cols = cols[-1:] + cols[0:-1]
    data = data[cols]

    X = data.iloc[:, 1:].values

    y = data.iloc[:, 0].values

    # transform the labels from strings to integers
    encoder = LabelEncoder()
    y = encoder.fit_transform(y)

    return data, X, y
Beispiel #21
0
def download_file(url, name, root_destination='~/data/', zipfile=False,
                  replace=False):
    """Download a file from dropbox, google drive, or a URL.

    This will download a file and store it in a '~/data/` folder,
    creating directories if need be. It will also work for zip
    files, in which case it will unzip all of the files to the
    desired location.

    Parameters
    ----------
    url : string
        The url of the file to download. This may be a dropbox
        or google drive "share link", or a regular URL. If it
        is a share link, then it should point to a single file and
        not a folder. To download folders, zip them first.
    name : string
        The name / path of the file for the downloaded file, or
        the folder to zip the data into if the file is a zipfile.
    root_destination : string
        The root folder where data will be downloaded.
    zipfile : bool
        Whether the URL points to a zip file. If yes, it will be
        unzipped to root_destination + name.
    replace : bool
        If True and the URL points to a single file, overwrite the
        old file if possible.
    """
    # Make sure we have directories to dump files
    home = op.expanduser('~')
    tmpfile = home + '/tmp/tmp'
    if not op.isdir(home + '/data/'):
        print('Creating data folder...')
        os.makedirs(home + '/data/')

    if not op.isdir(home + '/tmp/'):
        print('Creating tmp folder...')
        os.makedirs(home + '/tmp/')

    download_path = _convert_url_to_downloadable(url)

    # Now save to the new destination
    out_path = root_destination.replace('~', home) + name
    if not op.isdir(op.dirname(out_path)):
        print('Creating path {} for output data'.format(out_path))
        os.makedirs(op.dirname(out_path))

    if zipfile is True:
        _fetch_file(download_path, tmpfile)
        myzip = ZipFile(tmpfile)
        myzip.extractall(out_path)
        os.remove(tmpfile)
    else:
        if len(name) == 0:
            raise ValueError('Cannot overwrite the root data directory')
        if replace is False and op.exists(out_path):
            raise ValueError('Path {} exists, use `replace=True` to '
                             'overwrite'.format(out_path))
        _fetch_file(download_path, out_path)
    print('Successfully moved file to {}'.format(out_path))
Beispiel #22
0
def load_property_inspection():
    """
    Loads and returns several variables for the data set from Kaggle's Property Inspection Prediction competition.
    Link: https://www.kaggle.com/c/liberty-mutual-group-property-inspection-prediction

    Returns
    ----------
    data : array-like
        Pandas data frame containing the entire data set.

    X : array-like
        Training input samples.

    y : array-like
        Target values.
    """
    file_location = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'property_inspection.zip')
    z = ZipFile(file_location)
    data = pd.read_csv(z.open('train.csv'))
    data = data.set_index('Id')

    X = data.iloc[:, 1:].values
    y = data.iloc[:, 0].values

    # transform the categorical variables from strings to integers
    encoder = CategoryEncoder()
    X = encoder.fit_transform(X)

    return data, X, y
Beispiel #23
0
def parse_template(template_name):
    """Resolve template name into absolute path to the template
    and boolean if absolute path is temporary directory.
    """
    if template_name.startswith('http'):
        if '#' in template_name:
            url, subpath = template_name.rsplit('#', 1)
        else:
            url = template_name
            subpath = ''
        with tempfile.NamedTemporaryFile() as tmpfile:
            urlretrieve(url, tmpfile.name)
            if not is_zipfile(tmpfile.name):
                raise ConfigurationError("Not a zip file: %s" % tmpfile)
            zf = ZipFile(tmpfile)
            try:
                path = tempfile.mkdtemp()
                zf.extractall(path)
                return os.path.join(path, subpath), True
            finally:
                zf.close()

    registry = TemplatesRegistry()
    if registry.has_template(template_name):
        path = registry.path_of_template(template_name)
    elif ':' in template_name:
        path = resolve_dotted_path(template_name)
    else:
        path = os.path.realpath(template_name)

    if not os.path.isdir(path):
        raise ConfigurationError('Template directory does not exist: %s' % path)
    return path, False
Beispiel #24
0
def download_unzip(input_zip):
    url = urllib.urlopen(input_zip)
    unzipped_string = ''
    zipfile = ZipFile(StringIO(url.read()))
    for name in zipfile.namelist():
        unzipped_string += zipfile.open(name).read()
    return unzipped_string
Beispiel #25
0
    def export_zip(self, paths):
        stringio = StringIO()
        archive = ZipFile(stringio, mode='w')

        def _add_resource(resource):
            for filename in resource.get_files_to_archive(True):
                if filename.endswith('.metadata'):
                    continue
                path = Path(self.handler.key).get_pathto(filename)
                archive.writestr(str(path), resource.handler.to_str())

        for path in paths:
            child = self.get_resource(path, soft=True)
            if child is None:
                continue
            # A Folder => we add its content
            if isinstance(child, Folder):
                for subchild in child.traverse_resources():
                    if subchild is None or isinstance(subchild, Folder):
                        continue
                    _add_resource(subchild)
            else:
                _add_resource(child)

        archive.close()
        return stringio.getvalue()
Beispiel #26
0
def get_sightings_from_atlas(uri, species_ids):
    # Create a dict of sightings
    # Each species ID will have a list of sightings with [lat, long]
    sightings = dict()
    for species_id in species_ids:
        sightings[species_id] = []

    # The CSV headers
    LONG = 0
    LAT = 1
    LSID = 2
        
    # Download API call and unzip
    url = urlopen(uri)
    zipfile = ZipFile(StringIO(url.read()))

    # Skip the header row using [1:]
    for line in zipfile.open("data.csv").readlines()[1:]:
        sighting_record = line.split(",")
        sightings[sighting_record[LSID][1:-2]].append([sighting_record[LAT][1:-1],sighting_record[LONG][1:-1]])
        
    for species_id in species_ids:
        # Don't return too many sightings for a single species
        sightings[species_id] = sightings[species_id][0:species_sighting_limit]
        # Prune any empty entries
        if sightings[species_id] == []: del sightings[species_id]
        
    return sightings
Beispiel #27
0
def load_forest_cover():
    """
    Loads and returns several variables for the data set from Kaggle's Forest Cover Type Prediction competition.
    Link: https://www.kaggle.com/c/forest-cover-type-prediction

    Returns
    ----------
    data : array-like
        Pandas data frame containing the entire data set.

    X : array-like
        Training input samples.

    y : array-like
        Target values.
    """
    file_location = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'forest_cover.zip')
    z = ZipFile(file_location)
    data = pd.read_csv(z.open('train.csv'))
    data = data.set_index('Id')

    # move the label to the first position
    cols = data.columns.tolist()
    cols = cols[-1:] + cols[0:-1]
    data = data[cols]

    X = data.iloc[:, 1:].values
    y = data.iloc[:, 0].values

    return data, X, y
Beispiel #28
0
def createDevEnv(baseDir, type):
  fileBuffer = StringIO()
  createBuild(baseDir, type=type, outFile=fileBuffer, devenv=True, releaseBuild=True)

  from zipfile import ZipFile
  zip = ZipFile(StringIO(fileBuffer.getvalue()), 'r')
  zip.extractall(os.path.join(baseDir, 'devenv'))
  zip.close()

  print 'Development environment created, waiting for connections from active extensions...'
  metadata = readMetadata(baseDir, type)
  connections = [0]

  import SocketServer, time, thread

  class ConnectionHandler(SocketServer.BaseRequestHandler):
    def handle(self):
      connections[0] += 1
      self.request.sendall('HTTP/1.0 OK\nConnection: close\n\n%s' % metadata.get('general', 'basename'))

  server = SocketServer.TCPServer(('localhost', 43816), ConnectionHandler)

  def shutdown_server(server):
    time.sleep(10)
    server.shutdown()
  thread.start_new_thread(shutdown_server, (server,))
  server.serve_forever()

  if connections[0] == 0:
    print 'Warning: No incoming connections, extension probably not active in the browser yet'
  else:
    print 'Handled %i connection(s)' % connections[0]
Beispiel #29
0
    def upload_analyzer(self):
        """Upload analyzer to guest.
        @return: operation status.
        """
        zip_data = StringIO()
        zip_file = ZipFile(zip_data, "w", ZIP_DEFLATED)

        root = os.path.join("analyzer", self.platform)
        root_len = len(os.path.abspath(root))

        if not os.path.exists(root):
            log.error("No valid analyzer found at path: %s" % root)
            return False

        for root, dirs, files in os.walk(root):
            archive_root = os.path.abspath(root)[root_len:]
            for name in files:
                path = os.path.join(root, name)
                archive_name = os.path.join(archive_root, name)
                zip_file.write(path, archive_name, ZIP_DEFLATED)

        zip_file.close()
        data = xmlrpclib.Binary(zip_data.getvalue())
        zip_data.close()

        log.debug("Uploading analyzer to guest (ip=%s)" % self.ip)
        self.server.add_analyzer(data)
 def extract_zip(input_zip):
     input_zip = ZipFile(input_zip)
     return {name: input_zip.read(name) for name in input_zip.namelist()}
def make_capsule(rootdir=None, filename='project.zip'):
    '''Generate all the files used by running encrypted scripts, pack all
    of them to a zip file.

    rootdir        pyarmor root dir, where you can find license files,
                   auxiliary library and pyshield extension module.

    filename       output filename, the default value is project.zip

    Return True if sucess, otherwise False or raise exception
    '''
    try:
        if rootdir is None:
            rootdir = sys.rootdir
    except AttributeError:
        rootdir = os.path.dirname(os.path.abspath(sys.argv[0]))
    logging.info('Rootdir is %s', rootdir)
    filelist = 'public.key', 'pyimcore.py', 'pytransform.py'
    for x in filelist:
        src = os.path.join(rootdir, x)
        if not os.path.exists(src):
            raise RuntimeError('No %s found in the rootdir' % src)

    licfile = os.path.join(rootdir, 'license.lic')
    if not os.path.exists(licfile):
        raise RuntimeError('Missing license file %s' % licfile)

    logging.info('Generating project key ...')
    pri, pubx, capkey, lic = pytransform.generate_project_capsule(licfile)
    logging.info('Generating project OK.')
    logging.info('Writing capsule to %s ...', filename)
    myzip = ZipFile(filename, 'w')
    try:
        myzip.write(os.path.join(rootdir, 'public.key'), 'pyshield.key')
        myzip.writestr('pyshield.lic', capkey)
        myzip.write(os.path.join(rootdir, 'pyimcore.py'), 'pyimcore.py')
        myzip.write(os.path.join(rootdir, 'pytransform.py'), 'pytransform.py')
        myzip.writestr('private.key', pri)
        myzip.writestr('product.key', pubx)
        myzip.writestr('license.lic', lic)
    finally:
        myzip.close()
    logging.info('Write project capsule OK.')
Beispiel #32
0
import os, sys, wget, csv, redis
from datetime import date, timedelta
from zipfile import ZipFile

url = 'https://www.bseindia.com/download/BhavCopy/Equity/EQ' + (
    date.today() - timedelta(1)).strftime('%d%m%y') + '_CSV.ZIP'
try:
    wget.download(url, os.getcwd())
except:
    sys.exit()
file = 'EQ' + (date.today() - timedelta(1)).strftime('%d%m%y')
zip = ZipFile(file + '_CSV.ZIP').extractall()
file = open(file + '.csv')
file = csv.reader(file)

r = redis.Redis(host='localhost', port=6379, db=0, password=None)
r.set("total", 1)

for i in file:
    if file.line_num == 1:
        continue
    hash_key = "member:" + str(file.line_num - 1)
    r.hmset(
        hash_key, {
            "code": i[0],
            "name": i[1],
            "open": i[4],
            "high": i[5],
            "low": i[6],
            "close": i[7]
        })
Beispiel #33
0
def model_config(fuel_blob):
    with ZipFile(BytesIO(fuel_blob)) as model_file:
        with model_file.open("model.config", "r") as sdf_file:
            xml_string = sdf_file.read().decode("utf-8")

    return xml_string
Beispiel #34
0
def model_sdf(fuel_blob):
    with ZipFile(BytesIO(fuel_blob)) as model_file:
        with model_file.open("model.sdf", "r") as sdf_file:
            sdf_string = sdf_file.read().decode("utf-8")

    return sdf_string
Beispiel #35
0
from zipfile import ZipFile
import os

caminho = r'C:\Users\desen\Downloads\serie2'
with ZipFile('arquivo.zip', 'w') as zip:
    for file in os.listdir(
            caminho):  # Só os arquivos da pasta, não entra nas subpastas
        caminho_completo = os.path.join(caminho, file)
        zip.write(
            caminho_completo, file
        )  # o arquivo que quero salvar o nome que desejar ficar no arquivo

with ZipFile('arquivo.zip', 'r') as zip:
    for arquivo in zip.namelist():
        print(arquivo)

with ZipFile('arquivo.zip', 'r') as zip:
    zip.extractall('descompactado')
def do_encrypt(argv):
    '''Usage: pyarmor encrypt [OPTIONS] [File Patterns or @Filename]

Encrpty the files list in the command line, you can use a specified
pattern according to the rules used by the Unix shell. No tilde
expansion is done, but *, ?, and character ranges expressed with []
will be correctly matched.

You can either list file patterns in one file, one pattern one line,
then add a prefix '@' to the filename.

All the files will be encrypted and saved as orginal file name plus
'e'. By default, the encrypted scripts and all the auxiliary files
used to run the encrypted scripts are save in the path "dist".

Available options:

  -O, --output=DIR            Output path for runtime files and encrypted
                              files (if no --in-place)

                              The default value is "build".

  -C, --with-capsule=FILENAME Specify the filename of capsule generated
                              before.

                              The default value is "project.zip".

  -i, --in-place              [option], the encrypted scripts will be
                              saved in the original path (same as source).
                              Otherwise, save to --output specified.

  -s, --src=DIR               [option], the source path of python scripts.
                              The default value is current path.

  -p, --plat-name             [option] platform name to run encrypted
                              scripts. Only used when encrypted scripts
                              will be run in different platform.

  -m, --main=NAME             Generate wrapper file to run encrypted script

  -e, --mode=MODE             Encrypt mode, available value:
                                0     Encrypt both source and bytecode
                                1     Encrypt bytecode only.
                                2     Encrypt source code only.
                                3     Obfuscate bytecodes.
                                5     Obfuscate code object of module.
                                6     Combine mode 3 and 4
                                7     Obfuscate code object of module,
                                      output wrapper scripts
                                8     Obfuscate both code object and bytecode,
                                      output wrapper scripts
                              Mode 0, 1, 2 is deprecated from v3.2.0, this
                              option can be ignored in general.

  -d, --clean                 Clean output path at start.

  --manifest FILENAME         Write file list to FILENAME

For examples:

    - Encrypt a.py and b.py as a.pyx and b.pyx, saved in the path "dist":

      pyarmor encrypt a.py b.py

    - Use file pattern to specify files:

      pyarmor encrypt a.py *.py src/*.py lib/*.pyc

    - Save encrypted files in the directory "/tmp/build" other than "dist":

      pyarmor encrypt --output=/tmp/build a.py

    - Encrypt python scripts by project capsule "project.zip" in the
      current directory:

      pyarmor encrypt --with-capsule=project.zip src/*.py

    - Encrypt python scripts to run in different platform:

      pyarmor encrypt --plat-name=linux_x86_64 a.py b.py

Use MANIFEST.in to list files

      pyarmor encrypt --with-capsule=project.zip @myproject/MANIFEST.in

It's Following the Distutils’ own manifest template

    '''
    opts, args = getopt.getopt(
        argv, 'C:de:im:O:p:s:',
        ['in-place', 'output=', 'src=', 'with-capsule=', 'plat-name=',
         'main=', 'clean', 'mode=', 'manifest=']
    )

    output = 'build'
    srcpath = None
    capsule = 'project.zip'
    inplace = False
    pname = None
    extfile = None
    mainname = []
    clean = False
    mode = 8
    manifest = None

    for o, a in opts:
        if o in ('-O', '--output'):
            output = a
        elif o in ('-s', '--src'):
            srcpath = a
        elif o in ('-i', '--in-place'):
            inplace = True
        elif o in ('-C', '--with-capsule'):
            capsule = a
        elif o in ('-p', '--plat-name'):
            pname = a
        elif o in ('-d', '--clean'):
            clean = True
        elif o in ('-e', '--mode'):
            if a not in ('0', '1', '2', '3', '5', '6',
                         '7', '8', '9', '10', '11', '12',
                         '13', '14'):
                raise RuntimeError('Invalid mode "%s"' % a)
            mode = int(a)
        elif o in ('-m', '--main'):
            mainname.append(a)
        elif o in ('--manifest', ):
            manifest = a

    if srcpath is not None and not os.path.exists(srcpath):
        raise RuntimeError('No found specified source path "%s"' % srcpath)

    if capsule is None or not os.path.exists(capsule):
        raise RuntimeError('No found capsule file %s' % capsule)

    # Maybe user specify an empty path
    if output == '':
        output = 'build'

    logging.info('Output path is %s' % output)
    if os.path.exists(output) and clean:
        logging.info('Removing output path %s', output)
        shutil.rmtree(output)
        logging.info('Remove output path OK.')
    if not os.path.exists(output):
        logging.info('Make output path %s', output)
        os.makedirs(output)

    if pname is None:
        extfile = os.path.join(sys.rootdir, dll_name + dll_ext)
    else:
        logging.info("Cross publish, target platform is %s", pname)
        name = dll_name + ('.so' if pname.startswith('linux') else '.dll')
        extfile = os.path.join(sys.rootdir, 'platforms', pname, name)
        if not os.path.exists(extfile):
            # Need to download platforms/... from pyarmor homepage
            logging.info('You need download prebuilt library files '
                         'from pyarmor homepage first.')
            raise RuntimeError('Missing cross platform library %s' % extfile)
    logging.info('Copy %s to %s' % (extfile, output))
    shutil.copy(extfile, output)

    logging.info('Extract capsule %s ...', capsule)
    ZipFile(capsule).extractall(path=output)
    logging.info('Extract capsule to %s OK.', output)

    # Fix bootstrap restrict issue from v5.7.0
    make_license(capsule, os.path.join(output, 'license.lic'),
                 '*FLAGS:A*CODE:PyArmor')

    if mode >= 3:
        logging.info('Encrypt mode: %s', mode)
        with open(os.path.join(output, 'pyimcore.py'), 'w') as f:
            lines = 'from pytransform import old_init_runtime', \
                    'old_init_runtime(0, 0, 0, 0)', ''
            f.write('\n'.join(lines))
    elif mode:
        logging.info('Encrypt mode: %s', mode)
        with open(os.path.join(output, 'pyimcore.py'), 'r') as f:
            lines = f.read()
        with open(os.path.join(output, 'pyimcore.py'), 'w') as f:
            i = lines.rfind('\n\n')
            if i == -1:
                raise RuntimeError('Invalid pyimcore.py')
            f.write(lines[:i])
            if mode == 1:
                f.write('\n\nold_init_runtime()\n')
            elif mode == 2:
                f.write('\n\nsys.meta_path.append(PyshieldImporter())\n'
                        'old_init_runtime(0, 0, 0, 0)\n')

    prikey = os.path.join(output, 'private.key')
    if os.path.exists(prikey):
        logging.info('Remove private key %s in the output', prikey)
        os.remove(prikey)

    if mode not in (7, 8, 9, 10, 11, 12, 13, 14):
        for name in mainname:
            n = name.find(':')
            if n == -1:
                script = os.path.join(output, name + '.py')
            else:
                script = os.path.join(output, name[n+1:])
                name = name[:n]
            logging.info('Writing script wrapper %s ...', script)
            ch = 'c' if mode == 1 or mode == 3 else ext_char
            with open(script, 'w') as f:
                f.write(wrap_runner % (name + '.py' + ch))
            logging.info('Write script wrapper OK.')

    filelist = _parse_file_args(args, srcpath=srcpath)
    if manifest is not None:
        logging.info('Write filelist to %s', manifest)
        with open(manifest, 'w') as fp:
            fp.write('\n'.join([x[0] for x in filelist]))

    if len(filelist[:1]) == 0:
        logging.info('Generate extra files OK.')
    else:
        prokey = os.path.join(output, 'product.key')
        if not os.path.exists(prokey):
            raise RuntimeError('Missing project key %s' % prokey)
        logging.info('Encrypt files ...')
        encrypt_files(filelist, prokey, mode, None if inplace else output)
        if mode in (7, 8, 9, 10, 11, 12, 13, 14):
            for name in mainname:
                script = os.path.join(
                    output, name + ('' if name.endswith('.py') else '.py'))
                with open(script, 'r') as f:
                    source = f.read()
                logging.info('Patch entry script %s.', script)
                with open(script, 'w') as f:
                    f.write('import pyimcore\n')
                    f.write(source)
        logging.info('Encrypt files OK.')
Beispiel #37
0
    def _getAllObjectsData(self, objects_listing, context_path):
        """
        Returns the data in all files with a content object to be placed in a zipfile
        """
        # Use temporary IO object instead of writing to filesystem.
        out = StringIO()
        zipFile = ZipFile(out, 'w', ZIP_DEFLATED)

        for obj in objects_listing:
            object_path = str(obj.virtual_url_path())

            if self._objImplementsInterface(
                    obj, interfaces.IATFile) or self._objImplementsInterface(
                        obj, interfaces.IATImage):
                file_data = str(obj.data)
                object_path = object_path.replace(context_path + '/', '')

            elif self._objImplementsInterface(obj, interfaces.IATDocument):

                if "text/html" == obj.Format():
                    file_data = obj.getText()
                    if object_path[-5:] != ".html" and object_path[
                            -4:] != ".htm":
                        object_path += ".html"

                elif "text/x-rst" == obj.Format():
                    file_data = obj.getRawText()
                    if object_path[-4:] != ".rst":
                        object_path += ".rst"

                elif "text/structured" == obj.Format():
                    file_data = obj.getRawText()
                    if object_path[-4:] != ".stx":
                        object_path += ".stx"

                elif "text/plain" == obj.Format():
                    file_data = obj.getRawText()
                    if object_path[-4:] != ".txt":
                        object_path += ".txt"

                else:
                    file_data = obj.getRawText()

                object_path = object_path.replace(context_path + '/', '')

            elif self._objImplementsInterface(obj, interfaces.IATFolder):
                if hasattr(obj, 'getRawText'):
                    file_data = obj.getRawText()

                    if object_path == context_path:
                        object_path = object_path.split("/")[-1]
                    else:
                        object_path = object_path.replace(
                            context_path + '/', '')

                    if object_path[-5:] != ".html" and object_path[
                            -4:] != ".htm":
                        object_path += ".html"
            else:
                continue

            # start point for object path, adding 1 removes the initial '/'
            object_path = self.generateSafeFileName(object_path)
            if object_path:
                zipFile.writestr(object_path, file_data)

        zipFile.close()

        out.seek(0)
        content = out.read()
        out.close()

        return content
Beispiel #38
0
_author_ = "aman"

from zipfile import ZipFile

for i in range(17):
    print("{0:>2} in binary is {0:>08b}".format(i))

with ZipFile('C:\\Users\\aman.raj\\Documents\\GST\\GST_C_20200229.zip',
             'r') as zipObj:
    zipObj.extractall('C:\\Users\\aman.raj\\Documents\\GST\\temp')
Beispiel #39
0
    def importContent(self,
                      file,
                      context=None,
                      description=None,
                      contributors=None,
                      categories=None,
                      overwrite=False):
        """
        Import content from a zip file, creating the folder structure within a ZODB hierarchy.
        """
        self.bad_folders = []

        zf = ZipFile(file, 'r')

        files = [file.filename for file in zf.filelist]

        if len(files) < 1:
            return ('failure', 'The zip file was empty')

        for current_file in files:

            # If the current file is a folder move to the next file.
            if current_file[-1] == '/':
                continue

            if current_file[0] == '/':
                path_as_list = current_file[1:].split('/')
            else:
                path_as_list = current_file.split('/')

            file_name = path_as_list[-1]

            # Checks to make sure that the file path does not contain any previouslsy found bad folders.
            if not self._checkFilePath(current_file, path_as_list):
                continue

            folder = self._createFolderStructure(path_as_list, context)

            # no folder to add to? Then move on to next object.
            if not folder:
                continue

            code, msg = context.checkid(id=file_name,
                                        required=1,
                                        contained_by=folder)

            # Create an object if everything looks good
            if not code or (overwrite and 'existing' == code):

                fdata = zf.read(current_file)

                if "existing" == code:
                    folder.manage_delObjects([file_name])

                obj = self._createObject(file_name, fdata, folder)
                try:
                    obj.setFilename(file_name)
                except:
                    pass  #do nothing
                if hasattr(obj, 'description') and description:
                    obj.setDescription(description)
                if hasattr(obj, 'contributors') and contributors:
                    obj.setContributors(contributors)
                if hasattr(obj, 'subject') and categories:
                    obj.setSubject(categories)
                obj.reindexObject()

        zf.close()
Beispiel #40
0
    def processAlgorithm(self, parameters, context, feedback):
        self.parameters = parameters
        self.context = context
        filename = self.parameterAsFile(parameters, self.PrmInput, context)
        f, extension = os.path.splitext(filename)
        extension = extension.lower()
        try:
            if extension == '.kmz':
                kmz = ZipFile(filename, 'r')
                kml = kmz.open('doc.kml', 'r')
            elif extension == '.kml':
                kml = open(filename,
                           encoding="utf-8",
                           errors="backslashreplace")
            else:
                msg = "Invalid extension: Should be kml or kmz"
                feedback.reportError(msg)
                raise QgsProcessingException(msg)
        except:
            msg = "Failed to open file"
            feedback.reportError(msg)
            raise QgsProcessingException(msg)

        skipPt = True if self.PrmPointOutputLayer not in parameters or parameters[
            self.PrmPointOutputLayer] is None else False
        skipline = True if self.PrmLineOutputLayer not in parameters or parameters[
            self.PrmLineOutputLayer] is None else False
        skipPoly = True if self.PrmPolygonOutputLayer not in parameters or parameters[
            self.PrmPolygonOutputLayer] is None else False
        self.cntPt = 0
        self.cntLine = 0
        self.cntPoly = 0
        parser = xml.sax.make_parser()
        # Do a pre-pass through the KML to see if there are any extended data fields
        preprocess = PreProcessHandler()
        parser.setContentHandler(preprocess)
        try:
            input_source = xml.sax.xmlreader.InputSource()
            input_source.setByteStream(kml)
            input_source.setEncoding('utf-8')
            parser.parse(input_source)
        except:
            preprocess.endDocument()
        # Reset the KML/KMZ file to the beginning. The handler seems to automatically close the file
        # which is why we are reopening it.
        if extension == '.kmz':
            kmz.close()
        else:
            kml.close()
        if extension == '.kmz':
            kmz = ZipFile(filename, 'r')
            kml = kmz.open('doc.kml', 'r')
        else:
            kml = open(filename, encoding="utf-8", errors="backslashreplace")

        # Set up the handler for doing the main processing
        self.extData = preprocess.getExtendedDataFields()
        self.extData.sort()
        self.extDataMap = {}
        index = 0
        for item in self.extData:
            self.extDataMap[item] = index
            index += 1
        handler = PlacemarkHandler(skipPt, skipline, skipPoly, self.extDataMap,
                                   feedback)
        handler.addpoint.connect(self.addpoint)
        handler.addline.connect(self.addline)
        handler.addpolygon.connect(self.addpolygon)
        parser.setContentHandler(handler)
        try:
            input_source = xml.sax.xmlreader.InputSource()
            input_source.setByteStream(kml)
            input_source.setEncoding('utf-8')
            parser.parse(input_source)
        except:
            '''s = traceback.format_exc()
            feedback.pushInfo(s)'''
            feedback.pushInfo(
                tr('Failure in kml extraction - May return partial results.'))
            handler.endDocument()

        if extension == '.kmz':
            kmz.close()
        else:
            kml.close()

        feedback.pushInfo('{} points extracted'.format(self.cntPt))
        feedback.pushInfo('{} lines extracted'.format(self.cntLine))
        feedback.pushInfo('{} polygons extracted'.format(self.cntPoly))

        r = {}
        if self.cntPt > 0:
            r[self.PrmPointOutputLayer] = self.dest_id_pt
        if self.cntLine > 0:
            r[self.PrmLineOutputLayer] = self.dest_id_line
        if self.cntPoly > 0:
            r[self.PrmPolygonOutputLayer] = self.dest_id_poly

        return (r)
def source_dataset():
    bea_dataset_name = os.getenv('BEA_DATASET_NAME', 'regional')
    table_name = os.getenv('BEA_TABLE_NAME', 'CAINC1')
    source_dataset_url = "https://apps.bea.gov/{}/zip/{}.zip".format(bea_dataset_name, table_name)
    
    response = None
    retries = 5
    for attempt in range(retries):
        try:
            response = urlopen(source_dataset_url)
        except HTTPError as e:
            if attempt == retries:
                raise Exception('HTTPError: ', e.code)
            time.sleep(0.2 * attempt)
        except URLError as e:
            if attempt == retries:
                raise Exception('URLError: ', e.reason)
            time.sleep(0.2 * attempt)
        else:
            break
            
    if response == None:
        raise Exception('There was an issue downloading the dataset')
            
    data_set_name = os.environ['DATA_SET_NAME']

    data_dir = '/tmp'
    if not os.path.exists(data_dir):
        os.mkdir(data_dir)

    zip_location = os.path.join(data_dir, data_set_name+'.zip')

    with open(zip_location, 'wb') as f:
        f.write(response.read())

    with ZipFile(zip_location, 'r') as z:
        z.extractall(data_dir)

    os.remove(zip_location)

    s3_bucket = os.environ['S3_BUCKET']
    s3 = boto3.client('s3')

    unzipped_name = os.listdir(data_dir)[0]

    s3_uploads = []
    asset_list = []

    for r, d, f in os.walk(data_dir):
        for filename in f:
            obj_name = os.path.join(r, filename).split('/', 3).pop().replace(' ', '_').lower()
            file_location = os.path.join(r, filename)
            new_s3_key = data_set_name + '/dataset/' + obj_name

            has_changes = md5_compare(s3, s3_bucket, new_s3_key, file_location)
            if has_changes:
                s3.upload_file(file_location, s3_bucket, new_s3_key)
                print('Uploaded: ' + filename)
            else:
                print('No changes in: ' + filename)

            asset_source = {'Bucket': s3_bucket, 'Key': new_s3_key}
            s3_uploads.append({'has_changes': has_changes, 'asset_source': asset_source})

    count_updated_data = sum(upload['has_changes'] == True for upload in s3_uploads)
    if count_updated_data > 0:
        asset_list = list(map(lambda upload: upload['asset_source'], s3_uploads))
        if len(asset_list) == 0:
            raise Exception('Something went wrong when uploading files to s3')

    # asset_list is returned to be used in lamdba_handler function
    # if it is empty, lambda_handler will not republish
    return asset_list
Beispiel #42
0
'''
ZipFile-向已有的zip压缩包中添加文件(一)
'''

from zipfile import ZipFile


file_name= 'new.zip'
file_list=['Snipaste.png','test_fun.py']
with ZipFile(file=file_name,mode='w') as myzip:    #注意w会删除压缩包内已有的文件
    for f in file_list:
        myzip.write(f)
myzip.printdir()

Beispiel #43
0
def parseM3U(infile):#Traite les m3u local
    oGui = cGui()
    oInputParameterHandler = cInputParameterHandler()
    sUrl = oInputParameterHandler.getValue('siteUrl')

    if 'iptv4sat' in sUrl or '.zip' in sUrl:
        sHtmlContent = getHtml(sUrl)
        from zipfile import ZipFile
        import io
        zip_file = ZipFile(io.BytesIO(sHtmlContent))
        files = zip_file.namelist()
        with zip_file.open(files[0]) as f:
            sHtmlContent = []
            for line in f:
                sHtmlContent.append(line)
            inf = sHtmlContent

    elif not '#EXTM3U' in sUrl:
        site= infile
        user_agent = 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/48.0.2564.116 Chrome/48.0.2564.116 Safari/537.36'
        headers = {'User-Agent': user_agent}
        #req = urllib2.Request(site,headers=headers)
        #inf = urllib2.urlopen(req)

        oRequestHandler = cRequestHandler(sUrl)
        oRequestHandler.addHeaderEntry('User-Agent',user_agent)
        inf = oRequestHandler.request()

        inf = inf.split('\n')
    else:
        inf = infile

    try:
        line = inf.readline()
    except:
        pass

    #cConfig().log(str(line))
    #if not line.startswith('#EXTM3U'):
        #return

    playlist=[]
    song=track(None,None,None,None)
    ValidEntry = False

    for line in inf:

        line=line.strip()
        if line.startswith('#EXTINF:'):
            length,title=line.split('#EXTINF:')[1].split(',',1)
            try:
                licon = line.split('#EXTINF:')[1].partition('tvg-logo=')[2]
                icon = licon.split('"')[1]
            except:
                icon = "tv.png"
            ValidEntry = True

            song=track(length,title,None,icon)
        elif (len(line) != 0):
            if (ValidEntry) and (not (line.startswith('!') or line.startswith('#'))):
                ValidEntry = False
                song.path=line
                playlist.append(song)
                #VSlog(playlist)
                song=track(None,None,None,None)

    try:
        inf.close()
    except:
        pass

    return playlist
Beispiel #44
0
class XPCShellRemote(xpcshell.XPCShellTests, object):

    def __init__(self, devmgr, options, args, log=None):
        xpcshell.XPCShellTests.__init__(self, log)

        # Add Android version (SDK level) to mozinfo so that manifest entries
        # can be conditional on android_version.
        androidVersion = devmgr.shellCheckOutput(['getprop', 'ro.build.version.sdk'])
        mozinfo.info['android_version'] = androidVersion

        self.localLib = options.localLib
        self.localBin = options.localBin
        self.options = options
        self.device = devmgr
        self.pathMapping = []
        self.remoteTestRoot = self.device.getTestRoot("xpcshell")
        # remoteBinDir contains xpcshell and its wrapper script, both of which must
        # be executable. Since +x permissions cannot usually be set on /mnt/sdcard,
        # and the test root may be on /mnt/sdcard, remoteBinDir is set to be on
        # /data/local, always.
        self.remoteBinDir = "/data/local/xpcb"
        # Terse directory names are used here ("c" for the components directory)
        # to minimize the length of the command line used to execute
        # xpcshell on the remote device. adb has a limit to the number
        # of characters used in a shell command, and the xpcshell command
        # line can be quite complex.
        self.remoteTmpDir = remoteJoin(self.remoteTestRoot, "tmp")
        self.remoteScriptsDir = self.remoteTestRoot
        self.remoteComponentsDir = remoteJoin(self.remoteTestRoot, "c")
        self.remoteModulesDir = remoteJoin(self.remoteTestRoot, "m")
        self.remoteMinidumpDir = remoteJoin(self.remoteTestRoot, "minidumps")
        self.profileDir = remoteJoin(self.remoteTestRoot, "p")
        self.remoteDebugger = options.debugger
        self.remoteDebuggerArgs = options.debuggerArgs
        self.testingModulesDir = options.testingModulesDir

        self.env = {}

        if self.options.objdir:
            self.xpcDir = os.path.join(self.options.objdir, "_tests/xpcshell")
        elif os.path.isdir(os.path.join(here, 'tests')):
            self.xpcDir = os.path.join(here, 'tests')
        else:
            print >> sys.stderr, "Couldn't find local xpcshell test directory"
            sys.exit(1)

        if options.localAPK:
            self.localAPKContents = ZipFile(options.localAPK)
        if options.setup:
            self.setupUtilities()
            self.setupModules()
            self.setupTestDir()
        self.setupMinidumpDir()
        self.remoteAPK = None
        if options.localAPK:
            self.remoteAPK = remoteJoin(self.remoteBinDir, os.path.basename(options.localAPK))
            self.setAppRoot()

        # data that needs to be passed to the RemoteXPCShellTestThread
        self.mobileArgs = {
            'device': self.device,
            'remoteBinDir': self.remoteBinDir,
            'remoteScriptsDir': self.remoteScriptsDir,
            'remoteComponentsDir': self.remoteComponentsDir,
            'remoteModulesDir': self.remoteModulesDir,
            'options': self.options,
            'remoteDebugger': self.remoteDebugger,
            'pathMapping': self.pathMapping,
            'profileDir': self.profileDir,
            'remoteTmpDir': self.remoteTmpDir,
            'remoteMinidumpDir': self.remoteMinidumpDir,
        }
        if self.remoteAPK:
            self.mobileArgs['remoteAPK'] = self.remoteAPK

    def setLD_LIBRARY_PATH(self):
        self.env["LD_LIBRARY_PATH"] = self.remoteBinDir

    def pushWrapper(self):
        # Rather than executing xpcshell directly, this wrapper script is
        # used. By setting environment variables and the cwd in the script,
        # the length of the per-test command line is shortened. This is
        # often important when using ADB, as there is a limit to the length
        # of the ADB command line.
        localWrapper = tempfile.mktemp()
        f = open(localWrapper, "w")
        f.write("#!/system/bin/sh\n")
        for envkey, envval in self.env.iteritems():
            f.write("export %s=%s\n" % (envkey, envval))
        f.write("cd $1\n")
        f.write("echo xpcw: cd $1\n")
        f.write("shift\n")
        f.write("echo xpcw: xpcshell \"$@\"\n")
        f.write("%s/xpcshell \"$@\"\n" % self.remoteBinDir)
        f.close()
        remoteWrapper = remoteJoin(self.remoteBinDir, "xpcw")
        self.device.pushFile(localWrapper, remoteWrapper)
        os.remove(localWrapper)
        self.device.chmodDir(self.remoteBinDir)

    def buildEnvironment(self):
        self.buildCoreEnvironment()
        self.setLD_LIBRARY_PATH()
        self.env["MOZ_LINKER_CACHE"] = self.remoteBinDir
        if self.options.localAPK and self.appRoot:
            self.env["GRE_HOME"] = self.appRoot
        self.env["XPCSHELL_TEST_PROFILE_DIR"] = self.profileDir
        self.env["TMPDIR"] = self.remoteTmpDir
        self.env["HOME"] = self.profileDir
        self.env["XPCSHELL_TEST_TEMP_DIR"] = self.remoteTmpDir
        self.env["XPCSHELL_MINIDUMP_DIR"] = self.remoteMinidumpDir
        if self.options.setup:
            self.pushWrapper()

    def setAppRoot(self):
        # Determine the application root directory associated with the package
        # name used by the Fennec APK.
        self.appRoot = None
        packageName = None
        if self.options.localAPK:
            try:
                packageName = self.localAPKContents.read("package-name.txt")
                if packageName:
                    self.appRoot = self.device.getAppRoot(packageName.strip())
            except Exception as detail:
                print "unable to determine app root: " + str(detail)
                pass
        return None

    def setupUtilities(self):
        if (not self.device.dirExists(self.remoteBinDir)):
            # device.mkDir may fail here where shellCheckOutput may succeed -- see bug 817235
            try:
                self.device.shellCheckOutput(["mkdir", self.remoteBinDir]);
            except devicemanager.DMError:
                # Might get a permission error; try again as root, if available
                self.device.shellCheckOutput(["mkdir", self.remoteBinDir], root=True);
                self.device.shellCheckOutput(["chmod", "777", self.remoteBinDir], root=True);

        remotePrefDir = remoteJoin(self.remoteBinDir, "defaults/pref")
        if (self.device.dirExists(self.remoteTmpDir)):
            self.device.removeDir(self.remoteTmpDir)
        self.device.mkDir(self.remoteTmpDir)
        if (not self.device.dirExists(remotePrefDir)):
            self.device.mkDirs(remoteJoin(remotePrefDir, "extra"))
        if (not self.device.dirExists(self.remoteScriptsDir)):
            self.device.mkDir(self.remoteScriptsDir)
        if (not self.device.dirExists(self.remoteComponentsDir)):
            self.device.mkDir(self.remoteComponentsDir)

        local = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'head.js')
        remoteFile = remoteJoin(self.remoteScriptsDir, "head.js")
        self.device.pushFile(local, remoteFile)

        # The xpcshell binary is required for all tests. Additional binaries
        # are required for some tests. This list should be similar to
        # TEST_HARNESS_BINS in testing/mochitest/Makefile.in.
        binaries = ["xpcshell",
                    "ssltunnel",
                    "certutil",
                    "pk12util",
                    "BadCertServer",
                    "OCSPStaplingServer",
                    "GenerateOCSPResponse"]
        for fname in binaries:
            local = os.path.join(self.localBin, fname)
            if os.path.isfile(local):
                print >> sys.stderr, "Pushing %s.." % fname
                remoteFile = remoteJoin(self.remoteBinDir, fname)
                self.device.pushFile(local, remoteFile)
            else:
                print >> sys.stderr, "*** Expected binary %s not found in %s!" % (fname, self.localBin)

        local = os.path.join(self.localBin, "components/httpd.js")
        remoteFile = remoteJoin(self.remoteComponentsDir, "httpd.js")
        self.device.pushFile(local, remoteFile)

        local = os.path.join(self.localBin, "components/httpd.manifest")
        remoteFile = remoteJoin(self.remoteComponentsDir, "httpd.manifest")
        self.device.pushFile(local, remoteFile)

        local = os.path.join(self.localBin, "components/test_necko.xpt")
        remoteFile = remoteJoin(self.remoteComponentsDir, "test_necko.xpt")
        self.device.pushFile(local, remoteFile)

        if self.options.localAPK:
            remoteFile = remoteJoin(self.remoteBinDir, os.path.basename(self.options.localAPK))
            self.device.pushFile(self.options.localAPK, remoteFile)

        self.pushLibs()

    def pushLibs(self):
        pushed_libs_count = 0
        if self.options.localAPK:
            try:
                dir = tempfile.mkdtemp()
                szip = os.path.join(self.localBin, '..', 'host', 'bin', 'szip')
                if not os.path.exists(szip):
                    # Tinderbox builds must run szip from the test package
                    szip = os.path.join(self.localBin, 'host', 'szip')
                if not os.path.exists(szip):
                    # If the test package doesn't contain szip, it means files
                    # are not szipped in the test package.
                    szip = None
                for info in self.localAPKContents.infolist():
                    if info.filename.endswith(".so"):
                        print >> sys.stderr, "Pushing %s.." % info.filename
                        remoteFile = remoteJoin(self.remoteBinDir, os.path.basename(info.filename))
                        self.localAPKContents.extract(info, dir)
                        file = os.path.join(dir, info.filename)
                        if szip:
                            out = subprocess.check_output([szip, '-d', file], stderr=subprocess.STDOUT)
                        self.device.pushFile(os.path.join(dir, info.filename), remoteFile)
                        pushed_libs_count += 1
            finally:
                shutil.rmtree(dir)
            return pushed_libs_count

        for file in os.listdir(self.localLib):
            if (file.endswith(".so")):
                print >> sys.stderr, "Pushing %s.." % file
                if 'libxul' in file:
                    print >> sys.stderr, "This is a big file, it could take a while."
                remoteFile = remoteJoin(self.remoteBinDir, file)
                self.device.pushFile(os.path.join(self.localLib, file), remoteFile)
                pushed_libs_count += 1

        # Additional libraries may be found in a sub-directory such as "lib/armeabi-v7a"
        localArmLib = os.path.join(self.localLib, "lib")
        if os.path.exists(localArmLib):
            for root, dirs, files in os.walk(localArmLib):
                for file in files:
                    if (file.endswith(".so")):
                        print >> sys.stderr, "Pushing %s.." % file
                        remoteFile = remoteJoin(self.remoteBinDir, file)
                        self.device.pushFile(os.path.join(root, file), remoteFile)
                        pushed_libs_count += 1

        return pushed_libs_count

    def setupModules(self):
        if self.testingModulesDir:
            self.device.pushDir(self.testingModulesDir, self.remoteModulesDir)

    def setupTestDir(self):
        print 'pushing %s' % self.xpcDir
        try:
            self.device.pushDir(self.xpcDir, self.remoteScriptsDir, retryLimit=10)
        except TypeError:
            # Foopies have an older mozdevice ver without retryLimit
            self.device.pushDir(self.xpcDir, self.remoteScriptsDir)

    def setupMinidumpDir(self):
        if self.device.dirExists(self.remoteMinidumpDir):
            self.device.removeDir(self.remoteMinidumpDir)
        self.device.mkDir(self.remoteMinidumpDir)

    def buildTestList(self):
        xpcshell.XPCShellTests.buildTestList(self)
        uniqueTestPaths = set([])
        for test in self.alltests:
            uniqueTestPaths.add(test['here'])
        for testdir in uniqueTestPaths:
            abbrevTestDir = os.path.relpath(testdir, self.xpcDir)
            remoteScriptDir = remoteJoin(self.remoteScriptsDir, abbrevTestDir)
            self.pathMapping.append(PathMapping(testdir, remoteScriptDir))
Beispiel #45
0
else:
    box = None

fLOG('----------------------')
for date, mail in rev:
    fLOG(mail["Date"], mail['From'])
    if 'att' in mail:
        att = (mail['name'], mail['att'])
    else:
        for att_ in mail.enumerate_attachments():
            att = att_
            break
    if ".zip" in att[0]:
        content = att[1]
        st = io.BytesIO(content)
        with ZipFile(st) as myzip:
            infos = myzip.infolist()
            if len(infos) != 1:
                fLOG("skip mail '{0}' from '{1}'".format(i, mail['From']))
            else:
                name = infos[0].filename
                data = myzip.read(name)
                datas = data.decode('ascii')
                rec = compute_metrics(datas)
                rec["a_sender"] = mail['From']
                perf_both.append(rec)
                rec['a_date'] = mail['Date']
    elif ".csv" in att[0]:
        datas = att[1].decode('ascii', errors='ignore')
        try:
            df = pandas.read_csv(io.StringIO(datas), sep=";")
Beispiel #46
0
def _extract_seq_results(order, session):
    """
    Given an order (full metadata), saves the ABI trace files into a temporary zip file and
    returns some information about the order.

    Args:
        order: a dictionary containing the keys 'id', 'orderType',
            'orderName', and 'orderStatus' representing a certain GeneWiz order.
        session: a Requests session to download files with
    Returns:
        A tuple encoding (string_summary, zip_file_path). String_summary is a
        message fit for a Slack message summarizing the results and zip_file_path
        is a filesystem path to a temporary zip file containing all trace files.
    """
    if order['orderType'] != 'Sanger Sequencing':
        raise RuntimeError('Order is not a Sanger sequencing result')
    if order['orderStatus'] != 'Completed':
        raise RuntimeError('Selected order is not completed!')
    details_html = session.get('https://clims4.genewiz.com/SangerSequencing/ViewResults',
            data={'OrderId': order['id']})

    details_model_match = re.search("window.GWZ.CLIMS.model = (.*);", details_html.text)
    details = json.loads(details_model_match.group(1))

    reaction_list = list(details['OrdersResults'].values())[0]
    
    # Extract each sequencing reaction information
    ab1_files = []
    tempdir = tempfile.mkdtemp()
    names = []
    qualities = []
    lengths = []
    for reaction in reaction_list:
        quality = reaction['QS']
        length = reaction['CRL']
        order_id = order['id']
        sequencer = reaction['Sequencer']
        folder = reaction['ActualPlateFolder']
        file_name = reaction['AB1FileName']
        labwell = '_' + reaction['LabWellFormatted']
        query_string = {'orderId': order_id,
                    'sequencer': sequencer,
                    'folder': folder,
                    'fileName': file_name[:-4] + labwell + '.ab1',
                    'labwell': labwell}

        names.append(reaction['SamplePrimerName'])
        qualities.append(quality)
        lengths.append(length)

        ab1_file = session.get('https://clims4.genewiz.com/SangerSequencing/DownloadResult',
                data=query_string)
        # Assert that we actually got an AB1 files
        assert ab1_file.content[0:4] == b'ABIF'

        # Save the file into our temp directory
        new_ab1 = os.path.join(tempdir, reaction['SamplePrimerName'] + '.ab1')
        ab1_files.append(new_ab1)
        with open(new_ab1, 'wb') as outfile:
            outfile.write(ab1_file.content)

    # Zip all files together
    order_name = reaction_list[0]['OrderName']
    if order_name is None:
        order_name = 'no_order_name'
    zip_filename = os.path.join(tempdir, order_name + '.zip')
    with ZipFile(zip_filename, 'w') as outzip:
        for ab1 in ab1_files:
            outzip.write(ab1, arcname = os.path.basename(ab1))

    # Cleanup
    for filename in ab1_files:
        os.remove(filename)

    # Built output string:
    url = '<https://clims4.genewiz.com/SangerSequencing/ViewResults?OrderID={}|Sequencing results>'.format(order['id'])
    order_str = 'Order: {}'.format(order_name)
    table = ''
    for tup in zip(names, lengths, qualities):
        table += '\n\t{}: Length {}, quality {}'.format(*tup)
    
    return (order_str + '\n' + url + table, zip_filename)
Beispiel #47
0
    def __init__(self, devmgr, options, args, log=None):
        xpcshell.XPCShellTests.__init__(self, log)

        # Add Android version (SDK level) to mozinfo so that manifest entries
        # can be conditional on android_version.
        androidVersion = devmgr.shellCheckOutput(['getprop', 'ro.build.version.sdk'])
        mozinfo.info['android_version'] = androidVersion

        self.localLib = options.localLib
        self.localBin = options.localBin
        self.options = options
        self.device = devmgr
        self.pathMapping = []
        self.remoteTestRoot = self.device.getTestRoot("xpcshell")
        # remoteBinDir contains xpcshell and its wrapper script, both of which must
        # be executable. Since +x permissions cannot usually be set on /mnt/sdcard,
        # and the test root may be on /mnt/sdcard, remoteBinDir is set to be on
        # /data/local, always.
        self.remoteBinDir = "/data/local/xpcb"
        # Terse directory names are used here ("c" for the components directory)
        # to minimize the length of the command line used to execute
        # xpcshell on the remote device. adb has a limit to the number
        # of characters used in a shell command, and the xpcshell command
        # line can be quite complex.
        self.remoteTmpDir = remoteJoin(self.remoteTestRoot, "tmp")
        self.remoteScriptsDir = self.remoteTestRoot
        self.remoteComponentsDir = remoteJoin(self.remoteTestRoot, "c")
        self.remoteModulesDir = remoteJoin(self.remoteTestRoot, "m")
        self.remoteMinidumpDir = remoteJoin(self.remoteTestRoot, "minidumps")
        self.profileDir = remoteJoin(self.remoteTestRoot, "p")
        self.remoteDebugger = options.debugger
        self.remoteDebuggerArgs = options.debuggerArgs
        self.testingModulesDir = options.testingModulesDir

        self.env = {}

        if self.options.objdir:
            self.xpcDir = os.path.join(self.options.objdir, "_tests/xpcshell")
        elif os.path.isdir(os.path.join(here, 'tests')):
            self.xpcDir = os.path.join(here, 'tests')
        else:
            print >> sys.stderr, "Couldn't find local xpcshell test directory"
            sys.exit(1)

        if options.localAPK:
            self.localAPKContents = ZipFile(options.localAPK)
        if options.setup:
            self.setupUtilities()
            self.setupModules()
            self.setupTestDir()
        self.setupMinidumpDir()
        self.remoteAPK = None
        if options.localAPK:
            self.remoteAPK = remoteJoin(self.remoteBinDir, os.path.basename(options.localAPK))
            self.setAppRoot()

        # data that needs to be passed to the RemoteXPCShellTestThread
        self.mobileArgs = {
            'device': self.device,
            'remoteBinDir': self.remoteBinDir,
            'remoteScriptsDir': self.remoteScriptsDir,
            'remoteComponentsDir': self.remoteComponentsDir,
            'remoteModulesDir': self.remoteModulesDir,
            'options': self.options,
            'remoteDebugger': self.remoteDebugger,
            'pathMapping': self.pathMapping,
            'profileDir': self.profileDir,
            'remoteTmpDir': self.remoteTmpDir,
            'remoteMinidumpDir': self.remoteMinidumpDir,
        }
        if self.remoteAPK:
            self.mobileArgs['remoteAPK'] = self.remoteAPK
Beispiel #48
0
class MetadataLookup(GObject.GObject):
    """
    Look up the localized activity name and size of a bundle.

    This is useful when no previous version of the activity is installed,
    and there is no local source of the activity's name.
    """
    __gsignals__ = {
        'complete': (GObject.SignalFlags.RUN_FIRST,
                     None, (object, object, object)),
    }

    def __init__(self, url):
        GObject.GObject.__init__(self)
        self._url = url
        self._icon_file_name = None
        self._size = None

    def run(self):
        # Perform the name lookup, catch any exceptions, and report the result.
        try:
            name = self._do_name_lookup()
            self._complete(name)
        except Exception as e:
            self._complete(e)

    def _do_name_lookup(self):
        fd = httprange.open(self._url)
        self._size = fd.size()
        return self._name_from_fd(fd)

    def _name_from_fd(self, fd):
        self._zf = ZipFile(fd)
        self._namelist = self._zf.namelist()
        self._prefix = None
        have_activity_info = False
        have_library_info = False
        for path in self._namelist:
            if path.count('/') != 2:
                continue
            if path.endswith('/activity/activity.info'):
                have_activity_info = True
                self._prefix = path.split('/', 1)[0]
                break
            if path.endswith('/library/library.info'):
                have_library_info = True
                self._prefix = path.split('/', 1)[0]
                break

        if self._prefix is None:
            raise Exception("Couldn't find activity prefix")

        # To find the localized name, we first look for an activity.linfo file
        # in the current locale.
        # We fall back on pulling the non-localized name from activity.info,
        # if there is one.
        # The final fallback is pulling the name from library.info; in the
        # case of a content bundle, that name is expected to be already
        # localized according to the content.
        name = self._locale_data_lookup()
        icon_path = None
        if not name and have_activity_info:
            name = self._activity_info_lookup('name')
        if not name and have_library_info:
            name = self._library_info_lookup('name')

        # get icondata
        if have_activity_info:
            icon_name = self._activity_info_lookup('icon')
        if not name and have_library_info:
            icon_name = self._library_info_lookup('icon')
        icon_path = os.path.join(self._prefix, 'activity',
                                 '%s.svg' % icon_name)
        if icon_path is not None and icon_path in self._namelist:
            icon_data = self._zf.read(icon_path)
            # save the icon to a temporary file
            with NamedTemporaryFile(mode='w', suffix='.svg',
                                    delete=False) as iconfile:
                iconfile.write(icon_data)
                self._icon_file_name = iconfile.name
        return name

    def _locale_data_lookup(self):
        lang = locale.getdefaultlocale()[0]
        if lang is None:
            return None

        for f in ('locale/%s/activity.linfo' % lang,
                  'locale/%s/activity.linfo' % lang[:2]):
            filename = os.path.join(self._prefix, f)
            if filename not in self._namelist:
                continue
            cp = ConfigParser()
            cp.readfp(StringIO(self._zf.read(filename)))
            return cp.get('Activity', 'name')
        return None

    def _activity_info_lookup(self, parameter):
        filename = os.path.join(self._prefix, 'activity', 'activity.info')
        cp = ConfigParser()
        cp.readfp(StringIO(self._zf.read(filename)))
        if cp.has_option('Activity', parameter):
            return cp.get('Activity', parameter)
        else:
            return ''

    def _library_info_lookup(self, parameter):
        filename = os.path.join(self._prefix, 'library', 'library.info')
        cp = ConfigParser()
        cp.readfp(StringIO(self._zf.read(filename)))
        if cp.has_option('Library', parameter):
            return cp.get('Library', parameter)
        else:
            return ''

    def _complete(self, result):
        GLib.idle_add(self.emit, 'complete', result, self._size,
                      self._icon_file_name)
Beispiel #49
0
 def makezipfile(data):
     output_filename = '{}_zipped.zip'.format(data)
     with ZipFile(output_filename, "w") as z:
         z.write(data)
     return output_filename
Beispiel #50
0
def extractPdf(filePath):
    with ZipFile(filePath, 'r') as zipobject:
        listoffile = zipobject.namelist()
        for filename in listoffile:
            if filename.endswith('.pdf'):
                zipobject.extract(filename, 'Extracted_pdf')
def create_submission(model,
                      loader,
                      int_to_label,
                      config,
                      pred_zip=None,
                      tta=False,
                      base_h=1536,
                      base_w=1536):
    if pred_zip is not None and os.path.exists(pred_zip):
        os.remove(pred_zip)

    sub = []
    model.eval()
    image_ids, labels = [], []
    with th.no_grad():
        for ret in tqdm(loader):
            Xs = ret[:-1]
            fns = ret[-1]
            hm_preds, classes_embeddings = 0., 0.
            # scale TTA
            for X in Xs:
                X = X.to(config.device).float()
                hm_pred, classes_embedding = model(X, return_embeddings=True)
                hm_pred = th.nn.functional.interpolate(hm_pred,
                                                       size=(base_h // 4,
                                                             base_w // 4),
                                                       mode='bilinear',
                                                       align_corners=True)
                classes_embedding = th.nn.functional.interpolate(
                    classes_embedding,
                    size=(base_h // 4, base_w // 4),
                    mode='bilinear',
                    align_corners=True)
                hm_pred = th.sigmoid(hm_pred)
                hm_preds += hm_pred
                classes_embeddings += classes_embedding

            hm_pred = hm_preds / len(Xs)
            classes_embeddings /= len(Xs)
            hm_pred = nms(hm_pred).cpu().numpy()
            for j in range(len(hm_pred)):
                img_id = KuzushijiDataset.fn_to_id(fns[j])
                w, h = Image.open(fns[j]).size
                pad_top, pad_bottom, pad_left, pad_right = \
                    KuzushijiDataset.get_paddings(h, w, ratio=1.5)
                h = h + pad_top + pad_bottom
                w = w + pad_left + pad_right
                image_ids.append(img_id)
                hmp = hm_pred[j].squeeze(0)
                if pred_zip is not None:
                    pred_fn = img_id + '.png'
                    yp_img = np.uint8(hmp * 255)
                    img_bytes = cv2.imencode('.png', yp_img)[1].tobytes()
                    with ZipFile(pred_zip, 'a') as f:
                        f.writestr(pred_fn, img_bytes)

                hmp_thresholded = hmp > config.p_letter
                ys, xs = np.where(hmp_thresholded)
                if len(ys) == 0:
                    labels.append('')
                    continue

                centers = th.stack(
                    [th.tensor(xs), th.tensor(ys)], -1).unsqueeze(0)
                gathered_embeddings = model.gather_embeddings(
                    classes_embeddings[j:j + 1], centers)
                classes_preds = model.classes(
                    gathered_embeddings.unsqueeze(-1).unsqueeze(-1)).squeeze(
                        -1).squeeze(-1)
                # set prob for extra letters to zero
                classes_preds = classes_preds[:, :4212]
                classes_preds = th.nn.functional.softmax(classes_preds,
                                                         1).cpu().numpy()

                per_image_labels = []
                for center_ind in range(len(ys)):
                    x, y = xs[center_ind], ys[center_ind]
                    pred_probs = classes_preds[center_ind]
                    # scale to (padded) image coords
                    x, y = int(round(x * 4 * (w / base_w))), int(
                        round(y * 4 * (h / base_h)))
                    # undo padding
                    x -= pad_left
                    y -= pad_top
                    # pred_classes = np.where(pred_probs > config.p_class)[0]
                    pred_classes = [pred_probs.argmax()]
                    # print(len(pred_classes))
                    for pred_class in pred_classes:
                        ll = int_to_label[pred_class]
                        per_image_labels.extend([ll, str(x), str(y)])

                image_label_str = ' '.join(per_image_labels)
                labels.append(image_label_str)
                # img = cv2.imread('test_images/%s.jpg' % (img_id), 1)
                # plt.subplot(2, 1, 1)
                # plt.imshow(img)
                # plt.subplot(2, 1, 2)
                # plt.imshow(hmp.squeeze())
                # plt.show()

    sub = pd.DataFrame({'image_id': image_ids, 'labels': labels})
    return sub
Beispiel #52
0
 def __init__(self, *args, **kwargs):
     """
     Create a :class:`.ZipFileArchiver` instance. We create a new
     :class:`zipfile.ZipFile` and store it to the ``zipfile`` member. 
     """
     self.zipfile = ZipFile(*args, **kwargs)
def getZipNameInMemory(zipfile_obj: zipfile.ZipFile):
    files_in_zip = zipfile_obj.namelist()
    if len(files_in_zip) == 0: return None
    zip_name = zipfile_obj.filename if bool(zipfile_obj.filename) else 'nameless_file.zip'

    return pathlib.Path(zip_name).stem
Beispiel #54
0
    if not os.path.exists(APPENGINE_TARGET_DIR):
        print('Downloading the AppEngine SDK...')

        #First try and get it from the 'featured' folder
        sdk_file = urlopen(FEATURED_SDK_REPO + APPENGINE_SDK_FILENAME)
        if sdk_file.getcode() == 404:
            #Failing that, 'deprecated'
            sdk_file = urlopen(DEPRECATED_SDK_REPO + APPENGINE_SDK_FILENAME)

        #Handle other errors
        if sdk_file.getcode() >= 299:
            raise Exception(
                'App Engine SDK could not be found. {} returned code {}.'.
                format(sdk_file.geturl(), sdk_file.getcode()))

        zipfile = ZipFile(StringIO(sdk_file.read()))
        zipfile.extractall(TARGET_DIR)

        #Make sure the dev_appserver and appcfg are executable
        for module in ("dev_appserver.py", "appcfg.py"):
            app = os.path.join(APPENGINE_TARGET_DIR, module)
            st = os.stat(app)
            os.chmod(app, st.st_mode | stat.S_IEXEC)
    else:
        print(
            'Not updating SDK as it exists. Remove {} and re-run to get the latest SDK'
            .format(APPENGINE_TARGET_DIR))

    print("Running pip...")
    args = [
        "pip", "install", "--no-deps", "-r", REQUIREMENTS_FILE, "-t",
def main(args):
    if len(args) < 1:
        sys.stderr.write("Error - one required argument: <data directory>\n")
        sys.exit(-1)

    working_dir = args[0]
    
    print("Reading data...")
    Y, outcome_map, outcome_list, X, feature_alphabet = ctk_io.read_multitask_token_sequence_data(working_dir) # ('data_testing/multitask_assertion/train_and_test') 
    
    print("Shape of X is %s and Y is %s" % (str(X.shape), str(Y.shape)))
    
    num_examples, dimension = X.shape
    num_y_examples, num_labels = Y.shape
    assert num_examples == num_y_examples
    
    #print("Data has %d examples and dimension %d" % (num_examples, dimension) )
    #print("Output has %d dimensions" % (num_labels) )

    #X = np.reshape(X, (num_examples, 11, dimension / 11))
    
    Y_adj, indices = ctk_io.flatten_outputs(Y)
    stopper = nn_models.get_early_stopper()
    
    output_dims_list = []
    y_list = []
    
    for i in range(len(indices)-1):
        label_dims = indices[i+1] - indices[i]
        output_dims_list.append(label_dims)
        if label_dims == 1:
            y_list.append(Y_adj[:, indices[i]])
        else:
            y_list.append(Y_adj[:, indices[i]:indices[i+1]])
        
        print("Dimensions of label %d are %s" % (i, str(y_list[-1].shape) ) )

    model = nn_models.get_multitask_cnn(X.shape, len(feature_alphabet), output_dims_list, conv_layers=filters, fc_layers=layers, 
                                        embed_dim=embed_dim, filter_widths=widths)
    #model = nn_models.get_multitask_mlp(X.shape, len(feature_alphabet), output_dims_list, fc_layers=layers, embed_dim=embed_dim)
    
    model.fit(X, y_list,
                  nb_epoch=nb_epoch,
                  batch_size=batch_size,
                  verbose=1,
                  validation_split=0.2,
                  callbacks=[stopper])
                  
    model.summary()
    
    json_string = model.to_json()
    open(os.path.join(working_dir, 'model_0.json'), 'w').write(json_string)
    model.save_weights(os.path.join(working_dir, 'model_0.h5'), overwrite=True)
    
    #script_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
    fn = open(os.path.join(working_dir, 'alphabets.pkl'), 'w')
    pickle.dump( (feature_alphabet, outcome_map, outcome_list), fn)
    fn.close()
    
    with ZipFile(os.path.join(working_dir, 'script.model'), 'w') as myzip:
        myzip.write(os.path.join(working_dir, 'model_0.json'), 'model_0.json')
        myzip.write(os.path.join(working_dir, 'model_0.h5'), 'model_0.h5')
        myzip.write(os.path.join(working_dir, 'alphabets.pkl'), 'alphabets.pkl')
Beispiel #56
0
def get_liste_pays():
    with ZipFile('asia.zip', 'r') as fichier:
        return fichier.namelist()
Beispiel #57
0
 def __init__(self, archpath):
     super(ZIPArchive, self).__init__(ZipFile(archpath))
Beispiel #58
0
 def get_zipfile_attachment(self, email, attachment_file_name,
                            zipfile_file_name):
     attachment = self.get_attachment(email, attachment_file_name)
     zipfile = ZipFile(attachment, 'r')
     return zipfile.open(zipfile_file_name, 'r')
        return True
    else:
        return False


genrecounts = dict()

volumesread = dict()

for folder in folderlist:
    thispath = os.path.join(rootpath, folder)
    filelist = os.listdir(thispath)
    for afile in filelist:
        if afile.endswith("maps.zip"):
            filepath = os.path.join(thispath, afile)
            with ZipFile(filepath, mode='r') as zf:
                for member in zf.infolist():

                    if not member.filename.endswith(
                            '/') and not member.filename.endswith(
                                "_Store") and not member.filename.startswith(
                                    "_"):
                        datafile = ZipFile.open(zf, name=member, mode='r')
                        filelines = datafile.readlines()
                        filelines[0] = filelines[0].rstrip()
                        htid = filelines[0].decode(encoding="UTF-8")
                        thismap = list()
                        counter = 0
                        for line in filelines[1:]:
                            line = line.decode(encoding="UTF-8")
                            line = line.rstrip()
Beispiel #60
-1
def extractAll(zip):
    """
    Extrait tous les fichiers d'une archive word dans un répertoire temporaire.
    """

    myzip = ZipFile(zip, 'a')
    myzip.extractall('./../tmp/')