Example #1
0
 def getModuleInfoFileObjects(self):
     fs = FileSystemAdapter()
     fs.chdir(self.path)
     for filepath in fs.glob('*.info'):
         module = os.path.basename(filepath).split('.')[
             0]  # assume dot is not allowed for module filenames
         yield ModuleInfoFile(filepath, module, self.machine_name,
                              self.major_version, self.is_core_project)
Example #2
0
class ProjectsLinkedDropSelfLinkedPipeline(object):
    fs = FileSystemAdapter(settings.PARSLEY_CONTRIBUTED_PROJECTS_ROOT)

    def process_item(self, current_item, current_spider):
        if current_spider.name == "projects_linked" and current_item[
                'linked_from'] == current_item['linked_to']:
            raise DropItem("Self-linked item")
        return current_item
    def parse(self, response):
        """

        @url http://localhost/
        @returns items 33 33
        @returns requests 0 0
        """
        fs = FileSystemAdapter(PARSLEY_CORE_PROJECTS_ROOT)
        projects = fs.get_subdirectories_names()
        for project in projects:
            data = dict()
            data['machine_name'] = project
            data['project_uid'] = 0
            data['project_maintenance_status'] = 'Core module'
            data['project_development_status'] = 'Core module'

            item = ScrapyItemListWrapper(data.keys())
            for key, value in data.iteritems():
                item[key] = value
            yield item
Example #4
0
class ProjectsContributedGitClonePipeline(object):
    fs = FileSystemAdapter(settings.PARSLEY_CONTRIBUTED_PROJECTS_ROOT)

    def process_item(self, current_item, current_spider):
        if current_spider.name == "projects_releases":
            project_machine_name = current_item['release_project_machine_name']
            self._git_clone(project_machine_name)
        return current_item

    def _git_clone(self, project_machine_name):  # todo
        # self.fs.run("git clone --no-checkout http://git.drupal.org/project/%s.git" % project_machine_name)
        pass
 def getModuleInfoFileObjects(self):
     fs = FileSystemAdapter()
     fs.chdir(self.path)
     for filepath in fs.glob('*.info'):
         module = os.path.basename(filepath).split('.')[0]  # assume dot is not allowed for module filenames
         yield ModuleInfoFile(filepath, module, self.machine_name, self.major_version, self.is_core_project)