def getModuleInfoFileObjects(self): fs = FileSystemAdapter() fs.chdir(self.path) for filepath in fs.glob('*.info'): module = os.path.basename(filepath).split('.')[ 0] # assume dot is not allowed for module filenames yield ModuleInfoFile(filepath, module, self.machine_name, self.major_version, self.is_core_project)
class ProjectsLinkedDropSelfLinkedPipeline(object): fs = FileSystemAdapter(settings.PARSLEY_CONTRIBUTED_PROJECTS_ROOT) def process_item(self, current_item, current_spider): if current_spider.name == "projects_linked" and current_item[ 'linked_from'] == current_item['linked_to']: raise DropItem("Self-linked item") return current_item
def parse(self, response): """ @url http://localhost/ @returns items 33 33 @returns requests 0 0 """ fs = FileSystemAdapter(PARSLEY_CORE_PROJECTS_ROOT) projects = fs.get_subdirectories_names() for project in projects: data = dict() data['machine_name'] = project data['project_uid'] = 0 data['project_maintenance_status'] = 'Core module' data['project_development_status'] = 'Core module' item = ScrapyItemListWrapper(data.keys()) for key, value in data.iteritems(): item[key] = value yield item
class ProjectsContributedGitClonePipeline(object): fs = FileSystemAdapter(settings.PARSLEY_CONTRIBUTED_PROJECTS_ROOT) def process_item(self, current_item, current_spider): if current_spider.name == "projects_releases": project_machine_name = current_item['release_project_machine_name'] self._git_clone(project_machine_name) return current_item def _git_clone(self, project_machine_name): # todo # self.fs.run("git clone --no-checkout http://git.drupal.org/project/%s.git" % project_machine_name) pass
def getModuleInfoFileObjects(self): fs = FileSystemAdapter() fs.chdir(self.path) for filepath in fs.glob('*.info'): module = os.path.basename(filepath).split('.')[0] # assume dot is not allowed for module filenames yield ModuleInfoFile(filepath, module, self.machine_name, self.major_version, self.is_core_project)